Jaspers Hub

jaspers / research

pluginOfficialReviewed★ 0↓ 28

Jaspers Research: Research rooms where configurable analysts research companies in parallel over SEC filings data, answering with checked citations.

20 KB
1import type { BackendContext } from '@jaspers-ai/sdk'
2import { loadAnalysts } from './analyst-files'
3import { resolveAnalysts, type Analyst } from './analysts'
4import { systemPrompt, type PromptInput } from './prompt'
5import {
6 claimWorkspace,
7 nextSeq,
8 onWorkspace,
9 outputOf,
10 roomIdFor,
11 roomNews,
12 seqFile,
13 shownOf,
14 threadText,
15 transcript,
16 type AnalystMessage,
17 type Message,
18 type Room,
19 type RoomOutput,
20 type UserMessage,
21} from './room'
22import { runAnalyst } from './runner'
23
24// Rooms on disk, and the analysts' work on them. A room is rooms/<id>/room.json and one file per
25// message; this host keeps the rooms it has touched in memory, publishes each as live values (the
26// room with its message numbers and its output, and every message under its own key), and runs one
27// job per analyst per message. An analyst already at work on a room takes its next message after.
28// A room belongs to the workspace it was started on: a call from another workspace does not see it,
29// and its analysts read the views on its own workspace.
30
31/** What the view reads for a room: the room, its message numbers in order, and what it publishes. */
32export interface RoomLive {
33 room: Room
34 seqs: number[]
35 output: RoomOutput
36}
37
38const ROOM_ID = /^[a-z0-9][a-z0-9-]*$/
39
40const rooms = new Map<string, { room: Room; messages: Map<number, Message> }>()
41/** Messages waiting for their analyst to finish the one before, by room and analyst. */
42const queues = new Map<string, number[]>()
43/** For each message the user posted, the answers still out, so one notice goes out when all are in. */
44const outstanding = new Map<string, Set<number>>()
45/** Analyst names as last read, for rosters of analysts that have not answered yet. */
46const names = new Map<string, string>()
47
48/** Every analyst, noting their names on the way. */
49async function analystsNow(ctx: BackendContext): Promise<Analyst[]> {
50 const all = await loadAnalysts(ctx)
51 for (const analyst of all) names.set(analyst.id, analyst.name)
52 return all
53}
54
55/** The rooms of the workspace the call came from, most recently used first. */
56export async function listRooms(ctx: BackendContext): Promise<{ id: string; name: string; tickers: string[]; analysts: string[]; working: string[]; updatedAt: string }[]> {
57 const entries = await ctx.files.list('rooms')
58 const found: Room[] = []
59 for (const entry of entries) {
60 if (!entry.dir) continue
61 const room = await readJson<Room>(ctx, `${entry.path}/room.json`)
62 if (room && onWorkspace(room, ctx.workspace)) found.push(room)
63 }
64 return found
65 .sort((a, b) => b.updatedAt.localeCompare(a.updatedAt))
66 .map((room) => ({
67 id: room.id,
68 name: room.name,
69 tickers: room.tickers,
70 analysts: room.analysts,
71 working: [...(rooms.get(room.id)?.messages.values() ?? [])].filter(isBusy).map((m) => (m as AnalystMessage).analyst),
72 updatedAt: room.updatedAt,
73 }))
74}
75
76/** A new room. Analysts named loosely are found; any that match no analyst are left out and listed. */
77export async function createRoom(
78 ctx: BackendContext,
79 input: { name: string; analysts: string[]; tickers?: string[] },
80): Promise<{ room: Room; unmatched: string[] }> {
81 const name = input.name.trim()
82 if (!name) throw new Error('A room needs a name.')
83 const { ids: analysts, unmatched } = usable(await analystsNow(ctx), input.analysts)
84 const now = new Date().toISOString()
85 const room: Room = {
86 id: roomIdFor(name, Math.random().toString(36).slice(2, 6).padEnd(4, '0')),
87 name,
88 tickers: tickers(input.tickers),
89 analysts,
90 createdAt: now,
91 updatedAt: now,
92 ...(ctx.workspace ? { workspace: ctx.workspace } : {}),
93 }
94 rooms.set(room.id, { room, messages: new Map() })
95 await writeRoom(ctx, room)
96 return { room, unmatched }
97}
98
99export async function updateRoom(ctx: BackendContext, input: { room: string; name?: string; analysts?: string[]; tickers?: string[] }): Promise<Room> {
100 const loaded = await here(ctx, input.room)
101 const room: Room = { ...loaded.room, updatedAt: new Date().toISOString() }
102 if (input.name !== undefined && input.name.trim()) room.name = input.name.trim()
103 if (input.analysts !== undefined) room.analysts = usable(await analystsNow(ctx), input.analysts).ids
104 if (input.tickers !== undefined) room.tickers = tickers(input.tickers)
105 loaded.room = room
106 await writeRoom(ctx, room)
107 return room
108}
109
110/** Reads a room into this host and publishes it. The view asks for this when it opens one. */
111export async function openRoom(ctx: BackendContext, id: string): Promise<RoomLive> {
112 await analystsNow(ctx)
113 await here(ctx, id)
114 return publishRoom(ctx, id)
115}
116
117/**
118 * The user's message into the room, and a job for each analyst it is for. Analysts the room does not
119 * have yet join it. Returns at once with who is working; the answers arrive in the room.
120 */
121export async function post(ctx: BackendContext, input: { room: string; text: string; to: string[] }): Promise<string> {
122 const loaded = await here(ctx, input.room)
123 const text = input.text.trim()
124 if (!text) throw new Error('Nothing to post.')
125 const all = await analystsNow(ctx)
126 const { ids: known, unmatched: unknown } = resolveAnalysts(all, input.to)
127 if (known.length === 0) {
128 throw new Error(`No analyst to post to. The room has ${loaded.room.analysts.join(', ') || 'none'}; every analyst: ${all.filter((a) => !a.error).map((a) => a.id).join(', ')}.`)
129 }
130 const joining = known.filter((id) => !loaded.room.analysts.includes(id))
131 if (joining.length > 0) {
132 loaded.room = { ...loaded.room, analysts: [...loaded.room.analysts, ...joining] }
133 }
134 loaded.room = { ...loaded.room, updatedAt: new Date().toISOString() }
135 await writeRoom(ctx, loaded.room)
136
137 const seq = nextSeq([...loaded.messages.values()])
138 // The analysts read the views on the room's own workspace, which is the one the call came from.
139 const workspace = loaded.room.workspace ?? (await callerWorkspace(ctx))
140 const message: UserMessage = { seq, role: 'user', text, to: known, at: new Date().toISOString(), ...(workspace ? { workspace } : {}) }
141 await saveMessage(ctx, loaded.room.id, message)
142 const answers = new Set<number>()
143 const queued: string[] = []
144 for (const [i, id] of known.entries()) {
145 const analyst = all.find((a) => a.id === id)!
146 const answer: AnalystMessage = {
147 seq: seq + 1 + i,
148 role: 'analyst',
149 analyst: id,
150 name: analyst.name,
151 color: analyst.color,
152 inReplyTo: seq,
153 status: 'queued',
154 text: '',
155 steps: [],
156 files: [],
157 citations: [],
158 audit: null,
159 turns: 0,
160 error: null,
161 usage: { input: 0, output: 0 },
162 startedAt: null,
163 endedAt: null,
164 }
165 answers.add(answer.seq)
166 await saveMessage(ctx, loaded.room.id, answer)
167 if (!schedule(ctx, loaded.room.id, answer)) queued.push(analyst.name)
168 }
169 outstanding.set(`${loaded.room.id}/${seq}`, answers)
170 const addressed = known.map((id) => all.find((a) => a.id === id)!.name)
171 return [
172 `Posted to ${loaded.room.name}. Working: ${addressed.join(', ')}.`,
173 queued.length > 0 ? `${queued.join(', ')} ${queued.length === 1 ? 'is' : 'are'} finishing an earlier message first.` : '',
174 joining.length > 0 ? `Added to the room: ${joining.join(', ')}.` : '',
175 unknown.length > 0 ? `Not analysts: ${unknown.join(', ')}.` : '',
176 'The answers arrive in the room.',
177 ]
178 .filter(Boolean)
179 .join(' ')
180}
181
182/** Stops the room's work, or one analyst's: running answers keep what they have, queued ones end. */
183export async function stop(ctx: BackendContext, input: { room: string; analyst?: string }): Promise<string> {
184 const loaded = await here(ctx, input.room)
185 const reason = 'stopped by the user'
186 let stopped = 0
187 for (const message of loaded.messages.values()) {
188 if (message.role !== 'analyst' || (input.analyst && message.analyst !== input.analyst)) continue
189 if (message.status === 'queued') {
190 queues.delete(`${loaded.room.id}/${message.analyst}`)
191 await finish(ctx, loaded.room.id, { ...message, status: 'stopped', error: reason, endedAt: new Date().toISOString() })
192 stopped++
193 } else if (message.status === 'working') {
194 ctx.jobs.abort(`${loaded.room.id}/${message.analyst}`, reason)
195 stopped++
196 }
197 }
198 return stopped === 0 ? 'Nothing was running.' : `Stopped ${stopped} answer${stopped === 1 ? '' : 's'}.`
199}
200
201export async function readThread(ctx: BackendContext, input: { room: string; last?: number; analyst?: string }): Promise<string> {
202 const loaded = await here(ctx, input.room)
203 const byId = Object.fromEntries((await analystsNow(ctx)).map((a) => [a.id, a.name]))
204 return threadText(sorted(loaded.messages), byId, { last: input.last, analyst: input.analyst }) || 'The room has no messages yet.'
205}
206
207/** At start: answers a dead host left working or queued are over. */
208export async function markInterrupted(ctx: BackendContext): Promise<void> {
209 for (const entry of await ctx.files.list('rooms')) {
210 if (!entry.dir) continue
211 for (const file of await ctx.files.list(`${entry.path}/messages`)) {
212 const message = await readJson<Message>(ctx, file.path)
213 if (message?.role === 'analyst' && (message.status === 'working' || message.status === 'queued')) {
214 const ended: AnalystMessage = { ...message, status: 'stopped', error: 'interrupted: the app closed or the plugin restarted', endedAt: new Date().toISOString() }
215 await ctx.files.write(file.path, JSON.stringify(ended, null, 2))
216 }
217 }
218 }
219}
220
221/**
222 * At start: a room made before rooms belonged to a workspace is given the workspace its first message
223 * came from, while that workspace is still there, else the one on screen. It only adds the field.
224 */
225export async function claimRooms(ctx: BackendContext): Promise<void> {
226 const exists = new Map<string, boolean>()
227 let onScreen: string | null | undefined
228 for (const entry of await ctx.files.list('rooms')) {
229 if (!entry.dir) continue
230 const room = await readJson<Room>(ctx, `${entry.path}/room.json`)
231 if (!room || typeof room.workspace === 'string') continue
232 const messages: Message[] = []
233 for (const file of await ctx.files.list(`${entry.path}/messages`)) {
234 const message = await readJson<Message>(ctx, file.path)
235 if (message) messages.push(message)
236 }
237 for (const message of messages) {
238 if (message.role !== 'user' || !message.workspace || exists.has(message.workspace)) continue
239 exists.set(message.workspace, await workspaceExists(ctx, message.workspace))
240 }
241 const known = new Set([...exists].filter(([, found]) => found).map(([id]) => id))
242 onScreen ??= (await callerWorkspace(ctx)) ?? null
243 const workspace = claimWorkspace(messages, known, onScreen)
244 if (workspace) await ctx.files.write(`${entry.path}/room.json`, JSON.stringify({ ...room, workspace }, null, 2))
245 }
246}
247
248/** A room a call may reach: its workspace's own. Another's reads as no room, which the view takes as the list. */
249async function here(ctx: BackendContext, id: string): Promise<{ room: Room; messages: Map<number, Message> }> {
250 const loaded = await load(ctx, id)
251 if (!onWorkspace(loaded.room, ctx.workspace)) throw new Error(`No room ${id} on this workspace. Call research__rooms for the rooms here.`)
252 return loaded
253}
254
255/** Throws unless the call may reach the room, for a source that works on its folder rather than its thread. */
256export async function checkRoomHere(ctx: BackendContext, id: string): Promise<void> {
257 await here(ctx, id)
258}
259
260/** Starts an answer's job now, or queues it behind the analyst's current one. True when it started. */
261function schedule(ctx: BackendContext, roomId: string, message: AnalystMessage): boolean {
262 const key = `${roomId}/${message.analyst}`
263 if (ctx.jobs.running().includes(key)) {
264 queues.set(key, [...(queues.get(key) ?? []), message.seq])
265 return false
266 }
267 ctx.jobs.start(key, async (signal) => {
268 try {
269 await answer(ctx, roomId, message.seq, signal)
270 } finally {
271 // The job's key is released after this returns, so the next message starts a turn later.
272 setTimeout(() => {
273 const queue = queues.get(key) ?? []
274 const next = queue.shift()
275 if (queue.length === 0) queues.delete(key)
276 const waiting = next === undefined ? undefined : rooms.get(roomId)?.messages.get(next)
277 if (waiting?.role === 'analyst' && waiting.status === 'queued') schedule(ctx, roomId, waiting)
278 }, 0)
279 }
280 })
281 return true
282}
283
284/** One analyst answering one message, from the files as they are now. */
285async function answer(ctx: BackendContext, roomId: string, seq: number, signal: AbortSignal): Promise<void> {
286 const loaded = await load(ctx, roomId)
287 const message = loaded.messages.get(seq)
288 if (message?.role !== 'analyst' || message.status !== 'queued') return
289 const all = await analystsNow(ctx)
290 const analyst = all.find((a) => a.id === message.analyst)
291 const asked = loaded.messages.get(message.inReplyTo)
292 if (!analyst || analyst.error || asked?.role !== 'user') {
293 await finish(ctx, roomId, { ...message, status: 'error', error: analyst?.error ?? `No analyst ${message.analyst}.`, endedAt: new Date().toISOString() })
294 return
295 }
296 const connections = await ctx.tools.connections(analyst.connections.length > 0 ? { connections: analyst.connections } : undefined)
297 const roster = loaded.room.analysts.map((id) => all.find((a) => a.id === id)).filter((a): a is Analyst => a !== undefined)
298 const workspaceId = asked.workspace ?? null
299 const workspace = workspaceId ? await workspaceOf(ctx, workspaceId) : null
300 const system = systemPrompt({
301 analyst,
302 room: loaded.room,
303 roster: roster.map((a) => ({ id: a.id, name: a.name, description: a.description })),
304 connections,
305 workspace,
306 today: new Date().toISOString().slice(0, 10),
307 })
308 // The whole room now, and after that whatever lands in it while this analyst works.
309 const labels = Object.fromEntries(all.map((a) => [a.id, a.name]))
310 const thread = sorted(loaded.messages)
311 const firstTurn = transcript({ messages: thread, names: labels, analyst: analyst.id, message: asked, answer: seq })
312 const shown = shownOf(thread)
313 const results = `rooms/${roomId}/sandbox/data`
314 const ended = await runAnalyst(
315 ctx,
316 { message, analyst, system, firstTurn, title: loaded.room.name, workspace: workspace ? workspaceId : null },
317 {
318 save: (m) => saveMessage(ctx, roomId, m),
319 saveResult: (ref, text) => ctx.files.write(`${results}/${seq}-${ref}.txt`, text),
320 readResult: (ref) => ctx.files.read(`${results}/${seq}-${ref}.txt`).catch(() => ''),
321 news: async () => roomNews({ messages: sorted((rooms.get(roomId) ?? loaded).messages), names: labels, message: asked, answer: seq, shown }),
322 now: () => Date.now(),
323 iso: () => new Date().toISOString(),
324 },
325 signal,
326 )
327 await finish(ctx, roomId, ended)
328}
329
330/** An answer's last write, and the notice when it was the last one its question was waiting on. */
331async function finish(ctx: BackendContext, roomId: string, message: AnalystMessage): Promise<void> {
332 await saveMessage(ctx, roomId, message)
333 const key = `${roomId}/${message.inReplyTo}`
334 const waiting = outstanding.get(key)
335 if (!waiting) return
336 waiting.delete(message.seq)
337 if (waiting.size > 0) return
338 outstanding.delete(key)
339 const loaded = rooms.get(roomId)
340 if (!loaded) return
341 const answers = [...loaded.messages.values()].filter((m): m is AnalystMessage => m.role === 'analyst' && m.inReplyTo === message.inReplyTo)
342 // Stopping them was the user's own doing; that needs no notice.
343 if (answers.every((m) => m.status === 'stopped')) return
344 const named = (status: AnalystMessage['status']): string[] => answers.filter((m) => m.status === status).map((m) => m.name)
345 const parts = [
346 named('done').length > 0 ? `${list(named('done'))} answered` : '',
347 named('error').length > 0 ? `${list(named('error'))} failed` : '',
348 named('stopped').length > 0 ? `${list(named('stopped'))} stopped` : '',
349 ].filter(Boolean)
350 ctx.notify(`${loaded.room.name}: ${parts.join('; ')}.`)
351}
352
353async function saveMessage(ctx: BackendContext, roomId: string, message: Message): Promise<void> {
354 const loaded = rooms.get(roomId)
355 if (!loaded) return
356 loaded.messages.set(message.seq, message)
357 await ctx.files.write(`rooms/${roomId}/messages/${seqFile(message.seq)}`, JSON.stringify(message, null, 2))
358 ctx.live.set(`rooms/${roomId}/m/${message.seq}`, message)
359 publishRoom(ctx, roomId)
360}
361
362async function writeRoom(ctx: BackendContext, room: Room): Promise<void> {
363 await ctx.files.write(`rooms/${room.id}/room.json`, JSON.stringify(room, null, 2))
364 publishRoom(ctx, room.id)
365}
366
367function publishRoom(ctx: BackendContext, roomId: string): RoomLive {
368 const loaded = rooms.get(roomId)!
369 const messages = sorted(loaded.messages)
370 const roster = loaded.room.analysts.map((id) => ({ id, name: names.get(id) ?? id }))
371 const live: RoomLive = { room: loaded.room, seqs: messages.map((m) => m.seq), output: outputOf(loaded.room, messages, roster) }
372 ctx.live.set(`rooms/${roomId}`, live)
373 return live
374}
375
376async function load(ctx: BackendContext, id: string): Promise<{ room: Room; messages: Map<number, Message> }> {
377 if (typeof id !== 'string' || !ROOM_ID.test(id)) throw new Error(`No room ${String(id)}.`)
378 const cached = rooms.get(id)
379 if (cached) return cached
380 const room = await readJson<Room>(ctx, `rooms/${id}/room.json`)
381 if (!room) throw new Error(`No room ${id}. Call research__rooms for the rooms there are.`)
382 const messages = new Map<number, Message>()
383 for (const file of await ctx.files.list(`rooms/${id}/messages`)) {
384 const message = await readJson<Message>(ctx, file.path)
385 if (message && typeof message.seq === 'number') messages.set(message.seq, message)
386 }
387 const loaded = { room, messages }
388 rooms.set(id, loaded)
389 for (const message of messages.values()) ctx.live.set(`rooms/${id}/m/${message.seq}`, message)
390 return loaded
391}
392
393/**
394 * The workspace the call came from, or the one on screen when nobody said: undefined when the tree
395 * cannot be read, which leaves the analysts without it.
396 */
397async function callerWorkspace(ctx: BackendContext): Promise<string | undefined> {
398 if (ctx.workspace) return ctx.workspace
399 try {
400 const workspace = (await ctx.state.get('workspace')) as { id?: unknown } | null
401 return typeof workspace?.id === 'string' ? workspace.id : undefined
402 } catch {
403 return undefined
404 }
405}
406
407/** Whether a workspace is still there. */
408async function workspaceExists(ctx: BackendContext, id: string): Promise<boolean> {
409 try {
410 await ctx.state.get(`workspaces/${id}/workspace`)
411 return true
412 } catch {
413 return false
414 }
415}
416
417/** A workspace by id, with the views open on it now, for the prompt; null when it is gone or cannot be read. */
418async function workspaceOf(ctx: BackendContext, id: string): Promise<PromptInput['workspace']> {
419 try {
420 const { name } = (await ctx.state.get(`workspaces/${id}/workspace`)) as { name: string }
421 const panels = (await ctx.state.get(`workspaces/${id}/panels`)) as { elementId: string; view: string | null; summary: string | null; textLength?: number | null }[]
422 return { name, panels: panels.map((p) => ({ id: p.elementId, view: p.view, summary: p.summary, textLength: p.textLength ?? null })) }
423 } catch {
424 return null
425 }
426}
427
428/** The working analysts a list names, found loosely. A list that names none of them is refused, with the ones there are. */
429function usable(all: Analyst[], requested: string[]): { ids: string[]; unmatched: string[] } {
430 const found = resolveAnalysts(all, requested)
431 if (found.ids.length === 0) {
432 throw new Error(`No analyst matches ${requested.join(', ') || 'nothing'}. The analysts are ${all.filter((a) => !a.error).map((a) => a.id).join(', ')}.`)
433 }
434 return found
435}
436
437function list(names: string[]): string {
438 return names.length <= 1 ? (names[0] ?? '') : `${names.slice(0, -1).join(', ')} and ${names.at(-1)}`
439}
440
441function tickers(list: string[] | undefined): string[] {
442 return [...new Set((list ?? []).map((t) => t.trim().toUpperCase()).filter(Boolean))]
443}
444
445function sorted(messages: Map<number, Message>): Message[] {
446 return [...messages.values()].sort((a, b) => a.seq - b.seq)
447}
448
449function isBusy(message: Message): boolean {
450 return message.role === 'analyst' && (message.status === 'working' || message.status === 'queued')
451}
452
453async function readJson<T>(ctx: BackendContext, path: string): Promise<T | null> {
454 try {
455 return JSON.parse(await ctx.files.read(path)) as T
456 } catch {
457 return null
458 }
459}