Jaspers Jaspers Research: Research rooms where configurable analysts research companies in parallel over SEC filings data, answering with checked citations.
| 1 | // Citations: the tags an analyst wraps its sourced sentences in, the numbers they get, the verdicts |
| 2 | // the server gives them, and the links that open each source on the quoted words. An analyst writes <cite document_id chunk_id quote>fact</cite> |
| 3 | // (or <dbcitation document_id section_key quote> for a Filing Database section); the room sends |
| 4 | // every tag's ids and quote to the connection's show_citations tool, which finds the quote in the |
| 5 | // source and answers with the sec.gov page that opens on it highlighted. A quote from a view's text, |
| 6 | // like a transcript on the workspace, is <cite panel quote> instead, and the room checks it itself |
| 7 | // against the text. Pure: the reading and the numbering can be checked in a test, and the view |
| 8 | // numbers the same way the runner does. |
| 9 | |
| 10 | export interface CitationTag { |
| 11 | kind: 'cite' | 'dbcitation' |
| 12 | attrs: Record<string, string> |
| 13 | /** The sentence the tag wraps; empty for a self-closing or unclosed tag used as a marker. */ |
| 14 | inner: string |
| 15 | start: number |
| 16 | end: number |
| 17 | } |
| 18 | |
| 19 | /** One passage an answer cites, numbered in order of first appearance: a source and the quote from it. */ |
| 20 | export interface NumberedCitation { |
| 21 | n: number |
| 22 | /** The source: document and chunk, or document and section. Two citations may share one. */ |
| 23 | key: string |
| 24 | attrs: Record<string, string> |
| 25 | quote: string |
| 26 | claim: string |
| 27 | } |
| 28 | |
| 29 | /** A numbered citation after the server has looked at it, or before, with nothing known. */ |
| 30 | export interface Citation { |
| 31 | n: number |
| 32 | key: string |
| 33 | documentId: string |
| 34 | chunkId: string | null |
| 35 | sectionKey: string | null |
| 36 | /** The element whose text the quote is from, for a citation of a view; null or missing for a filing, and in answers saved before views could be cited. */ |
| 37 | panel?: string | null |
| 38 | quote: string |
| 39 | claim: string |
| 40 | /** The words the server found the quote in, as its source has them; null when it found none. */ |
| 41 | passage: string | null |
| 42 | /** The server's link to the source, as it gave it; `citationLink` is the one to open. Null until resolved. */ |
| 43 | url: string | null |
| 44 | /** "10-K · 2025-09-05 · FLWS", or null until resolved. */ |
| 45 | title: string | null |
| 46 | /** null until the server has answered, or when no server can. */ |
| 47 | verified: boolean | null |
| 48 | match: 'exact' | 'prefix' | 'none' | null |
| 49 | } |
| 50 | |
| 51 | /** What the audit of an answer found, and whether the analyst got a repair turn for it. */ |
| 52 | export interface Audit { |
| 53 | uncited: number |
| 54 | unverified: number |
| 55 | repaired: boolean |
| 56 | } |
| 57 | |
| 58 | /** What the server says about one source, keyed the way the tags are. */ |
| 59 | export interface ServerRef { |
| 60 | key: string |
| 61 | verified: boolean |
| 62 | match: 'exact' | 'prefix' | 'none' |
| 63 | url: string | null |
| 64 | title: string | null |
| 65 | sourceQuote: string | null |
| 66 | } |
| 67 | |
| 68 | const OPEN_TAG = /<(cite|dbcitation)\b([^>]*)>/gi |
| 69 | const ATTR = /([A-Za-z_][\w-]*)\s*=\s*(?:"((?:[^"\\]|\\.)*)"|'((?:[^'\\]|\\.)*)'|([^\s"'>]+))/g |
| 70 | const CLOSE_TAG = /<\/(?:cite|dbcitation)\s*>/gi |
| 71 | |
| 72 | /** Every tag in the text, in order, with where it sits. A closing tag with no opening one is not a tag. */ |
| 73 | export function parseCitationTags(text: string): CitationTag[] { |
| 74 | const tags: CitationTag[] = [] |
| 75 | const opens = [...text.matchAll(OPEN_TAG)] |
| 76 | for (const [i, open] of opens.entries()) { |
| 77 | const start = open.index ?? 0 |
| 78 | const kind = open[1]!.toLowerCase() as CitationTag['kind'] |
| 79 | const rawAttrs = open[2] ?? '' |
| 80 | const selfClosing = /\/\s*$/.test(rawAttrs) |
| 81 | const attrs = readAttrs(rawAttrs) |
| 82 | const openEnd = start + open[0].length |
| 83 | if (selfClosing) { |
| 84 | tags.push({ kind, attrs, inner: '', start, end: openEnd }) |
| 85 | continue |
| 86 | } |
| 87 | // The inner text runs to this tag's closing tag, unless another tag opens first: then it never closed. |
| 88 | const nextOpen = opens[i + 1]?.index ?? text.length |
| 89 | CLOSE_TAG.lastIndex = openEnd |
| 90 | const close = CLOSE_TAG.exec(text) |
| 91 | if (close && close.index <= nextOpen) { |
| 92 | tags.push({ kind, attrs, inner: text.slice(openEnd, close.index), start, end: close.index + close[0].length }) |
| 93 | } else { |
| 94 | tags.push({ kind, attrs, inner: '', start, end: openEnd }) |
| 95 | } |
| 96 | } |
| 97 | return tags |
| 98 | } |
| 99 | |
| 100 | /** The panel a tag quotes a view's text from, when it names one and no document. */ |
| 101 | export function citedPanel(attrs: Record<string, string>): string | null { |
| 102 | return !(attrs['document_id'] ?? attrs['article_id'] ?? attrs['call_id']) && attrs['panel'] ? attrs['panel'] : null |
| 103 | } |
| 104 | |
| 105 | /** What names a source: the document and its chunk, the document and its section, or a view's panel. Null for a tag with no id. */ |
| 106 | export function citationKey(attrs: Record<string, string>): string | null { |
| 107 | const panel = citedPanel(attrs) |
| 108 | if (panel) return `panel:${panel}` |
| 109 | const id = attrs['document_id'] ?? attrs['article_id'] ?? attrs['call_id'] |
| 110 | if (!id) return null |
| 111 | const part = attrs['section_key'] ?? attrs['chunk_id'] ?? '' |
| 112 | return `${id}:${part === 'N/A' ? '' : part}` |
| 113 | } |
| 114 | |
| 115 | /** What gets a number of its own: the source and the quote. Two tags quoting the same words share one. */ |
| 116 | export function numberKey(attrs: Record<string, string>): string | null { |
| 117 | const key = citationKey(attrs) |
| 118 | return key === null ? null : `${key}\n${(attrs['quote'] ?? '').trim()}` |
| 119 | } |
| 120 | |
| 121 | /** The passages an answer cites, each once, numbered as they first appear. Each is its own link, on its own words. */ |
| 122 | export function numberCitations(text: string): NumberedCitation[] { |
| 123 | const seen = new Map<string, NumberedCitation>() |
| 124 | for (const tag of parseCitationTags(text)) { |
| 125 | const key = citationKey(tag.attrs) |
| 126 | const slot = numberKey(tag.attrs) |
| 127 | if (key === null || slot === null) continue |
| 128 | const claim = tag.inner.replace(/\s+/g, ' ').trim() |
| 129 | const found = seen.get(slot) |
| 130 | if (found) { |
| 131 | if (!found.claim && claim) found.claim = claim |
| 132 | continue |
| 133 | } |
| 134 | seen.set(slot, { n: seen.size + 1, key, attrs: tag.attrs, quote: (tag.attrs['quote'] ?? '').trim(), claim }) |
| 135 | } |
| 136 | return [...seen.values()] |
| 137 | } |
| 138 | |
| 139 | /** |
| 140 | * The text with each tag turned into its sentence and a [n] marker, for a reader without the view. |
| 141 | * A piece of an answer takes its numbers from the whole answer, so its markers match the view's. |
| 142 | */ |
| 143 | export function toMarkers(text: string, numbered: NumberedCitation[] = numberCitations(text)): string { |
| 144 | const numbers = new Map(numbered.map((c) => [numberKey(c.attrs), c.n])) |
| 145 | return rewrite(text, (tag) => { |
| 146 | const slot = numberKey(tag.attrs) |
| 147 | const n = slot === null ? undefined : numbers.get(slot) |
| 148 | const inner = tag.inner.trim() |
| 149 | if (n === undefined) return inner |
| 150 | return inner ? `${inner} [${n}]` : `[${n}]` |
| 151 | }) |
| 152 | } |
| 153 | |
| 154 | /** What a source is called: the server's title for it, or its ids until the server has named it. */ |
| 155 | export function sourceTitle(numbered: NumberedCitation, citation: Citation | undefined): string { |
| 156 | const { attrs } = numbered |
| 157 | const panel = citedPanel(attrs) |
| 158 | if (panel) return citation?.title ?? `View ${panel}` |
| 159 | return ( |
| 160 | citation?.title ?? |
| 161 | `Document ${attrs['document_id'] ?? '?'}${attrs['chunk_id'] ? ` · chunk ${attrs['chunk_id']}` : ''}${attrs['section_key'] ? ` · ${attrs['section_key']}` : ''}` |
| 162 | ) |
| 163 | } |
| 164 | |
| 165 | /** The text with the tags gone and their sentences kept. */ |
| 166 | export function stripCitations(text: string): string { |
| 167 | return rewrite(text, (tag) => tag.inner.trim()) |
| 168 | } |
| 169 | |
| 170 | /** Every tag replaced by what `render` makes of it; a stray closing tag goes too. */ |
| 171 | function rewrite(text: string, render: (tag: CitationTag) => string): string { |
| 172 | let out = '' |
| 173 | let at = 0 |
| 174 | for (const tag of parseCitationTags(text)) { |
| 175 | out += text.slice(at, tag.start) + render(tag) |
| 176 | at = tag.end |
| 177 | } |
| 178 | return (out + text.slice(at)).replace(CLOSE_TAG, '') |
| 179 | } |
| 180 | |
| 181 | /** A sentence with this many words states something, cited or not. */ |
| 182 | const MIN_WORDS = 8 |
| 183 | const MAX_UNCITED = 25 |
| 184 | const MAX_SENTENCE = 400 |
| 185 | const FIGURE = /\d|\$|%/ |
| 186 | const HEADING = /^\s{0,3}#{1,6}\s/ |
| 187 | const TABLE_ROW = /^\s*\|/ |
| 188 | const TABLE_SEPARATOR = /^\s*\|?\s*:?-{2,}/ |
| 189 | const BULLET = /^\s*(?:[-*+•]|\d+[.)])\s+/ |
| 190 | const FENCE = /^\s*```/ |
| 191 | const TURN_LIMIT = /^Stopped at the \d+-turn limit\./ |
| 192 | const BOUNDARY = /[.!?]["'”’)\]]*\s+(?=[A-Z0-9$(["“])/g |
| 193 | const ABBREVIATIONS = new Set( |
| 194 | 'inc corp co ltd llc plc mr mrs ms dr vs no st u.s e.g i.e jan feb mar apr jun jul aug sep sept oct nov dec fig approx est'.split(' '), |
| 195 | ) |
| 196 | |
| 197 | /** |
| 198 | * The sentences of a draft that state something without a citation: any with a figure in it, or |
| 199 | * long enough to be a claim. Headings, tables, code, and the turn-limit line are not sentences. A |
| 200 | * sentence anywhere inside a tag counts as cited, so a tag around two sentences covers both. |
| 201 | */ |
| 202 | export function uncitedSentences(draft: string): string[] { |
| 203 | const tags = parseCitationTags(draft) |
| 204 | const masked = maskTags(draft, tags) |
| 205 | const cited = (start: number, end: number): boolean => tags.some((t) => t.start < end && t.end > start) |
| 206 | const found: string[] = [] |
| 207 | let at = 0 |
| 208 | let inFence = false |
| 209 | for (const line of masked.split('\n')) { |
| 210 | const lineStart = at |
| 211 | at += line.length + 1 |
| 212 | if (FENCE.test(line)) { |
| 213 | inFence = !inFence |
| 214 | continue |
| 215 | } |
| 216 | if (inFence || HEADING.test(line) || TABLE_SEPARATOR.test(line) || TABLE_ROW.test(line) || TURN_LIMIT.test(line.trim())) continue |
| 217 | const bullet = BULLET.exec(line) |
| 218 | const offset = bullet ? bullet[0].length : 0 |
| 219 | for (const [s, e] of sentenceSpans(line.slice(offset))) { |
| 220 | const start = lineStart + offset + s |
| 221 | const end = lineStart + offset + e |
| 222 | if (cited(start, end)) continue |
| 223 | const text = masked.slice(start, end).replace(/\s+/g, ' ').trim() |
| 224 | if (!text) continue |
| 225 | const words = text.split(' ').filter((w) => /[A-Za-z0-9]/.test(w)).length |
| 226 | if (!FIGURE.test(text) && words < MIN_WORDS) continue |
| 227 | found.push(text.length > MAX_SENTENCE ? `${text.slice(0, MAX_SENTENCE)}…` : text) |
| 228 | if (found.length >= MAX_UNCITED) return found |
| 229 | } |
| 230 | } |
| 231 | return found |
| 232 | } |
| 233 | |
| 234 | /** Tags replaced by spaces of the same length, so offsets hold and attribute text ends no sentence. */ |
| 235 | function maskTags(text: string, tags: CitationTag[]): string { |
| 236 | let out = text |
| 237 | for (const tag of tags) { |
| 238 | const openEnd = text.indexOf('>', tag.start) + 1 |
| 239 | out = blank(out, tag.start, openEnd) |
| 240 | if (tag.end > openEnd + tag.inner.length) out = blank(out, openEnd + tag.inner.length, tag.end) |
| 241 | } |
| 242 | return out.replace(CLOSE_TAG, (m) => ' '.repeat(m.length)) |
| 243 | } |
| 244 | |
| 245 | function blank(text: string, start: number, end: number): string { |
| 246 | return text.slice(0, start) + ' '.repeat(Math.max(0, end - start)) + text.slice(end) |
| 247 | } |
| 248 | |
| 249 | /** Where each sentence of a segment starts and ends, splitting at sentence ends that are not abbreviations. */ |
| 250 | function sentenceSpans(segment: string): [number, number][] { |
| 251 | const spans: [number, number][] = [] |
| 252 | let start = 0 |
| 253 | for (const match of segment.matchAll(BOUNDARY)) { |
| 254 | const index = match.index ?? 0 |
| 255 | const before = segment.slice(start, index + 1) |
| 256 | const lastWord = /([A-Za-z.]+)\.$/.exec(before)?.[1]?.toLowerCase().replace(/\.$/, '') |
| 257 | if (lastWord && (ABBREVIATIONS.has(lastWord) || /^[a-z]$/.test(lastWord))) continue |
| 258 | spans.push([start, index + 1]) |
| 259 | start = index + match[0].length |
| 260 | } |
| 261 | if (start < segment.length) spans.push([start, segment.length]) |
| 262 | return spans |
| 263 | } |
| 264 | |
| 265 | /** |
| 266 | * What show_citations takes: one claim per numbered citation, ids as the server's schema types them. |
| 267 | * A tag with no quote offers its sentence as the quote: a sentence copied from the source verifies |
| 268 | * on its opening words, and one that was not comes back unverified, which the repair then says. |
| 269 | */ |
| 270 | export function claimsFor(numbered: NumberedCitation[]): Record<string, unknown>[] { |
| 271 | return numbered.map((c) => { |
| 272 | const claim: Record<string, unknown> = { claim: c.claim || c.quote, quote: c.quote || c.claim } |
| 273 | const document = c.attrs['document_id'] |
| 274 | if (document !== undefined) claim['document_id'] = /^\d+$/.test(document) ? Number(document) : document |
| 275 | if (c.attrs['section_key']) claim['section_key'] = c.attrs['section_key'] |
| 276 | else if (c.attrs['chunk_id'] && c.attrs['chunk_id'] !== 'N/A') claim['chunk_id'] = /^\d+$/.test(c.attrs['chunk_id']) ? Number(c.attrs['chunk_id']) : c.attrs['chunk_id'] |
| 277 | return claim |
| 278 | }) |
| 279 | } |
| 280 | |
| 281 | /** The refs in show_citations' answer, keyed like the tags. Anything that is not that answer reads as none. */ |
| 282 | export function readShowCitations(text: string): ServerRef[] { |
| 283 | let parsed: unknown |
| 284 | try { |
| 285 | parsed = JSON.parse(text) |
| 286 | } catch { |
| 287 | return [] |
| 288 | } |
| 289 | if (typeof parsed !== 'object' || parsed === null || !Array.isArray((parsed as { citations?: unknown }).citations)) return [] |
| 290 | const refs: ServerRef[] = [] |
| 291 | for (const raw of (parsed as { citations: unknown[] }).citations) { |
| 292 | if (typeof raw !== 'object' || raw === null) continue |
| 293 | const ref = raw as Record<string, unknown> |
| 294 | const key = citationKey({ |
| 295 | ...(ref['document_id'] !== undefined && ref['document_id'] !== null ? { document_id: String(ref['document_id']) } : {}), |
| 296 | ...(ref['chunk_id'] !== undefined && ref['chunk_id'] !== null ? { chunk_id: String(ref['chunk_id']) } : {}), |
| 297 | ...(typeof ref['section_key'] === 'string' ? { section_key: ref['section_key'] } : {}), |
| 298 | }) |
| 299 | if (key === null) continue |
| 300 | const match = ref['match'] |
| 301 | const title = [ref['form_type'], ref['filing_date'], ref['ticker']].filter((v) => typeof v === 'string' && v).join(' · ') |
| 302 | refs.push({ |
| 303 | key, |
| 304 | verified: ref['verified'] === true, |
| 305 | match: match === 'exact' || match === 'prefix' ? match : 'none', |
| 306 | url: typeof ref['url'] === 'string' ? ref['url'] : typeof ref['source_url'] === 'string' ? ref['source_url'] : typeof ref['index_url'] === 'string' ? ref['index_url'] : null, |
| 307 | title: title || null, |
| 308 | sourceQuote: typeof ref['source_quote'] === 'string' ? ref['source_quote'] : null, |
| 309 | }) |
| 310 | } |
| 311 | return refs |
| 312 | } |
| 313 | |
| 314 | /** |
| 315 | * The numbered citations with what the server said about each; one it did not mention stays unknown. |
| 316 | * The server answers per source, so the caller sends it citations with distinct sources at a time. |
| 317 | */ |
| 318 | export function resolveCitations(numbered: NumberedCitation[], refs: ServerRef[]): Citation[] { |
| 319 | const byKey = new Map(refs.map((r) => [r.key, r])) |
| 320 | return numbered.map((c) => { |
| 321 | const ref = byKey.get(c.key) |
| 322 | return { |
| 323 | n: c.n, |
| 324 | key: c.key, |
| 325 | documentId: c.attrs['document_id'] ?? c.attrs['article_id'] ?? c.attrs['call_id'] ?? '', |
| 326 | chunkId: c.attrs['chunk_id'] && c.attrs['chunk_id'] !== 'N/A' ? c.attrs['chunk_id'] : null, |
| 327 | sectionKey: c.attrs['section_key'] ?? null, |
| 328 | panel: citedPanel(c.attrs), |
| 329 | quote: c.quote, |
| 330 | claim: c.claim, |
| 331 | passage: ref?.sourceQuote ?? null, |
| 332 | url: ref?.url ?? null, |
| 333 | title: ref?.title ?? null, |
| 334 | verified: ref ? ref.verified : null, |
| 335 | match: ref ? ref.match : null, |
| 336 | } |
| 337 | }) |
| 338 | } |
| 339 | |
| 340 | /** |
| 341 | * The citations of views' texts, checked here, since no server holds a view: each quote has to be in |
| 342 | * the text of the panel it names, whatever the spacing, the case, or the quote marks, which models |
| 343 | * change when they copy. `texts` is each panel's text by the id a tag names it by. The title is the |
| 344 | * text's first line, which names what it is; a panel with no text is only its id. |
| 345 | */ |
| 346 | export function resolveViewCitations(numbered: NumberedCitation[], texts: ReadonlyMap<string, string>): Citation[] { |
| 347 | const citations: Citation[] = [] |
| 348 | for (const c of numbered) { |
| 349 | const panel = citedPanel(c.attrs) |
| 350 | if (!panel) continue |
| 351 | const text = texts.get(panel) |
| 352 | const found = text !== undefined && c.quote !== '' && looseText(text).includes(looseText(c.quote)) |
| 353 | citations.push({ |
| 354 | n: c.n, |
| 355 | key: c.key, |
| 356 | documentId: '', |
| 357 | chunkId: null, |
| 358 | sectionKey: null, |
| 359 | panel, |
| 360 | quote: c.quote, |
| 361 | claim: c.claim, |
| 362 | passage: null, |
| 363 | url: null, |
| 364 | title: text?.split('\n', 1)[0]?.trim() || `View ${panel}`, |
| 365 | verified: found, |
| 366 | match: found ? 'exact' : 'none', |
| 367 | }) |
| 368 | } |
| 369 | return citations |
| 370 | } |
| 371 | |
| 372 | /** Text as a quote of it is compared: one kind of quote mark and dash, single spaces, lower case. */ |
| 373 | function looseText(text: string): string { |
| 374 | return text |
| 375 | .replace(/[‘’‚‛′`´"“”„‟″]/g, "'") |
| 376 | .replace(/[‐‑‒–—―]/g, '-') |
| 377 | .replace(/\s+/g, ' ') |
| 378 | .trim() |
| 379 | .toLowerCase() |
| 380 | } |
| 381 | |
| 382 | /** |
| 383 | * The link a citation opens: the server's page for its source, with a text directive that highlights |
| 384 | * the passage. The server's own directive takes the passage's first and last words as they are, and |
| 385 | * a browser misses those where table cells ran together in the source text or a word sits against a |
| 386 | * symbol the page sets apart; this one takes words that read the same on the page. An index page, a |
| 387 | * link with some other fragment, or a passage with no such words keeps the server's link. |
| 388 | */ |
| 389 | export function citationLink(citation: { url: string | null; passage?: string | null; quote: string }): string | null { |
| 390 | const { url } = citation |
| 391 | if (!url) return null |
| 392 | const hash = url.indexOf('#') |
| 393 | if (hash >= 0 && !url.startsWith('#:~:', hash)) return url |
| 394 | const page = hash >= 0 ? url.slice(0, hash) : url |
| 395 | if (/-index\.html?$/i.test(page)) return url |
| 396 | const fragment = textFragment(citation.passage || citation.quote) |
| 397 | return fragment ? page + fragment : url |
| 398 | } |
| 399 | |
| 400 | /** How many words the opening and closing terms take, and the longest passage highlighted as one term. */ |
| 401 | const START_WORDS = 8 |
| 402 | const END_WORDS = 5 |
| 403 | const WHOLE_WORDS = 10 |
| 404 | const JSON_FIELD = /"[\w ]+"\s*:/ |
| 405 | const JSON_STRING_VALUE = /"[^"]*"\s*:\s*"([^"]+)"/g |
| 406 | |
| 407 | /** |
| 408 | * The `#:~:text=` directive that highlights a passage on its rendered page, or '' when none of its |
| 409 | * words can be trusted to read the same there. A browser finds each term inside one block of text |
| 410 | * and at word boundaries, so terms come from runs of plain words (`wordRuns`): prose runs from its |
| 411 | * first run with three words that say something to the end of its last, or is one term when that is |
| 412 | * a single short run; a table row, whose cells the page sets apart, runs from its first figure to its |
| 413 | * last; and structured data is found by its longest quoted value of two or more words, like a name. |
| 414 | */ |
| 415 | export function textFragment(passage: string): string { |
| 416 | let text = plainText(passage) |
| 417 | if (JSON_FIELD.test(text)) { |
| 418 | const values = [...text.matchAll(JSON_STRING_VALUE)].map((m) => m[1]!).filter((v) => v.trim().split(/\s+/).length >= 2) |
| 419 | if (values.length === 0) return '' |
| 420 | text = values.reduce((a, b) => (b.length > a.length ? b : a)) |
| 421 | } |
| 422 | const runs = wordRuns(text) |
| 423 | const prose = runs.filter((run) => run.length >= 3 && run.filter((word) => /[A-Za-z]/.test(word)).length >= 2) |
| 424 | let terms = isTableRow(text, runs, prose) || prose.length === 0 ? figureTerms(runs) : null |
| 425 | if (!terms && prose.length > 0) { |
| 426 | const start = prose.find((run) => contentWords(run) >= 3) ?? prose.reduce((a, b) => (contentWords(b) > contentWords(a) ? b : a)) |
| 427 | const end = prose[prose.length - 1]! |
| 428 | if (end !== start) terms = [start.slice(0, START_WORDS), end.slice(-END_WORDS)] |
| 429 | else if (start.length <= WHOLE_WORDS) terms = [start] |
| 430 | else terms = [start.slice(0, Math.min(START_WORDS, start.length - END_WORDS)), start.slice(-END_WORDS)] |
| 431 | } |
| 432 | return terms ? `#:~:text=${terms.map((words) => encodeTerm(words.join(' '))).join(',')}` : '' |
| 433 | } |
| 434 | |
| 435 | /** Words too common to find a passage by: small words, and month names, since a date opens so many sentences in a filing. */ |
| 436 | const COMMON_WORDS = new Set( |
| 437 | 'a an and as at by for from in into is it of on or the to was were with its their this that than be been has have had which who per january february march april may june july august september october november december'.split(' '), |
| 438 | ) |
| 439 | |
| 440 | /** How many of a run's words say something: words with letters that are not common words. */ |
| 441 | function contentWords(run: string[]): number { |
| 442 | return run.filter((word) => { |
| 443 | const bare = word.toLowerCase().replace(/[.,;:]+$/, '') |
| 444 | return /[a-z]/.test(bare) && !COMMON_WORDS.has(bare) |
| 445 | }).length |
| 446 | } |
| 447 | |
| 448 | /** Marks that start a run of plain words and marks that end one, since a page may set either apart. */ |
| 449 | const OPENERS = new Set(['"', '“', '”', "'", '‘', '’', '(', '[', '{', '$', '€', '£']) |
| 450 | const CLOSERS = new Set(['"', '“', '”', "'", '‘', '’', ')', ']', '}', '%', '®', '™', '*', '†', '‡']) |
| 451 | const PUNCTUATION = new Set([',', '.', ';', ':', '!', '?']) |
| 452 | const PLAIN_WORD = /^(?:[A-Za-z0-9](?:[A-Za-z0-9.,'’&/-]*[A-Za-z0-9])?|&)$/ |
| 453 | /** Where table cells ran together in source text: a pipe or a line, or letters against digits or a dollar sign. */ |
| 454 | const CELL_BREAK = /[|\n\r\t]+/ |
| 455 | const GLUED = /(?<=[A-Za-z)])(?=\$)|(?<=[a-z])(?=\d)|(?<=\d)(?=[a-z]{2,})/ |
| 456 | const GLUED_ANYWHERE = /[A-Za-z)]\$|[a-z]\d|\d[a-z]{2,}/ |
| 457 | const EMPTY_CELL = /(^|\s)\$?[-—–](\s|$)/ |
| 458 | |
| 459 | /** |
| 460 | * The runs of words in a passage that should read on the page exactly as they read here. A run ends |
| 461 | * at a table cell, at a quote mark, bracket, currency or percent sign, and at a word holding anything |
| 462 | * else a page may render differently, so a term taken from one run is text a browser finds whole. |
| 463 | */ |
| 464 | export function wordRuns(text: string): string[][] { |
| 465 | const runs: string[][] = [] |
| 466 | for (const segment of text.split(CELL_BREAK).flatMap((part) => part.split(GLUED))) { |
| 467 | let run: string[] = [] |
| 468 | const close = (): void => { |
| 469 | if (run.length > 0) runs.push(run) |
| 470 | run = [] |
| 471 | } |
| 472 | for (const raw of segment.trim().split(/\s+/)) { |
| 473 | let word = raw |
| 474 | let opened = false |
| 475 | while (word && OPENERS.has(word[0]!)) { |
| 476 | word = word.slice(1) |
| 477 | opened = true |
| 478 | } |
| 479 | let end = word.length |
| 480 | let closed = false |
| 481 | while (end > 0 && (CLOSERS.has(word[end - 1]!) || PUNCTUATION.has(word[end - 1]!))) { |
| 482 | if (CLOSERS.has(word[end - 1]!)) closed = true |
| 483 | end-- |
| 484 | } |
| 485 | const core = word.slice(0, end) |
| 486 | if (!PLAIN_WORD.test(core)) { |
| 487 | close() |
| 488 | continue |
| 489 | } |
| 490 | if (opened) close() |
| 491 | run.push(closed ? core : word) |
| 492 | if (closed) close() |
| 493 | } |
| 494 | close() |
| 495 | } |
| 496 | return runs |
| 497 | } |
| 498 | |
| 499 | /** A table row: pipes, cells run together, an empty cell beside a figure, or figures with too few words for prose. */ |
| 500 | function isTableRow(text: string, runs: string[][], prose: string[][]): boolean { |
| 501 | const figures = runs.flat().filter(isFigure).length |
| 502 | const longest = Math.max(0, ...prose.map((run) => run.length)) |
| 503 | return text.includes('|') || GLUED_ANYWHERE.test(text) || (EMPTY_CELL.test(text) && figures >= 1) || (figures >= 2 && longest < 5) |
| 504 | } |
| 505 | |
| 506 | /** A figure distinctive enough to find a row by: grouped thousands, or four digits or more that are not a year. */ |
| 507 | function isFigure(word: string): boolean { |
| 508 | const figure = word.replace(/[.,;:]+$/, '') |
| 509 | return (/^\d{1,3}(,\d{3})+(\.\d+)?$/.test(figure) || /^\d{4,}(\.\d+)?$/.test(figure)) && !/^(19|20)\d{2}$/.test(figure) |
| 510 | } |
| 511 | |
| 512 | /** A row's first and last figures as terms, one when they are the same. */ |
| 513 | function figureTerms(runs: string[][]): string[][] | null { |
| 514 | const figures = runs.flat().filter(isFigure).map((word) => word.replace(/[.,;:]+$/, '')) |
| 515 | if (figures.length === 0) return null |
| 516 | const first = figures[0]! |
| 517 | const last = figures[figures.length - 1]! |
| 518 | return last === first ? [[first]] : [[first], [last]] |
| 519 | } |
| 520 | |
| 521 | /** A term as a text directive needs it: `-`, `,`, and `&` are the directive's own syntax, so they are encoded. */ |
| 522 | function encodeTerm(term: string): string { |
| 523 | return encodeURIComponent(term).replace(/-/g, '%2D') |
| 524 | } |
| 525 | |
| 526 | /** Text with what a model or a converter added taken back out: backslash escapes, HTML entities, Markdown marks. */ |
| 527 | function plainText(text: string): string { |
| 528 | return unescapeText(text).replace(/\*\*|__|`/g, '').replace(/\u00a0/g, ' ') |
| 529 | } |
| 530 | |
| 531 | const ENTITIES: Record<string, string> = { quot: '"', amp: '&', apos: "'", lt: '<', gt: '>', nbsp: ' ' } |
| 532 | |
| 533 | /** A value with JSON's backslash escapes and HTML's entities undone, as a model may have copied them from a tool result. */ |
| 534 | function unescapeText(value: string): string { |
| 535 | return value |
| 536 | .replace(/\\[nrt]/g, ' ') |
| 537 | .replace(/\\([^\sA-Za-z0-9])/g, '$1') |
| 538 | .replace(/&(#x[0-9a-f]+|#\d+|[a-z]+);/gi, (entity, name: string) => { |
| 539 | if (name[0] !== '#') return ENTITIES[name.toLowerCase()] ?? entity |
| 540 | const code = name[1] === 'x' || name[1] === 'X' ? parseInt(name.slice(2), 16) : Number(name.slice(1)) |
| 541 | return Number.isInteger(code) && code > 0 && code <= 0x10ffff ? String.fromCodePoint(code) : entity |
| 542 | }) |
| 543 | } |
| 544 | |
| 545 | /** The message that sends a draft back for repair, or null when the audit found nothing. */ |
| 546 | export function buildRepairMessage(report: { uncited: string[]; unverified: { n: number; tag: string; quote: string; reason: string }[] }): string | null { |
| 547 | if (report.uncited.length === 0 && report.unverified.length === 0) return null |
| 548 | const parts = [ |
| 549 | 'A citation audit of your draft found the problems below. Reply with the COMPLETE corrected response - the full text, not a diff or a summary of changes - keeping everything that was not flagged identical.', |
| 550 | ] |
| 551 | if (report.uncited.length > 0) { |
| 552 | parts.push( |
| 553 | 'Sentences that state tool-derived facts without a citation. For each one either add a citation tag whose quote="…" is copied verbatim from a tool result already in this conversation, remove the claim, or rewrite it so it is clearly your own reasoning:\n' + |
| 554 | report.uncited.map((s, i) => `${i + 1}. "${s}"`).join('\n'), |
| 555 | ) |
| 556 | } |
| 557 | if (report.unverified.length > 0) { |
| 558 | parts.push( |
| 559 | 'Citations whose quote could not be found in the cited source. Replace quote="…" with an exact excerpt copied from that source (same ids), or drop the citation together with the claim it supported:\n' + |
| 560 | report.unverified.map((u, i) => `${i + 1}. ${u.tag} quote="${u.quote}" - ${u.reason}`).join('\n'), |
| 561 | ) |
| 562 | } |
| 563 | parts.push( |
| 564 | 'Rules: use only ids that appear in tool results; every <cite>/<dbcitation> tag carries quote="…" copied character-for-character (5-40 words, no double quotes inside); add no new claims; call a tool only when you must re-read a source to copy an exact quote; keep the structure, tables and formatting of the answer.', |
| 565 | ) |
| 566 | return parts.join('\n\n') |
| 567 | } |
| 568 | |
| 569 | function readAttrs(raw: string): Record<string, string> { |
| 570 | const attrs: Record<string, string> = {} |
| 571 | for (const match of raw.replace(/\/\s*$/, '').matchAll(ATTR)) { |
| 572 | attrs[match[1]!.toLowerCase()] = unescapeText(match[2] ?? match[3] ?? match[4] ?? '').trim() |
| 573 | } |
| 574 | return attrs |
| 575 | } |