Jaspers Research Agents: Agents you define that answer research chats from SEC filings, your documents, and the web, citing every passage.
| 1 | // Documents the user uploads, and the pages agents read: their text in the research store under an |
| 2 | // id the store gives, which an agent's documents name. Pure: the names, which files are read, and a |
| 3 | // PDF's text as pdf.js gives it. |
| 4 | |
| 5 | export interface Doc { |
| 6 | name: string |
| 7 | /** Its id in the store, which an agent's documents name. */ |
| 8 | id: string |
| 9 | /** Characters of text. */ |
| 10 | size: number |
| 11 | /** Milliseconds since the epoch. */ |
| 12 | modified: number |
| 13 | } |
| 14 | |
| 15 | export const FOLDER = 'docs' |
| 16 | /** Files that are read: text, and PDFs with a text layer. Anything else (Word, images, scans) is not read yet. */ |
| 17 | export const READ_TYPES = ['txt', 'md', 'markdown', 'csv', 'tsv', 'json', 'html', 'htm', 'xml', 'log', 'pdf'] |
| 18 | const NAME_MAX = 120 |
| 19 | |
| 20 | /** A file's name as a document's: letters, digits, dot, dash, underscore; the rest become dashes. */ |
| 21 | export function docName(fileName: string): string { |
| 22 | const base = fileName.slice(fileName.lastIndexOf('/') + 1).trim() |
| 23 | const name = base |
| 24 | .replace(/[^\w.-]+/g, '-') |
| 25 | .replace(/^[.-]+/, '') |
| 26 | .slice(0, NAME_MAX) |
| 27 | if (!name) throw new Error(`${fileName} has no usable name.`) |
| 28 | return name |
| 29 | } |
| 30 | |
| 31 | export function extensionOf(name: string): string { |
| 32 | const dot = name.lastIndexOf('.') |
| 33 | return dot === -1 ? '' : name.slice(dot + 1).toLowerCase() |
| 34 | } |
| 35 | |
| 36 | export function isReadable(name: string): boolean { |
| 37 | return READ_TYPES.includes(extensionOf(name)) |
| 38 | } |
| 39 | |
| 40 | export function isPdf(name: string): boolean { |
| 41 | return extensionOf(name) === 'pdf' |
| 42 | } |
| 43 | |
| 44 | export function isMarkup(name: string): boolean { |
| 45 | return ['html', 'htm'].includes(extensionOf(name)) |
| 46 | } |
| 47 | |
| 48 | /** One run of text on a PDF page, as pdf.js's getTextContent gives it; marked-content items have no `str`. */ |
| 49 | export interface PdfItem { |
| 50 | str?: string |
| 51 | hasEOL?: boolean |
| 52 | } |
| 53 | |
| 54 | /** |
| 55 | * A PDF's text, page by page, each page under a `[page N]` line so a passage says where it came from. |
| 56 | * Runs are joined as pdf.js's own viewer joins them: pdf.js puts the spaces in, and a line ends where it |
| 57 | * says one does. |
| 58 | */ |
| 59 | export function pdfText(pages: PdfItem[][]): string { |
| 60 | return pages |
| 61 | .map((items, i) => { |
| 62 | const text = items |
| 63 | .map((item) => (item.str ?? '') + (item.hasEOL ? '\n' : '')) |
| 64 | .join('') |
| 65 | .replace(/[ \t]+\n/g, '\n') |
| 66 | .trim() |
| 67 | return text ? `[page ${i + 1}]\n${text}` : '' |
| 68 | }) |
| 69 | .filter(Boolean) |
| 70 | .join('\n\n') |
| 71 | } |