From cb9dd58152b52e0fcf0bde7d6daf68d7f4617a05 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Sat, 5 Apr 2025 23:03:40 -0700 Subject: [PATCH 1/2] added URL upload for our file parser --- sim/app/api/files/parse/route.ts | 99 +++++++++++++++++++++++++ sim/blocks/blocks/file.ts | 120 ++++++++++++++++++++++++------- sim/tools/file/parser.ts | 109 +++++++++++----------------- sim/tools/file/types.ts | 26 +++++++ 4 files changed, 264 insertions(+), 90 deletions(-) create mode 100644 sim/tools/file/types.ts diff --git a/sim/app/api/files/parse/route.ts b/sim/app/api/files/parse/route.ts index d137fa6abee..5b6d3980888 100644 --- a/sim/app/api/files/parse/route.ts +++ b/sim/app/api/files/parse/route.ts @@ -1,7 +1,9 @@ import { NextRequest, NextResponse } from 'next/server' +import { createHash } from 'crypto' import { existsSync } from 'fs' import fs from 'fs' import { readFile, unlink, writeFile } from 'fs/promises' +import { tmpdir } from 'os' import { join } from 'path' import path from 'path' import { isSupportedFileType, parseFile } from '@/lib/file-parsers' @@ -140,6 +142,11 @@ export async function POST(request: NextRequest) { async function parseFileSingle(filePath: string, fileType?: string): Promise { logger.info('Parsing file:', filePath) + // Check if this is an external URL + if (filePath.startsWith('http://') || filePath.startsWith('https://')) { + return handleExternalUrl(filePath, fileType) + } + // Check if this is an S3 path const isS3Path = filePath.includes('/api/files/serve/s3/') @@ -152,6 +159,98 @@ async function parseFileSingle(filePath: string, fileType?: string): Promise { + logger.info(`Handling external URL: ${url}`) + + try { + // Create a unique filename for the temporary file + const urlHash = createHash('md5').update(url).digest('hex') + const urlObj = new URL(url) + const originalFilename = urlObj.pathname.split('/').pop() || 'download' + const tmpFilename = `${urlHash}-${originalFilename}` + const tmpFilePath = path.join(tmpdir(), tmpFilename) + + // Download the file using native fetch + logger.info(`Downloading file from URL to ${tmpFilePath}`) + const response = await fetch(url, { + method: 'GET', + headers: { + 'User-Agent': 'SimStudio/1.0', + }, + }) + + if (!response.ok) { + throw new Error(`Failed to download file: ${response.status} ${response.statusText}`) + } + + // Get the file buffer from response + const arrayBuffer = await response.arrayBuffer() + const fileBuffer = Buffer.from(arrayBuffer) + + // Write to temporary file + await writeFile(tmpFilePath, fileBuffer) + logger.info(`Downloaded ${fileBuffer.length} bytes to ${tmpFilePath}`) + + // Determine file extension and type + const contentType = response.headers.get('content-type') || '' + const extension = + path.extname(originalFilename).toLowerCase().substring(1) || + (contentType ? contentType.split('/').pop() || 'unknown' : 'unknown') + + try { + // Process based on file type + let result: ParseResult + + if (extension === 'pdf') { + result = await handlePdfBuffer(fileBuffer, originalFilename, fileType, url) + } else if (extension === 'csv') { + result = await handleCsvBuffer(fileBuffer, originalFilename, fileType, url) + } else if (isSupportedFileType(extension)) { + result = await handleGenericTextBuffer( + fileBuffer, + originalFilename, + extension, + fileType, + url + ) + } else { + result = handleGenericBuffer(fileBuffer, originalFilename, extension, fileType) + } + + // Clean up temporary file + try { + await unlink(tmpFilePath) + logger.info(`Deleted temporary file: ${tmpFilePath}`) + } catch (cleanupError) { + logger.warn(`Failed to delete temporary file ${tmpFilePath}:`, cleanupError) + } + + return result + } catch (parseError) { + logger.error(`Error parsing downloaded file: ${url}`, parseError) + + // Clean up temporary file on error + try { + await unlink(tmpFilePath) + } catch (cleanupError) { + // Ignore cleanup errors on parse failure + } + + throw parseError + } + } catch (error) { + logger.error(`Error handling external URL ${url}:`, error) + return { + success: false, + error: `Failed to download or process file from URL: ${(error as Error).message}`, + filePath: url, + } + } +} + /** * Handle file stored in S3 */ diff --git a/sim/blocks/blocks/file.ts b/sim/blocks/blocks/file.ts index d2a9117668b..de44ffa2073 100644 --- a/sim/blocks/blocks/file.ts +++ b/sim/blocks/blocks/file.ts @@ -1,25 +1,66 @@ import { DocumentIcon } from '@/components/icons' -import { FileParserOutput } from '@/tools/file/parser' -import { BlockConfig } from '../types' +import { createLogger } from '@/lib/logs/console-logger' +import { FileParserOutput } from '@/tools/file/types' +import { BlockConfig, SubBlockConfig, SubBlockLayout, SubBlockType } from '../types' + +const logger = createLogger('FileBlock') + +const isProduction = process.env.NODE_ENV === 'production' +const isS3Enabled = process.env.USE_S3 === 'true' +const shouldEnableURLInput = isProduction || isS3Enabled + +// Define sub-blocks conditionally +const inputMethodBlock: SubBlockConfig = { + id: 'inputMethod', + title: 'Select Input Method', + type: 'dropdown' as SubBlockType, + layout: 'full' as SubBlockLayout, + options: [ + { id: 'url', label: 'File URL' }, + { id: 'upload', label: 'Upload Files' }, + ], +} + +const fileUrlBlock: SubBlockConfig = { + id: 'filePath', + title: 'File URL', + type: 'short-input' as SubBlockType, + layout: 'full' as SubBlockLayout, + placeholder: 'Enter URL to a file (https://example.com/document.pdf)', + condition: { + field: 'inputMethod', + value: 'url', + }, +} + +const fileUploadBlock: SubBlockConfig = { + id: 'file', + title: 'Upload Files', + type: 'file-upload' as SubBlockType, + layout: 'full' as SubBlockLayout, + acceptedTypes: '.pdf,.csv,.docx', + multiple: true, + maxSize: 100, +} export const FileBlock: BlockConfig = { type: 'file', name: 'File', description: 'Read and parse multiple files', longDescription: - 'Upload and extract contents from structured file formats including PDFs, CSV spreadsheets, and Word documents (DOCX). Specialized parsers extract text and metadata from each format. You can upload multiple files at once and access them individually or as a combined document.', + 'Upload and extract contents from structured file formats including PDFs, CSV spreadsheets, and Word documents (DOCX). ' + + (shouldEnableURLInput + ? 'You can either provide a URL to a file or upload files directly. ' + : 'Upload files directly. ') + + 'Specialized parsers extract text and metadata from each format. You can upload multiple files at once and access them individually or as a combined document.', category: 'tools', bgColor: '#40916C', icon: DocumentIcon, subBlocks: [ + ...(shouldEnableURLInput ? [inputMethodBlock, fileUrlBlock] : []), { - id: 'file', - title: 'Upload Files', - type: 'file-upload', - layout: 'full', - acceptedTypes: '.pdf,.csv,.docx', - multiple: true, - maxSize: 100, + ...fileUploadBlock, + ...(shouldEnableURLInput ? { condition: { field: 'inputMethod', value: 'upload' } } : {}), }, ], tools: { @@ -27,34 +68,65 @@ export const FileBlock: BlockConfig = { config: { tool: () => 'file_parser', params: (params) => { - console.log('File block params:', params) + // Determine input method based on whether URL input is enabled + const inputMethod = shouldEnableURLInput ? params.inputMethod || 'url' : 'upload' + + if (inputMethod === 'url') { + if (!params.filePath || params.filePath.trim() === '') { + logger.error('Missing file URL') + throw new Error('File URL is required') + } + + const fileUrl = params.filePath.trim() - // Handle case where 'file' is an array (multiple files) - if (params.file && Array.isArray(params.file) && params.file.length > 0) { - // Process all files by sending array of paths - const filePaths = params.file.map((file) => file.path) return { - filePath: filePaths.length === 1 ? filePaths[0] : filePaths, + filePath: fileUrl, fileType: params.fileType || 'auto', } } - // Handle case where 'file' is a single file object - if (params.file && params.file.path) { - return { - filePath: params.file.path, - fileType: params.fileType || 'auto', + // Handle file upload input (always possible, default if URL input disabled) + if (inputMethod === 'upload') { + // Handle case where 'file' is an array (multiple files) + if (params.file && Array.isArray(params.file) && params.file.length > 0) { + const filePaths = params.file.map((file) => file.path) + + return { + filePath: filePaths.length === 1 ? filePaths[0] : filePaths, + fileType: params.fileType || 'auto', + } + } + + // Handle case where 'file' is a single file object + if (params.file && params.file.path) { + return { + filePath: params.file.path, + fileType: params.fileType || 'auto', + } } + + // If no files, return error + logger.error('No files provided for upload method') + throw new Error('Please upload a file') // Changed error message slightly } - // If no files, return empty params - return { filePath: '', fileType: params.fileType || 'auto' } + // This part should ideally not be reached if logic above is correct + logger.error(`Invalid configuration or state: ${inputMethod}`) + throw new Error('Invalid configuration: Unable to determine input method') }, }, }, inputs: { + // Conditionally require inputMethod and filePath only if URL input is enabled + ...(shouldEnableURLInput + ? { + inputMethod: { type: 'string', required: false }, // Not strictly required as it defaults + filePath: { type: 'string', required: false }, // Required only if inputMethod is 'url' (validated in params) + } + : {}), fileType: { type: 'string', required: false }, - file: { type: 'json', required: true }, + // File input is always potentially needed, but only required if method is 'upload' (validated in params) + file: { type: 'json', required: false }, }, outputs: { response: { diff --git a/sim/tools/file/parser.ts b/sim/tools/file/parser.ts index 999c9af1c5b..5504ae97f66 100644 --- a/sim/tools/file/parser.ts +++ b/sim/tools/file/parser.ts @@ -1,40 +1,20 @@ -import { ToolConfig, ToolResponse } from '../types' +import { createLogger } from '@/lib/logs/console-logger' +import { ToolConfig } from '../types' +import { FileParseResult, FileParserInput, FileParserOutput, FileParserOutputData } from './types' -export interface FileParserInput { - filePath: string | string[] - fileType?: string -} - -export interface FileParseResult { - content: string - fileType: string - size: number - name: string - binary: boolean - metadata?: Record -} - -export interface FileParserOutputData { - files: FileParseResult[] - combinedContent: string - [key: string]: any -} - -export interface FileParserOutput extends ToolResponse { - output: FileParserOutputData -} +const logger = createLogger('FileParserTool') export const fileParserTool: ToolConfig = { id: 'file_parser', name: 'File Parser', - description: 'Parse one or more uploaded files (text, PDF, CSV, images, etc.)', + description: 'Parse one or more uploaded files or files from URLs (text, PDF, CSV, images, etc.)', version: '1.0.0', params: { filePath: { type: 'string', required: true, - description: 'Path to the uploaded file(s). Can be a single path or an array of paths.', + description: 'Path to the file(s). Can be a single path, URL, or an array of paths.', }, fileType: { type: 'string', @@ -50,77 +30,74 @@ export const fileParserTool: ToolConfig = { 'Content-Type': 'application/json', }), body: (params: any) => { - console.log('[fileParserTool] Request parameters:', params) + logger.info('Request parameters received by tool body:', params) - // Check for valid input if (!params) { - console.error('[fileParserTool] No parameters provided') - throw new Error('No parameters provided') + logger.error('Tool body received no parameters') + throw new Error('No parameters provided to tool body') } - // Handle various input formats - let filePath = null + let determinedFilePath: string | string[] | null = null + let determinedFileType: string | undefined = params.fileType - // Handle multiple files case from block output - if (params.files && Array.isArray(params.files)) { - console.log('[fileParserTool] Processing multiple files:', params.files.length) - filePath = params.files.map((file: any) => file.path) + // 1. Check for direct filePath (URL or single path from upload) + if (params.filePath) { + logger.info('Tool body found direct filePath:', params.filePath) + determinedFilePath = params.filePath } - // Handle the case where params is an object with file property - else if (params.file) { - if (Array.isArray(params.file)) { - console.log( - '[fileParserTool] Processing multiple files from file array:', - params.file.length - ) - filePath = params.file.map((file: any) => file.path) - } else if (params.file.path) { - console.log('[fileParserTool] Extracted file path from file object:', params.file.path) - filePath = params.file.path - } + // 2. Check for file upload (array) + else if (params.file && Array.isArray(params.file) && params.file.length > 0) { + logger.info('Tool body processing file array upload') + const filePaths = params.file.map((file: any) => file.path) + determinedFilePath = filePaths.length === 1 ? filePaths[0] : filePaths + } + // 3. Check for file upload (single object) + else if (params.file && params.file.path) { + logger.info('Tool body processing single file object upload') + determinedFilePath = params.file.path } - // Handle direct filePath parameter - else if (params.filePath) { - console.log('[fileParserTool] Using direct filePath parameter:', params.filePath) - filePath = params.filePath + // 4. Check for deprecated multiple files case (from older blocks?) + else if (params.files && Array.isArray(params.files)) { + logger.info('Tool body processing legacy files array:', params.files.length) + determinedFilePath = params.files.map((file: any) => file.path) } - if (!filePath) { - console.error('[fileParserTool] Missing required parameter: filePath') + // Final check if filePath was determined + if (!determinedFilePath) { + logger.error('Tool body could not determine filePath from parameters:', params) throw new Error('Missing required parameter: filePath') } + logger.info('Tool body determined filePath:', determinedFilePath) return { - filePath, - fileType: params.fileType, + filePath: determinedFilePath, + fileType: determinedFileType, } }, isInternalRoute: true, }, transformResponse: async (response: Response): Promise => { - console.log('[fileParserTool] Received response status:', response.status) + logger.info('Received response status:', response.status) try { const result = await response.json() - console.log('[fileParserTool] Response parsed successfully') + logger.info('Response parsed successfully') if (!response.ok) { const errorMsg = result.error || 'File parsing failed' - console.error('[fileParserTool] Error in response:', errorMsg) + logger.error('Error in response:', errorMsg) throw new Error(errorMsg) } // Handle multiple files response if (result.results) { - console.log('[fileParserTool] Processing multiple files response') + logger.info('Processing multiple files response') // Extract individual file results const fileResults = result.results.map((fileResult: any) => { if (!fileResult.success) { - console.warn( - `[fileParserTool] Error parsing file ${fileResult.filePath}: ${fileResult.error}` - ) + logger.warn(`Error parsing file ${fileResult.filePath}: ${fileResult.error}`) return { content: `Error parsing file: ${fileResult.error || 'Unknown error'}`, fileType: 'text/plain', @@ -161,7 +138,7 @@ export const fileParserTool: ToolConfig = { // Handle single file response if (result.success) { - console.log('[fileParserTool] Successfully parsed file:', result.output.name) + logger.info('Successfully parsed file:', result.output.name) // For a single file, create the output with both array and named property const output: FileParserOutputData = { @@ -179,13 +156,13 @@ export const fileParserTool: ToolConfig = { // Handle error response throw new Error(result.error || 'File parsing failed') } catch (error) { - console.error('[fileParserTool] Error processing response:', error) + logger.error('Error processing response:', error) throw error } }, transformError: (error: any) => { - console.error('[fileParserTool] Error occurred:', error) + logger.error('Error occurred:', error) return error.message || 'File parsing failed' }, } diff --git a/sim/tools/file/types.ts b/sim/tools/file/types.ts new file mode 100644 index 00000000000..73f4d3c436c --- /dev/null +++ b/sim/tools/file/types.ts @@ -0,0 +1,26 @@ +import { ToolResponse } from '../types' + +export interface FileParserInput { + filePath: string | string[] + fileType?: string + fileUrl?: string +} + +export interface FileParseResult { + content: string + fileType: string + size: number + name: string + binary: boolean + metadata?: Record +} + +export interface FileParserOutputData { + files: FileParseResult[] + combinedContent: string + [key: string]: any +} + +export interface FileParserOutput extends ToolResponse { + output: FileParserOutputData +} From 880e35dc52eaa4db4af09bcb464e1ac9ef61444e Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Sun, 6 Apr 2025 00:58:40 -0700 Subject: [PATCH 2/2] resolved PR comments --- sim/app/api/files/parse/route.ts | 51 ++++++++++++++++++++++++-------- sim/tools/file/parser.ts | 10 +++++-- sim/tools/file/types.ts | 1 - 3 files changed, 47 insertions(+), 15 deletions(-) diff --git a/sim/app/api/files/parse/route.ts b/sim/app/api/files/parse/route.ts index 5b6d3980888..77c2d1f8af2 100644 --- a/sim/app/api/files/parse/route.ts +++ b/sim/app/api/files/parse/route.ts @@ -1,8 +1,8 @@ import { NextRequest, NextResponse } from 'next/server' +import * as binExt from 'binary-extensions' +import { Buffer } from 'buffer' import { createHash } from 'crypto' -import { existsSync } from 'fs' -import fs from 'fs' -import { readFile, unlink, writeFile } from 'fs/promises' +import fsPromises, { readFile, unlink, writeFile } from 'fs/promises' import { tmpdir } from 'os' import { join } from 'path' import path from 'path' @@ -14,6 +14,10 @@ import '@/lib/uploads/setup.server' const logger = createLogger('FilesParseAPI') +// Constants for URL downloads +const MAX_DOWNLOAD_SIZE_BYTES = 100 * 1024 * 1024 // 100 MB +const DOWNLOAD_TIMEOUT_MS = 30000 // 30 seconds + interface ParseSuccessResult { success: true output: { @@ -180,12 +184,28 @@ async function handleExternalUrl(url: string, fileType?: string): Promise MAX_DOWNLOAD_SIZE_BYTES) { + throw new Error( + `File size (${prettySize(fileSize)}) exceeds the limit of ${prettySize( + MAX_DOWNLOAD_SIZE_BYTES + )}.` + ) + } + } else { + logger.warn('Content-Length header missing, cannot verify file size before download.') + } + // Get the file buffer from response const arrayBuffer = await response.arrayBuffer() const fileBuffer = Buffer.from(arrayBuffer) @@ -243,9 +263,13 @@ async function handleExternalUrl(url: string, fileType?: string): Promise = { let determinedFilePath: string | string[] | null = null let determinedFileType: string | undefined = params.fileType + // Determine the file path(s) based on input parameters. + // Precedence: direct filePath > file array > single file object > legacy files array // 1. Check for direct filePath (URL or single path from upload) if (params.filePath) { logger.info('Tool body found direct filePath:', params.filePath) @@ -49,7 +51,7 @@ export const fileParserTool: ToolConfig = { else if (params.file && Array.isArray(params.file) && params.file.length > 0) { logger.info('Tool body processing file array upload') const filePaths = params.file.map((file: any) => file.path) - determinedFilePath = filePaths.length === 1 ? filePaths[0] : filePaths + determinedFilePath = filePaths // Always send as array } // 3. Check for file upload (single object) else if (params.file && params.file.path) { @@ -59,7 +61,11 @@ export const fileParserTool: ToolConfig = { // 4. Check for deprecated multiple files case (from older blocks?) else if (params.files && Array.isArray(params.files)) { logger.info('Tool body processing legacy files array:', params.files.length) - determinedFilePath = params.files.map((file: any) => file.path) + if (params.files.length > 0) { + determinedFilePath = params.files.map((file: any) => file.path) + } else { + logger.warn('Legacy files array provided but is empty') + } } // Final check if filePath was determined diff --git a/sim/tools/file/types.ts b/sim/tools/file/types.ts index 73f4d3c436c..3a033934e35 100644 --- a/sim/tools/file/types.ts +++ b/sim/tools/file/types.ts @@ -3,7 +3,6 @@ import { ToolResponse } from '../types' export interface FileParserInput { filePath: string | string[] fileType?: string - fileUrl?: string } export interface FileParseResult {