Merge branch 'FlowiseAI:main' into main

Author: Mhydrosmart

Dockerfile

@@ -5,34 +5,41 @@
 # docker run -d -p 3000:3000 flowise
 
 FROM node:20-alpine
-RUN apk add --update libc6-compat python3 make g++
-# needed for pdfjs-dist
-RUN apk add --no-cache build-base cairo-dev pango-dev
 
-# Install Chromium
-RUN apk add --no-cache chromium
-
-# Install curl for container-level health checks
-# Fixes: https://github.com/FlowiseAI/Flowise/issues/4126
-RUN apk add --no-cache curl
-
-#install PNPM globaly
-RUN npm install -g pnpm
+# Install system dependencies and build tools
+RUN apk update && \
+    apk add --no-cache \
+        libc6-compat \
+        python3 \
+        make \
+        g++ \
+        build-base \
+        cairo-dev \
+        pango-dev \
+        chromium \
+        curl && \
+    npm install -g pnpm
 
 ENV PUPPETEER_SKIP_DOWNLOAD=true
 ENV PUPPETEER_EXECUTABLE_PATH=/usr/bin/chromium-browser
 
 ENV NODE_OPTIONS=--max-old-space-size=8192
 
-WORKDIR /usr/src
+WORKDIR /usr/src/flowise
 
 # Copy app source
 COPY . .
 
-RUN pnpm install
+# Install dependencies and build
+RUN pnpm install && \
+    pnpm build
+
+# Give the node user ownership of the application files
+RUN chown -R node:node .
 
-RUN pnpm build
+# Switch to non-root user (node user already exists in node:20-alpine)
+USER node
 
 EXPOSE 3000
 
-CMD [ "pnpm", "start" ]
+CMD [ "pnpm", "start" ]

package.json

@@ -1,6 +1,6 @@
 {
     "name": "flowise",
-    "version": "10",
+    "version": "3.0.11",
     "private": true,
     "homepage": "https://flowiseai.com",
     "workspaces": [

packages/components/nodes/documentloaders/Json/Json.ts

@@ -47,7 +47,7 @@ class Json_DocumentLoaders implements INode {
     constructor() {
         this.label = 'Json File'
         this.name = 'jsonFile'
-        this.version = 3.0
+        this.version = 3.1
         this.type = 'Document'
         this.icon = 'json.svg'
         this.category = 'Document Loaders'
@@ -66,14 +66,25 @@ class Json_DocumentLoaders implements INode {
                 type: 'TextSplitter',
                 optional: true
             },
+            {
+                label: 'Separate by JSON Object (JSON Array)',
+                name: 'separateByObject',
+                type: 'boolean',
+                description: 'If enabled and the file is a JSON Array, each JSON object will be extracted as a chunk',
+                optional: true,
+                additionalParams: true
+            },
             {
                 label: 'Pointers Extraction (separated by commas)',
                 name: 'pointersName',
                 type: 'string',
                 description:
                     'Ex: { "key": "value" }, Pointer Extraction = "key", "value" will be extracted as pageContent of the chunk. Use comma to separate multiple pointers',
                 placeholder: 'key1, key2',
-                optional: true
+                optional: true,
+                hide: {
+                    separateByObject: true
+                }
             },
             {
                 label: 'Additional Metadata',
@@ -122,6 +133,7 @@ class Json_DocumentLoaders implements INode {
         const pointersName = nodeData.inputs?.pointersName as string
         const metadata = nodeData.inputs?.metadata
         const _omitMetadataKeys = nodeData.inputs?.omitMetadataKeys as string
+        const separateByObject = nodeData.inputs?.separateByObject as boolean
         const output = nodeData.outputs?.output as string
 
         let omitMetadataKeys: string[] = []
@@ -153,7 +165,7 @@ class Json_DocumentLoaders implements INode {
                 if (!file) continue
                 const fileData = await getFileFromStorage(file, orgId, chatflowid)
                 const blob = new Blob([fileData])
-                const loader = new JSONLoader(blob, pointers.length != 0 ? pointers : undefined, metadata)
+                const loader = new JSONLoader(blob, pointers.length != 0 ? pointers : undefined, metadata, separateByObject)
 
                 if (textSplitter) {
                     let splittedDocs = await loader.load()
@@ -176,7 +188,7 @@ class Json_DocumentLoaders implements INode {
                 splitDataURI.pop()
                 const bf = Buffer.from(splitDataURI.pop() || '', 'base64')
                 const blob = new Blob([bf])
-                const loader = new JSONLoader(blob, pointers.length != 0 ? pointers : undefined, metadata)
+                const loader = new JSONLoader(blob, pointers.length != 0 ? pointers : undefined, metadata, separateByObject)
 
                 if (textSplitter) {
                     let splittedDocs = await loader.load()
@@ -306,13 +318,20 @@ class TextLoader extends BaseDocumentLoader {
 class JSONLoader extends TextLoader {
     public pointers: string[]
     private metadataMapping: Record<string, string>
-
-    constructor(filePathOrBlob: string | Blob, pointers: string | string[] = [], metadataMapping: Record<string, string> = {}) {
+    private separateByObject: boolean
+
+    constructor(
+        filePathOrBlob: string | Blob,
+        pointers: string | string[] = [],
+        metadataMapping: Record<string, string> = {},
+        separateByObject: boolean = false
+    ) {
         super(filePathOrBlob)
         this.pointers = Array.isArray(pointers) ? pointers : [pointers]
         if (metadataMapping) {
             this.metadataMapping = typeof metadataMapping === 'object' ? metadataMapping : JSON.parse(metadataMapping)
         }
+        this.separateByObject = separateByObject
     }
 
     protected async parse(raw: string): Promise<Document[]> {
@@ -323,14 +342,24 @@ class JSONLoader extends TextLoader {
         const jsonArray = Array.isArray(json) ? json : [json]
 
         for (const item of jsonArray) {
-            const content = this.extractContent(item)
-            const metadata = this.extractMetadata(item)
-
-            for (const pageContent of content) {
-                documents.push({
-                    pageContent,
-                    metadata
-                })
+            if (this.separateByObject) {
+                if (typeof item === 'object' && item !== null && !Array.isArray(item)) {
+                    const metadata = this.extractMetadata(item)
+                    const pageContent = this.formatObjectAsKeyValue(item)
+                    documents.push({
+                        pageContent,
+                        metadata
+                    })
+                }
+            } else {
+                const content = this.extractContent(item)
+                const metadata = this.extractMetadata(item)
+                for (const pageContent of content) {
+                    documents.push({
+                        pageContent,
+                        metadata
+                    })
+                }
             }
         }
 
@@ -370,6 +399,30 @@ class JSONLoader extends TextLoader {
         return metadata
     }
 
+    /**
+     * Formats a JSON object as readable key-value pairs
+     */
+    private formatObjectAsKeyValue(obj: any, prefix: string = ''): string {
+        const lines: string[] = []
+
+        for (const [key, value] of Object.entries(obj)) {
+            const fullKey = prefix ? `${prefix}.${key}` : key
+
+            if (value === null || value === undefined) {
+                lines.push(`${fullKey}: ${value}`)
+            } else if (Array.isArray(value)) {
+                lines.push(`${fullKey}: ${JSON.stringify(value)}`)
+            } else if (typeof value === 'object') {
+                // Recursively format nested objects
+                lines.push(this.formatObjectAsKeyValue(value, fullKey))
+            } else {
+                lines.push(`${fullKey}: ${value}`)
+            }
+        }
+
+        return lines.join('\n')
+    }
+
     /**
      * If JSON pointers are specified, return all strings below any of them
      * and exclude all other nodes expect if they match a JSON pointer.

packages/components/nodes/documentloaders/Playwright/Playwright.ts

@@ -190,11 +190,14 @@ class Playwright_DocumentLoaders implements INode {
         async function playwrightLoader(url: string): Promise<Document[] | undefined> {
             try {
                 let docs = []
+
+                const executablePath = process.env.PLAYWRIGHT_EXECUTABLE_PATH
+
                 const config: PlaywrightWebBaseLoaderOptions = {
                     launchOptions: {
                         args: ['--no-sandbox'],
                         headless: true,
-                        executablePath: process.env.PLAYWRIGHT_EXECUTABLE_FILE_PATH
+                        executablePath: executablePath
                     }
                 }
                 if (waitUntilGoToOption) {

packages/components/nodes/documentloaders/Puppeteer/Puppeteer.ts

@@ -181,11 +181,14 @@ class Puppeteer_DocumentLoaders implements INode {
         async function puppeteerLoader(url: string): Promise<Document[] | undefined> {
             try {
                 let docs: Document[] = []
+
+                const executablePath = process.env.PUPPETEER_EXECUTABLE_PATH
+
                 const config: PuppeteerWebBaseLoaderOptions = {
                     launchOptions: {
                         args: ['--no-sandbox'],
                         headless: 'new',
-                        executablePath: process.env.PUPPETEER_EXECUTABLE_FILE_PATH
+                        executablePath: executablePath
                     }
                 }
                 if (waitUntilGoToOption) {

packages/components/nodes/documentloaders/Unstructured/Unstructured.ts

@@ -27,8 +27,6 @@ type Element = {
 }
 
 export class UnstructuredLoader extends BaseDocumentLoader {
-    public filePath: string
-
     private apiUrl = process.env.UNSTRUCTURED_API_URL || 'https://api.unstructuredapp.io/general/v0/general'
 
     private apiKey: string | undefined = process.env.UNSTRUCTURED_API_KEY
@@ -138,7 +136,7 @@ export class UnstructuredLoader extends BaseDocumentLoader {
         })
 
         if (!response.ok) {
-            throw new Error(`Failed to partition file ${this.filePath} with error ${response.status} and message ${await response.text()}`)
+            throw new Error(`Failed to partition file with error ${response.status} and message ${await response.text()}`)
         }
 
         const elements = await response.json()

packages/components/nodes/documentloaders/Unstructured/UnstructuredFile.ts

@@ -4,15 +4,11 @@ import {
     UnstructuredLoaderOptions,
     UnstructuredLoaderStrategy,
     SkipInferTableTypes,
-    HiResModelName,
-    UnstructuredLoader as LCUnstructuredLoader
+    HiResModelName
 } from '@langchain/community/document_loaders/fs/unstructured'
 import { getCredentialData, getCredentialParam, handleEscapeCharacters } from '../../../src/utils'
 import { getFileFromStorage, INodeOutputsValue } from '../../../src'
 import { UnstructuredLoader } from './Unstructured'
-import { isPathTraversal } from '../../../src/validator'
-import sanitize from 'sanitize-filename'
-import path from 'path'
 
 class UnstructuredFile_DocumentLoaders implements INode {
     label: string
@@ -44,17 +40,6 @@ class UnstructuredFile_DocumentLoaders implements INode {
             optional: true
         }
         this.inputs = [
-            /** Deprecated
-            {
-                label: 'File Path',
-                name: 'filePath',
-                type: 'string',
-                placeholder: '',
-                optional: true,
-                warning:
-                    'Use the File Upload instead of File path. If file is uploaded, this path is ignored. Path will be deprecated in future releases.'
-            },
-             */
             {
                 label: 'Files Upload',
                 name: 'fileObject',
@@ -455,7 +440,6 @@ class UnstructuredFile_DocumentLoaders implements INode {
     }
 
     async init(nodeData: INodeData, _: string, options: ICommonObject): Promise<any> {
-        const filePath = nodeData.inputs?.filePath as string
         const unstructuredAPIUrl = nodeData.inputs?.unstructuredAPIUrl as string
         const strategy = nodeData.inputs?.strategy as UnstructuredLoaderStrategy
         const encoding = nodeData.inputs?.encoding as string
@@ -560,37 +544,8 @@ class UnstructuredFile_DocumentLoaders implements INode {
                     docs.push(...loaderDocs)
                 }
             }
-        } else if (filePath) {
-            if (!filePath || typeof filePath !== 'string') {
-                throw new Error('Invalid file path format')
-            }
-
-            if (isPathTraversal(filePath)) {
-                throw new Error('Invalid path characters detected in filePath - path traversal not allowed')
-            }
-
-            const parsedPath = path.parse(filePath)
-            const sanitizedFilename = sanitize(parsedPath.base)
-
-            if (!sanitizedFilename || sanitizedFilename.trim() === '') {
-                throw new Error('Invalid filename after sanitization')
-            }
-
-            const sanitizedFilePath = path.join(parsedPath.dir, sanitizedFilename)
-
-            if (!path.isAbsolute(sanitizedFilePath)) {
-                throw new Error('File path must be absolute')
-            }
-
-            if (sanitizedFilePath.includes('..')) {
-                throw new Error('Invalid file path - directory traversal not allowed')
-            }
-
-            const loader = new LCUnstructuredLoader(sanitizedFilePath, obj)
-            const loaderDocs = await loader.load()
-            docs.push(...loaderDocs)
         } else {
-            throw new Error('File path or File upload is required')
+            throw new Error('File upload is required')
         }
 
         if (metadata) {

packages/components/nodes/documentloaders/Unstructured/UnstructuredFolder.ts

@@ -1,3 +1,6 @@
+/*
+* Uncomment this if you want to use the UnstructuredFolder to load a folder from the file system
+
 import { omit } from 'lodash'
 import { ICommonObject, INode, INodeData, INodeOutputsValue, INodeParams } from '../../../src/Interface'
 import {
@@ -516,3 +519,4 @@ class UnstructuredFolder_DocumentLoaders implements INode {
 }
 
 module.exports = { nodeClass: UnstructuredFolder_DocumentLoaders }
+*/