diff --git a/.devcontainer/docker-compose.yml b/.devcontainer/docker-compose.yml index 277ac84f856..e7c36c55352 100644 --- a/.devcontainer/docker-compose.yml +++ b/.devcontainer/docker-compose.yml @@ -1,5 +1,3 @@ -version: "3.8" - services: app: build: diff --git a/.env.example b/.env.example index 5571fe46cf1..dd99dce7567 100644 --- a/.env.example +++ b/.env.example @@ -82,7 +82,7 @@ PROXY= #============# ANTHROPIC_API_KEY=user_provided -# ANTHROPIC_MODELS=claude-3-5-sonnet-20240620,claude-3-opus-20240229,claude-3-sonnet-20240229,claude-3-haiku-20240307,claude-2.1,claude-2,claude-1.2,claude-1,claude-1-100k,claude-instant-1,claude-instant-1-100k +# ANTHROPIC_MODELS=claude-3-5-sonnet-20241022,claude-3-5-sonnet-latest,claude-3-5-sonnet-20240620,claude-3-opus-20240229,claude-3-sonnet-20240229,claude-3-haiku-20240307,claude-2.1,claude-2,claude-1.2,claude-1,claude-1-100k,claude-instant-1,claude-instant-1-100k # ANTHROPIC_REVERSE_PROXY= #============# @@ -146,6 +146,8 @@ GOOGLE_KEY=user_provided # GOOGLE_TITLE_MODEL=gemini-pro +# GOOGLE_LOC=us-central1 + # Google Safety Settings # NOTE: These settings apply to both Vertex AI and Gemini API (AI Studio) # @@ -412,6 +414,7 @@ LDAP_CA_CERT_PATH= # LDAP_LOGIN_USES_USERNAME=true # LDAP_ID= # LDAP_USERNAME= +# LDAP_EMAIL= # LDAP_FULL_NAME= #========================# diff --git a/.github/dependabot.yml b/.github/dependabot.yml deleted file mode 100644 index ccdc68d81b3..00000000000 --- a/.github/dependabot.yml +++ /dev/null @@ -1,47 +0,0 @@ -# To get started with Dependabot version updates, you'll need to specify which -# package ecosystems to update and where the package manifests are located. -# Please see the documentation for all configuration options: -# https://docs.github.com/github/administering-a-repository/configuration-options-for-dependency-updates - -version: 2 -updates: - - package-ecosystem: "npm" # See documentation for possible values - directory: "/api" # Location of package manifests - target-branch: "dev" - versioning-strategy: increase-if-necessary - schedule: - interval: "weekly" - allow: - # Allow both direct and indirect updates for all packages - - dependency-type: "all" - commit-message: - prefix: "npm api prod" - prefix-development: "npm api dev" - include: "scope" - - package-ecosystem: "npm" # See documentation for possible values - directory: "/client" # Location of package manifests - target-branch: "dev" - versioning-strategy: increase-if-necessary - schedule: - interval: "weekly" - allow: - # Allow both direct and indirect updates for all packages - - dependency-type: "all" - commit-message: - prefix: "npm client prod" - prefix-development: "npm client dev" - include: "scope" - - package-ecosystem: "npm" # See documentation for possible values - directory: "/" # Location of package manifests - target-branch: "dev" - versioning-strategy: increase-if-necessary - schedule: - interval: "weekly" - allow: - # Allow both direct and indirect updates for all packages - - dependency-type: "all" - commit-message: - prefix: "npm all prod" - prefix-development: "npm all dev" - include: "scope" - diff --git a/.github/workflows/helmcharts.yml b/.github/workflows/helmcharts.yml index fcd8bc7df64..bc715557e4a 100644 --- a/.github/workflows/helmcharts.yml +++ b/.github/workflows/helmcharts.yml @@ -25,11 +25,9 @@ jobs: - name: Install Helm uses: azure/setup-helm@v4 env: - GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" + GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" - name: Run chart-releaser uses: helm/chart-releaser-action@v1.6.0 - with: - charts_dir: helmchart env: - CR_TOKEN: "${{ secrets.GITHUB_TOKEN }}" \ No newline at end of file + CR_TOKEN: "${{ secrets.GITHUB_TOKEN }}" diff --git a/Dockerfile b/Dockerfile index 344ec0297b1..40d13ef56c3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -# v0.7.5-rc2 +# v0.7.5 # Base node image FROM node:20-alpine AS node diff --git a/Dockerfile.multi b/Dockerfile.multi index a32183d82f3..4d58de0c838 100644 --- a/Dockerfile.multi +++ b/Dockerfile.multi @@ -1,5 +1,5 @@ # Dockerfile.multi -# v0.7.5-rc2 +# v0.7.5 # Base for all builds FROM node:20-alpine AS base diff --git a/api/app/clients/AnthropicClient.js b/api/app/clients/AnthropicClient.js index 873f2615695..ff0dc4de5b7 100644 --- a/api/app/clients/AnthropicClient.js +++ b/api/app/clients/AnthropicClient.js @@ -17,8 +17,8 @@ const { parseParamFromPrompt, createContextHandlers, } = require('./prompts'); +const { getModelMaxTokens, getModelMaxOutputTokens, matchModelName } = require('~/utils'); const { spendTokens, spendStructuredTokens } = require('~/models/spendTokens'); -const { getModelMaxTokens, matchModelName } = require('~/utils'); const { sleep } = require('~/server/utils'); const BaseClient = require('./BaseClient'); const { logger } = require('~/config'); @@ -64,6 +64,12 @@ class AnthropicClient extends BaseClient { /** Whether or not the model supports Prompt Caching * @type {boolean} */ this.supportsCacheControl; + /** The key for the usage object's input tokens + * @type {string} */ + this.inputTokensKey = 'input_tokens'; + /** The key for the usage object's output tokens + * @type {string} */ + this.outputTokensKey = 'output_tokens'; } setOptions(options) { @@ -114,7 +120,14 @@ class AnthropicClient extends BaseClient { this.options.maxContextTokens ?? getModelMaxTokens(this.modelOptions.model, EModelEndpoint.anthropic) ?? 100000; - this.maxResponseTokens = this.modelOptions.maxOutputTokens || 1500; + this.maxResponseTokens = + this.modelOptions.maxOutputTokens ?? + getModelMaxOutputTokens( + this.modelOptions.model, + this.options.endpointType ?? this.options.endpoint, + this.options.endpointTokenConfig, + ) ?? + 1500; this.maxPromptTokens = this.options.maxPromptTokens || this.maxContextTokens - this.maxResponseTokens; @@ -138,17 +151,6 @@ class AnthropicClient extends BaseClient { this.endToken = ''; this.gptEncoder = this.constructor.getTokenizer('cl100k_base'); - if (!this.modelOptions.stop) { - const stopTokens = [this.startToken]; - if (this.endToken && this.endToken !== this.startToken) { - stopTokens.push(this.endToken); - } - stopTokens.push(`${this.userLabel}`); - stopTokens.push('<|diff_marker|>'); - - this.modelOptions.stop = stopTokens; - } - return this; } @@ -200,7 +202,7 @@ class AnthropicClient extends BaseClient { } /** - * Calculates the correct token count for the current message based on the token count map and API usage. + * Calculates the correct token count for the current user message based on the token count map and API usage. * Edge case: If the calculation results in a negative value, it returns the original estimate. * If revisiting a conversation with a chat history entirely composed of token estimates, * the cumulative token count going forward should become more accurate as the conversation progresses. @@ -208,7 +210,7 @@ class AnthropicClient extends BaseClient { * @param {Record} params.tokenCountMap - A map of message IDs to their token counts. * @param {string} params.currentMessageId - The ID of the current message to calculate. * @param {AnthropicStreamUsage} params.usage - The usage object returned by the API. - * @returns {number} The correct token count for the current message. + * @returns {number} The correct token count for the current user message. */ calculateCurrentTokenCount({ tokenCountMap, currentMessageId, usage }) { const originalEstimate = tokenCountMap[currentMessageId] || 0; @@ -680,7 +682,14 @@ class AnthropicClient extends BaseClient { */ checkPromptCacheSupport(modelName) { const modelMatch = matchModelName(modelName, EModelEndpoint.anthropic); - if (modelMatch === 'claude-3-5-sonnet' || modelMatch === 'claude-3-haiku') { + if (modelMatch.includes('claude-3-5-sonnet-latest')) { + return false; + } + if ( + modelMatch === 'claude-3-5-sonnet' || + modelMatch === 'claude-3-haiku' || + modelMatch === 'claude-3-opus' + ) { return true; } return false; diff --git a/api/app/clients/BaseClient.js b/api/app/clients/BaseClient.js index 1d36d81d1af..33e3df3ac6e 100644 --- a/api/app/clients/BaseClient.js +++ b/api/app/clients/BaseClient.js @@ -3,7 +3,7 @@ const fetch = require('node-fetch'); const { supportsBalanceCheck, isAgentsEndpoint, - paramEndpoints, + isParamEndpoint, ErrorTypes, Constants, CacheKeys, @@ -42,6 +42,14 @@ class BaseClient { this.conversationId; /** @type {string} */ this.responseMessageId; + /** @type {TAttachment[]} */ + this.attachments; + /** The key for the usage object's input tokens + * @type {string} */ + this.inputTokensKey = 'prompt_tokens'; + /** The key for the usage object's output tokens + * @type {string} */ + this.outputTokensKey = 'completion_tokens'; } setOptions() { @@ -582,7 +590,10 @@ class BaseClient { if (typeof completion === 'string') { responseMessage.text = addSpaceIfNeeded(generation) + completion; - } else if (Array.isArray(completion) && paramEndpoints.has(this.options.endpoint)) { + } else if ( + Array.isArray(completion) && + isParamEndpoint(this.options.endpoint, this.options.endpointType) + ) { responseMessage.text = ''; responseMessage.content = completion; } else if (Array.isArray(completion)) { @@ -604,8 +615,8 @@ class BaseClient { * @type {StreamUsage | null} */ const usage = this.getStreamUsage != null ? this.getStreamUsage() : null; - if (usage != null && Number(usage.output_tokens) > 0) { - responseMessage.tokenCount = usage.output_tokens; + if (usage != null && Number(usage[this.outputTokensKey]) > 0) { + responseMessage.tokenCount = usage[this.outputTokensKey]; completionTokens = responseMessage.tokenCount; await this.updateUserMessageTokenCount({ usage, tokenCountMap, userMessage, opts }); } else { @@ -620,6 +631,10 @@ class BaseClient { await this.userMessagePromise; } + if (this.artifactPromises) { + responseMessage.attachments = (await Promise.all(this.artifactPromises)).filter((a) => a); + } + this.responsePromise = this.saveMessageToDatabase(responseMessage, saveOptions, user); const messageCache = getLogStores(CacheKeys.MESSAGES); messageCache.set( @@ -655,7 +670,7 @@ class BaseClient { /** @type {boolean} */ const shouldUpdateCount = this.calculateCurrentTokenCount != null && - Number(usage.input_tokens) > 0 && + Number(usage[this.inputTokensKey]) > 0 && (this.options.resendFiles || (!this.options.resendFiles && !this.options.attachments?.length)) && !this.options.promptPrefix; diff --git a/api/app/clients/ChatGPTClient.js b/api/app/clients/ChatGPTClient.js index 0a7f6fc7d88..104e9e5ac3f 100644 --- a/api/app/clients/ChatGPTClient.js +++ b/api/app/clients/ChatGPTClient.js @@ -1,19 +1,21 @@ const Keyv = require('keyv'); const crypto = require('crypto'); +const { CohereClient } = require('cohere-ai'); +const { fetchEventSource } = require('@waylaidwanderer/fetch-event-source'); +const { encoding_for_model: encodingForModel, get_encoding: getEncoding } = require('tiktoken'); const { + ImageDetail, EModelEndpoint, resolveHeaders, CohereConstants, mapModelToAzureConfig, } = require('librechat-data-provider'); -const { CohereClient } = require('cohere-ai'); -const { encoding_for_model: encodingForModel, get_encoding: getEncoding } = require('tiktoken'); -const { fetchEventSource } = require('@waylaidwanderer/fetch-event-source'); +const { extractBaseURL, constructAzureURL, genAzureChatCompletion } = require('~/utils'); +const { createContextHandlers } = require('./prompts'); const { createCoherePayload } = require('./llm'); const { Agent, ProxyAgent } = require('undici'); const BaseClient = require('./BaseClient'); const { logger } = require('~/config'); -const { extractBaseURL, constructAzureURL, genAzureChatCompletion } = require('~/utils'); const CHATGPT_MODEL = 'gpt-3.5-turbo'; const tokenizersCache = {}; @@ -612,21 +614,66 @@ ${botMessage.message} async buildPrompt(messages, { isChatGptModel = false, promptPrefix = null }) { promptPrefix = (promptPrefix || this.options.promptPrefix || '').trim(); + + // Handle attachments and create augmentedPrompt + if (this.options.attachments) { + const attachments = await this.options.attachments; + const lastMessage = messages[messages.length - 1]; + + if (this.message_file_map) { + this.message_file_map[lastMessage.messageId] = attachments; + } else { + this.message_file_map = { + [lastMessage.messageId]: attachments, + }; + } + + const files = await this.addImageURLs(lastMessage, attachments); + this.options.attachments = files; + + this.contextHandlers = createContextHandlers(this.options.req, lastMessage.text); + } + + if (this.message_file_map) { + this.contextHandlers = createContextHandlers( + this.options.req, + messages[messages.length - 1].text, + ); + } + + // Calculate image token cost and process embedded files + messages.forEach((message, i) => { + if (this.message_file_map && this.message_file_map[message.messageId]) { + const attachments = this.message_file_map[message.messageId]; + for (const file of attachments) { + if (file.embedded) { + this.contextHandlers?.processFile(file); + continue; + } + + messages[i].tokenCount = + (messages[i].tokenCount || 0) + + this.calculateImageTokenCost({ + width: file.width, + height: file.height, + detail: this.options.imageDetail ?? ImageDetail.auto, + }); + } + } + }); + + if (this.contextHandlers) { + this.augmentedPrompt = await this.contextHandlers.createContext(); + promptPrefix = this.augmentedPrompt + promptPrefix; + } + if (promptPrefix) { // If the prompt prefix doesn't end with the end token, add it. if (!promptPrefix.endsWith(`${this.endToken}`)) { promptPrefix = `${promptPrefix.trim()}${this.endToken}\n\n`; } promptPrefix = `${this.startToken}Instructions:\n${promptPrefix}`; - } else { - const currentDateString = new Date().toLocaleDateString('en-us', { - year: 'numeric', - month: 'long', - day: 'numeric', - }); - promptPrefix = `${this.startToken}Instructions:\nYou are ChatGPT, a large language model trained by OpenAI. Respond conversationally.\nCurrent date: ${currentDateString}${this.endToken}\n\n`; } - const promptSuffix = `${this.startToken}${this.chatGptLabel}:\n`; // Prompt ChatGPT to respond. const instructionsPayload = { @@ -714,10 +761,6 @@ ${botMessage.message} this.maxResponseTokens, ); - if (this.options.debug) { - console.debug(`Prompt : ${prompt}`); - } - if (isChatGptModel) { return { prompt: [instructionsPayload, messagePayload], context }; } diff --git a/api/app/clients/GoogleClient.js b/api/app/clients/GoogleClient.js index 92dd54be8d5..99b12428375 100644 --- a/api/app/clients/GoogleClient.js +++ b/api/app/clients/GoogleClient.js @@ -28,7 +28,7 @@ const { } = require('./prompts'); const BaseClient = require('./BaseClient'); -const loc = 'us-central1'; +const loc = process.env.GOOGLE_LOC || 'us-central1'; const publisher = 'google'; const endpointPrefix = `https://${loc}-aiplatform.googleapis.com`; // const apiEndpoint = loc + '-aiplatform.googleapis.com'; @@ -593,6 +593,8 @@ class GoogleClient extends BaseClient { createLLM(clientOptions) { const model = clientOptions.modelName ?? clientOptions.model; + clientOptions.location = loc; + clientOptions.endpoint = `${loc}-aiplatform.googleapis.com`; if (this.project_id && this.isTextModel) { logger.debug('Creating Google VertexAI client'); return new GoogleVertexAI(clientOptions); diff --git a/api/app/clients/OllamaClient.js b/api/app/clients/OllamaClient.js index c88ef72d580..d86e120f437 100644 --- a/api/app/clients/OllamaClient.js +++ b/api/app/clients/OllamaClient.js @@ -60,7 +60,9 @@ class OllamaClient { try { const ollamaEndpoint = deriveBaseURL(baseURL); /** @type {Promise>} */ - const response = await axios.get(`${ollamaEndpoint}/api/tags`); + const response = await axios.get(`${ollamaEndpoint}/api/tags`, { + timeout: 5000, + }); models = response.data.models.map((tag) => tag.name); return models; } catch (error) { diff --git a/api/app/clients/OpenAIClient.js b/api/app/clients/OpenAIClient.js index 79582d1d389..d2762ca1d79 100644 --- a/api/app/clients/OpenAIClient.js +++ b/api/app/clients/OpenAIClient.js @@ -19,6 +19,7 @@ const { constructAzureURL, getModelMaxTokens, genAzureChatCompletion, + getModelMaxOutputTokens, } = require('~/utils'); const { truncateText, @@ -64,6 +65,11 @@ class OpenAIClient extends BaseClient { /** @type {string | undefined} - The API Completions URL */ this.completionsUrl; + + /** @type {OpenAIUsageMetadata | undefined} */ + this.usage; + /** @type {boolean|undefined} */ + this.isO1Model; } // TODO: PluginsClient calls this 3x, unneeded @@ -94,6 +100,8 @@ class OpenAIClient extends BaseClient { this.options.modelOptions, ); + this.isO1Model = /\bo1\b/i.test(this.modelOptions.model); + this.defaultVisionModel = this.options.visionModel ?? 'gpt-4-vision-preview'; if (typeof this.options.attachments?.then === 'function') { this.options.attachments.then((attachments) => this.checkVisionRequest(attachments)); @@ -138,7 +146,8 @@ class OpenAIClient extends BaseClient { const { model } = this.modelOptions; - this.isChatCompletion = this.useOpenRouter || !!reverseProxy || model.includes('gpt'); + this.isChatCompletion = + /\bo1\b/i.test(model) || model.includes('gpt') || this.useOpenRouter || !!reverseProxy; this.isChatGptModel = this.isChatCompletion; if ( model.includes('text-davinci') || @@ -169,7 +178,14 @@ class OpenAIClient extends BaseClient { logger.debug('[OpenAIClient] maxContextTokens', this.maxContextTokens); } - this.maxResponseTokens = this.modelOptions.max_tokens || 1024; + this.maxResponseTokens = + this.modelOptions.max_tokens ?? + getModelMaxOutputTokens( + model, + this.options.endpointType ?? this.options.endpoint, + this.options.endpointTokenConfig, + ) ?? + 1024; this.maxPromptTokens = this.options.maxPromptTokens || this.maxContextTokens - this.maxResponseTokens; @@ -187,8 +203,8 @@ class OpenAIClient extends BaseClient { model: this.modelOptions.model, endpoint: this.options.endpoint, endpointType: this.options.endpointType, - chatGptLabel: this.options.chatGptLabel, modelDisplayLabel: this.options.modelDisplayLabel, + chatGptLabel: this.options.chatGptLabel || this.options.modelLabel, }); this.userLabel = this.options.userLabel || 'User'; @@ -533,7 +549,7 @@ class OpenAIClient extends BaseClient { promptPrefix = this.augmentedPrompt + promptPrefix; } - if (promptPrefix) { + if (promptPrefix && this.isO1Model !== true) { promptPrefix = `Instructions:\n${promptPrefix.trim()}`; instructions = { role: 'system', @@ -561,6 +577,16 @@ class OpenAIClient extends BaseClient { messages, }; + /** EXPERIMENTAL */ + if (promptPrefix && this.isO1Model === true) { + const lastUserMessageIndex = payload.findLastIndex((message) => message.role === 'user'); + if (lastUserMessageIndex !== -1) { + payload[ + lastUserMessageIndex + ].content = `${promptPrefix}\n${payload[lastUserMessageIndex].content}`; + } + } + if (tokenCountMap) { tokenCountMap.instructions = instructions?.tokenCount; result.tokenCountMap = tokenCountMap; @@ -621,6 +647,12 @@ class OpenAIClient extends BaseClient { if (completionResult && typeof completionResult === 'string') { reply = completionResult; + } else if ( + completionResult && + typeof completionResult === 'object' && + Array.isArray(completionResult.choices) + ) { + reply = completionResult.choices[0]?.text?.replace(this.endToken, ''); } } else if (typeof opts.onProgress === 'function' || this.options.useChatCompletion) { reply = await this.chatCompletion({ @@ -810,27 +842,27 @@ class OpenAIClient extends BaseClient { } const titleChatCompletion = async () => { - modelOptions.model = model; + try { + modelOptions.model = model; - if (this.azure) { - modelOptions.model = process.env.AZURE_OPENAI_DEFAULT_MODEL ?? modelOptions.model; - this.azureEndpoint = genAzureChatCompletion(this.azure, modelOptions.model, this); - } + if (this.azure) { + modelOptions.model = process.env.AZURE_OPENAI_DEFAULT_MODEL ?? modelOptions.model; + this.azureEndpoint = genAzureChatCompletion(this.azure, modelOptions.model, this); + } - const instructionsPayload = [ - { - role: this.options.titleMessageRole ?? (this.isOllama ? 'user' : 'system'), - content: `Please generate ${titleInstruction} + const instructionsPayload = [ + { + role: this.options.titleMessageRole ?? (this.isOllama ? 'user' : 'system'), + content: `Please generate ${titleInstruction} ${convo} ||>Title:`, - }, - ]; + }, + ]; - const promptTokens = this.getTokenCountForMessage(instructionsPayload[0]); + const promptTokens = this.getTokenCountForMessage(instructionsPayload[0]); - try { let useChatCompletion = true; if (this.options.reverseProxyUrl === CohereConstants.API_URL) { @@ -885,6 +917,60 @@ ${convo} return title; } + /** + * Get stream usage as returned by this client's API response. + * @returns {OpenAIUsageMetadata} The stream usage object. + */ + getStreamUsage() { + if ( + this.usage && + typeof this.usage === 'object' && + 'completion_tokens_details' in this.usage && + this.usage.completion_tokens_details && + typeof this.usage.completion_tokens_details === 'object' && + 'reasoning_tokens' in this.usage.completion_tokens_details + ) { + const outputTokens = Math.abs( + this.usage.completion_tokens_details.reasoning_tokens - this.usage[this.outputTokensKey], + ); + return { + ...this.usage.completion_tokens_details, + [this.inputTokensKey]: this.usage[this.inputTokensKey], + [this.outputTokensKey]: outputTokens, + }; + } + return this.usage; + } + + /** + * Calculates the correct token count for the current user message based on the token count map and API usage. + * Edge case: If the calculation results in a negative value, it returns the original estimate. + * If revisiting a conversation with a chat history entirely composed of token estimates, + * the cumulative token count going forward should become more accurate as the conversation progresses. + * @param {Object} params - The parameters for the calculation. + * @param {Record} params.tokenCountMap - A map of message IDs to their token counts. + * @param {string} params.currentMessageId - The ID of the current message to calculate. + * @param {OpenAIUsageMetadata} params.usage - The usage object returned by the API. + * @returns {number} The correct token count for the current user message. + */ + calculateCurrentTokenCount({ tokenCountMap, currentMessageId, usage }) { + const originalEstimate = tokenCountMap[currentMessageId] || 0; + + if (!usage || typeof usage[this.inputTokensKey] !== 'number') { + return originalEstimate; + } + + tokenCountMap[currentMessageId] = 0; + const totalTokensFromMap = Object.values(tokenCountMap).reduce((sum, count) => { + const numCount = Number(count); + return sum + (isNaN(numCount) ? 0 : numCount); + }, 0); + const totalInputTokens = usage[this.inputTokensKey] ?? 0; + + const currentMessageTokens = totalInputTokens - totalTokensFromMap; + return currentMessageTokens > 0 ? currentMessageTokens : originalEstimate; + } + async summarizeMessages({ messagesToRefine, remainingContextTokens }) { logger.debug('[OpenAIClient] Summarizing messages...'); let context = messagesToRefine; @@ -1000,7 +1086,16 @@ ${convo} } } - async recordTokenUsage({ promptTokens, completionTokens, context = 'message' }) { + /** + * @param {object} params + * @param {number} params.promptTokens + * @param {number} params.completionTokens + * @param {OpenAIUsageMetadata} [params.usage] + * @param {string} [params.model] + * @param {string} [params.context='message'] + * @returns {Promise} + */ + async recordTokenUsage({ promptTokens, completionTokens, usage, context = 'message' }) { await spendTokens( { context, @@ -1011,6 +1106,24 @@ ${convo} }, { promptTokens, completionTokens }, ); + + if ( + usage && + typeof usage === 'object' && + 'reasoning_tokens' in usage && + typeof usage.reasoning_tokens === 'number' + ) { + await spendTokens( + { + context: 'reasoning', + model: this.modelOptions.model, + conversationId: this.conversationId, + user: this.user ?? this.options.req.user?.id, + endpointTokenConfig: this.options.endpointTokenConfig, + }, + { completionTokens: usage.reasoning_tokens }, + ); + } } getTokenCountForResponse(response) { @@ -1117,6 +1230,11 @@ ${convo} opts.defaultHeaders = { ...opts.defaultHeaders, 'api-key': this.apiKey }; } + if (this.isO1Model === true && modelOptions.max_tokens != null) { + modelOptions.max_completion_tokens = modelOptions.max_tokens; + delete modelOptions.max_tokens; + } + if (process.env.OPENAI_ORGANIZATION) { opts.organization = process.env.OPENAI_ORGANIZATION; } @@ -1191,6 +1309,11 @@ ${convo} /** @type {(value: void | PromiseLike) => void} */ let streamResolve; + if (modelOptions.stream && this.isO1Model) { + delete modelOptions.stream; + delete modelOptions.stop; + } + if (modelOptions.stream) { streamPromise = new Promise((resolve) => { streamResolve = resolve; @@ -1269,6 +1392,8 @@ ${convo} } const { choices } = chatCompletion; + this.usage = chatCompletion.usage; + if (!Array.isArray(choices) || choices.length === 0) { logger.warn('[OpenAIClient] Chat completion response has no choices'); return intermediateReply.join(''); diff --git a/api/app/clients/prompts/formatAgentMessages.spec.js b/api/app/clients/prompts/formatAgentMessages.spec.js new file mode 100644 index 00000000000..fe0f1e02281 --- /dev/null +++ b/api/app/clients/prompts/formatAgentMessages.spec.js @@ -0,0 +1,285 @@ +const { ToolMessage } = require('@langchain/core/messages'); +const { ContentTypes } = require('librechat-data-provider'); +const { HumanMessage, AIMessage, SystemMessage } = require('langchain/schema'); +const { formatAgentMessages } = require('./formatMessages'); + +describe('formatAgentMessages', () => { + it('should format simple user and AI messages', () => { + const payload = [ + { role: 'user', content: 'Hello' }, + { role: 'assistant', content: 'Hi there!' }, + ]; + const result = formatAgentMessages(payload); + expect(result).toHaveLength(2); + expect(result[0]).toBeInstanceOf(HumanMessage); + expect(result[1]).toBeInstanceOf(AIMessage); + }); + + it('should handle system messages', () => { + const payload = [{ role: 'system', content: 'You are a helpful assistant.' }]; + const result = formatAgentMessages(payload); + expect(result).toHaveLength(1); + expect(result[0]).toBeInstanceOf(SystemMessage); + }); + + it('should format messages with content arrays', () => { + const payload = [ + { + role: 'user', + content: [{ type: ContentTypes.TEXT, [ContentTypes.TEXT]: 'Hello' }], + }, + ]; + const result = formatAgentMessages(payload); + expect(result).toHaveLength(1); + expect(result[0]).toBeInstanceOf(HumanMessage); + }); + + it('should handle tool calls and create ToolMessages', () => { + const payload = [ + { + role: 'assistant', + content: [ + { + type: ContentTypes.TEXT, + [ContentTypes.TEXT]: 'Let me check that for you.', + tool_call_ids: ['123'], + }, + { + type: ContentTypes.TOOL_CALL, + tool_call: { + id: '123', + name: 'search', + args: '{"query":"weather"}', + output: 'The weather is sunny.', + }, + }, + ], + }, + ]; + const result = formatAgentMessages(payload); + expect(result).toHaveLength(2); + expect(result[0]).toBeInstanceOf(AIMessage); + expect(result[1]).toBeInstanceOf(ToolMessage); + expect(result[0].tool_calls).toHaveLength(1); + expect(result[1].tool_call_id).toBe('123'); + }); + + it('should handle multiple content parts in assistant messages', () => { + const payload = [ + { + role: 'assistant', + content: [ + { type: ContentTypes.TEXT, [ContentTypes.TEXT]: 'Part 1' }, + { type: ContentTypes.TEXT, [ContentTypes.TEXT]: 'Part 2' }, + ], + }, + ]; + const result = formatAgentMessages(payload); + expect(result).toHaveLength(1); + expect(result[0]).toBeInstanceOf(AIMessage); + expect(result[0].content).toHaveLength(2); + }); + + it('should throw an error for invalid tool call structure', () => { + const payload = [ + { + role: 'assistant', + content: [ + { + type: ContentTypes.TOOL_CALL, + tool_call: { + id: '123', + name: 'search', + args: '{"query":"weather"}', + output: 'The weather is sunny.', + }, + }, + ], + }, + ]; + expect(() => formatAgentMessages(payload)).toThrow('Invalid tool call structure'); + }); + + it('should handle tool calls with non-JSON args', () => { + const payload = [ + { + role: 'assistant', + content: [ + { type: ContentTypes.TEXT, [ContentTypes.TEXT]: 'Checking...', tool_call_ids: ['123'] }, + { + type: ContentTypes.TOOL_CALL, + tool_call: { + id: '123', + name: 'search', + args: 'non-json-string', + output: 'Result', + }, + }, + ], + }, + ]; + const result = formatAgentMessages(payload); + expect(result).toHaveLength(2); + expect(result[0].tool_calls[0].args).toBe('non-json-string'); + }); + + it('should handle complex tool calls with multiple steps', () => { + const payload = [ + { + role: 'assistant', + content: [ + { + type: ContentTypes.TEXT, + [ContentTypes.TEXT]: 'I\'ll search for that information.', + tool_call_ids: ['search_1'], + }, + { + type: ContentTypes.TOOL_CALL, + tool_call: { + id: 'search_1', + name: 'search', + args: '{"query":"weather in New York"}', + output: 'The weather in New York is currently sunny with a temperature of 75°F.', + }, + }, + { + type: ContentTypes.TEXT, + [ContentTypes.TEXT]: 'Now, I\'ll convert the temperature.', + tool_call_ids: ['convert_1'], + }, + { + type: ContentTypes.TOOL_CALL, + tool_call: { + id: 'convert_1', + name: 'convert_temperature', + args: '{"temperature": 75, "from": "F", "to": "C"}', + output: '23.89°C', + }, + }, + { type: ContentTypes.TEXT, [ContentTypes.TEXT]: 'Here\'s your answer.' }, + ], + }, + ]; + + const result = formatAgentMessages(payload); + + expect(result).toHaveLength(5); + expect(result[0]).toBeInstanceOf(AIMessage); + expect(result[1]).toBeInstanceOf(ToolMessage); + expect(result[2]).toBeInstanceOf(AIMessage); + expect(result[3]).toBeInstanceOf(ToolMessage); + expect(result[4]).toBeInstanceOf(AIMessage); + + // Check first AIMessage + expect(result[0].content).toBe('I\'ll search for that information.'); + expect(result[0].tool_calls).toHaveLength(1); + expect(result[0].tool_calls[0]).toEqual({ + id: 'search_1', + name: 'search', + args: { query: 'weather in New York' }, + }); + + // Check first ToolMessage + expect(result[1].tool_call_id).toBe('search_1'); + expect(result[1].name).toBe('search'); + expect(result[1].content).toBe( + 'The weather in New York is currently sunny with a temperature of 75°F.', + ); + + // Check second AIMessage + expect(result[2].content).toBe('Now, I\'ll convert the temperature.'); + expect(result[2].tool_calls).toHaveLength(1); + expect(result[2].tool_calls[0]).toEqual({ + id: 'convert_1', + name: 'convert_temperature', + args: { temperature: 75, from: 'F', to: 'C' }, + }); + + // Check second ToolMessage + expect(result[3].tool_call_id).toBe('convert_1'); + expect(result[3].name).toBe('convert_temperature'); + expect(result[3].content).toBe('23.89°C'); + + // Check final AIMessage + expect(result[4].content).toStrictEqual([ + { [ContentTypes.TEXT]: 'Here\'s your answer.', type: ContentTypes.TEXT }, + ]); + }); + + it.skip('should not produce two consecutive assistant messages and format content correctly', () => { + const payload = [ + { role: 'user', content: 'Hello' }, + { + role: 'assistant', + content: [{ type: ContentTypes.TEXT, [ContentTypes.TEXT]: 'Hi there!' }], + }, + { + role: 'assistant', + content: [{ type: ContentTypes.TEXT, [ContentTypes.TEXT]: 'How can I help you?' }], + }, + { role: 'user', content: 'What\'s the weather?' }, + { + role: 'assistant', + content: [ + { + type: ContentTypes.TEXT, + [ContentTypes.TEXT]: 'Let me check that for you.', + tool_call_ids: ['weather_1'], + }, + { + type: ContentTypes.TOOL_CALL, + tool_call: { + id: 'weather_1', + name: 'check_weather', + args: '{"location":"New York"}', + output: 'Sunny, 75°F', + }, + }, + ], + }, + { + role: 'assistant', + content: [ + { type: ContentTypes.TEXT, [ContentTypes.TEXT]: 'Here\'s the weather information.' }, + ], + }, + ]; + + const result = formatAgentMessages(payload); + + // Check correct message count and types + expect(result).toHaveLength(6); + expect(result[0]).toBeInstanceOf(HumanMessage); + expect(result[1]).toBeInstanceOf(AIMessage); + expect(result[2]).toBeInstanceOf(HumanMessage); + expect(result[3]).toBeInstanceOf(AIMessage); + expect(result[4]).toBeInstanceOf(ToolMessage); + expect(result[5]).toBeInstanceOf(AIMessage); + + // Check content of messages + expect(result[0].content).toStrictEqual([ + { [ContentTypes.TEXT]: 'Hello', type: ContentTypes.TEXT }, + ]); + expect(result[1].content).toStrictEqual([ + { [ContentTypes.TEXT]: 'Hi there!', type: ContentTypes.TEXT }, + { [ContentTypes.TEXT]: 'How can I help you?', type: ContentTypes.TEXT }, + ]); + expect(result[2].content).toStrictEqual([ + { [ContentTypes.TEXT]: 'What\'s the weather?', type: ContentTypes.TEXT }, + ]); + expect(result[3].content).toBe('Let me check that for you.'); + expect(result[4].content).toBe('Sunny, 75°F'); + expect(result[5].content).toStrictEqual([ + { [ContentTypes.TEXT]: 'Here\'s the weather information.', type: ContentTypes.TEXT }, + ]); + + // Check that there are no consecutive AIMessages + const messageTypes = result.map((message) => message.constructor); + for (let i = 0; i < messageTypes.length - 1; i++) { + expect(messageTypes[i] === AIMessage && messageTypes[i + 1] === AIMessage).toBe(false); + } + + // Additional check to ensure the consecutive assistant messages were combined + expect(result[1].content).toHaveLength(2); + }); +}); diff --git a/api/app/clients/prompts/formatMessages.js b/api/app/clients/prompts/formatMessages.js index a5ed05c6458..29784d65319 100644 --- a/api/app/clients/prompts/formatMessages.js +++ b/api/app/clients/prompts/formatMessages.js @@ -142,6 +142,9 @@ const formatAgentMessages = (payload) => { const messages = []; for (const message of payload) { + if (typeof message.content === 'string') { + message.content = [{ type: ContentTypes.TEXT, [ContentTypes.TEXT]: message.content }]; + } if (message.role !== 'assistant') { messages.push(formatMessage({ message, langChain: true })); continue; @@ -152,10 +155,22 @@ const formatAgentMessages = (payload) => { for (const part of message.content) { if (part.type === ContentTypes.TEXT && part.tool_call_ids) { - // If there's pending content, add it as an AIMessage + /* + If there's pending content, it needs to be aggregated as a single string to prepare for tool calls. + For Anthropic models, the "tool_calls" field on a message is only respected if content is a string. + */ if (currentContent.length > 0) { - messages.push(new AIMessage({ content: currentContent })); + let content = currentContent.reduce((acc, curr) => { + if (curr.type === ContentTypes.TEXT) { + return `${acc}${curr[ContentTypes.TEXT]}\n`; + } + return acc; + }, ''); + content = `${content}\n${part[ContentTypes.TEXT] ?? ''}`.trim(); + lastAIMessage = new AIMessage({ content }); + messages.push(lastAIMessage); currentContent = []; + continue; } // Create a new AIMessage with this text and prepare for tool calls diff --git a/api/app/clients/specs/AnthropicClient.test.js b/api/app/clients/specs/AnthropicClient.test.js index 29267de462b..eef6bb6748a 100644 --- a/api/app/clients/specs/AnthropicClient.test.js +++ b/api/app/clients/specs/AnthropicClient.test.js @@ -201,10 +201,10 @@ describe('AnthropicClient', () => { ); }); - it('should add beta header for claude-3-5-sonnet model', () => { + it('should add "max-tokens" & "prompt-caching" beta header for claude-3-5-sonnet model', () => { const client = new AnthropicClient('test-api-key'); const modelOptions = { - model: 'claude-3-5-sonnet-20240307', + model: 'claude-3-5-sonnet-20241022', }; client.setOptions({ modelOptions, promptCache: true }); const anthropicClient = client.getClient(modelOptions); @@ -215,7 +215,7 @@ describe('AnthropicClient', () => { ); }); - it('should add beta header for claude-3-haiku model', () => { + it('should add "prompt-caching" beta header for claude-3-haiku model', () => { const client = new AnthropicClient('test-api-key'); const modelOptions = { model: 'claude-3-haiku-2028', @@ -229,6 +229,30 @@ describe('AnthropicClient', () => { ); }); + it('should add "prompt-caching" beta header for claude-3-opus model', () => { + const client = new AnthropicClient('test-api-key'); + const modelOptions = { + model: 'claude-3-opus-2028', + }; + client.setOptions({ modelOptions, promptCache: true }); + const anthropicClient = client.getClient(modelOptions); + expect(anthropicClient._options.defaultHeaders).toBeDefined(); + expect(anthropicClient._options.defaultHeaders).toHaveProperty('anthropic-beta'); + expect(anthropicClient._options.defaultHeaders['anthropic-beta']).toBe( + 'prompt-caching-2024-07-31', + ); + }); + + it('should not add beta header for claude-3-5-sonnet-latest model', () => { + const client = new AnthropicClient('test-api-key'); + const modelOptions = { + model: 'anthropic/claude-3-5-sonnet-latest', + }; + client.setOptions({ modelOptions, promptCache: true }); + const anthropicClient = client.getClient(modelOptions); + expect(anthropicClient.defaultHeaders).not.toHaveProperty('anthropic-beta'); + }); + it('should not add beta header for other models', () => { const client = new AnthropicClient('test-api-key'); client.setOptions({ diff --git a/api/app/clients/specs/OpenAIClient.test.js b/api/app/clients/specs/OpenAIClient.test.js index 45903984193..0725efd9d83 100644 --- a/api/app/clients/specs/OpenAIClient.test.js +++ b/api/app/clients/specs/OpenAIClient.test.js @@ -611,15 +611,7 @@ describe('OpenAIClient', () => { expect(getCompletion).toHaveBeenCalled(); expect(getCompletion.mock.calls.length).toBe(1); - const currentDateString = new Date().toLocaleDateString('en-us', { - year: 'numeric', - month: 'long', - day: 'numeric', - }); - - expect(getCompletion.mock.calls[0][0]).toBe( - `||>Instructions:\nYou are ChatGPT, a large language model trained by OpenAI. Respond conversationally.\nCurrent date: ${currentDateString}\n\n||>User:\nHi mom!\n||>Assistant:\n`, - ); + expect(getCompletion.mock.calls[0][0]).toBe('||>User:\nHi mom!\n||>Assistant:\n'); expect(fetchEventSource).toHaveBeenCalled(); expect(fetchEventSource.mock.calls.length).toBe(1); @@ -701,4 +693,70 @@ describe('OpenAIClient', () => { expect(client.modelOptions.stop).toBeUndefined(); }); }); + + describe('getStreamUsage', () => { + it('should return this.usage when completion_tokens_details is null', () => { + const client = new OpenAIClient('test-api-key', defaultOptions); + client.usage = { + completion_tokens_details: null, + prompt_tokens: 10, + completion_tokens: 20, + }; + client.inputTokensKey = 'prompt_tokens'; + client.outputTokensKey = 'completion_tokens'; + + const result = client.getStreamUsage(); + + expect(result).toEqual(client.usage); + }); + + it('should return this.usage when completion_tokens_details is missing reasoning_tokens', () => { + const client = new OpenAIClient('test-api-key', defaultOptions); + client.usage = { + completion_tokens_details: { + other_tokens: 5, + }, + prompt_tokens: 10, + completion_tokens: 20, + }; + client.inputTokensKey = 'prompt_tokens'; + client.outputTokensKey = 'completion_tokens'; + + const result = client.getStreamUsage(); + + expect(result).toEqual(client.usage); + }); + + it('should calculate output tokens correctly when completion_tokens_details is present with reasoning_tokens', () => { + const client = new OpenAIClient('test-api-key', defaultOptions); + client.usage = { + completion_tokens_details: { + reasoning_tokens: 30, + other_tokens: 5, + }, + prompt_tokens: 10, + completion_tokens: 20, + }; + client.inputTokensKey = 'prompt_tokens'; + client.outputTokensKey = 'completion_tokens'; + + const result = client.getStreamUsage(); + + expect(result).toEqual({ + reasoning_tokens: 30, + other_tokens: 5, + prompt_tokens: 10, + completion_tokens: 10, // |30 - 20| = 10 + }); + }); + + it('should return this.usage when it is undefined', () => { + const client = new OpenAIClient('test-api-key', defaultOptions); + client.usage = undefined; + + const result = client.getStreamUsage(); + + expect(result).toBeUndefined(); + }); + }); }); diff --git a/api/app/clients/tools/AzureAiSearch.js b/api/app/clients/tools/AzureAiSearch.js index 9b50aa2c433..1e20b9ce81d 100644 --- a/api/app/clients/tools/AzureAiSearch.js +++ b/api/app/clients/tools/AzureAiSearch.js @@ -77,7 +77,7 @@ class AzureAISearch extends StructuredTool { try { const searchOption = { queryType: this.queryType, - top: this.top, + top: typeof this.top === 'string' ? Number(this.top) : this.top, }; if (this.select) { searchOption.select = this.select.split(','); diff --git a/api/app/clients/tools/index.js b/api/app/clients/tools/index.js index f16d229e6b7..6cc31237df0 100644 --- a/api/app/clients/tools/index.js +++ b/api/app/clients/tools/index.js @@ -25,7 +25,6 @@ module.exports = { // Basic Tools CodeBrew, AzureAiSearch, - GoogleSearchAPI, WolframAlphaAPI, OpenAICreateImage, StableDiffusionAPI, @@ -37,8 +36,9 @@ module.exports = { CodeSherpa, StructuredSD, StructuredACS, + GoogleSearchAPI, CodeSherpaTools, + TraversaalSearch, StructuredWolfram, TavilySearchResults, - TraversaalSearch, }; diff --git a/api/app/clients/tools/structured/AzureAISearch.js b/api/app/clients/tools/structured/AzureAISearch.js index 0ce7b43fb21..e25da94426d 100644 --- a/api/app/clients/tools/structured/AzureAISearch.js +++ b/api/app/clients/tools/structured/AzureAISearch.js @@ -1,9 +1,9 @@ const { z } = require('zod'); -const { StructuredTool } = require('langchain/tools'); +const { Tool } = require('@langchain/core/tools'); const { SearchClient, AzureKeyCredential } = require('@azure/search-documents'); const { logger } = require('~/config'); -class AzureAISearch extends StructuredTool { +class AzureAISearch extends Tool { // Constants for default values static DEFAULT_API_VERSION = '2023-11-01'; static DEFAULT_QUERY_TYPE = 'simple'; @@ -83,7 +83,7 @@ class AzureAISearch extends StructuredTool { try { const searchOption = { queryType: this.queryType, - top: this.top, + top: typeof this.top === 'string' ? Number(this.top) : this.top, }; if (this.select) { searchOption.select = this.select.split(','); diff --git a/api/app/clients/tools/structured/DALLE3.js b/api/app/clients/tools/structured/DALLE3.js index 3155992ca9b..8cfeaf84164 100644 --- a/api/app/clients/tools/structured/DALLE3.js +++ b/api/app/clients/tools/structured/DALLE3.js @@ -2,7 +2,7 @@ const { z } = require('zod'); const path = require('path'); const OpenAI = require('openai'); const { v4: uuidv4 } = require('uuid'); -const { Tool } = require('langchain/tools'); +const { Tool } = require('@langchain/core/tools'); const { HttpsProxyAgent } = require('https-proxy-agent'); const { FileContext } = require('librechat-data-provider'); const { getImageBasename } = require('~/server/services/Files/images'); diff --git a/api/app/clients/tools/structured/GoogleSearch.js b/api/app/clients/tools/structured/GoogleSearch.js index c2cf72b9d66..d703d56f83e 100644 --- a/api/app/clients/tools/structured/GoogleSearch.js +++ b/api/app/clients/tools/structured/GoogleSearch.js @@ -4,11 +4,12 @@ const { getEnvironmentVariable } = require('@langchain/core/utils/env'); class GoogleSearchResults extends Tool { static lc_name() { - return 'GoogleSearchResults'; + return 'google'; } constructor(fields = {}) { super(fields); + this.name = 'google'; this.envVarApiKey = 'GOOGLE_SEARCH_API_KEY'; this.envVarSearchEngineId = 'GOOGLE_CSE_ID'; this.override = fields.override ?? false; diff --git a/api/app/clients/tools/structured/StableDiffusion.js b/api/app/clients/tools/structured/StableDiffusion.js index cfcbf73ac4b..6309da35d87 100644 --- a/api/app/clients/tools/structured/StableDiffusion.js +++ b/api/app/clients/tools/structured/StableDiffusion.js @@ -5,12 +5,12 @@ const path = require('path'); const axios = require('axios'); const sharp = require('sharp'); const { v4: uuidv4 } = require('uuid'); -const { StructuredTool } = require('langchain/tools'); +const { Tool } = require('@langchain/core/tools'); const { FileContext } = require('librechat-data-provider'); const paths = require('~/config/paths'); const { logger } = require('~/config'); -class StableDiffusionAPI extends StructuredTool { +class StableDiffusionAPI extends Tool { constructor(fields) { super(); /** @type {string} User ID */ diff --git a/api/app/clients/tools/structured/Wolfram.js b/api/app/clients/tools/structured/Wolfram.js index fc857b35cb2..1b426298cc0 100644 --- a/api/app/clients/tools/structured/Wolfram.js +++ b/api/app/clients/tools/structured/Wolfram.js @@ -1,10 +1,10 @@ /* eslint-disable no-useless-escape */ const axios = require('axios'); const { z } = require('zod'); -const { StructuredTool } = require('langchain/tools'); +const { Tool } = require('@langchain/core/tools'); const { logger } = require('~/config'); -class WolframAlphaAPI extends StructuredTool { +class WolframAlphaAPI extends Tool { constructor(fields) { super(); /* Used to initialize the Tool without necessary variables. */ diff --git a/api/app/clients/tools/util/createFileSearchTool.js b/api/app/clients/tools/util/createFileSearchTool.js new file mode 100644 index 00000000000..f00e4757f60 --- /dev/null +++ b/api/app/clients/tools/util/createFileSearchTool.js @@ -0,0 +1,104 @@ +const { z } = require('zod'); +const axios = require('axios'); +const { tool } = require('@langchain/core/tools'); +const { Tools, EToolResources } = require('librechat-data-provider'); +const { getFiles } = require('~/models/File'); +const { logger } = require('~/config'); + +/** + * + * @param {Object} options + * @param {ServerRequest} options.req + * @param {Agent['tool_resources']} options.tool_resources + * @returns + */ +const createFileSearchTool = async (options) => { + const { req, tool_resources } = options; + const file_ids = tool_resources?.[EToolResources.file_search]?.file_ids ?? []; + const files = (await getFiles({ file_id: { $in: file_ids } })).map((file) => ({ + file_id: file.file_id, + filename: file.filename, + })); + + const fileList = files.map((file) => `- ${file.filename}`).join('\n'); + const toolDescription = `Performs a semantic search based on a natural language query across the following files:\n${fileList}`; + + const FileSearch = tool( + async ({ query }) => { + if (files.length === 0) { + return 'No files to search. Instruct the user to add files for the search.'; + } + const jwtToken = req.headers.authorization.split(' ')[1]; + if (!jwtToken) { + return 'There was an error authenticating the file search request.'; + } + const queryPromises = files.map((file) => + axios + .post( + `${process.env.RAG_API_URL}/query`, + { + file_id: file.file_id, + query, + k: 5, + }, + { + headers: { + Authorization: `Bearer ${jwtToken}`, + 'Content-Type': 'application/json', + }, + }, + ) + .catch((error) => { + logger.error( + `Error encountered in \`file_search\` while querying file_id ${file._id}:`, + error, + ); + return null; + }), + ); + + const results = await Promise.all(queryPromises); + const validResults = results.filter((result) => result !== null); + + if (validResults.length === 0) { + return 'No results found or errors occurred while searching the files.'; + } + + const formattedResults = validResults + .flatMap((result) => + result.data.map(([docInfo, relevanceScore]) => ({ + filename: docInfo.metadata.source.split('/').pop(), + content: docInfo.page_content, + relevanceScore, + })), + ) + .sort((a, b) => b.relevanceScore - a.relevanceScore); + + const formattedString = formattedResults + .map( + (result) => + `File: ${result.filename}\nRelevance: ${result.relevanceScore.toFixed(4)}\nContent: ${ + result.content + }\n`, + ) + .join('\n---\n'); + + return formattedString; + }, + { + name: Tools.file_search, + description: toolDescription, + schema: z.object({ + query: z + .string() + .describe( + 'A natural language query to search for relevant information in the files. Be specific and use keywords related to the information you\'re looking for. The query will be used for semantic similarity matching against the file contents.', + ), + }), + }, + ); + + return FileSearch; +}; + +module.exports = createFileSearchTool; diff --git a/api/app/clients/tools/util/handleTools.js b/api/app/clients/tools/util/handleTools.js index 7ed18658711..7a7be797d56 100644 --- a/api/app/clients/tools/util/handleTools.js +++ b/api/app/clients/tools/util/handleTools.js @@ -1,8 +1,8 @@ +const { Tools } = require('librechat-data-provider'); const { ZapierToolKit } = require('langchain/agents'); const { Calculator } = require('langchain/tools/calculator'); -const { WebBrowser } = require('langchain/tools/webbrowser'); const { SerpAPI, ZapierNLAWrapper } = require('langchain/tools'); -const { OpenAIEmbeddings } = require('langchain/embeddings/openai'); +const { createCodeExecutionTool, EnvVar } = require('@librechat/agents'); const { getUserPluginAuthValue } = require('~/server/services/PluginService'); const { availableTools, @@ -24,16 +24,11 @@ const { StructuredWolfram, TavilySearchResults, } = require('../'); +const createFileSearchTool = require('./createFileSearchTool'); const { loadToolSuite } = require('./loadToolSuite'); const { loadSpecs } = require('./loadSpecs'); const { logger } = require('~/config'); -const getOpenAIKey = async (options, user) => { - let openAIApiKey = options.openAIApiKey ?? process.env.OPENAI_API_KEY; - openAIApiKey = openAIApiKey === 'user_provided' ? null : openAIApiKey; - return openAIApiKey || (await getUserPluginAuthValue(user, 'OPENAI_API_KEY')); -}; - /** * Validates the availability and authentication of tools for a user based on environment variables or user-specific plugin authentication values. * Tools without required authentication or with valid authentication are considered valid. @@ -97,6 +92,45 @@ const validateTools = async (user, tools = []) => { } }; +const loadAuthValues = async ({ userId, authFields }) => { + let authValues = {}; + + /** + * Finds the first non-empty value for the given authentication field, supporting alternate fields. + * @param {string[]} fields Array of strings representing the authentication fields. Supports alternate fields delimited by "||". + * @returns {Promise<{ authField: string, authValue: string} | null>} An object containing the authentication field and value, or null if not found. + */ + const findAuthValue = async (fields) => { + for (const field of fields) { + let value = process.env[field]; + if (value) { + return { authField: field, authValue: value }; + } + try { + value = await getUserPluginAuthValue(userId, field); + } catch (err) { + if (field === fields[fields.length - 1] && !value) { + throw err; + } + } + if (value) { + return { authField: field, authValue: value }; + } + } + return null; + }; + + for (let authField of authFields) { + const fields = authField.split('||'); + const result = await findAuthValue(fields); + if (result) { + authValues[result.authField] = result.authValue; + } + } + + return authValues; +}; + /** * Initializes a tool with authentication values for the given user, supporting alternate authentication fields. * Authentication fields can have alternates separated by "||", and the first defined variable will be used. @@ -109,41 +143,7 @@ const validateTools = async (user, tools = []) => { */ const loadToolWithAuth = (userId, authFields, ToolConstructor, options = {}) => { return async function () { - let authValues = {}; - - /** - * Finds the first non-empty value for the given authentication field, supporting alternate fields. - * @param {string[]} fields Array of strings representing the authentication fields. Supports alternate fields delimited by "||". - * @returns {Promise<{ authField: string, authValue: string} | null>} An object containing the authentication field and value, or null if not found. - */ - const findAuthValue = async (fields) => { - for (const field of fields) { - let value = process.env[field]; - if (value) { - return { authField: field, authValue: value }; - } - try { - value = await getUserPluginAuthValue(userId, field); - } catch (err) { - if (field === fields[fields.length - 1] && !value) { - throw err; - } - } - if (value) { - return { authField: field, authValue: value }; - } - } - return null; - }; - - for (let authField of authFields) { - const fields = authField.split('||'); - const result = await findAuthValue(fields); - if (result) { - authValues[result.authField] = result.authValue; - } - } - + const authValues = await loadAuthValues({ userId, authFields }); return new ToolConstructor({ ...options, ...authValues, userId }); }; }; @@ -169,8 +169,6 @@ const loadTools = async ({ traversaal_search: TraversaalSearch, }; - const openAIApiKey = await getOpenAIKey(options, user); - const customConstructors = { e2b_code_interpreter: async () => { if (!functions) { @@ -183,7 +181,6 @@ const loadTools = async ({ user, options: { model, - openAIApiKey, ...options, }, }); @@ -200,14 +197,6 @@ const loadTools = async ({ options, }); }, - 'web-browser': async () => { - // let openAIApiKey = options.openAIApiKey ?? process.env.OPENAI_API_KEY; - // openAIApiKey = openAIApiKey === 'user_provided' ? null : openAIApiKey; - // openAIApiKey = openAIApiKey || (await getUserPluginAuthValue(user, 'OPENAI_API_KEY')); - const browser = new WebBrowser({ model, embeddings: new OpenAIEmbeddings({ openAIApiKey }) }); - browser.description_for_model = browser.description; - return browser; - }, serpapi: async () => { let apiKey = process.env.SERPAPI_API_KEY; if (!apiKey) { @@ -264,6 +253,22 @@ const loadTools = async ({ const remainingTools = []; for (const tool of tools) { + if (tool === Tools.execute_code) { + const authValues = await loadAuthValues({ + userId: user.id, + authFields: [EnvVar.CODE_API_KEY], + }); + requestedTools[tool] = () => + createCodeExecutionTool({ + user_id: user.id, + ...authValues, + }); + continue; + } else if (tool === Tools.file_search) { + requestedTools[tool] = () => createFileSearchTool(options); + continue; + } + if (customConstructors[tool]) { requestedTools[tool] = customConstructors[tool]; continue; @@ -331,6 +336,7 @@ const loadTools = async ({ module.exports = { loadToolWithAuth, + loadAuthValues, validateTools, loadTools, }; diff --git a/api/app/clients/tools/util/index.js b/api/app/clients/tools/util/index.js index ea67bb4ced2..73d10270b61 100644 --- a/api/app/clients/tools/util/index.js +++ b/api/app/clients/tools/util/index.js @@ -1,8 +1,9 @@ -const { validateTools, loadTools } = require('./handleTools'); +const { validateTools, loadTools, loadAuthValues } = require('./handleTools'); const handleOpenAIErrors = require('./handleOpenAIErrors'); module.exports = { handleOpenAIErrors, + loadAuthValues, validateTools, loadTools, }; diff --git a/api/models/Agent.js b/api/models/Agent.js index 2112a44991f..7d599d3032c 100644 --- a/api/models/Agent.js +++ b/api/models/Agent.js @@ -1,11 +1,14 @@ const mongoose = require('mongoose'); +const { SystemRoles } = require('librechat-data-provider'); const { GLOBAL_PROJECT_NAME } = require('librechat-data-provider').Constants; +const { CONFIG_STORE, STARTUP_CONFIG } = require('librechat-data-provider').CacheKeys; const { getProjectByName, addAgentIdsToProject, removeAgentIdsFromProject, removeAgentFromAllProjects, } = require('./Project'); +const getLogStores = require('~/cache/getLogStores'); const agentSchema = require('./schema/agent'); const Agent = mongoose.model('agent', agentSchema); @@ -30,6 +33,43 @@ const createAgent = async (agentData) => { */ const getAgent = async (searchParameter) => await Agent.findOne(searchParameter).lean(); +/** + * Load an agent based on the provided ID + * + * @param {Object} params + * @param {ServerRequest} params.req + * @param {string} params.agent_id + * @returns {Promise} The agent document as a plain object, or null if not found. + */ +const loadAgent = async ({ req, agent_id }) => { + const agent = await getAgent({ + id: agent_id, + }); + + if (agent.author.toString() === req.user.id) { + return agent; + } + + if (!agent.projectIds) { + return null; + } + + const cache = getLogStores(CONFIG_STORE); + /** @type {TStartupConfig} */ + const cachedStartupConfig = await cache.get(STARTUP_CONFIG); + let { instanceProjectId } = cachedStartupConfig ?? {}; + if (!instanceProjectId) { + instanceProjectId = (await getProjectByName(GLOBAL_PROJECT_NAME, '_id'))._id.toString(); + } + + for (const projectObjectId of agent.projectIds) { + const projectId = projectObjectId.toString(); + if (projectId === instanceProjectId) { + return agent; + } + } +}; + /** * Update an agent with new data without overwriting existing * properties, or create a new agent if it doesn't exist. @@ -41,10 +81,76 @@ const getAgent = async (searchParameter) => await Agent.findOne(searchParameter) * @returns {Promise} The updated or newly created agent document as a plain object. */ const updateAgent = async (searchParameter, updateData) => { - const options = { new: true, upsert: true }; + const options = { new: true, upsert: false }; return await Agent.findOneAndUpdate(searchParameter, updateData, options).lean(); }; +/** + * Modifies an agent with the resource file id. + * @param {object} params + * @param {ServerRequest} params.req + * @param {string} params.agent_id + * @param {string} params.tool_resource + * @param {string} params.file_id + * @returns {Promise} The updated agent. + */ +const addAgentResourceFile = async ({ agent_id, tool_resource, file_id }) => { + const searchParameter = { id: agent_id }; + const agent = await getAgent(searchParameter); + + if (!agent) { + throw new Error('Agent not found for adding resource file'); + } + + const tool_resources = agent.tool_resources || {}; + + if (!tool_resources[tool_resource]) { + tool_resources[tool_resource] = { file_ids: [] }; + } + + if (!tool_resources[tool_resource].file_ids.includes(file_id)) { + tool_resources[tool_resource].file_ids.push(file_id); + } + + const updateData = { tool_resources }; + + return await updateAgent(searchParameter, updateData); +}; + +/** + * Removes a resource file id from an agent. + * @param {object} params + * @param {ServerRequest} params.req + * @param {string} params.agent_id + * @param {string} params.tool_resource + * @param {string} params.file_id + * @returns {Promise} The updated agent. + */ +const removeAgentResourceFile = async ({ agent_id, tool_resource, file_id }) => { + const searchParameter = { id: agent_id }; + const agent = await getAgent(searchParameter); + + if (!agent) { + throw new Error('Agent not found for removing resource file'); + } + + const tool_resources = agent.tool_resources || {}; + + if (tool_resources[tool_resource] && tool_resources[tool_resource].file_ids) { + tool_resources[tool_resource].file_ids = tool_resources[tool_resource].file_ids.filter( + (id) => id !== file_id, + ); + + if (tool_resources[tool_resource].file_ids.length === 0) { + delete tool_resources[tool_resource]; + } + } + + const updateData = { tool_resources }; + + return await updateAgent(searchParameter, updateData); +}; + /** * Deletes an agent based on the provided ID. * @@ -79,12 +185,25 @@ const getListAgents = async (searchParameter) => { query = { $or: [globalQuery, query] }; } - const agents = await Agent.find(query, { - id: 1, - name: 1, - avatar: 1, - projectIds: 1, - }).lean(); + const agents = ( + await Agent.find(query, { + id: 1, + _id: 0, + name: 1, + avatar: 1, + author: 1, + projectIds: 1, + isCollaborative: 1, + }).lean() + ).map((agent) => { + if (agent.author?.toString() !== author) { + delete agent.author; + } + if (agent.author) { + agent.author = agent.author.toString(); + } + return agent; + }); const hasMore = agents.length > 0; const firstId = agents.length > 0 ? agents[0].id : null; @@ -102,13 +221,15 @@ const getListAgents = async (searchParameter) => { * Updates the projects associated with an agent, adding and removing project IDs as specified. * This function also updates the corresponding projects to include or exclude the agent ID. * - * @param {string} agentId - The ID of the agent to update. - * @param {string[]} [projectIds] - Array of project IDs to add to the agent. - * @param {string[]} [removeProjectIds] - Array of project IDs to remove from the agent. + * @param {Object} params - Parameters for updating the agent's projects. + * @param {import('librechat-data-provider').TUser} params.user - Parameters for updating the agent's projects. + * @param {string} params.agentId - The ID of the agent to update. + * @param {string[]} [params.projectIds] - Array of project IDs to add to the agent. + * @param {string[]} [params.removeProjectIds] - Array of project IDs to remove from the agent. * @returns {Promise} The updated agent document. * @throws {Error} If there's an error updating the agent or projects. */ -const updateAgentProjects = async (agentId, projectIds, removeProjectIds) => { +const updateAgentProjects = async ({ user, agentId, projectIds, removeProjectIds }) => { const updateOps = {}; if (removeProjectIds && removeProjectIds.length > 0) { @@ -129,14 +250,36 @@ const updateAgentProjects = async (agentId, projectIds, removeProjectIds) => { return await getAgent({ id: agentId }); } - return await updateAgent({ id: agentId }, updateOps); + const updateQuery = { id: agentId, author: user.id }; + if (user.role === SystemRoles.ADMIN) { + delete updateQuery.author; + } + + const updatedAgent = await updateAgent(updateQuery, updateOps); + if (updatedAgent) { + return updatedAgent; + } + if (updateOps.$addToSet) { + for (const projectId of projectIds) { + await removeAgentIdsFromProject(projectId, [agentId]); + } + } else if (updateOps.$pull) { + for (const projectId of removeProjectIds) { + await addAgentIdsToProject(projectId, [agentId]); + } + } + + return await getAgent({ id: agentId }); }; module.exports = { - createAgent, getAgent, + loadAgent, + createAgent, updateAgent, deleteAgent, getListAgents, updateAgentProjects, + addAgentResourceFile, + removeAgentResourceFile, }; diff --git a/api/models/Banner.js b/api/models/Banner.js new file mode 100644 index 00000000000..8d439dae289 --- /dev/null +++ b/api/models/Banner.js @@ -0,0 +1,27 @@ +const Banner = require('./schema/banner'); +const logger = require('~/config/winston'); +/** + * Retrieves the current active banner. + * @returns {Promise} The active banner object or null if no active banner is found. + */ +const getBanner = async (user) => { + try { + const now = new Date(); + const banner = await Banner.findOne({ + displayFrom: { $lte: now }, + $or: [{ displayTo: { $gte: now } }, { displayTo: null }], + type: 'banner', + }).lean(); + + if (!banner || banner.isPublic || user) { + return banner; + } + + return null; + } catch (error) { + logger.error('[getBanners] Error getting banners', error); + throw new Error('Error getting banners'); + } +}; + +module.exports = { getBanner }; diff --git a/api/models/Conversation.js b/api/models/Conversation.js index 19622ba7962..0850ed0a71b 100644 --- a/api/models/Conversation.js +++ b/api/models/Conversation.js @@ -31,9 +31,39 @@ const getConvo = async (user, conversationId) => { } }; +const deleteNullOrEmptyConversations = async () => { + try { + const filter = { + $or: [ + { conversationId: null }, + { conversationId: '' }, + { conversationId: { $exists: false } }, + ], + }; + + const result = await Conversation.deleteMany(filter); + + // Delete associated messages + const messageDeleteResult = await deleteMessages(filter); + + logger.info( + `[deleteNullOrEmptyConversations] Deleted ${result.deletedCount} conversations and ${messageDeleteResult.deletedCount} messages`, + ); + + return { + conversations: result, + messages: messageDeleteResult, + }; + } catch (error) { + logger.error('[deleteNullOrEmptyConversations] Error deleting conversations', error); + throw new Error('Error deleting conversations with null or empty conversationId'); + } +}; + module.exports = { Conversation, searchConversation, + deleteNullOrEmptyConversations, /** * Saves a conversation to the database. * @param {Object} req - The request object. diff --git a/api/models/Preset.js b/api/models/Preset.js index c0134eca6a2..970b2958fb8 100644 --- a/api/models/Preset.js +++ b/api/models/Preset.js @@ -38,7 +38,8 @@ module.exports = { savePreset: async (user, { presetId, newPresetId, defaultPreset, ...preset }) => { try { const setter = { $set: {} }; - const update = { presetId, ...preset }; + const { user: _, ...cleanPreset } = preset; + const update = { presetId, ...cleanPreset }; if (preset.tools && Array.isArray(preset.tools)) { update.tools = preset.tools diff --git a/api/models/Prompt.js b/api/models/Prompt.js index 548589b4d7e..742ced0f728 100644 --- a/api/models/Prompt.js +++ b/api/models/Prompt.js @@ -7,6 +7,7 @@ const { removeGroupFromAllProjects, } = require('./Project'); const { Prompt, PromptGroup } = require('./schema/promptSchema'); +const { escapeRegExp } = require('~/server/utils'); const { logger } = require('~/config'); /** @@ -106,7 +107,7 @@ const getAllPromptGroups = async (req, filter) => { let searchShared = true; let searchSharedOnly = false; if (name) { - query.name = new RegExp(name, 'i'); + query.name = new RegExp(escapeRegExp(name), 'i'); } if (!query.category) { delete query.category; @@ -159,7 +160,7 @@ const getPromptGroups = async (req, filter) => { let searchShared = true; let searchSharedOnly = false; if (name) { - query.name = new RegExp(name, 'i'); + query.name = new RegExp(escapeRegExp(name), 'i'); } if (!query.category) { delete query.category; diff --git a/api/models/inviteUser.js b/api/models/inviteUser.js index c04bd9467a5..6cd699fd668 100644 --- a/api/models/inviteUser.js +++ b/api/models/inviteUser.js @@ -1,6 +1,5 @@ -const crypto = require('crypto'); -const bcrypt = require('bcryptjs'); const mongoose = require('mongoose'); +const { getRandomValues, hashToken } = require('~/server/utils/crypto'); const { createToken, findToken } = require('./Token'); const logger = require('~/config/winston'); @@ -18,8 +17,8 @@ const logger = require('~/config/winston'); */ const createInvite = async (email) => { try { - let token = crypto.randomBytes(32).toString('hex'); - const hash = bcrypt.hashSync(token, 10); + const token = await getRandomValues(32); + const hash = await hashToken(token); const encodedToken = encodeURIComponent(token); const fakeUserId = new mongoose.Types.ObjectId(); @@ -50,7 +49,7 @@ const createInvite = async (email) => { const getInvite = async (encodedToken, email) => { try { const token = decodeURIComponent(encodedToken); - const hash = bcrypt.hashSync(token, 10); + const hash = await hashToken(token); const invite = await findToken({ token: hash, email }); if (!invite) { @@ -59,7 +58,7 @@ const getInvite = async (encodedToken, email) => { return invite; } catch (error) { - logger.error('[getInvite] Error getting invite', error); + logger.error('[getInvite] Error getting invite:', error); return { error: true, message: error.message }; } }; diff --git a/api/models/schema/agent.js b/api/models/schema/agent.js index 819398ee7cb..d7c5762b531 100644 --- a/api/models/schema/agent.js +++ b/api/models/schema/agent.js @@ -5,6 +5,7 @@ const agentSchema = mongoose.Schema( id: { type: String, index: true, + unique: true, required: true, }, name: { @@ -44,10 +45,6 @@ const agentSchema = mongoose.Schema( tool_kwargs: { type: [{ type: mongoose.Schema.Types.Mixed }], }, - file_ids: { - type: [String], - default: undefined, - }, actions: { type: [String], default: undefined, @@ -57,6 +54,22 @@ const agentSchema = mongoose.Schema( ref: 'User', required: true, }, + authorName: { + type: String, + default: undefined, + }, + isCollaborative: { + type: Boolean, + default: undefined, + }, + conversation_starters: { + type: [String], + default: [], + }, + tool_resources: { + type: mongoose.Schema.Types.Mixed, + default: {}, + }, projectIds: { type: [mongoose.Schema.Types.ObjectId], ref: 'Project', diff --git a/api/models/schema/banner.js b/api/models/schema/banner.js new file mode 100644 index 00000000000..7fd86c1b677 --- /dev/null +++ b/api/models/schema/banner.js @@ -0,0 +1,36 @@ +const mongoose = require('mongoose'); + +const bannerSchema = mongoose.Schema( + { + bannerId: { + type: String, + required: true, + }, + message: { + type: String, + required: true, + }, + displayFrom: { + type: Date, + required: true, + default: Date.now, + }, + displayTo: { + type: Date, + }, + type: { + type: String, + enum: ['banner', 'popup'], + default: 'banner', + }, + isPublic: { + type: Boolean, + default: false, + }, + }, + + { timestamps: true }, +); + +const Banner = mongoose.model('Banner', bannerSchema); +module.exports = Banner; diff --git a/api/models/schema/conversationTagSchema.js b/api/models/schema/conversationTagSchema.js index f00a0cfce95..9b2a98c6d83 100644 --- a/api/models/schema/conversationTagSchema.js +++ b/api/models/schema/conversationTagSchema.js @@ -21,6 +21,7 @@ const conversationTagSchema = mongoose.Schema( position: { type: Number, default: 0, + index: true, }, }, { timestamps: true }, diff --git a/api/models/schema/messageSchema.js b/api/models/schema/messageSchema.js index dddbdb6ebe0..5d2e43a3ee3 100644 --- a/api/models/schema/messageSchema.js +++ b/api/models/schema/messageSchema.js @@ -115,6 +115,29 @@ const messageSchema = mongoose.Schema( iconURL: { type: String, }, + attachments: { type: [{ type: mongoose.Schema.Types.Mixed }], default: undefined }, + /* + attachments: { + type: [ + { + file_id: String, + filename: String, + filepath: String, + expiresAt: Date, + width: Number, + height: Number, + type: String, + conversationId: String, + messageId: { + type: String, + required: true, + }, + toolCallId: String, + }, + ], + default: undefined, + }, + */ }, { timestamps: true }, ); diff --git a/api/models/tx.js b/api/models/tx.js index 2de31f1b3be..e7960482c47 100644 --- a/api/models/tx.js +++ b/api/models/tx.js @@ -37,9 +37,12 @@ const tokenValues = Object.assign( '4k': { prompt: 1.5, completion: 2 }, '16k': { prompt: 3, completion: 4 }, 'gpt-3.5-turbo-1106': { prompt: 1, completion: 2 }, - 'gpt-4o-2024-08-06': { prompt: 2.5, completion: 10 }, + 'o1-preview': { prompt: 15, completion: 60 }, + 'o1-mini': { prompt: 3, completion: 12 }, + o1: { prompt: 15, completion: 60 }, 'gpt-4o-mini': { prompt: 0.15, completion: 0.6 }, - 'gpt-4o': { prompt: 5, completion: 15 }, + 'gpt-4o': { prompt: 2.5, completion: 10 }, + 'gpt-4o-2024-05-13': { prompt: 5, completion: 15 }, 'gpt-4-1106': { prompt: 10, completion: 30 }, 'gpt-3.5-turbo-0125': { prompt: 0.5, completion: 1.5 }, 'claude-3-opus': { prompt: 15, completion: 75 }, @@ -95,8 +98,14 @@ const getValueKey = (model, endpoint) => { return 'gpt-3.5-turbo-1106'; } else if (modelName.includes('gpt-3.5')) { return '4k'; - } else if (modelName.includes('gpt-4o-2024-08-06')) { - return 'gpt-4o-2024-08-06'; + } else if (modelName.includes('o1-preview')) { + return 'o1-preview'; + } else if (modelName.includes('o1-mini')) { + return 'o1-mini'; + } else if (modelName.includes('o1')) { + return 'o1'; + } else if (modelName.includes('gpt-4o-2024-05-13')) { + return 'gpt-4o-2024-05-13'; } else if (modelName.includes('gpt-4o-mini')) { return 'gpt-4o-mini'; } else if (modelName.includes('gpt-4o')) { diff --git a/api/models/tx.spec.js b/api/models/tx.spec.js index c8a8b335e3c..730c33a05d7 100644 --- a/api/models/tx.spec.js +++ b/api/models/tx.spec.js @@ -50,8 +50,10 @@ describe('getValueKey', () => { }); it('should return "gpt-4o" for model type of "gpt-4o"', () => { - expect(getValueKey('gpt-4o-2024-05-13')).toBe('gpt-4o'); + expect(getValueKey('gpt-4o-2024-08-06')).toBe('gpt-4o'); + expect(getValueKey('gpt-4o-2024-08-06-0718')).toBe('gpt-4o'); expect(getValueKey('openai/gpt-4o')).toBe('gpt-4o'); + expect(getValueKey('openai/gpt-4o-2024-08-06')).toBe('gpt-4o'); expect(getValueKey('gpt-4o-turbo')).toBe('gpt-4o'); expect(getValueKey('gpt-4o-0125')).toBe('gpt-4o'); }); @@ -60,14 +62,14 @@ describe('getValueKey', () => { expect(getValueKey('gpt-4o-mini-2024-07-18')).toBe('gpt-4o-mini'); expect(getValueKey('openai/gpt-4o-mini')).toBe('gpt-4o-mini'); expect(getValueKey('gpt-4o-mini-0718')).toBe('gpt-4o-mini'); - expect(getValueKey('gpt-4o-2024-08-06-0718')).not.toBe('gpt-4o'); + expect(getValueKey('gpt-4o-2024-08-06-0718')).not.toBe('gpt-4o-mini'); }); - it('should return "gpt-4o-2024-08-06" for model type of "gpt-4o-2024-08-06"', () => { - expect(getValueKey('gpt-4o-2024-08-06-2024-07-18')).toBe('gpt-4o-2024-08-06'); - expect(getValueKey('openai/gpt-4o-2024-08-06')).toBe('gpt-4o-2024-08-06'); - expect(getValueKey('gpt-4o-2024-08-06-0718')).toBe('gpt-4o-2024-08-06'); - expect(getValueKey('gpt-4o-2024-08-06-0718')).not.toBe('gpt-4o'); + it('should return "gpt-4o-2024-05-13" for model type of "gpt-4o-2024-05-13"', () => { + expect(getValueKey('gpt-4o-2024-05-13')).toBe('gpt-4o-2024-05-13'); + expect(getValueKey('openai/gpt-4o-2024-05-13')).toBe('gpt-4o-2024-05-13'); + expect(getValueKey('gpt-4o-2024-05-13-0718')).toBe('gpt-4o-2024-05-13'); + expect(getValueKey('gpt-4o-2024-05-13-0718')).not.toBe('gpt-4o'); }); it('should return "gpt-4o" for model type of "chatgpt-4o"', () => { @@ -134,7 +136,7 @@ describe('getMultiplier', () => { }); it('should return the correct multiplier for gpt-4o', () => { - const valueKey = getValueKey('gpt-4o-2024-05-13'); + const valueKey = getValueKey('gpt-4o-2024-08-06'); expect(getMultiplier({ valueKey, tokenType: 'prompt' })).toBe(tokenValues['gpt-4o'].prompt); expect(getMultiplier({ valueKey, tokenType: 'completion' })).toBe( tokenValues['gpt-4o'].completion, diff --git a/api/package.json b/api/package.json index dc9b8c092f6..68f6a3c823b 100644 --- a/api/package.json +++ b/api/package.json @@ -1,6 +1,6 @@ { "name": "@librechat/backend", - "version": "v0.7.5-rc2", + "version": "v0.7.5", "description": "", "scripts": { "start": "echo 'please run this from the root directory'", @@ -43,22 +43,22 @@ "@langchain/core": "^0.2.18", "@langchain/google-genai": "^0.0.11", "@langchain/google-vertexai": "^0.0.17", - "@librechat/agents": "^1.5.2", - "axios": "^1.3.4", + "@librechat/agents": "^1.6.9", + "axios": "^1.7.7", "bcryptjs": "^2.4.3", "cheerio": "^1.0.0-rc.12", "cohere-ai": "^7.9.1", "compression": "^1.7.4", "connect-redis": "^7.1.0", - "cookie": "^0.5.0", - "cookie-parser": "^1.4.6", + "cookie": "^0.7.2", + "cookie-parser": "^1.4.7", "cors": "^2.8.5", "dedent": "^1.5.3", "dotenv": "^16.0.3", - "express": "^4.18.2", + "express": "^4.21.1", "express-mongo-sanitize": "^2.2.0", - "express-rate-limit": "^6.9.0", - "express-session": "^1.17.3", + "express-rate-limit": "^7.4.1", + "express-session": "^1.18.1", "file-type": "^18.7.0", "firebase": "^10.6.0", "googleapis": "^126.0.1", @@ -76,11 +76,11 @@ "meilisearch": "^0.38.0", "mime": "^3.0.0", "module-alias": "^2.2.3", - "mongoose": "^7.1.1", + "mongoose": "^7.3.3", "multer": "^1.4.5-lts.1", "nanoid": "^3.3.7", "nodejs-gpt": "^1.37.4", - "nodemailer": "^6.9.4", + "nodemailer": "^6.9.15", "ollama": "^0.5.0", "openai": "^4.47.1", "openai-chat-tokens": "^0.2.8", @@ -101,7 +101,6 @@ "ua-parser-js": "^1.0.36", "winston": "^3.11.0", "winston-daily-rotate-file": "^4.7.1", - "ws": "^8.17.0", "zod": "^3.22.4" }, "devDependencies": { diff --git a/api/server/controllers/AskController.js b/api/server/controllers/AskController.js index dd5c8d65709..d2d774b0092 100644 --- a/api/server/controllers/AskController.js +++ b/api/server/controllers/AskController.js @@ -16,7 +16,12 @@ const AskController = async (req, res, next, initializeClient, addTitle) => { overrideParentMessageId = null, } = req.body; - logger.debug('[AskController]', { text, conversationId, ...endpointOption }); + logger.debug('[AskController]', { + text, + conversationId, + ...endpointOption, + modelsConfig: endpointOption.modelsConfig ? 'exists' : '', + }); let userMessage; let userMessagePromise; diff --git a/api/server/controllers/EditController.js b/api/server/controllers/EditController.js index b3b94fcebba..28fe2c4fea1 100644 --- a/api/server/controllers/EditController.js +++ b/api/server/controllers/EditController.js @@ -25,6 +25,7 @@ const EditController = async (req, res, next, initializeClient) => { isContinued, conversationId, ...endpointOption, + modelsConfig: endpointOption.modelsConfig ? 'exists' : '', }); let userMessage; diff --git a/api/server/controllers/agents/callbacks.js b/api/server/controllers/agents/callbacks.js index f6c1972b4fe..7bbb2212aa6 100644 --- a/api/server/controllers/agents/callbacks.js +++ b/api/server/controllers/agents/callbacks.js @@ -1,8 +1,13 @@ +const { Tools } = require('librechat-data-provider'); const { GraphEvents, ToolEndHandler, ChatModelStreamHandler } = require('@librechat/agents'); +const { processCodeOutput } = require('~/server/services/Files/Code/process'); +const { logger } = require('~/config'); /** @typedef {import('@librechat/agents').Graph} Graph */ /** @typedef {import('@librechat/agents').EventHandler} EventHandler */ /** @typedef {import('@librechat/agents').ModelEndData} ModelEndData */ +/** @typedef {import('@librechat/agents').ToolEndData} ToolEndData */ +/** @typedef {import('@librechat/agents').ToolEndCallback} ToolEndCallback */ /** @typedef {import('@librechat/agents').ChatModelStreamHandler} ChatModelStreamHandler */ /** @typedef {import('@librechat/agents').ContentAggregatorResult['aggregateContent']} ContentAggregator */ /** @typedef {import('@librechat/agents').GraphEvents} GraphEvents */ @@ -58,11 +63,12 @@ class ModelEndHandler { * @param {Object} options - The options object. * @param {ServerResponse} options.res - The options object. * @param {ContentAggregator} options.aggregateContent - The options object. + * @param {ToolEndCallback} options.toolEndCallback - Callback to use when tool ends. * @param {Array} options.collectedUsage - The list of collected usage metadata. * @returns {Record} The default handlers. * @throws {Error} If the request is not found. */ -function getDefaultHandlers({ res, aggregateContent, collectedUsage }) { +function getDefaultHandlers({ res, aggregateContent, toolEndCallback, collectedUsage }) { if (!res || !aggregateContent) { throw new Error( `[getDefaultHandlers] Missing required options: res: ${!res}, aggregateContent: ${!aggregateContent}`, @@ -70,7 +76,7 @@ function getDefaultHandlers({ res, aggregateContent, collectedUsage }) { } const handlers = { [GraphEvents.CHAT_MODEL_END]: new ModelEndHandler(collectedUsage), - [GraphEvents.TOOL_END]: new ToolEndHandler(), + [GraphEvents.TOOL_END]: new ToolEndHandler(toolEndCallback), [GraphEvents.CHAT_MODEL_STREAM]: new ChatModelStreamHandler(), [GraphEvents.ON_RUN_STEP]: { /** @@ -121,7 +127,67 @@ function getDefaultHandlers({ res, aggregateContent, collectedUsage }) { return handlers; } +/** + * + * @param {Object} params + * @param {ServerRequest} params.req + * @param {ServerResponse} params.res + * @param {Promise[]} params.artifactPromises + * @returns {ToolEndCallback} The tool end callback. + */ +function createToolEndCallback({ req, res, artifactPromises }) { + /** + * @type {ToolEndCallback} + */ + return async (data, metadata) => { + const output = data?.output; + if (!output) { + return; + } + + if (output.name !== Tools.execute_code) { + return; + } + + const { tool_call_id, artifact } = output; + if (!artifact.files) { + return; + } + + for (const file of artifact.files) { + const { id, name } = file; + artifactPromises.push( + (async () => { + const fileMetadata = await processCodeOutput({ + req, + id, + name, + toolCallId: tool_call_id, + messageId: metadata.run_id, + sessionId: artifact.session_id, + conversationId: metadata.thread_id, + }); + if (!res.headersSent) { + return fileMetadata; + } + + if (!fileMetadata) { + return null; + } + + res.write(`event: attachment\ndata: ${JSON.stringify(fileMetadata)}\n\n`); + return fileMetadata; + })().catch((error) => { + logger.error('Error processing code output:', error); + return null; + }), + ); + } + }; +} + module.exports = { sendEvent, getDefaultHandlers, + createToolEndCallback, }; diff --git a/api/server/controllers/agents/client.js b/api/server/controllers/agents/client.js index 137068ddd6a..f035d546f28 100644 --- a/api/server/controllers/agents/client.js +++ b/api/server/controllers/agents/client.js @@ -10,7 +10,10 @@ const { Callback, createMetadataAggregator } = require('@librechat/agents'); const { Constants, + VisionModes, + openAISchema, EModelEndpoint, + anthropicSchema, bedrockOutputParser, providerEndpointMap, removeNullishValues, @@ -35,11 +38,10 @@ const { logger } = require('~/config'); /** @typedef {import('@librechat/agents').MessageContentComplex} MessageContentComplex */ -// const providerSchemas = { -// [EModelEndpoint.bedrock]: true, -// }; - const providerParsers = { + [EModelEndpoint.openAI]: openAISchema, + [EModelEndpoint.azureOpenAI]: openAISchema, + [EModelEndpoint.anthropic]: anthropicSchema, [EModelEndpoint.bedrock]: bedrockOutputParser, }; @@ -57,10 +59,11 @@ class AgentClient extends BaseClient { this.run; const { - maxContextTokens, - modelOptions = {}, contentParts, collectedUsage, + artifactPromises, + maxContextTokens, + modelOptions = {}, ...clientOptions } = options; @@ -70,6 +73,8 @@ class AgentClient extends BaseClient { this.contentParts = contentParts; /** @type {Array} */ this.collectedUsage = collectedUsage; + /** @type {ArtifactPromises} */ + this.artifactPromises = artifactPromises; this.options = Object.assign({ endpoint: options.endpoint }, clientOptions); } @@ -180,10 +185,10 @@ class AgentClient extends BaseClient { ); } - getBuildMessagesOptions(opts) { + getBuildMessagesOptions() { return { - instructions: opts.instructions, - additional_instructions: opts.additional_instructions, + instructions: this.options.agent.instructions, + additional_instructions: this.options.agent.additional_instructions, }; } @@ -192,6 +197,7 @@ class AgentClient extends BaseClient { this.options.req, attachments, this.options.agent.provider, + VisionModes.agents, ); message.image_urls = image_urls.length ? image_urls : undefined; return files; @@ -210,8 +216,6 @@ class AgentClient extends BaseClient { }); let payload; - /** @type {{ role: string; name: string; content: string } | undefined} */ - let systemMessage; /** @type {number | undefined} */ let promptTokens; @@ -259,21 +263,21 @@ class AgentClient extends BaseClient { } /* If message has files, calculate image token cost */ - // if (this.message_file_map && this.message_file_map[message.messageId]) { - // const attachments = this.message_file_map[message.messageId]; - // for (const file of attachments) { - // if (file.embedded) { - // this.contextHandlers?.processFile(file); - // continue; - // } - - // orderedMessages[i].tokenCount += this.calculateImageTokenCost({ - // width: file.width, - // height: file.height, - // detail: this.options.imageDetail ?? ImageDetail.auto, - // }); - // } - // } + if (this.message_file_map && this.message_file_map[message.messageId]) { + const attachments = this.message_file_map[message.messageId]; + for (const file of attachments) { + if (file.embedded) { + this.contextHandlers?.processFile(file); + continue; + } + + // orderedMessages[i].tokenCount += this.calculateImageTokenCost({ + // width: file.width, + // height: file.height, + // detail: this.options.imageDetail ?? ImageDetail.auto, + // }); + } + } return formattedMessage; }); @@ -284,20 +288,7 @@ class AgentClient extends BaseClient { } if (systemContent) { - systemContent = `${systemContent.trim()}`; - systemMessage = { - role: 'system', - name: 'instructions', - content: systemContent, - }; - - if (this.contextStrategy) { - const instructionTokens = this.getTokenCountForMessage(systemMessage); - if (instructionTokens >= 0) { - const firstMessageTokens = orderedMessages[0].tokenCount ?? 0; - orderedMessages[0].tokenCount = firstMessageTokens + instructionTokens; - } - } + this.options.agent.instructions = systemContent; } if (this.contextStrategy) { @@ -477,7 +468,6 @@ class AgentClient extends BaseClient { provider: providerEndpointMap[this.options.agent.provider], thread_id: this.conversationId, }, - run_id: this.responseMessageId, signal: abortController.signal, streamMode: 'values', version: 'v2', diff --git a/api/server/controllers/agents/run.js b/api/server/controllers/agents/run.js index 5aeefa122db..2377ce64f13 100644 --- a/api/server/controllers/agents/run.js +++ b/api/server/controllers/agents/run.js @@ -45,10 +45,9 @@ async function createRun({ ); const graphConfig = { - runId, - llmConfig, tools, toolMap, + llmConfig, instructions: agent.instructions, additional_instructions: agent.additional_instructions, }; @@ -59,6 +58,7 @@ async function createRun({ } return Run.create({ + runId, graphConfig, customHandlers, }); diff --git a/api/server/controllers/agents/v1.js b/api/server/controllers/agents/v1.js index 65e37f2618c..bc92a4b663b 100644 --- a/api/server/controllers/agents/v1.js +++ b/api/server/controllers/agents/v1.js @@ -1,5 +1,5 @@ const { nanoid } = require('nanoid'); -const { FileContext, Constants } = require('librechat-data-provider'); +const { FileContext, Constants, Tools, SystemRoles } = require('librechat-data-provider'); const { getAgent, createAgent, @@ -14,6 +14,11 @@ const { updateAgentProjects } = require('~/models/Agent'); const { deleteFileByFilter } = require('~/models/File'); const { logger } = require('~/config'); +const systemTools = { + [Tools.execute_code]: true, + [Tools.file_search]: true, +}; + /** * Creates an Agent. * @route POST /Agents @@ -27,9 +32,17 @@ const createAgentHandler = async (req, res) => { const { tools = [], provider, name, description, instructions, model, ...agentData } = req.body; const { id: userId } = req.user; - agentData.tools = tools - .map((tool) => (typeof tool === 'string' ? req.app.locals.availableTools[tool] : tool)) - .filter(Boolean); + agentData.tools = []; + + for (const tool of tools) { + if (req.app.locals.availableTools[tool]) { + agentData.tools.push(tool); + } + + if (systemTools[tool]) { + agentData.tools.push(tool); + } + } Object.assign(agentData, { author: userId, @@ -80,10 +93,24 @@ const getAgentHandler = async (req, res) => { return res.status(404).json({ error: 'Agent not found' }); } + agent.author = agent.author.toString(); + agent.isCollaborative = !!agent.isCollaborative; + if (agent.author !== author) { delete agent.author; } + if (!agent.isCollaborative && agent.author !== author && req.user.role !== SystemRoles.ADMIN) { + return res.status(200).json({ + id: agent.id, + name: agent.name, + avatar: agent.avatar, + author: agent.author, + projectIds: agent.projectIds, + isCollaborative: agent.isCollaborative, + }); + } + return res.status(200).json(agent); } catch (error) { logger.error('[/Agents/:id] Error retrieving agent', error); @@ -106,12 +133,29 @@ const updateAgentHandler = async (req, res) => { const { projectIds, removeProjectIds, ...updateData } = req.body; let updatedAgent; + const query = { id, author: req.user.id }; + if (req.user.role === SystemRoles.ADMIN) { + delete query.author; + } if (Object.keys(updateData).length > 0) { - updatedAgent = await updateAgent({ id, author: req.user.id }, updateData); + updatedAgent = await updateAgent(query, updateData); } if (projectIds || removeProjectIds) { - updatedAgent = await updateAgentProjects(id, projectIds, removeProjectIds); + updatedAgent = await updateAgentProjects({ + user: req.user, + agentId: id, + projectIds, + removeProjectIds, + }); + } + + if (updatedAgent.author) { + updatedAgent.author = updatedAgent.author.toString(); + } + + if (updatedAgent.author !== req.user.id) { + delete updatedAgent.author; } return res.json(updatedAgent); @@ -182,8 +226,6 @@ const uploadAgentAvatarHandler = async (req, res) => { return res.status(400).json({ message: 'Agent ID is required' }); } - let { avatar: _avatar = '{}' } = req.body; - const image = await uploadImageBuffer({ req, context: FileContext.avatar, @@ -192,10 +234,12 @@ const uploadAgentAvatarHandler = async (req, res) => { }, }); + let _avatar; try { - _avatar = JSON.parse(_avatar); + const agent = await getAgent({ id: agent_id }); + _avatar = agent.avatar; } catch (error) { - logger.error('[/avatar/:agent_id] Error parsing avatar', error); + logger.error('[/avatar/:agent_id] Error fetching agent', error); _avatar = {}; } @@ -203,7 +247,7 @@ const uploadAgentAvatarHandler = async (req, res) => { const { deleteFile } = getStrategyFunctions(_avatar.source); try { await deleteFile(req, { filepath: _avatar.filepath }); - await deleteFileByFilter({ filepath: _avatar.filepath }); + await deleteFileByFilter({ user: req.user.id, filepath: _avatar.filepath }); } catch (error) { logger.error('[/avatar/:agent_id] Error deleting old avatar', error); } diff --git a/api/server/controllers/assistants/chatV1.js b/api/server/controllers/assistants/chatV1.js index dcf76a44522..a1a023a4476 100644 --- a/api/server/controllers/assistants/chatV1.js +++ b/api/server/controllers/assistants/chatV1.js @@ -314,7 +314,9 @@ const chatV1 = async (req, res) => { } if (typeof endpointOption.artifactsPrompt === 'string' && endpointOption.artifactsPrompt) { - body.additional_instructions = `${body.additional_instructions ?? ''}\n${endpointOption.artifactsPrompt}`.trim(); + body.additional_instructions = `${body.additional_instructions ?? ''}\n${ + endpointOption.artifactsPrompt + }`.trim(); } if (instructions) { @@ -371,11 +373,14 @@ const chatV1 = async (req, res) => { visionMessage.content = createVisionPrompt(plural); visionMessage = formatMessage({ message: visionMessage, endpoint: EModelEndpoint.openAI }); - visionPromise = openai.chat.completions.create({ - model: 'gpt-4-vision-preview', - messages: [visionMessage], - max_tokens: 4000, - }); + visionPromise = openai.chat.completions + .create({ + messages: [visionMessage], + max_tokens: 4000, + }) + .catch((error) => { + logger.error('[/assistants/chat/] Error creating vision prompt', error); + }); const pluralized = plural ? 's' : ''; body.additional_instructions = `${ diff --git a/api/server/controllers/assistants/v1.js b/api/server/controllers/assistants/v1.js index 5a922cec6b1..982e212b7e3 100644 --- a/api/server/controllers/assistants/v1.js +++ b/api/server/controllers/assistants/v1.js @@ -241,7 +241,6 @@ const getAssistantDocuments = async (req, res) => { * @param {string} req.params.assistant_id - The ID of the assistant. * @param {Express.Multer.File} req.file - The avatar image file. * @param {object} req.body - Request body - * @param {string} [req.body.metadata] - Optional metadata for the assistant's avatar. * @returns {Object} 200 - success response - application/json */ const uploadAssistantAvatar = async (req, res) => { @@ -251,7 +250,6 @@ const uploadAssistantAvatar = async (req, res) => { return res.status(400).json({ message: 'Assistant ID is required' }); } - let { metadata: _metadata = '{}' } = req.body; const { openai } = await getOpenAIClient({ req, res }); await validateAuthor({ req, openai }); @@ -263,10 +261,15 @@ const uploadAssistantAvatar = async (req, res) => { }, }); + let _metadata; + try { - _metadata = JSON.parse(_metadata); + const assistant = await openai.beta.assistants.retrieve(assistant_id); + if (assistant) { + _metadata = assistant.metadata; + } } catch (error) { - logger.error('[/avatar/:assistant_id] Error parsing metadata', error); + logger.error('[/avatar/:assistant_id] Error fetching assistant', error); _metadata = {}; } @@ -274,7 +277,7 @@ const uploadAssistantAvatar = async (req, res) => { const { deleteFile } = getStrategyFunctions(_metadata.avatar_source); try { await deleteFile(req, { filepath: _metadata.avatar }); - await deleteFileByFilter({ filepath: _metadata.avatar }); + await deleteFileByFilter({ user: req.user.id, filepath: _metadata.avatar }); } catch (error) { logger.error('[/avatar/:assistant_id] Error deleting old avatar', error); } diff --git a/api/server/controllers/assistants/v2.js b/api/server/controllers/assistants/v2.js index 0e1b4a64122..7d91e55b984 100644 --- a/api/server/controllers/assistants/v2.js +++ b/api/server/controllers/assistants/v2.js @@ -149,7 +149,6 @@ const updateAssistant = async ({ req, openai, assistant_id, updateData }) => { * @param {string} params.assistant_id * @param {string} params.tool_resource * @param {string} params.file_id - * @param {AssistantUpdateParams} params.updateData * @returns {Promise} The updated assistant. */ const addResourceFileId = async ({ req, openai, assistant_id, tool_resource, file_id }) => { diff --git a/api/server/index.js b/api/server/index.js index 47ce354f2a0..3bc0a050031 100644 --- a/api/server/index.js +++ b/api/server/index.js @@ -106,6 +106,7 @@ const startServer = async () => { app.use('/api/share', routes.share); app.use('/api/roles', routes.roles); app.use('/api/agents', routes.agents); + app.use('/api/banner', routes.banner); app.use('/api/bedrock', routes.bedrock); app.use('/api/tags', routes.tags); @@ -113,7 +114,8 @@ const startServer = async () => { app.use((req, res) => { // Replace lang attribute in index.html with lang from cookies or accept-language header const lang = req.cookies.lang || req.headers['accept-language']?.split(',')[0] || 'en-US'; - const updatedIndexHtml = indexHTML.replace(/lang="en-US"/g, `lang="${lang}"`); + const saneLang = lang.replace(/"/g, '"'); // sanitize untrusted user input + const updatedIndexHtml = indexHTML.replace(/lang="en-US"/g, `lang="${saneLang}"`); res.send(updatedIndexHtml); }); diff --git a/api/server/middleware/abortMiddleware.js b/api/server/middleware/abortMiddleware.js index e9a12e2bf35..6e608e1cc1e 100644 --- a/api/server/middleware/abortMiddleware.js +++ b/api/server/middleware/abortMiddleware.js @@ -173,6 +173,10 @@ const handleAbortError = async (res, req, error, data) => { errorText = `{"type":"${ErrorTypes.INVALID_REQUEST}"}`; } + if (error?.message?.includes('does not support \'system\'')) { + errorText = `{"type":"${ErrorTypes.NO_SYSTEM_MESSAGES}"}`; + } + const respondWithError = async (partialText) => { let options = { sender, diff --git a/api/server/middleware/buildEndpointOption.js b/api/server/middleware/buildEndpointOption.js index 2b4ba40172d..a85c55c06bb 100644 --- a/api/server/middleware/buildEndpointOption.js +++ b/api/server/middleware/buildEndpointOption.js @@ -10,7 +10,6 @@ const openAI = require('~/server/services/Endpoints/openAI'); const agents = require('~/server/services/Endpoints/agents'); const custom = require('~/server/services/Endpoints/custom'); const google = require('~/server/services/Endpoints/google'); -const enforceModelSpec = require('./enforceModelSpec'); const { handleError } = require('~/server/utils'); const buildFunction = { @@ -28,7 +27,7 @@ const buildFunction = { async function buildEndpointOption(req, res, next) { const { endpoint, endpointType } = req.body; - const parsedBody = parseCompactConvo({ endpoint, endpointType, conversation: req.body }); + let parsedBody = parseCompactConvo({ endpoint, endpointType, conversation: req.body }); if (req.app.locals.modelSpecs?.list && req.app.locals.modelSpecs?.enforce) { /** @type {{ list: TModelSpec[] }}*/ @@ -57,10 +56,11 @@ async function buildEndpointOption(req, res, next) { }); } - const isValidModelSpec = enforceModelSpec(currentModelSpec, parsedBody); - if (!isValidModelSpec) { - return handleError(res, { text: 'Model spec mismatch' }); - } + parsedBody = parseCompactConvo({ + endpoint, + endpointType, + conversation: currentModelSpec.preset, + }); } const endpointFn = buildFunction[endpointType ?? endpoint]; diff --git a/api/server/middleware/enforceModelSpec.js b/api/server/middleware/enforceModelSpec.js deleted file mode 100644 index 17270a5cf8f..00000000000 --- a/api/server/middleware/enforceModelSpec.js +++ /dev/null @@ -1,58 +0,0 @@ -const interchangeableKeys = new Map([ - ['chatGptLabel', ['modelLabel']], - ['modelLabel', ['chatGptLabel']], -]); - -/** - * Middleware to enforce the model spec for a conversation - * @param {TModelSpec} modelSpec - The model spec to enforce - * @param {TConversation} parsedBody - The parsed body of the conversation - * @returns {boolean} - Whether the model spec is enforced - */ -const enforceModelSpec = (modelSpec, parsedBody) => { - for (const [key, value] of Object.entries(modelSpec.preset)) { - if (key === 'endpoint') { - continue; - } - - if (!checkMatch(key, value, parsedBody)) { - return false; - } - } - return true; -}; - -/** - * Checks if there is a match for the given key and value in the parsed body - * or any of its interchangeable keys, including deep comparison for objects and arrays. - * @param {string} key - * @param {any} value - * @param {object} parsedBody - * @returns {boolean} - */ -const checkMatch = (key, value, parsedBody) => { - const isEqual = (a, b) => { - if (Array.isArray(a) && Array.isArray(b)) { - return a.length === b.length && a.every((val, index) => isEqual(val, b[index])); - } else if (typeof a === 'object' && typeof b === 'object' && a !== null && b !== null) { - const keysA = Object.keys(a); - const keysB = Object.keys(b); - return keysA.length === keysB.length && keysA.every((k) => isEqual(a[k], b[k])); - } - return a === b; - }; - - if (isEqual(parsedBody[key], value)) { - return true; - } - - if (interchangeableKeys.has(key)) { - return interchangeableKeys - .get(key) - .some((interchangeableKey) => isEqual(parsedBody[interchangeableKey], value)); - } - - return false; -}; - -module.exports = enforceModelSpec; diff --git a/api/server/middleware/enforceModelSpec.spec.js b/api/server/middleware/enforceModelSpec.spec.js deleted file mode 100644 index 04a8e5b35fb..00000000000 --- a/api/server/middleware/enforceModelSpec.spec.js +++ /dev/null @@ -1,47 +0,0 @@ -// enforceModelSpec.test.js - -const enforceModelSpec = require('./enforceModelSpec'); - -describe('enforceModelSpec function', () => { - test('returns true when all model specs match parsed body directly', () => { - const modelSpec = { preset: { title: 'Dialog', status: 'Active' } }; - const parsedBody = { title: 'Dialog', status: 'Active' }; - expect(enforceModelSpec(modelSpec, parsedBody)).toBe(true); - }); - - test('returns true when model specs match via interchangeable keys', () => { - const modelSpec = { preset: { chatGptLabel: 'GPT-4' } }; - const parsedBody = { modelLabel: 'GPT-4' }; - expect(enforceModelSpec(modelSpec, parsedBody)).toBe(true); - }); - - test('returns false if any key value does not match', () => { - const modelSpec = { preset: { language: 'English', level: 'Advanced' } }; - const parsedBody = { language: 'Spanish', level: 'Advanced' }; - expect(enforceModelSpec(modelSpec, parsedBody)).toBe(false); - }); - - test('ignores the \'endpoint\' key in model spec', () => { - const modelSpec = { preset: { endpoint: 'ignored', feature: 'Special' } }; - const parsedBody = { feature: 'Special' }; - expect(enforceModelSpec(modelSpec, parsedBody)).toBe(true); - }); - - test('handles nested objects correctly', () => { - const modelSpec = { preset: { details: { time: 'noon', location: 'park' } } }; - const parsedBody = { details: { time: 'noon', location: 'park' } }; - expect(enforceModelSpec(modelSpec, parsedBody)).toBe(true); - }); - - test('handles arrays within objects', () => { - const modelSpec = { preset: { tags: ['urgent', 'important'] } }; - const parsedBody = { tags: ['urgent', 'important'] }; - expect(enforceModelSpec(modelSpec, parsedBody)).toBe(true); - }); - - test('fails when arrays in objects do not match', () => { - const modelSpec = { preset: { tags: ['urgent', 'important'] } }; - const parsedBody = { tags: ['important', 'urgent'] }; // Different order - expect(enforceModelSpec(modelSpec, parsedBody)).toBe(false); - }); -}); diff --git a/api/server/middleware/optionalJwtAuth.js b/api/server/middleware/optionalJwtAuth.js new file mode 100644 index 00000000000..8aa1c27e007 --- /dev/null +++ b/api/server/middleware/optionalJwtAuth.js @@ -0,0 +1,17 @@ +const passport = require('passport'); + +// This middleware does not require authentication, +// but if the user is authenticated, it will set the user object. +const optionalJwtAuth = (req, res, next) => { + passport.authenticate('jwt', { session: false }, (err, user) => { + if (err) { + return next(err); + } + if (user) { + req.user = user; + } + next(); + })(req, res, next); +}; + +module.exports = optionalJwtAuth; diff --git a/api/server/middleware/spec/validateImages.spec.js b/api/server/middleware/spec/validateImages.spec.js index ebc6fa60a30..8b04ac931f2 100644 --- a/api/server/middleware/spec/validateImages.spec.js +++ b/api/server/middleware/spec/validateImages.spec.js @@ -3,6 +3,7 @@ const validateImageRequest = require('~/server/middleware/validateImageRequest') describe('validateImageRequest middleware', () => { let req, res, next; + const validObjectId = '65cfb246f7ecadb8b1e8036b'; beforeEach(() => { req = { @@ -43,7 +44,7 @@ describe('validateImageRequest middleware', () => { test('should return 403 if refresh token is expired', () => { const expiredToken = jwt.sign( - { id: '123', exp: Math.floor(Date.now() / 1000) - 3600 }, + { id: validObjectId, exp: Math.floor(Date.now() / 1000) - 3600 }, process.env.JWT_REFRESH_SECRET, ); req.headers.cookie = `refreshToken=${expiredToken}`; @@ -54,22 +55,34 @@ describe('validateImageRequest middleware', () => { test('should call next() for valid image path', () => { const validToken = jwt.sign( - { id: '123', exp: Math.floor(Date.now() / 1000) + 3600 }, + { id: validObjectId, exp: Math.floor(Date.now() / 1000) + 3600 }, process.env.JWT_REFRESH_SECRET, ); req.headers.cookie = `refreshToken=${validToken}`; - req.originalUrl = '/images/123/example.jpg'; + req.originalUrl = `/images/${validObjectId}/example.jpg`; validateImageRequest(req, res, next); expect(next).toHaveBeenCalled(); }); test('should return 403 for invalid image path', () => { const validToken = jwt.sign( - { id: '123', exp: Math.floor(Date.now() / 1000) + 3600 }, + { id: validObjectId, exp: Math.floor(Date.now() / 1000) + 3600 }, process.env.JWT_REFRESH_SECRET, ); req.headers.cookie = `refreshToken=${validToken}`; - req.originalUrl = '/images/456/example.jpg'; + req.originalUrl = '/images/65cfb246f7ecadb8b1e8036c/example.jpg'; // Different ObjectId + validateImageRequest(req, res, next); + expect(res.status).toHaveBeenCalledWith(403); + expect(res.send).toHaveBeenCalledWith('Access Denied'); + }); + + test('should return 403 for invalid ObjectId format', () => { + const validToken = jwt.sign( + { id: validObjectId, exp: Math.floor(Date.now() / 1000) + 3600 }, + process.env.JWT_REFRESH_SECRET, + ); + req.headers.cookie = `refreshToken=${validToken}`; + req.originalUrl = '/images/123/example.jpg'; // Invalid ObjectId validateImageRequest(req, res, next); expect(res.status).toHaveBeenCalledWith(403); expect(res.send).toHaveBeenCalledWith('Access Denied'); @@ -78,16 +91,16 @@ describe('validateImageRequest middleware', () => { // File traversal tests test('should prevent file traversal attempts', () => { const validToken = jwt.sign( - { id: '123', exp: Math.floor(Date.now() / 1000) + 3600 }, + { id: validObjectId, exp: Math.floor(Date.now() / 1000) + 3600 }, process.env.JWT_REFRESH_SECRET, ); req.headers.cookie = `refreshToken=${validToken}`; const traversalAttempts = [ - '/images/123/../../../etc/passwd', - '/images/123/..%2F..%2F..%2Fetc%2Fpasswd', - '/images/123/image.jpg/../../../etc/passwd', - '/images/123/%2e%2e%2f%2e%2e%2f%2e%2e%2fetc%2fpasswd', + `/images/${validObjectId}/../../../etc/passwd`, + `/images/${validObjectId}/..%2F..%2F..%2Fetc%2Fpasswd`, + `/images/${validObjectId}/image.jpg/../../../etc/passwd`, + `/images/${validObjectId}/%2e%2e%2f%2e%2e%2f%2e%2e%2fetc%2fpasswd`, ]; traversalAttempts.forEach((attempt) => { @@ -101,11 +114,11 @@ describe('validateImageRequest middleware', () => { test('should handle URL encoded characters in valid paths', () => { const validToken = jwt.sign( - { id: '123', exp: Math.floor(Date.now() / 1000) + 3600 }, + { id: validObjectId, exp: Math.floor(Date.now() / 1000) + 3600 }, process.env.JWT_REFRESH_SECRET, ); req.headers.cookie = `refreshToken=${validToken}`; - req.originalUrl = '/images/123/image%20with%20spaces.jpg'; + req.originalUrl = `/images/${validObjectId}/image%20with%20spaces.jpg`; validateImageRequest(req, res, next); expect(next).toHaveBeenCalled(); }); diff --git a/api/server/middleware/validateImageRequest.js b/api/server/middleware/validateImageRequest.js index e07e48cc716..eb37b9dbb56 100644 --- a/api/server/middleware/validateImageRequest.js +++ b/api/server/middleware/validateImageRequest.js @@ -2,6 +2,24 @@ const cookies = require('cookie'); const jwt = require('jsonwebtoken'); const { logger } = require('~/config'); +const OBJECT_ID_LENGTH = 24; +const OBJECT_ID_PATTERN = /^[0-9a-f]{24}$/i; + +/** + * Validates if a string is a valid MongoDB ObjectId + * @param {string} id - String to validate + * @returns {boolean} - Whether string is a valid ObjectId format + */ +function isValidObjectId(id) { + if (typeof id !== 'string') { + return false; + } + if (id.length !== OBJECT_ID_LENGTH) { + return false; + } + return OBJECT_ID_PATTERN.test(id); +} + /** * Middleware to validate image request. * Must be set by `secureImageLinks` via custom config file. @@ -25,6 +43,11 @@ function validateImageRequest(req, res, next) { return res.status(403).send('Access Denied'); } + if (!isValidObjectId(payload.id)) { + logger.warn('[validateImageRequest] Invalid User ID'); + return res.status(403).send('Access Denied'); + } + const currentTimeInSeconds = Math.floor(Date.now() / 1000); if (payload.exp < currentTimeInSeconds) { logger.warn('[validateImageRequest] Refresh token expired'); diff --git a/api/server/routes/agents/chat.js b/api/server/routes/agents/chat.js index 353c2fc7b1f..8302abcde02 100644 --- a/api/server/routes/agents/chat.js +++ b/api/server/routes/agents/chat.js @@ -10,6 +10,7 @@ const { } = require('~/server/middleware'); const { initializeClient } = require('~/server/services/Endpoints/agents'); const AgentController = require('~/server/controllers/agents/request'); +const addTitle = require('~/server/services/Endpoints/agents/title'); router.post('/abort', handleAbort()); @@ -28,7 +29,7 @@ router.post( buildEndpointOption, setHeaders, async (req, res, next) => { - await AgentController(req, res, next, initializeClient); + await AgentController(req, res, next, initializeClient, addTitle); }, ); diff --git a/api/server/routes/agents/v1.js b/api/server/routes/agents/v1.js index d3a3005bd55..8e347bbe2a1 100644 --- a/api/server/routes/agents/v1.js +++ b/api/server/routes/agents/v1.js @@ -2,6 +2,7 @@ const multer = require('multer'); const express = require('express'); const { PermissionTypes, Permissions } = require('librechat-data-provider'); const { requireJwtAuth, generateCheckAccess } = require('~/server/middleware'); +const { getAvailableTools } = require('~/server/controllers/PluginController'); const v1 = require('~/server/controllers/agents/v1'); const actions = require('./actions'); @@ -36,9 +37,7 @@ router.use('/actions', actions); * @route GET /agents/tools * @returns {TPlugin[]} 200 - application/json */ -router.use('/tools', (req, res) => { - res.json([]); -}); +router.use('/tools', getAvailableTools); /** * Creates an agent. diff --git a/api/server/routes/banner.js b/api/server/routes/banner.js new file mode 100644 index 00000000000..cf7eafd017d --- /dev/null +++ b/api/server/routes/banner.js @@ -0,0 +1,15 @@ +const express = require('express'); + +const { getBanner } = require('~/models/Banner'); +const optionalJwtAuth = require('~/server/middleware/optionalJwtAuth'); +const router = express.Router(); + +router.get('/', optionalJwtAuth, async (req, res) => { + try { + res.status(200).send(await getBanner(req.user)); + } catch (error) { + res.status(500).json({ message: 'Error getting banner' }); + } +}); + +module.exports = router; diff --git a/api/server/routes/bedrock/chat.js b/api/server/routes/bedrock/chat.js index 605a012710c..c8d6be35de7 100644 --- a/api/server/routes/bedrock/chat.js +++ b/api/server/routes/bedrock/chat.js @@ -10,7 +10,7 @@ const { } = require('~/server/middleware'); const { initializeClient } = require('~/server/services/Endpoints/bedrock'); const AgentController = require('~/server/controllers/agents/request'); -const addTitle = require('~/server/services/Endpoints/bedrock/title'); +const addTitle = require('~/server/services/Endpoints/agents/title'); router.post('/abort', handleAbort()); diff --git a/api/server/routes/convos.js b/api/server/routes/convos.js index 104b0616f81..47a8ef19a8f 100644 --- a/api/server/routes/convos.js +++ b/api/server/routes/convos.js @@ -109,8 +109,14 @@ router.post('/clear', async (req, res) => { router.post('/update', async (req, res) => { const update = req.body.arg; + if (!update.conversationId) { + return res.status(400).json({ error: 'conversationId is required' }); + } + try { - const dbResponse = await saveConvo(req, update, { context: 'POST /api/convos/update' }); + const dbResponse = await saveConvo(req, update, { + context: `POST /api/convos/update ${update.conversationId}`, + }); res.status(201).json(dbResponse); } catch (error) { logger.error('Error updating conversation', error); diff --git a/api/server/routes/files/files.js b/api/server/routes/files/files.js index 1804244ae5a..6d4ab7fefb7 100644 --- a/api/server/routes/files/files.js +++ b/api/server/routes/files/files.js @@ -1,18 +1,22 @@ const fs = require('fs').promises; const express = require('express'); +const { EnvVar } = require('@librechat/agents'); const { isUUID, - checkOpenAIStorage, FileSources, EModelEndpoint, + isAgentsEndpoint, + checkOpenAIStorage, } = require('librechat-data-provider'); const { filterFile, processFileUpload, processDeleteRequest, + processAgentFileUpload, } = require('~/server/services/Files/process'); const { getStrategyFunctions } = require('~/server/services/Files/strategies'); const { getOpenAIClient } = require('~/server/controllers/assistants/helpers'); +const { loadAuthValues } = require('~/app/clients/tools/util'); const { getFiles } = require('~/models/File'); const { logger } = require('~/config'); @@ -62,8 +66,20 @@ router.delete('/', async (req, res) => { return; } - await processDeleteRequest({ req, files }); + const fileIds = files.map((file) => file.file_id); + const userFiles = await getFiles({ file_id: { $in: fileIds }, user: req.user.id }); + if (userFiles.length !== files.length) { + return res.status(403).json({ message: 'You can only delete your own files' }); + } + + await processDeleteRequest({ req, files: userFiles }); + logger.debug( + `[/files] Files deleted successfully: ${files + .filter((f) => f.file_id) + .map((f) => f.file_id) + .join(', ')}`, + ); res.status(200).json({ message: 'Files deleted successfully' }); } catch (error) { logger.error('[/files] Error deleting files:', error); @@ -71,6 +87,36 @@ router.delete('/', async (req, res) => { } }); +router.get('/code/download/:sessionId/:fileId', async (req, res) => { + try { + const { sessionId, fileId } = req.params; + const logPrefix = `Session ID: ${sessionId} | File ID: ${fileId} | Code output download requested by user `; + logger.debug(logPrefix); + + if (!sessionId || !fileId) { + return res.status(400).send('Bad request'); + } + + const { getDownloadStream } = getStrategyFunctions(FileSources.execute_code); + if (!getDownloadStream) { + logger.warn( + `${logPrefix} has no stream method implemented for ${FileSources.execute_code} source`, + ); + return res.status(501).send('Not Implemented'); + } + + const result = await loadAuthValues({ userId: req.user.id, authFields: [EnvVar.CODE_API_KEY] }); + + /** @type {AxiosResponse | undefined} */ + const response = await getDownloadStream(`${sessionId}/${fileId}`, result[EnvVar.CODE_API_KEY]); + res.set(response.headers); + response.data.pipe(res); + } catch (error) { + logger.error('Error downloading file:', error); + res.status(500).send('Error downloading file'); + } +}); + router.get('/download/:userId/:file_id', async (req, res) => { try { const { userId, file_id } = req.params; @@ -154,6 +200,10 @@ router.post('/', async (req, res) => { metadata.temp_file_id = metadata.file_id; metadata.file_id = req.file_id; + if (isAgentsEndpoint(metadata.endpoint)) { + return await processAgentFileUpload({ req, res, file, metadata }); + } + await processFileUpload({ req, res, file, metadata }); } catch (error) { let message = 'Error processing file'; @@ -177,7 +227,7 @@ router.post('/', async (req, res) => { try { await fs.unlink(file.path); } catch (error) { - logger.error('[/files/images] Error deleting file after file processing:', error); + logger.error('[/files] Error deleting file after file processing:', error); } } }); diff --git a/api/server/routes/index.js b/api/server/routes/index.js index 3790aacd24f..4aba91e9548 100644 --- a/api/server/routes/index.js +++ b/api/server/routes/index.js @@ -24,6 +24,7 @@ const edit = require('./edit'); const keys = require('./keys'); const user = require('./user'); const ask = require('./ask'); +const banner = require('./banner'); module.exports = { ask, @@ -52,4 +53,5 @@ module.exports = { assistants, categories, staticRoute, + banner, }; diff --git a/api/server/routes/prompts.js b/api/server/routes/prompts.js index 54128d3b395..c78591265d4 100644 --- a/api/server/routes/prompts.js +++ b/api/server/routes/prompts.js @@ -134,7 +134,7 @@ const createPrompt = async (req, res) => { } }; -router.post('/', createPrompt); +router.post('/', checkPromptCreate, createPrompt); /** * Updates a prompt group diff --git a/api/server/routes/tags.js b/api/server/routes/tags.js index c9f637c473b..d3e27d37110 100644 --- a/api/server/routes/tags.js +++ b/api/server/routes/tags.js @@ -61,7 +61,8 @@ router.post('/', async (req, res) => { */ router.put('/:tag', async (req, res) => { try { - const tag = await updateConversationTag(req.user.id, req.params.tag, req.body); + const decodedTag = decodeURIComponent(req.params.tag); + const tag = await updateConversationTag(req.user.id, decodedTag, req.body); if (tag) { res.status(200).json(tag); } else { @@ -81,7 +82,8 @@ router.put('/:tag', async (req, res) => { */ router.delete('/:tag', async (req, res) => { try { - const tag = await deleteConversationTag(req.user.id, req.params.tag); + const decodedTag = decodeURIComponent(req.params.tag); + const tag = await deleteConversationTag(req.user.id, decodedTag); if (tag) { res.status(200).json(tag); } else { diff --git a/api/server/services/AppService.js b/api/server/services/AppService.js index 19a9fc91a99..f99e9628711 100644 --- a/api/server/services/AppService.js +++ b/api/server/services/AppService.js @@ -8,6 +8,7 @@ const { loadDefaultInterface } = require('./start/interface'); const { azureConfigSetup } = require('./start/azureOpenAI'); const { loadAndFormatTools } = require('./ToolService'); const { initializeRoles } = require('~/models/Role'); +const { cleanup } = require('./cleanup'); const paths = require('~/config/paths'); /** @@ -17,6 +18,7 @@ const paths = require('~/config/paths'); * @param {Express.Application} app - The Express application object. */ const AppService = async (app) => { + cleanup(); await initializeRoles(); /** @type {TCustomConfig}*/ const config = (await loadCustomConfig()) ?? {}; diff --git a/api/server/services/AuthService.js b/api/server/services/AuthService.js index 326b0ed420b..5812dd26f99 100644 --- a/api/server/services/AuthService.js +++ b/api/server/services/AuthService.js @@ -11,7 +11,7 @@ const { deleteUserById, } = require('~/models/userMethods'); const { createToken, findToken, deleteTokens, Session } = require('~/models'); -const { sendEmail, checkEmailConfig } = require('~/server/utils'); +const { isEnabled, checkEmailConfig, sendEmail } = require('~/server/utils'); const { registerSchema } = require('~/strategies/validators'); const { hashToken } = require('~/server/utils/crypto'); const isDomainAllowed = require('./isDomainAllowed'); @@ -188,7 +188,8 @@ const registerUser = async (user, additionalData = {}) => { }; const emailEnabled = checkEmailConfig(); - const newUser = await createUser(newUserData, false, true); + const disableTTL = isEnabled(process.env.ALLOW_UNVERIFIED_EMAIL_LOGIN); + const newUser = await createUser(newUserData, disableTTL, true); newUserId = newUser._id; if (emailEnabled && !newUser.emailVerified) { await sendVerificationEmail({ diff --git a/api/server/services/Config/EndpointService.js b/api/server/services/Config/EndpointService.js index 485c99f3731..49f9d8f5489 100644 --- a/api/server/services/Config/EndpointService.js +++ b/api/server/services/Config/EndpointService.js @@ -45,8 +45,14 @@ module.exports = { AZURE_ASSISTANTS_BASE_URL, EModelEndpoint.azureAssistants, ), - [EModelEndpoint.bedrock]: generateConfig(process.env.BEDROCK_AWS_SECRET_ACCESS_KEY), + [EModelEndpoint.bedrock]: generateConfig( + process.env.BEDROCK_AWS_SECRET_ACCESS_KEY ?? process.env.BEDROCK_AWS_DEFAULT_REGION, + ), /* key will be part of separate config */ - [EModelEndpoint.agents]: generateConfig(process.env.I_AM_A_TEAPOT), + [EModelEndpoint.agents]: generateConfig( + process.env.EXPERIMENTAL_AGENTS, + undefined, + EModelEndpoint.agents, + ), }, }; diff --git a/api/server/services/Endpoints/agents/build.js b/api/server/services/Endpoints/agents/build.js index d04dee9a06d..853c9ba2669 100644 --- a/api/server/services/Endpoints/agents/build.js +++ b/api/server/services/Endpoints/agents/build.js @@ -1,13 +1,12 @@ -const { getAgent } = require('~/models/Agent'); +const { loadAgent } = require('~/models/Agent'); const { logger } = require('~/config'); const buildOptions = (req, endpoint, parsedBody) => { const { agent_id, instructions, spec, ...model_parameters } = parsedBody; - const agentPromise = getAgent({ - id: agent_id, - // TODO: better author handling - author: req.user.id, + const agentPromise = loadAgent({ + req, + agent_id, }).catch((error) => { logger.error(`[/agents/:${agent_id}] Error retrieving agent during build options step`, error); return undefined; diff --git a/api/server/services/Endpoints/agents/initialize.js b/api/server/services/Endpoints/agents/initialize.js index a079e2145fb..b756b781ee7 100644 --- a/api/server/services/Endpoints/agents/initialize.js +++ b/api/server/services/Endpoints/agents/initialize.js @@ -14,14 +14,16 @@ const { tool } = require('@langchain/core/tools'); const { createContentAggregator } = require('@librechat/agents'); const { EModelEndpoint, - providerEndpointMap, getResponseSender, + providerEndpointMap, } = require('librechat-data-provider'); -const { getDefaultHandlers } = require('~/server/controllers/agents/callbacks'); -// for testing purposes -// const createTavilySearchTool = require('~/app/clients/tools/structured/TavilySearch'); -const initAnthropic = require('~/server/services/Endpoints/anthropic/initializeClient'); -const initOpenAI = require('~/server/services/Endpoints/openAI/initializeClient'); +const { + getDefaultHandlers, + createToolEndCallback, +} = require('~/server/controllers/agents/callbacks'); +const initAnthropic = require('~/server/services/Endpoints/anthropic/initialize'); +const initOpenAI = require('~/server/services/Endpoints/openAI/initialize'); +const getBedrockOptions = require('~/server/services/Endpoints/bedrock/options'); const { loadAgentTools } = require('~/server/services/ToolService'); const AgentClient = require('~/server/controllers/agents/client'); const { getModelMaxTokens } = require('~/utils'); @@ -50,6 +52,7 @@ const providerConfigMap = { [EModelEndpoint.openAI]: initOpenAI, [EModelEndpoint.azureOpenAI]: initOpenAI, [EModelEndpoint.anthropic]: initAnthropic, + [EModelEndpoint.bedrock]: getBedrockOptions, }; const initializeClient = async ({ req, res, endpointOption }) => { @@ -58,34 +61,33 @@ const initializeClient = async ({ req, res, endpointOption }) => { } // TODO: use endpointOption to determine options/modelOptions + /** @type {Array} */ + const collectedUsage = []; + /** @type {ArtifactPromises} */ + const artifactPromises = []; const { contentParts, aggregateContent } = createContentAggregator(); - const eventHandlers = getDefaultHandlers({ res, aggregateContent }); - - // const tools = [createTavilySearchTool()]; - // const tools = [_getWeather]; - // const tool_calls = [{ name: 'getPeople_action_swapi---dev' }]; - // const tool_calls = [{ name: 'dalle' }]; - // const tool_calls = [{ name: 'getItmOptions_action_YWlhcGkzLn' }]; - // const tool_calls = [{ name: 'tavily_search_results_json' }]; - // const tool_calls = [ - // { name: 'searchListings_action_emlsbG93NT' }, - // { name: 'searchAddress_action_emlsbG93NT' }, - // { name: 'searchMLS_action_emlsbG93NT' }, - // { name: 'searchCoordinates_action_emlsbG93NT' }, - // { name: 'searchUrl_action_emlsbG93NT' }, - // { name: 'getPropertyDetails_action_emlsbG93NT' }, - // ]; + const toolEndCallback = createToolEndCallback({ req, res, artifactPromises }); + const eventHandlers = getDefaultHandlers({ + res, + aggregateContent, + toolEndCallback, + collectedUsage, + }); if (!endpointOption.agent) { throw new Error('No agent promise provided'); } - /** @type {Agent} */ + /** @type {Agent | null} */ const agent = await endpointOption.agent; + if (!agent) { + throw new Error('Agent not found'); + } const { tools, toolMap } = await loadAgentTools({ req, tools: agent.tools, agent_id: agent.id, + tool_resources: agent.tool_resources, // openAIApiKey: process.env.OPENAI_API_KEY, }); @@ -121,8 +123,11 @@ const initializeClient = async ({ req, res, endpointOption }) => { contentParts, modelOptions, eventHandlers, + collectedUsage, + artifactPromises, endpoint: EModelEndpoint.agents, configOptions: options.configOptions, + attachments: endpointOption.attachments, maxContextTokens: agent.max_context_tokens ?? getModelMaxTokens(modelOptions.model, providerEndpointMap[agent.provider]), diff --git a/api/server/services/Endpoints/bedrock/title.js b/api/server/services/Endpoints/agents/title.js similarity index 93% rename from api/server/services/Endpoints/bedrock/title.js rename to api/server/services/Endpoints/agents/title.js index 520b9f78c43..56fd28668df 100644 --- a/api/server/services/Endpoints/bedrock/title.js +++ b/api/server/services/Endpoints/agents/title.js @@ -33,7 +33,7 @@ const addTitle = async (req, { text, response, client }) => { conversationId: response.conversationId, title, }, - { context: 'api/server/services/Endpoints/bedrock/title.js' }, + { context: 'api/server/services/Endpoints/agents/title.js' }, ); }; diff --git a/api/server/services/Endpoints/anthropic/buildOptions.js b/api/server/services/Endpoints/anthropic/build.js similarity index 100% rename from api/server/services/Endpoints/anthropic/buildOptions.js rename to api/server/services/Endpoints/anthropic/build.js diff --git a/api/server/services/Endpoints/anthropic/index.js b/api/server/services/Endpoints/anthropic/index.js index 772b1efb118..c4e7533c5d0 100644 --- a/api/server/services/Endpoints/anthropic/index.js +++ b/api/server/services/Endpoints/anthropic/index.js @@ -1,6 +1,6 @@ -const addTitle = require('./addTitle'); -const buildOptions = require('./buildOptions'); -const initializeClient = require('./initializeClient'); +const addTitle = require('./title'); +const buildOptions = require('./build'); +const initializeClient = require('./initialize'); module.exports = { addTitle, diff --git a/api/server/services/Endpoints/anthropic/initializeClient.js b/api/server/services/Endpoints/anthropic/initialize.js similarity index 100% rename from api/server/services/Endpoints/anthropic/initializeClient.js rename to api/server/services/Endpoints/anthropic/initialize.js diff --git a/api/server/services/Endpoints/anthropic/addTitle.js b/api/server/services/Endpoints/anthropic/title.js similarity index 100% rename from api/server/services/Endpoints/anthropic/addTitle.js rename to api/server/services/Endpoints/anthropic/title.js diff --git a/api/server/services/Endpoints/assistants/buildOptions.js b/api/server/services/Endpoints/assistants/build.js similarity index 100% rename from api/server/services/Endpoints/assistants/buildOptions.js rename to api/server/services/Endpoints/assistants/build.js diff --git a/api/server/services/Endpoints/assistants/index.js b/api/server/services/Endpoints/assistants/index.js index 772b1efb118..15fca45a34c 100644 --- a/api/server/services/Endpoints/assistants/index.js +++ b/api/server/services/Endpoints/assistants/index.js @@ -1,6 +1,6 @@ -const addTitle = require('./addTitle'); -const buildOptions = require('./buildOptions'); -const initializeClient = require('./initializeClient'); +const addTitle = require('./title'); +const buildOptions = require('./build'); +const initializeClient = require('./initalize'); module.exports = { addTitle, diff --git a/api/server/services/Endpoints/assistants/initializeClient.js b/api/server/services/Endpoints/assistants/initalize.js similarity index 100% rename from api/server/services/Endpoints/assistants/initializeClient.js rename to api/server/services/Endpoints/assistants/initalize.js diff --git a/api/server/services/Endpoints/assistants/initializeClient.spec.js b/api/server/services/Endpoints/assistants/initialize.spec.js similarity index 98% rename from api/server/services/Endpoints/assistants/initializeClient.spec.js rename to api/server/services/Endpoints/assistants/initialize.spec.js index 3879fc0ffce..261f37e9d17 100644 --- a/api/server/services/Endpoints/assistants/initializeClient.spec.js +++ b/api/server/services/Endpoints/assistants/initialize.spec.js @@ -2,7 +2,7 @@ const { HttpsProxyAgent } = require('https-proxy-agent'); const { ErrorTypes } = require('librechat-data-provider'); const { getUserKey, getUserKeyExpiry, getUserKeyValues } = require('~/server/services/UserService'); -const initializeClient = require('./initializeClient'); +const initializeClient = require('./initalize'); // const { OpenAIClient } = require('~/app'); jest.mock('~/server/services/UserService', () => ({ diff --git a/api/server/services/Endpoints/assistants/addTitle.js b/api/server/services/Endpoints/assistants/title.js similarity index 100% rename from api/server/services/Endpoints/assistants/addTitle.js rename to api/server/services/Endpoints/assistants/title.js diff --git a/api/server/services/Endpoints/azureAssistants/buildOptions.js b/api/server/services/Endpoints/azureAssistants/build.js similarity index 100% rename from api/server/services/Endpoints/azureAssistants/buildOptions.js rename to api/server/services/Endpoints/azureAssistants/build.js diff --git a/api/server/services/Endpoints/azureAssistants/index.js b/api/server/services/Endpoints/azureAssistants/index.js index 39944683067..202cb0e4d75 100644 --- a/api/server/services/Endpoints/azureAssistants/index.js +++ b/api/server/services/Endpoints/azureAssistants/index.js @@ -1,5 +1,5 @@ -const buildOptions = require('./buildOptions'); -const initializeClient = require('./initializeClient'); +const buildOptions = require('./build'); +const initializeClient = require('./initialize'); module.exports = { buildOptions, diff --git a/api/server/services/Endpoints/azureAssistants/initializeClient.js b/api/server/services/Endpoints/azureAssistants/initialize.js similarity index 100% rename from api/server/services/Endpoints/azureAssistants/initializeClient.js rename to api/server/services/Endpoints/azureAssistants/initialize.js diff --git a/api/server/services/Endpoints/azureAssistants/initializeClient.spec.js b/api/server/services/Endpoints/azureAssistants/initialize.spec.js similarity index 98% rename from api/server/services/Endpoints/azureAssistants/initializeClient.spec.js rename to api/server/services/Endpoints/azureAssistants/initialize.spec.js index 6dc4a6d47a3..d0c8a364eb3 100644 --- a/api/server/services/Endpoints/azureAssistants/initializeClient.spec.js +++ b/api/server/services/Endpoints/azureAssistants/initialize.spec.js @@ -2,7 +2,7 @@ const { HttpsProxyAgent } = require('https-proxy-agent'); const { ErrorTypes } = require('librechat-data-provider'); const { getUserKey, getUserKeyExpiry, getUserKeyValues } = require('~/server/services/UserService'); -const initializeClient = require('./initializeClient'); +const initializeClient = require('./initialize'); // const { OpenAIClient } = require('~/app'); jest.mock('~/server/services/UserService', () => ({ diff --git a/api/server/services/Endpoints/bedrock/options.js b/api/server/services/Endpoints/bedrock/options.js index 0839d033ce0..405d76fe4df 100644 --- a/api/server/services/Endpoints/bedrock/options.js +++ b/api/server/services/Endpoints/bedrock/options.js @@ -19,7 +19,7 @@ const getOptions = async ({ req, endpointOption }) => { const expiresAt = req.body.key; const isUserProvided = BEDROCK_AWS_SECRET_ACCESS_KEY === AuthType.USER_PROVIDED; - const credentials = isUserProvided + let credentials = isUserProvided ? await getUserKey({ userId: req.user.id, name: EModelEndpoint.bedrock }) : { accessKeyId: BEDROCK_AWS_ACCESS_KEY_ID, @@ -30,6 +30,14 @@ const getOptions = async ({ req, endpointOption }) => { throw new Error('Bedrock credentials not provided. Please provide them again.'); } + if ( + !isUserProvided && + (credentials.accessKeyId === undefined || credentials.accessKeyId === '') && + (credentials.secretAccessKey === undefined || credentials.secretAccessKey === '') + ) { + credentials = undefined; + } + if (expiresAt && isUserProvided) { checkUserKeyExpiry(expiresAt, EModelEndpoint.bedrock); } @@ -53,7 +61,6 @@ const getOptions = async ({ req, endpointOption }) => { /** @type {import('@librechat/agents').BedrockConverseClientOptions} */ const requestOptions = Object.assign( { - credentials, model: endpointOption.model, region: BEDROCK_AWS_DEFAULT_REGION, streaming: true, @@ -72,6 +79,10 @@ const getOptions = async ({ req, endpointOption }) => { endpointOption.model_parameters, ); + if (credentials) { + requestOptions.credentials = credentials; + } + const configOptions = {}; if (PROXY) { configOptions.httpAgent = new HttpsProxyAgent(PROXY); diff --git a/api/server/services/Endpoints/custom/buildOptions.js b/api/server/services/Endpoints/custom/build.js similarity index 96% rename from api/server/services/Endpoints/custom/buildOptions.js rename to api/server/services/Endpoints/custom/build.js index 3458d317b0f..add78470f5d 100644 --- a/api/server/services/Endpoints/custom/buildOptions.js +++ b/api/server/services/Endpoints/custom/build.js @@ -3,6 +3,7 @@ const generateArtifactsPrompt = require('~/app/clients/prompts/artifacts'); const buildOptions = (endpoint, parsedBody, endpointType) => { const { + modelLabel, chatGptLabel, promptPrefix, maxContextTokens, @@ -17,6 +18,7 @@ const buildOptions = (endpoint, parsedBody, endpointType) => { const endpointOption = removeNullishValues({ endpoint, endpointType, + modelLabel, chatGptLabel, promptPrefix, resendFiles, diff --git a/api/server/services/Endpoints/custom/index.js b/api/server/services/Endpoints/custom/index.js index 3cda8d5fece..5a70d78749b 100644 --- a/api/server/services/Endpoints/custom/index.js +++ b/api/server/services/Endpoints/custom/index.js @@ -1,5 +1,5 @@ -const initializeClient = require('./initializeClient'); -const buildOptions = require('./buildOptions'); +const initializeClient = require('./initialize'); +const buildOptions = require('./build'); module.exports = { initializeClient, diff --git a/api/server/services/Endpoints/custom/initializeClient.js b/api/server/services/Endpoints/custom/initialize.js similarity index 100% rename from api/server/services/Endpoints/custom/initializeClient.js rename to api/server/services/Endpoints/custom/initialize.js diff --git a/api/server/services/Endpoints/google/buildOptions.js b/api/server/services/Endpoints/google/build.js similarity index 100% rename from api/server/services/Endpoints/google/buildOptions.js rename to api/server/services/Endpoints/google/build.js diff --git a/api/server/services/Endpoints/google/index.js b/api/server/services/Endpoints/google/index.js index 772b1efb118..c4e7533c5d0 100644 --- a/api/server/services/Endpoints/google/index.js +++ b/api/server/services/Endpoints/google/index.js @@ -1,6 +1,6 @@ -const addTitle = require('./addTitle'); -const buildOptions = require('./buildOptions'); -const initializeClient = require('./initializeClient'); +const addTitle = require('./title'); +const buildOptions = require('./build'); +const initializeClient = require('./initialize'); module.exports = { addTitle, diff --git a/api/server/services/Endpoints/google/initializeClient.js b/api/server/services/Endpoints/google/initialize.js similarity index 100% rename from api/server/services/Endpoints/google/initializeClient.js rename to api/server/services/Endpoints/google/initialize.js diff --git a/api/server/services/Endpoints/google/initializeClient.spec.js b/api/server/services/Endpoints/google/initialize.spec.js similarity index 97% rename from api/server/services/Endpoints/google/initializeClient.spec.js rename to api/server/services/Endpoints/google/initialize.spec.js index 657dcbcaa83..e5391107bd2 100644 --- a/api/server/services/Endpoints/google/initializeClient.spec.js +++ b/api/server/services/Endpoints/google/initialize.spec.js @@ -1,6 +1,6 @@ // file deepcode ignore HardcodedNonCryptoSecret: No hardcoded secrets const { getUserKey } = require('~/server/services/UserService'); -const initializeClient = require('./initializeClient'); +const initializeClient = require('./initialize'); const { GoogleClient } = require('~/app'); jest.mock('~/server/services/UserService', () => ({ diff --git a/api/server/services/Endpoints/google/addTitle.js b/api/server/services/Endpoints/google/title.js similarity index 97% rename from api/server/services/Endpoints/google/addTitle.js rename to api/server/services/Endpoints/google/title.js index f21d123214b..b93f13797ff 100644 --- a/api/server/services/Endpoints/google/addTitle.js +++ b/api/server/services/Endpoints/google/title.js @@ -3,7 +3,7 @@ const getLogStores = require('~/cache/getLogStores'); const { isEnabled } = require('~/server/utils'); const { saveConvo } = require('~/models'); const { logger } = require('~/config'); -const initializeClient = require('./initializeClient'); +const initializeClient = require('./initialize'); const addTitle = async (req, { text, response, client }) => { const { TITLE_CONVO = 'true' } = process.env ?? {}; diff --git a/api/server/services/Endpoints/gptPlugins/buildOptions.js b/api/server/services/Endpoints/gptPlugins/build.js similarity index 83% rename from api/server/services/Endpoints/gptPlugins/buildOptions.js rename to api/server/services/Endpoints/gptPlugins/build.js index 1a7f26beb68..0d1ec097ad6 100644 --- a/api/server/services/Endpoints/gptPlugins/buildOptions.js +++ b/api/server/services/Endpoints/gptPlugins/build.js @@ -3,6 +3,7 @@ const generateArtifactsPrompt = require('~/app/clients/prompts/artifacts'); const buildOptions = (endpoint, parsedBody) => { const { + modelLabel, chatGptLabel, promptPrefix, agentOptions, @@ -16,10 +17,10 @@ const buildOptions = (endpoint, parsedBody) => { } = parsedBody; const endpointOption = removeNullishValues({ endpoint, - tools: - tools - .map((tool) => tool?.pluginKey ?? tool) - .filter((toolName) => typeof toolName === 'string'), + tools: tools + .map((tool) => tool?.pluginKey ?? tool) + .filter((toolName) => typeof toolName === 'string'), + modelLabel, chatGptLabel, promptPrefix, agentOptions, diff --git a/api/server/services/Endpoints/gptPlugins/index.js b/api/server/services/Endpoints/gptPlugins/index.js index 39944683067..202cb0e4d75 100644 --- a/api/server/services/Endpoints/gptPlugins/index.js +++ b/api/server/services/Endpoints/gptPlugins/index.js @@ -1,5 +1,5 @@ -const buildOptions = require('./buildOptions'); -const initializeClient = require('./initializeClient'); +const buildOptions = require('./build'); +const initializeClient = require('./initialize'); module.exports = { buildOptions, diff --git a/api/server/services/Endpoints/gptPlugins/initializeClient.js b/api/server/services/Endpoints/gptPlugins/initialize.js similarity index 100% rename from api/server/services/Endpoints/gptPlugins/initializeClient.js rename to api/server/services/Endpoints/gptPlugins/initialize.js diff --git a/api/server/services/Endpoints/gptPlugins/initializeClient.spec.js b/api/server/services/Endpoints/gptPlugins/initialize.spec.js similarity index 99% rename from api/server/services/Endpoints/gptPlugins/initializeClient.spec.js rename to api/server/services/Endpoints/gptPlugins/initialize.spec.js index 2dc5bc0653e..54dfffc7958 100644 --- a/api/server/services/Endpoints/gptPlugins/initializeClient.spec.js +++ b/api/server/services/Endpoints/gptPlugins/initialize.spec.js @@ -1,7 +1,7 @@ // gptPlugins/initializeClient.spec.js const { EModelEndpoint, ErrorTypes, validateAzureGroups } = require('librechat-data-provider'); const { getUserKey, getUserKeyValues } = require('~/server/services/UserService'); -const initializeClient = require('./initializeClient'); +const initializeClient = require('./initialize'); const { PluginsClient } = require('~/app'); // Mock getUserKey since it's the only function we want to mock diff --git a/api/server/services/Endpoints/openAI/buildOptions.js b/api/server/services/Endpoints/openAI/build.js similarity index 96% rename from api/server/services/Endpoints/openAI/buildOptions.js rename to api/server/services/Endpoints/openAI/build.js index 9f305939fb6..ff9cc484e79 100644 --- a/api/server/services/Endpoints/openAI/buildOptions.js +++ b/api/server/services/Endpoints/openAI/build.js @@ -3,6 +3,7 @@ const generateArtifactsPrompt = require('~/app/clients/prompts/artifacts'); const buildOptions = (endpoint, parsedBody) => { const { + modelLabel, chatGptLabel, promptPrefix, maxContextTokens, @@ -17,6 +18,7 @@ const buildOptions = (endpoint, parsedBody) => { const endpointOption = removeNullishValues({ endpoint, + modelLabel, chatGptLabel, promptPrefix, resendFiles, diff --git a/api/server/services/Endpoints/openAI/index.js b/api/server/services/Endpoints/openAI/index.js index 772b1efb118..c4e7533c5d0 100644 --- a/api/server/services/Endpoints/openAI/index.js +++ b/api/server/services/Endpoints/openAI/index.js @@ -1,6 +1,6 @@ -const addTitle = require('./addTitle'); -const buildOptions = require('./buildOptions'); -const initializeClient = require('./initializeClient'); +const addTitle = require('./title'); +const buildOptions = require('./build'); +const initializeClient = require('./initialize'); module.exports = { addTitle, diff --git a/api/server/services/Endpoints/openAI/initializeClient.js b/api/server/services/Endpoints/openAI/initialize.js similarity index 98% rename from api/server/services/Endpoints/openAI/initializeClient.js rename to api/server/services/Endpoints/openAI/initialize.js index b72b3d32c44..215b9437309 100644 --- a/api/server/services/Endpoints/openAI/initializeClient.js +++ b/api/server/services/Endpoints/openAI/initialize.js @@ -130,7 +130,7 @@ const initializeClient = async ({ if (optionsOnly) { const requestOptions = Object.assign( { - modelOptions: endpointOption.modelOptions, + modelOptions: endpointOption.model_parameters, }, clientOptions, ); diff --git a/api/server/services/Endpoints/openAI/initializeClient.spec.js b/api/server/services/Endpoints/openAI/initialize.spec.js similarity index 99% rename from api/server/services/Endpoints/openAI/initializeClient.spec.js rename to api/server/services/Endpoints/openAI/initialize.spec.js index 0988a0fcb79..b1a702e995c 100644 --- a/api/server/services/Endpoints/openAI/initializeClient.spec.js +++ b/api/server/services/Endpoints/openAI/initialize.spec.js @@ -1,6 +1,6 @@ const { EModelEndpoint, ErrorTypes, validateAzureGroups } = require('librechat-data-provider'); const { getUserKey, getUserKeyValues } = require('~/server/services/UserService'); -const initializeClient = require('./initializeClient'); +const initializeClient = require('./initialize'); const { OpenAIClient } = require('~/app'); // Mock getUserKey since it's the only function we want to mock diff --git a/api/server/services/Endpoints/openAI/llm.js b/api/server/services/Endpoints/openAI/llm.js index 3817224a4ba..bd51679e1b6 100644 --- a/api/server/services/Endpoints/openAI/llm.js +++ b/api/server/services/Endpoints/openAI/llm.js @@ -38,7 +38,6 @@ function getLLMConfig(apiKey, options = {}) { } = options; let llmConfig = { - model: 'gpt-4o-mini', streaming, }; diff --git a/api/server/services/Endpoints/openAI/addTitle.js b/api/server/services/Endpoints/openAI/title.js similarity index 100% rename from api/server/services/Endpoints/openAI/addTitle.js rename to api/server/services/Endpoints/openAI/title.js diff --git a/api/server/services/Files/Audio/streamAudio.js b/api/server/services/Files/Audio/streamAudio.js index 4d1157bd349..7b6bef03f84 100644 --- a/api/server/services/Files/Audio/streamAudio.js +++ b/api/server/services/Files/Audio/streamAudio.js @@ -1,4 +1,3 @@ -const WebSocket = require('ws'); const { CacheKeys, findLastSeparatorIndex, SEPARATORS } = require('librechat-data-provider'); const { getLogStores } = require('~/cache'); @@ -44,33 +43,6 @@ function getRandomVoiceId(voiceIds) { * @property {string[]} normalizedAlignment.chars */ -/** - * - * @param {Record} parameters - * @returns - */ -function assembleQuery(parameters) { - let query = ''; - let hasQuestionMark = false; - - for (const [key, value] of Object.entries(parameters)) { - if (value == null) { - continue; - } - - if (!hasQuestionMark) { - query += '?'; - hasQuestionMark = true; - } else { - query += '&'; - } - - query += `${key}=${value}`; - } - - return query; -} - const MAX_NOT_FOUND_COUNT = 6; const MAX_NO_CHANGE_COUNT = 10; @@ -197,144 +169,6 @@ function splitTextIntoChunks(text, chunkSize = 4000) { return chunks; } -/** - * Input stream text to speech - * @param {Express.Response} res - * @param {AsyncIterable} textStream - * @param {(token: string) => Promise} callback - Whether to continue the stream or not - * @returns {AsyncGenerator} - */ -function inputStreamTextToSpeech(res, textStream, callback) { - const model = 'eleven_monolingual_v1'; - const wsUrl = `wss://api.elevenlabs.io/v1/text-to-speech/${getRandomVoiceId()}/stream-input${assembleQuery( - { - model_id: model, - // flush: true, - // optimize_streaming_latency: this.settings.optimizeStreamingLatency, - optimize_streaming_latency: 1, - // output_format: this.settings.outputFormat, - }, - )}`; - const socket = new WebSocket(wsUrl); - - socket.onopen = function () { - const streamStart = { - text: ' ', - voice_settings: { - stability: 0.5, - similarity_boost: 0.8, - }, - xi_api_key: process.env.ELEVENLABS_API_KEY, - // generation_config: { chunk_length_schedule: [50, 90, 120, 150, 200] }, - }; - - socket.send(JSON.stringify(streamStart)); - - // send stream until done - const streamComplete = new Promise((resolve, reject) => { - (async () => { - let textBuffer = ''; - let shouldContinue = true; - for await (const textDelta of textStream) { - textBuffer += textDelta; - - // using ". " as separator: sending in full sentences improves the quality - // of the audio output significantly. - const separatorIndex = findLastSeparatorIndex(textBuffer); - - // Callback for textStream (will return false if signal is aborted) - shouldContinue = await callback(textDelta); - - if (separatorIndex === -1) { - continue; - } - - if (!shouldContinue) { - break; - } - - const textToProcess = textBuffer.slice(0, separatorIndex); - textBuffer = textBuffer.slice(separatorIndex + 1); - - const request = { - text: textToProcess, - try_trigger_generation: true, - }; - - socket.send(JSON.stringify(request)); - } - - // send remaining text: - if (shouldContinue && textBuffer.length > 0) { - socket.send( - JSON.stringify({ - text: `${textBuffer} `, // append space - try_trigger_generation: true, - }), - ); - } - })() - .then(resolve) - .catch(reject); - }); - - streamComplete - .then(() => { - const endStream = { - text: '', - }; - - socket.send(JSON.stringify(endStream)); - }) - .catch((e) => { - console.error('Error streaming text to speech:', e); - throw e; - }); - }; - - return (async function* audioStream() { - let isDone = false; - let chunks = []; - let resolve; - let waitForMessage = new Promise((r) => (resolve = r)); - - socket.onmessage = function (event) { - // console.log(event); - const audioChunk = JSON.parse(event.data); - if (audioChunk.audio && audioChunk.alignment) { - res.write(`event: audio\ndata: ${event.data}\n\n`); - chunks.push(audioChunk); - resolve(null); - waitForMessage = new Promise((r) => (resolve = r)); - } else if (audioChunk.isFinal) { - isDone = true; - resolve(null); - } else if (audioChunk.message) { - console.warn('Received Elevenlabs message:', audioChunk.message); - resolve(null); - } - }; - - socket.onerror = function (error) { - console.error('WebSocket error:', error); - // throw error; - }; - - socket.onclose = function () { - isDone = true; - resolve(null); - }; - - while (!isDone) { - await waitForMessage; - yield* chunks; - chunks = []; - } - - res.write('event: end\ndata: \n\n'); - })(); -} - /** * * @param {AsyncIterable} llmStream @@ -349,7 +183,6 @@ async function* llmMessageSource(llmStream) { } module.exports = { - inputStreamTextToSpeech, findLastSeparatorIndex, createChunkProcessor, splitTextIntoChunks, diff --git a/api/server/services/Files/Code/crud.js b/api/server/services/Files/Code/crud.js new file mode 100644 index 00000000000..10aa130d2ea --- /dev/null +++ b/api/server/services/Files/Code/crud.js @@ -0,0 +1,34 @@ +// downloadStream.js + +const axios = require('axios'); +const { getCodeBaseURL } = require('@librechat/agents'); + +const baseURL = getCodeBaseURL(); + +/** + * Retrieves a download stream for a specified file. + * @param {string} fileIdentifier - The identifier for the file (e.g., "sessionId/fileId"). + * @param {string} apiKey - The API key for authentication. + * @returns {Promise} A promise that resolves to a readable stream of the file content. + * @throws {Error} If there's an error during the download process. + */ +async function getCodeOutputDownloadStream(fileIdentifier, apiKey) { + try { + const response = await axios({ + method: 'get', + url: `${baseURL}/download/${fileIdentifier}`, + responseType: 'stream', + headers: { + 'User-Agent': 'LibreChat/1.0', + 'X-API-Key': apiKey, + }, + timeout: 15000, + }); + + return response; + } catch (error) { + throw new Error(`Error downloading file: ${error.message}`); + } +} + +module.exports = { getCodeOutputDownloadStream }; diff --git a/api/server/services/Files/Code/index.js b/api/server/services/Files/Code/index.js new file mode 100644 index 00000000000..a6223d1ee5d --- /dev/null +++ b/api/server/services/Files/Code/index.js @@ -0,0 +1,5 @@ +const crud = require('./crud'); + +module.exports = { + ...crud, +}; diff --git a/api/server/services/Files/Code/process.js b/api/server/services/Files/Code/process.js new file mode 100644 index 00000000000..d8e8119367e --- /dev/null +++ b/api/server/services/Files/Code/process.js @@ -0,0 +1,87 @@ +const path = require('path'); +const { v4 } = require('uuid'); +const axios = require('axios'); +const { getCodeBaseURL, EnvVar } = require('@librechat/agents'); +const { FileContext, imageExtRegex } = require('librechat-data-provider'); +const { convertImage } = require('~/server/services/Files/images/convert'); +const { loadAuthValues } = require('~/app/clients/tools/util'); +const { createFile } = require('~/models/File'); +const { logger } = require('~/config'); + +/** + * Process OpenAI image files, convert to target format, save and return file metadata. + * @param {ServerRequest} params.req - The Express request object. + * @param {string} params.id - The file ID. + * @param {string} params.name - The filename. + * @param {string} params.toolCallId - The tool call ID that generated the file. + * @param {string} params.sessionId - The code execution session ID. + * @param {string} params.conversationId - The current conversation ID. + * @param {string} params.messageId - The current message ID. + * @returns {Promise} The file metadata or undefined if an error occurs. + */ +const processCodeOutput = async ({ + req, + id, + name, + toolCallId, + conversationId, + messageId, + sessionId, +}) => { + const currentDate = new Date(); + const baseURL = getCodeBaseURL(); + const fileExt = path.extname(name); + if (!fileExt || !imageExtRegex.test(name)) { + return { + filename: name, + filepath: `/api/files/code/download/${sessionId}/${id}`, + /** Note: expires 24 hours after creation */ + expiresAt: currentDate.getTime() + 86400000, + conversationId, + toolCallId, + messageId, + }; + } + + try { + const formattedDate = currentDate.toISOString(); + const result = await loadAuthValues({ userId: req.user.id, authFields: [EnvVar.CODE_API_KEY] }); + const response = await axios({ + method: 'get', + url: `${baseURL}/download/${sessionId}/${id}`, + responseType: 'arraybuffer', + headers: { + 'User-Agent': 'LibreChat/1.0', + 'X-API-Key': result[EnvVar.CODE_API_KEY], + }, + timeout: 15000, + }); + + const buffer = Buffer.from(response.data, 'binary'); + + const file_id = v4(); + const _file = await convertImage(req, buffer, 'high', `${file_id}${fileExt}`); + const file = { + ..._file, + file_id, + usage: 1, + filename: name, + conversationId, + user: req.user.id, + type: `image/${req.app.locals.imageOutputType}`, + createdAt: formattedDate, + updatedAt: formattedDate, + source: req.app.locals.fileStrategy, + context: FileContext.execute_code, + }; + createFile(file, true); + /** Note: `messageId` & `toolCallId` are not part of file DB schema; message object records associated file ID */ + return Object.assign(file, { messageId, toolCallId }); + } catch (error) { + logger.error('Error downloading file:', error); + } +}; + +module.exports = { + processCodeOutput, +}; diff --git a/api/server/services/Files/Local/crud.js b/api/server/services/Files/Local/crud.js index 18bf5127fd4..12b7738828f 100644 --- a/api/server/services/Files/Local/crud.js +++ b/api/server/services/Files/Local/crud.js @@ -202,8 +202,20 @@ const deleteLocalFile = async (req, file) => { } if (file.filepath.startsWith(`/uploads/${req.user.id}`)) { - const basePath = file.filepath.split('/uploads/')[1]; - const filepath = path.join(uploads, basePath); + const userUploadDir = path.join(uploads, req.user.id); + const basePath = file.filepath.split(`/uploads/${req.user.id}/`)[1]; + + if (!basePath) { + throw new Error(`Invalid file path: ${file.filepath}`); + } + + const filepath = path.join(userUploadDir, basePath); + + const rel = path.relative(userUploadDir, filepath); + if (rel.startsWith('..') || path.isAbsolute(rel) || rel.includes(`..${path.sep}`)) { + throw new Error(`Invalid file path: ${file.filepath}`); + } + await fs.promises.unlink(filepath); return; } diff --git a/api/server/services/Files/images/encode.js b/api/server/services/Files/images/encode.js index 05c9fc1d33f..f4579270199 100644 --- a/api/server/services/Files/images/encode.js +++ b/api/server/services/Files/images/encode.js @@ -1,6 +1,12 @@ const axios = require('axios'); -const { EModelEndpoint, FileSources, VisionModes } = require('librechat-data-provider'); -const { getStrategyFunctions } = require('../strategies'); +const { + FileSources, + VisionModes, + ImageDetail, + ContentTypes, + EModelEndpoint, +} = require('librechat-data-provider'); +const { getStrategyFunctions } = require('~/server/services/Files/strategies'); const { logger } = require('~/config'); /** @@ -79,7 +85,7 @@ async function encodeAndFormat(req, files, endpoint, mode) { promises.push(preparePayload(req, file)); } - const detail = req.body.imageDetail ?? 'auto'; + const detail = req.body.imageDetail ?? ImageDetail.auto; /** @type {Array<[MongoFile, string]>} */ const formattedImages = await Promise.all(promises); @@ -104,7 +110,7 @@ async function encodeAndFormat(req, files, endpoint, mode) { } const imagePart = { - type: 'image_url', + type: ContentTypes.IMAGE_URL, image_url: { url: imageContent.startsWith('http') ? imageContent @@ -113,6 +119,12 @@ async function encodeAndFormat(req, files, endpoint, mode) { }, }; + if (mode === VisionModes.agents) { + result.image_urls.push(imagePart); + result.files.push(fileMetadata); + continue; + } + if (endpoint && endpoint === EModelEndpoint.google && mode === VisionModes.generative) { delete imagePart.image_url; imagePart.inlineData = { diff --git a/api/server/services/Files/process.js b/api/server/services/Files/process.js index 197fd160cfe..00d98bbc20b 100644 --- a/api/server/services/Files/process.js +++ b/api/server/services/Files/process.js @@ -8,6 +8,7 @@ const { FileSources, imageExtRegex, EModelEndpoint, + EToolResources, mergeFileConfig, hostImageIdSuffix, checkOpenAIStorage, @@ -16,6 +17,7 @@ const { } = require('librechat-data-provider'); const { addResourceFileId, deleteResourceFileId } = require('~/server/controllers/assistants/v2'); const { convertImage, resizeAndConvert } = require('~/server/services/Files/images'); +const { addAgentResourceFile, removeAgentResourceFile } = require('~/models/Agent'); const { getOpenAIClient } = require('~/server/controllers/assistants/helpers'); const { createFile, updateFileUsage, deleteFiles } = require('~/models/File'); const { LB_QueueAsyncCall } = require('~/server/utils/queue'); @@ -124,6 +126,17 @@ const processDeleteRequest = async ({ req, files }) => { for (const file of files) { const source = file.source ?? FileSources.local; + if (req.body.agent_id && req.body.tool_resource) { + promises.push( + removeAgentResourceFile({ + req, + file_id: file.file_id, + agent_id: req.body.agent_id, + tool_resource: req.body.tool_resource, + }), + ); + } + if (checkOpenAIStorage(source) && !client[source]) { await initializeClients(); } @@ -398,6 +411,95 @@ const processFileUpload = async ({ req, res, file, metadata }) => { res.status(200).json({ message: 'File uploaded and processed successfully', ...result }); }; +/** + * Applies the current strategy for file uploads. + * Saves file metadata to the database with an expiry TTL. + * Files must be deleted from the server filesystem manually. + * + * @param {Object} params - The parameters object. + * @param {Express.Request} params.req - The Express request object. + * @param {Express.Response} params.res - The Express response object. + * @param {Express.Multer.File} params.file - The uploaded file. + * @param {FileMetadata} params.metadata - Additional metadata for the file. + * @returns {Promise} + */ +const processAgentFileUpload = async ({ req, res, file, metadata }) => { + const { agent_id, tool_resource } = metadata; + if (agent_id && !tool_resource) { + throw new Error('No tool resource provided for agent file upload'); + } + + if (tool_resource === EToolResources.file_search && file.mimetype.startsWith('image')) { + throw new Error('Image uploads are not supported for file search tool resources'); + } + + let messageAttachment = !!metadata.message_file; + if (!messageAttachment && !agent_id) { + throw new Error('No agent ID provided for agent file upload'); + } + + const source = + tool_resource === EToolResources.file_search + ? FileSources.vectordb + : req.app.locals.fileStrategy; + const { handleFileUpload } = getStrategyFunctions(source); + const { file_id, temp_file_id } = metadata; + + const { + bytes, + filename, + filepath: _filepath, + embedded, + height, + width, + } = await handleFileUpload({ + req, + file, + file_id, + }); + + let filepath = _filepath; + + if (!messageAttachment && tool_resource) { + await addAgentResourceFile({ + req, + agent_id, + file_id, + tool_resource: tool_resource, + }); + } + + if (file.mimetype.startsWith('image')) { + const result = await processImageFile({ + req, + file, + metadata: { file_id: v4() }, + returnFile: true, + }); + filepath = result.filepath; + } + + const result = await createFile( + { + user: req.user.id, + file_id, + temp_file_id, + bytes, + filepath, + filename: filename ?? file.originalname, + context: messageAttachment ? FileContext.message_attachment : FileContext.agents, + model: messageAttachment ? undefined : req.body.model, + type: file.mimetype, + embedded, + source, + height, + width, + }, + true, + ); + res.status(200).json({ message: 'Agent file uploaded and processed successfully', ...result }); +}; + /** * @param {object} params - The params object. * @param {OpenAI} params.openai - The OpenAI client instance. @@ -654,5 +756,6 @@ module.exports = { uploadImageBuffer, processFileUpload, processDeleteRequest, + processAgentFileUpload, retrieveAndProcessFile, }; diff --git a/api/server/services/Files/strategies.js b/api/server/services/Files/strategies.js index fa4e456fc9f..cfb837a50ed 100644 --- a/api/server/services/Files/strategies.js +++ b/api/server/services/Files/strategies.js @@ -21,6 +21,7 @@ const { } = require('./Local'); const { uploadOpenAIFile, deleteOpenAIFile, getOpenAIFileStream } = require('./OpenAI'); const { uploadVectors, deleteVectors } = require('./VectorDB'); +const { getCodeOutputDownloadStream } = require('./Code'); /** * Firebase Storage Strategy Functions @@ -103,6 +104,31 @@ const openAIStrategy = () => ({ getDownloadStream: getOpenAIFileStream, }); +/** + * Code Output Strategy Functions + * + * Note: null values mean that the strategy is not supported. + * */ +const codeOutputStrategy = () => ({ + /** @type {typeof saveFileFromURL | null} */ + saveURL: null, + /** @type {typeof getLocalFileURL | null} */ + getFileURL: null, + /** @type {typeof saveLocalBuffer | null} */ + saveBuffer: null, + /** @type {typeof processLocalAvatar | null} */ + processAvatar: null, + /** @type {typeof uploadLocalImage | null} */ + handleImageUpload: null, + /** @type {typeof prepareImagesLocal | null} */ + prepareImagePayload: null, + /** @type {typeof deleteLocalFile | null} */ + deleteFile: null, + /** @type {typeof uploadVectors | null} */ + handleFileUpload: null, + getDownloadStream: getCodeOutputDownloadStream, +}); + // Strategy Selector const getStrategyFunctions = (fileSource) => { if (fileSource === FileSources.firebase) { @@ -115,6 +141,8 @@ const getStrategyFunctions = (fileSource) => { return openAIStrategy(); } else if (fileSource === FileSources.vectordb) { return vectorStrategy(); + } else if (fileSource === FileSources.execute_code) { + return codeOutputStrategy(); } else { throw new Error('Invalid file source'); } diff --git a/api/server/services/ModelService.js b/api/server/services/ModelService.js index 7d2a3ae9ec9..0547d03187e 100644 --- a/api/server/services/ModelService.js +++ b/api/server/services/ModelService.js @@ -1,7 +1,7 @@ const axios = require('axios'); const { HttpsProxyAgent } = require('https-proxy-agent'); const { EModelEndpoint, defaultModels, CacheKeys } = require('librechat-data-provider'); -const { extractBaseURL, inputSchema, processModelData, logAxiosError } = require('~/utils'); +const { inputSchema, logAxiosError, extractBaseURL, processModelData } = require('~/utils'); const { OllamaClient } = require('~/app/clients/OllamaClient'); const getLogStores = require('~/cache/getLogStores'); @@ -66,6 +66,7 @@ const fetchModels = async ({ headers: { Authorization: `Bearer ${apiKey}`, }, + timeout: 5000, }; if (process.env.PROXY) { @@ -149,6 +150,7 @@ const fetchOpenAIModels = async (opts, _models = []) => { baseURL, azure: opts.azure, user: opts.user, + name: baseURL, }); } @@ -157,7 +159,7 @@ const fetchOpenAIModels = async (opts, _models = []) => { } if (baseURL === openaiBaseURL) { - const regex = /(text-davinci-003|gpt-)/; + const regex = /(text-davinci-003|gpt-|o1-)/; models = models.filter((model) => regex.test(model)); const instructModels = models.filter((model) => model.includes('instruct')); const otherModels = models.filter((model) => !model.includes('instruct')); @@ -175,7 +177,8 @@ const fetchOpenAIModels = async (opts, _models = []) => { * @param {object} opts - The options for fetching the models. * @param {string} opts.user - The user ID to send to the API. * @param {boolean} [opts.azure=false] - Whether to fetch models from Azure. - * @param {boolean} [opts.plugins=false] - Whether to fetch models from the plugins. + * @param {boolean} [opts.plugins=false] - Whether to fetch models for the plugins endpoint. + * @param {boolean} [opts.assistants=false] - Whether to fetch models for the Assistants endpoint. */ const getOpenAIModels = async (opts) => { let models = defaultModels[EModelEndpoint.openAI]; diff --git a/api/server/services/ModelService.spec.js b/api/server/services/ModelService.spec.js index 4e4647ee35d..a383db1e3c9 100644 --- a/api/server/services/ModelService.spec.js +++ b/api/server/services/ModelService.spec.js @@ -291,7 +291,9 @@ describe('fetchModels with Ollama specific logic', () => { }); expect(models).toEqual(['Ollama-Base', 'Ollama-Advanced']); - expect(axios.get).toHaveBeenCalledWith('https://api.ollama.test.com/api/tags'); // Adjusted to expect only one argument if no options are passed + expect(axios.get).toHaveBeenCalledWith('https://api.ollama.test.com/api/tags', { + timeout: 5000, + }); }); it('should handle errors gracefully when fetching Ollama models fails', async () => { diff --git a/api/server/services/Runs/StreamRunManager.js b/api/server/services/Runs/StreamRunManager.js index 951818bb6f1..ab1217939fd 100644 --- a/api/server/services/Runs/StreamRunManager.js +++ b/api/server/services/Runs/StreamRunManager.js @@ -329,7 +329,7 @@ class StreamRunManager { } } else if (typeof delta[key] === 'string' && typeof data[key] === 'string') { // Concatenate strings - data[key] += delta[key]; + // data[key] += delta[key]; } else if ( typeof delta[key] === 'object' && delta[key] !== null && diff --git a/api/server/services/Threads/manage.js b/api/server/services/Threads/manage.js index 5155d611092..8dbac189ab4 100644 --- a/api/server/services/Threads/manage.js +++ b/api/server/services/Threads/manage.js @@ -8,9 +8,9 @@ const { } = require('librechat-data-provider'); const { retrieveAndProcessFile } = require('~/server/services/Files/process'); const { recordMessage, getMessages } = require('~/models/Message'); +const { countTokens, escapeRegExp } = require('~/server/utils'); const { spendTokens } = require('~/models/spendTokens'); const { saveConvo } = require('~/models/Conversation'); -const { countTokens } = require('~/server/utils'); /** * Initializes a new thread or adds messages to an existing thread. @@ -518,14 +518,6 @@ const recordUsage = async ({ const uniqueCitationStart = '^====||==='; const uniqueCitationEnd = '==|||||^'; -/** Helper function to escape special characters in regex - * @param {string} string - The string to escape. - * @returns {string} The escaped string. - */ -function escapeRegExp(string) { - return string.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); -} - /** * Sorts, processes, and flattens messages to a single string. * diff --git a/api/server/services/ToolService.js b/api/server/services/ToolService.js index 109fbeddea4..778b29ff068 100644 --- a/api/server/services/ToolService.js +++ b/api/server/services/ToolService.js @@ -1,9 +1,8 @@ const fs = require('fs'); const path = require('path'); -const { StructuredTool } = require('langchain/tools'); -const { tool: toolFn } = require('@langchain/core/tools'); const { zodToJsonSchema } = require('zod-to-json-schema'); const { Calculator } = require('langchain/tools/calculator'); +const { tool: toolFn, Tool } = require('@langchain/core/tools'); const { Tools, ContentTypes, @@ -70,7 +69,7 @@ function loadAndFormatTools({ directory, adminFilter = [], adminIncluded = [] }) continue; } - if (!ToolClass || !(ToolClass.prototype instanceof StructuredTool)) { + if (!ToolClass || !(ToolClass.prototype instanceof Tool)) { continue; } @@ -152,7 +151,7 @@ const processVisionRequest = async (client, currentAction) => { /** @type {ChatCompletion | undefined} */ const completion = await client.visionPromise; - if (completion.usage) { + if (completion && completion.usage) { recordUsage({ user: client.req.user.id, model: client.req.body.model, @@ -378,11 +377,12 @@ async function processRequiredActions(client, requiredActions) { * @param {Object} params - Run params containing user and request information. * @param {ServerRequest} params.req - The request object. * @param {string} params.agent_id - The agent ID. - * @param {string[]} params.tools - The agent's available tools. + * @param {Agent['tools']} params.tools - The agent's available tools. + * @param {Agent['tool_resources']} params.tool_resources - The agent's available tool resources. * @param {string | undefined} [params.openAIApiKey] - The OpenAI API key. * @returns {Promise<{ tools?: StructuredTool[]; toolMap?: Record}>} The combined toolMap. */ -async function loadAgentTools({ req, agent_id, tools, openAIApiKey }) { +async function loadAgentTools({ req, agent_id, tools, tool_resources, openAIApiKey }) { if (!tools || tools.length === 0) { return {}; } @@ -394,6 +394,7 @@ async function loadAgentTools({ req, agent_id, tools, openAIApiKey }) { options: { req, openAIApiKey, + tool_resources, returnMetadata: true, processFileURL, uploadImageBuffer, @@ -405,6 +406,10 @@ async function loadAgentTools({ req, agent_id, tools, openAIApiKey }) { const agentTools = []; for (let i = 0; i < loadedTools.length; i++) { const tool = loadedTools[i]; + if (tool.name && (tool.name === Tools.execute_code || tool.name === Tools.file_search)) { + agentTools.push(tool); + continue; + } const toolInstance = toolFn( async (...args) => { diff --git a/api/server/services/UserService.js b/api/server/services/UserService.js index 30b54c74068..91d772477bf 100644 --- a/api/server/services/UserService.js +++ b/api/server/services/UserService.js @@ -103,10 +103,10 @@ const getUserKeyExpiry = async ({ userId, name }) => { * @param {string} params.userId - The unique identifier for the user. * @param {string} params.name - The name associated with the key. * @param {string} params.value - The value to be encrypted and stored as the key's value. - * @param {Date} params.expiresAt - The expiry date for the key. + * @param {Date} params.expiresAt - The expiry date for the key [optional] * @returns {Promise} The updated or newly inserted key document. * @description This function either updates an existing user key or inserts a new one into the database, - * after encrypting the provided value. It sets the provided expiry date for the key. + * after encrypting the provided value. It sets the provided expiry date for the key (or unsets for no expiry). */ const updateUserKey = async ({ userId, name, value, expiresAt = null }) => { const encryptedValue = await encrypt(value); @@ -115,13 +115,15 @@ const updateUserKey = async ({ userId, name, value, expiresAt = null }) => { name, value: encryptedValue, }; - - // Only add expiresAt to the update object if it's not null + const updateQuery = { $set: updateObject }; + // add expiresAt to the update object if it's not null if (expiresAt) { updateObject.expiresAt = new Date(expiresAt); + } else { + // make sure to remove if already present + updateQuery.$unset = { expiresAt }; } - - return await Key.findOneAndUpdate({ userId, name }, updateObject, { + return await Key.findOneAndUpdate({ userId, name }, updateQuery, { upsert: true, new: true, }).lean(); diff --git a/api/server/services/cleanup.js b/api/server/services/cleanup.js new file mode 100644 index 00000000000..814c0ecc94c --- /dev/null +++ b/api/server/services/cleanup.js @@ -0,0 +1,13 @@ +const { logger } = require('~/config'); +const { deleteNullOrEmptyConversations } = require('~/models/Conversation'); +const cleanup = async () => { + try { + await deleteNullOrEmptyConversations(); + } catch (error) { + logger.error('[cleanup] Error during app cleanup', error); + } finally { + logger.debug('Startup cleanup complete'); + } +}; + +module.exports = { cleanup }; diff --git a/api/server/utils/citations.js b/api/server/utils/citations.js index 33136c18b8d..97cb69aca77 100644 --- a/api/server/utils/citations.js +++ b/api/server/utils/citations.js @@ -1,6 +1,14 @@ const citationRegex = /\[\^\d+?\^\]/g; const regex = / \[.*?]\(.*?\)/g; +/** Helper function to escape special characters in regex + * @param {string} string - The string to escape. + * @returns {string} The escaped string. + */ +function escapeRegExp(string) { + return string.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); +} + const getCitations = (res) => { const adaptiveCards = res.details.adaptiveCards; const textBlocks = adaptiveCards && adaptiveCards[0].body; @@ -47,4 +55,4 @@ const citeText = (res, noLinks = false) => { return result; }; -module.exports = { getCitations, citeText }; +module.exports = { getCitations, citeText, escapeRegExp }; diff --git a/api/server/utils/crypto.js b/api/server/utils/crypto.js index c143506cc54..ea71df51ad0 100644 --- a/api/server/utils/crypto.js +++ b/api/server/utils/crypto.js @@ -102,4 +102,14 @@ async function hashToken(str) { return Buffer.from(hashBuffer).toString('hex'); } -module.exports = { encrypt, decrypt, encryptV2, decryptV2, hashToken }; +async function getRandomValues(length) { + if (!Number.isInteger(length) || length <= 0) { + throw new Error('Length must be a positive integer'); + } + + const randomValues = new Uint8Array(length); + webcrypto.getRandomValues(randomValues); + return Buffer.from(randomValues).toString('hex'); +} + +module.exports = { encrypt, decrypt, encryptV2, decryptV2, hashToken, getRandomValues }; diff --git a/api/server/utils/handleText.js b/api/server/utils/handleText.js index 17ab5697269..2668569834f 100644 --- a/api/server/utils/handleText.js +++ b/api/server/utils/handleText.js @@ -1,6 +1,8 @@ const { Capabilities, EModelEndpoint, + isAgentsEndpoint, + AgentCapabilities, isAssistantsEndpoint, defaultRetrievalModels, defaultAssistantsVersion, @@ -160,8 +162,8 @@ const isUserProvided = (value) => value === 'user_provided'; /** * Generate the configuration for a given key and base URL. * @param {string} key - * @param {string} baseURL - * @param {string} endpoint + * @param {string} [baseURL] + * @param {string} [endpoint] * @returns {boolean | { userProvide: boolean, userProvideURL?: boolean }} */ function generateConfig(key, baseURL, endpoint) { @@ -177,7 +179,7 @@ function generateConfig(key, baseURL, endpoint) { } const assistants = isAssistantsEndpoint(endpoint); - + const agents = isAgentsEndpoint(endpoint); if (assistants) { config.retrievalModels = defaultRetrievalModels; config.capabilities = [ @@ -189,6 +191,18 @@ function generateConfig(key, baseURL, endpoint) { ]; } + if (agents) { + config.capabilities = [ + AgentCapabilities.file_search, + AgentCapabilities.actions, + AgentCapabilities.tools, + ]; + + if (key === 'EXPERIMENTAL_RUN_CODE') { + config.capabilities.push(AgentCapabilities.execute_code); + } + } + if (assistants && endpoint === EModelEndpoint.azureAssistants) { config.version = defaultAssistantsVersion.azureAssistants; } else if (assistants) { diff --git a/api/server/utils/import/__data__/chatgpt-citations.json b/api/server/utils/import/__data__/chatgpt-citations.json new file mode 100644 index 00000000000..5a08631a6f9 --- /dev/null +++ b/api/server/utils/import/__data__/chatgpt-citations.json @@ -0,0 +1,345 @@ +[ + { + "title": "Citation Convo", + "create_time": 1704629915.775304, + "update_time": 1704717442.442031, + "mapping": { + "9e874379-5008-4a2d-aa2e-628d1d705a04": { + "id": "9e874379-5008-4a2d-aa2e-628d1d705a04", + "message": { + "id": "9e874379-5008-4a2d-aa2e-628d1d705a04", + "author": { + "role": "tool", + "name": "browser", + "metadata": {} + }, + "create_time": 1708580164.172007, + "update_time": null, + "content": { + "content_type": "tether_browsing_display", + "result": "# 【0†Signal Sciences - Crunchbase Company Profile & Funding†www.crunchbase.com】\nFounded Date Mar 10, 2014. Founders Andrew Peterson, Nick Galbreath, Zane Lackey. Operating Status Active. Last Funding Type Series C. Legal Name Signal Sciences Corp. Company Type For Profit. Contact Email info@signalsciences.com. Phone Number +1 424-404-1139. Signal Sciences is the fastest growing web application security company in the world.\n# 【1†Demand More from Your WAF - Signal Sciences now part of Fastly†www.signalsciences.com】\nModernize Your Application/API Protection While Lowering Your TCO. Legacy WAFs weren't designed for today's web apps that are distributed across cloud, on-premise, edge or hybrid environments. Our next-gen web application firewall (NGWAF) and runtime application self protection (RASP) increase security and maintain reliability without ...\n# 【2†Security at Scale: Fastly Announces Intent to Acquire Signal Science†www.fastly.com】\nSignal Sciences' technology combined with Fastly's current solutions will form Fastly's upcoming new security offering, called Secure@Edge. Secure@Edge will be a modern, unified web application and API security solution that will integrate with our Compute platform, and power and protect companies looking to further or begin their digital ...\n# 【3†Web Application and API Protection - Signal Sciences now part of Fastly†www.signalsciences.com】\nOur customers have spoken: Signal Sciences is the highest rated Web Application Firewalls (WAF) vendor. We were named a 2021 Gartner Peer Insights Customers' Choice for WAF based on detailed end-user testimonials. In addition, we are the only vendor to be named a Customers' Choice for three consecutive times and have an overall rating of 5 ...\n# 【4†Fastly Completes Acquisition of Signal Sciences | Fastly†www.fastly.com】\nSAN FRANCISCO, October 1, 2020 – Fastly, Inc. (NYSE: FSLY), provider of an edge cloud platform, today announced it has successfully completed the acquisition of Signal Sciences (“Signal Sciences”). The transaction combines Signal Sciences' powerful web application and API security solutions with Fastly's edge cloud platform and existing security offerings to form a unified suite of ...\n# 【5†Signal Sciences: Enterprises still overlooking web app security†www.techtarget.com】\nRansomware threats are everywhere these days, but one startup is focusing on the most common cause behind data breaches. Signal Sciences is a Web Application Firewall (WAF) and Runtime Application Self-Protection (RASP) company that protects customers' web applications and APIs, regardless of the software architecture or deployment model used, such as on-premises, cloud, containers or hybrid ...\n# 【6†Signal Sciences Launches Cloud WAF for Easy Web Application Security†www.signalsciences.com】\nToday we are proud to announce the launch of the new Signal Sciences Cloud Web Application Firewall (WAF) deployment option for our award-winning WAF and RASP solution.As the fastest growing web application security company in the world, we know organizations must quickly detect and stop web application layer attacks wherever their apps, APIs, or microservices operate—whether that be in the ...\n# 【7†Fastly Agrees to Acquire Signal Sciences for $775 Million - Cooley†www.cooley.com】\nSan Francisco – August 27, 2020 – Cooley advised edge cloud platform provider Fastly on its agreement to acquire Signal Sciences, one of the fastest growing web application security companies in the world, for approximately $775 million in cash and stock.The acquisition, which is expected to close this year, will expand Fastly's robust security portfolio at a time when security at the ...\n# 【8†Signal Sciences†info.signalsciences.com】\nSignal Sciences next-gen WAF and RASP technology is designed to work quickly and effectively, enabling application developers and operations teams to deliver modern, business-critical web applications and APIs while ensuring they're well protected and running performantly.. There are many vendors claiming to provide effective and scalable offerings to protect applications and APIs, so we ...\n# 【9†Gartner names Signal Sciences a Visionary in the 2020 Magic ... - Fastly†www.fastly.com】\nWe believe Signal Sciences' innovation earned them recognition in the 2019 Gartner Magic Quadrant for WAF, ... Gartner research publications consist of the opinions of Gartner's research organization and should not be construed as statements of fact. Gartner disclaims all warranties, express or implied, with respect to this research ...\n# 【10†Fastly acquires Signal Sciences - 2020-08-27 - Crunchbase†www.crunchbase.com】\nSignal Sciences Signal Sciences is a software as a service platform providing security monitoring and defense for your web applications. Acquiring Organization: Fastly Fastly helps digital businesses keep pace with their customer expectations by delivering secure and online experiences. Announced Date Aug 27, 2020;\n# 【11†Signal Sciences - Funding, Financials, Valuation & Investors - Crunchbase†www.crunchbase.com】\nFunding. Signal Sciences has raised a total of. $61.7M. in funding over 4 rounds. Their latest funding was raised on Feb 5, 2019 from a Series C round. Signal Sciences is funded by 8 investors. CRV and Harrison Metal are the most recent investors. Signal Sciences has invested in Open Raven on Feb 11, 2020. This investment - Seed Round - Open ...\n# 【12†Signal Sciences WAF Review and Alternatives - Comparitech†www.comparitech.com】\nSignal Sciences WAF is completely agentless as it works in the cloud to apply both managed and custom rulesets to your application traffic. Protections extend beyond the OWASP Top 10 and include defense from botnet attacks, account takeovers, credential stuffing, API abuse, and DDoS mitigation. Automatic blocking and scanning are applied to ...\n# 【13†Integrations - Web Application Security - Signal Sciences now part of ...†www.signalsciences.com】\nThe Signal Sciences site extension protects any IIS web application hosted on AAS and provides production visibility necessary to detect and block malicious web requests before they can compromise your business-critical apps, APIs and microservices. ... and efficiently into an organization's existing hybrid, or cloud architecture ...\n# 【14†Fastly Next-Gen WAF professional services | Fastly Products†docs.fastly.com】\nFastly Next-Gen WAF (powered by Signal Sciences) (Next-Gen WAF) professional services provide your organization with training, implementation, and maintenance services for the Next-Gen WAF. Depending on the service offerings you select, our team will provide training and work with you to plan, test, deploy, and maintain a solution to protect ...\n# 【15†Andrew Peterson - Founder & CEO @ Signal Sciences - Crunchbase†www.crunchbase.com】\nAndrew Peterson is the founder and CEO of Signal Sciences, a Software-as-a-Service platform that provides security monitoring and defense solutions for web applications. He was previously the group product manager of international growth at Etsy. Prior to Etsy, Peterson was a health information management officer at the Clinton Foundation\n# 【16†Signal Sciences†info.signalsciences.com】\nThe webinar will cover: The most common attacks targeting distributed web apps. How a web application firewall (WAF) can inspect and decision on both north-south (client-to-app origin) and east-west (service-to-service) web requests. How to detect and prevent malicious client-side activity originating from website supply chain vendors. A live ...\n# 【17†Next-Generation Cloud Web Application Firewall (WAF) - Signal Sciences ...†www.signalsciences.com】\nWith a single change to a DNS record, web requests are routed to the Signal Sciences Cloud WAF to inspect and decision on those requests and block bad requests. Our unique solution provides robust detection and protection capabilities no matter where your organization's applications operate or how rapidly they scale. No complex software ...\n# 【18†How to Configure SAML 2.0 for Signal Sciences - UserDocs†saml-doc.okta.com】\nConfiguration Steps. Log in to Signal Sciences as an Organization Administrator. Navigate to organization > Settings.. Click the Switch to SAML button:. Click the I understand, configure SSO button:. On the Configure SSO with SAML page, enter the following:. Copy and paste the following IDP Login page into the SAML endpoint field:. Sign into the Okta Admin Dashboard to generate this variable.\n# 【19†Powering the best of the internet | Fastly†www.fastly.com】\nEverybody needs speed, reliability, security, savings, and scale – but different industries have different needs. Fastly's powerful network and smarter solutions can be tailored to your organization. We partner with you to guarantee a smooth migration, so you can deliver the best possible user experiences.\nVisible: 0% - 100%", + "summary": null, + "assets": [], + "tether_id": null + }, + "status": "finished_successfully", + "end_turn": null, + "weight": 0, + "metadata": { + "_cite_metadata": { + "citation_format": { + "name": "tether_og", + "regex": "【(\\d+)(?::(\\d+))?†([^†【】]*)[^【】]*?】" + }, + "metadata_list": [ + { + "type": "webpage", + "title": "Signal Sciences - Crunchbase Company Profile & Funding", + "url": "https://www.crunchbase.com/organization/signal-sciences", + "text": "\nFounded Date Mar 10, 2014. Founders Andrew Peterson, Nick Galbreath, Zane Lackey. Operating Status Active. Last Funding Type Series C. Legal Name Signal Sciences Corp. Company Type For Profit. Contact Email info@signalsciences.com. Phone Number +1 424-404-1139. Signal Sciences is the fastest growing web application security company in the world.\n", + "pub_date": null, + "extra": null + }, + { + "type": "webpage", + "title": "Demand More from Your WAF - Signal Sciences now part of Fastly", + "url": "https://www.signalsciences.com/", + "text": "\nModernize Your Application/API Protection While Lowering Your TCO. Legacy WAFs weren't designed for today's web apps that are distributed across cloud, on-premise, edge or hybrid environments. Our next-gen web application firewall (NGWAF) and runtime application self protection (RASP) increase security and maintain reliability without ...\n", + "pub_date": null, + "extra": null + }, + { + "type": "webpage", + "title": "Security at Scale: Fastly Announces Intent to Acquire Signal Science", + "url": "https://www.fastly.com/blog/fastly-intent-to-acquire-signal-sciences", + "text": "\nSignal Sciences' technology combined with Fastly's current solutions will form Fastly's upcoming new security offering, called Secure@Edge. Secure@Edge will be a modern, unified web application and API security solution that will integrate with our Compute platform, and power and protect companies looking to further or begin their digital ...\n", + "pub_date": null, + "extra": null + }, + { + "type": "webpage", + "title": "Web Application and API Protection - Signal Sciences now part of Fastly", + "url": "https://www.signalsciences.com/products/", + "text": "\nOur customers have spoken: Signal Sciences is the highest rated Web Application Firewalls (WAF) vendor. We were named a 2021 Gartner Peer Insights Customers' Choice for WAF based on detailed end-user testimonials. In addition, we are the only vendor to be named a Customers' Choice for three consecutive times and have an overall rating of 5 ...\n", + "pub_date": null, + "extra": null + }, + { + "type": "webpage", + "title": "Fastly Completes Acquisition of Signal Sciences | Fastly", + "url": "https://www.fastly.com/press/press-releases/fastly-completes-acquisition-signal-sciences", + "text": "\nSAN FRANCISCO, October 1, 2020 – Fastly, Inc. (NYSE: FSLY), provider of an edge cloud platform, today announced it has successfully completed the acquisition of Signal Sciences (“Signal Sciences”). The transaction combines Signal Sciences' powerful web application and API security solutions with Fastly's edge cloud platform and existing security offerings to form a unified suite of ...\n", + "pub_date": null, + "extra": null + }, + { + "type": "webpage", + "title": "Signal Sciences: Enterprises still overlooking web app security", + "url": "https://www.techtarget.com/searchsecurity/news/252476659/Signal-Sciences-Enterprises-still-overlooking-web-app-security", + "text": "\nRansomware threats are everywhere these days, but one startup is focusing on the most common cause behind data breaches. Signal Sciences is a Web Application Firewall (WAF) and Runtime Application Self-Protection (RASP) company that protects customers' web applications and APIs, regardless of the software architecture or deployment model used, such as on-premises, cloud, containers or hybrid ...\n", + "pub_date": null, + "extra": null + }, + { + "type": "webpage", + "title": "Signal Sciences Launches Cloud WAF for Easy Web Application Security", + "url": "https://www.signalsciences.com/blog/cloud-waf-web-applications/", + "text": "\nToday we are proud to announce the launch of the new Signal Sciences Cloud Web Application Firewall (WAF) deployment option for our award-winning WAF and RASP solution.As the fastest growing web application security company in the world, we know organizations must quickly detect and stop web application layer attacks wherever their apps, APIs, or microservices operate—whether that be in the ...\n", + "pub_date": null, + "extra": null + }, + { + "type": "webpage", + "title": "Fastly Agrees to Acquire Signal Sciences for $775 Million - Cooley", + "url": "https://www.cooley.com/news/coverage/2020/2020-08-27-fastly-agrees-to-acquire-signal-sciences-for-775-million", + "text": "\nSan Francisco – August 27, 2020 – Cooley advised edge cloud platform provider Fastly on its agreement to acquire Signal Sciences, one of the fastest growing web application security companies in the world, for approximately $775 million in cash and stock.The acquisition, which is expected to close this year, will expand Fastly's robust security portfolio at a time when security at the ...\n", + "pub_date": null, + "extra": null + }, + { + "type": "webpage", + "title": "Signal Sciences", + "url": "https://info.signalsciences.com/10-key-capabilities-of-signal-sciences", + "text": "\nSignal Sciences next-gen WAF and RASP technology is designed to work quickly and effectively, enabling application developers and operations teams to deliver modern, business-critical web applications and APIs while ensuring they're well protected and running performantly.. There are many vendors claiming to provide effective and scalable offerings to protect applications and APIs, so we ...\n", + "pub_date": null, + "extra": null + }, + { + "type": "webpage", + "title": "Gartner names Signal Sciences a Visionary in the 2020 Magic ... - Fastly", + "url": "https://www.fastly.com/blog/gartner-names-signal-sciences-a-visionary-in-the-2020-magic-quadrant-for-waf", + "text": "\nWe believe Signal Sciences' innovation earned them recognition in the 2019 Gartner Magic Quadrant for WAF, ... Gartner research publications consist of the opinions of Gartner's research organization and should not be construed as statements of fact. Gartner disclaims all warranties, express or implied, with respect to this research ...\n", + "pub_date": null, + "extra": null + }, + { + "type": "webpage", + "title": "Fastly acquires Signal Sciences - 2020-08-27 - Crunchbase", + "url": "https://www.crunchbase.com/acquisition/fastly-acquires-signal-sciences--2b836efb", + "text": "\nSignal Sciences Signal Sciences is a software as a service platform providing security monitoring and defense for your web applications. Acquiring Organization: Fastly Fastly helps digital businesses keep pace with their customer expectations by delivering secure and online experiences. Announced Date Aug 27, 2020;\n", + "pub_date": null, + "extra": null + }, + { + "type": "webpage", + "title": "Signal Sciences - Funding, Financials, Valuation & Investors - Crunchbase", + "url": "https://www.crunchbase.com/organization/signal-sciences/company_financials", + "text": "\nFunding. Signal Sciences has raised a total of. $61.7M. in funding over 4 rounds. Their latest funding was raised on Feb 5, 2019 from a Series C round. Signal Sciences is funded by 8 investors. CRV and Harrison Metal are the most recent investors. Signal Sciences has invested in Open Raven on Feb 11, 2020. This investment - Seed Round - Open ...\n", + "pub_date": null, + "extra": null + }, + { + "type": "webpage", + "title": "Signal Sciences WAF Review and Alternatives - Comparitech", + "url": "https://www.comparitech.com/net-admin/signal-sciences-waf-review/", + "text": "\nSignal Sciences WAF is completely agentless as it works in the cloud to apply both managed and custom rulesets to your application traffic. Protections extend beyond the OWASP Top 10 and include defense from botnet attacks, account takeovers, credential stuffing, API abuse, and DDoS mitigation. Automatic blocking and scanning are applied to ...\n", + "pub_date": null, + "extra": null + }, + { + "type": "webpage", + "title": "Integrations - Web Application Security - Signal Sciences now part of ...", + "url": "https://www.signalsciences.com/integrations/", + "text": "\nThe Signal Sciences site extension protects any IIS web application hosted on AAS and provides production visibility necessary to detect and block malicious web requests before they can compromise your business-critical apps, APIs and microservices. ... and efficiently into an organization's existing hybrid, or cloud architecture ...\n", + "pub_date": null, + "extra": null + }, + { + "type": "webpage", + "title": "Fastly Next-Gen WAF professional services | Fastly Products", + "url": "https://docs.fastly.com/products/fastly-next-gen-waf-professional-services", + "text": "\nFastly Next-Gen WAF (powered by Signal Sciences) (Next-Gen WAF) professional services provide your organization with training, implementation, and maintenance services for the Next-Gen WAF. Depending on the service offerings you select, our team will provide training and work with you to plan, test, deploy, and maintain a solution to protect ...\n", + "pub_date": null, + "extra": null + }, + { + "type": "webpage", + "title": "Andrew Peterson - Founder & CEO @ Signal Sciences - Crunchbase", + "url": "https://www.crunchbase.com/person/andrew-peterson", + "text": "\nAndrew Peterson is the founder and CEO of Signal Sciences, a Software-as-a-Service platform that provides security monitoring and defense solutions for web applications. He was previously the group product manager of international growth at Etsy. Prior to Etsy, Peterson was a health information management officer at the Clinton Foundation\n", + "pub_date": null, + "extra": null + }, + { + "type": "webpage", + "title": "Signal Sciences", + "url": "https://info.signalsciences.com/preventing-modern-web-app-attacks-source-defense", + "text": "\nThe webinar will cover: The most common attacks targeting distributed web apps. How a web application firewall (WAF) can inspect and decision on both north-south (client-to-app origin) and east-west (service-to-service) web requests. How to detect and prevent malicious client-side activity originating from website supply chain vendors. A live ...\n", + "pub_date": null, + "extra": null + }, + { + "type": "webpage", + "title": "Next-Generation Cloud Web Application Firewall (WAF) - Signal Sciences ...", + "url": "https://www.signalsciences.com/products/cloud-waf/", + "text": "\nWith a single change to a DNS record, web requests are routed to the Signal Sciences Cloud WAF to inspect and decision on those requests and block bad requests. Our unique solution provides robust detection and protection capabilities no matter where your organization's applications operate or how rapidly they scale. No complex software ...\n", + "pub_date": null, + "extra": null + }, + { + "type": "webpage", + "title": "How to Configure SAML 2.0 for Signal Sciences - UserDocs", + "url": "https://saml-doc.okta.com/SAML_Docs/How-to-Configure-SAML-2.0-for-Signal-Sciences.html", + "text": "\nConfiguration Steps. Log in to Signal Sciences as an Organization Administrator. Navigate to organization > Settings.. Click the Switch to SAML button:. Click the I understand, configure SSO button:. On the Configure SSO with SAML page, enter the following:. Copy and paste the following IDP Login page into the SAML endpoint field:. Sign into the Okta Admin Dashboard to generate this variable.\n", + "pub_date": null, + "extra": null + }, + { + "type": "webpage", + "title": "Powering the best of the internet | Fastly", + "url": "https://www.fastly.com/", + "text": "\nEverybody needs speed, reliability, security, savings, and scale – but different industries have different needs. Fastly's powerful network and smarter solutions can be tailored to your organization. We partner with you to guarantee a smooth migration, so you can deliver the best possible user experiences.\nVisible: 0% - 100%", + "pub_date": null, + "extra": null + }, + { + "type": "webpage", + "title": "Search results for query: 'Signal Sciences organization'", + "url": "", + "text": "# 【0†Signal Sciences - Crunchbase Company Profile & Funding†www.crunchbase.com】\nFounded Date Mar 10, 2014. Founders Andrew Peterson, Nick Galbreath, Zane Lackey. Operating Status Active. Last Funding Type Series C. Legal Name Signal Sciences Corp. Company Type For Profit. Contact Email info@signalsciences.com. Phone Number +1 424-404-1139. Signal Sciences is the fastest growing web application security company in the world.\n# 【1†Demand More from Your WAF - Signal Sciences now part of Fastly†www.signalsciences.com】\nModernize Your Application/API Protection While Lowering Your TCO. Legacy WAFs weren't designed for today's web apps that are distributed across cloud, on-premise, edge or hybrid environments. Our next-gen web application firewall (NGWAF) and runtime application self protection (RASP) increase security and maintain reliability without ...\n# 【2†Security at Scale: Fastly Announces Intent to Acquire Signal Science†www.fastly.com】\nSignal Sciences' technology combined with Fastly's current solutions will form Fastly's upcoming new security offering, called Secure@Edge. Secure@Edge will be a modern, unified web application and API security solution that will integrate with our Compute platform, and power and protect companies looking to further or begin their digital ...\n# 【3†Web Application and API Protection - Signal Sciences now part of Fastly†www.signalsciences.com】\nOur customers have spoken: Signal Sciences is the highest rated Web Application Firewalls (WAF) vendor. We were named a 2021 Gartner Peer Insights Customers' Choice for WAF based on detailed end-user testimonials. In addition, we are the only vendor to be named a Customers' Choice for three consecutive times and have an overall rating of 5 ...\n# 【4†Fastly Completes Acquisition of Signal Sciences | Fastly†www.fastly.com】\nSAN FRANCISCO, October 1, 2020 – Fastly, Inc. (NYSE: FSLY), provider of an edge cloud platform, today announced it has successfully completed the acquisition of Signal Sciences (“Signal Sciences”). The transaction combines Signal Sciences' powerful web application and API security solutions with Fastly's edge cloud platform and existing security offerings to form a unified suite of ...\n# 【5†Signal Sciences: Enterprises still overlooking web app security†www.techtarget.com】\nRansomware threats are everywhere these days, but one startup is focusing on the most common cause behind data breaches. Signal Sciences is a Web Application Firewall (WAF) and Runtime Application Self-Protection (RASP) company that protects customers' web applications and APIs, regardless of the software architecture or deployment model used, such as on-premises, cloud, containers or hybrid ...\n# 【6†Signal Sciences Launches Cloud WAF for Easy Web Application Security†www.signalsciences.com】\nToday we are proud to announce the launch of the new Signal Sciences Cloud Web Application Firewall (WAF) deployment option for our award-winning WAF and RASP solution.As the fastest growing web application security company in the world, we know organizations must quickly detect and stop web application layer attacks wherever their apps, APIs, or microservices operate—whether that be in the ...\n# 【7†Fastly Agrees to Acquire Signal Sciences for $775 Million - Cooley†www.cooley.com】\nSan Francisco – August 27, 2020 – Cooley advised edge cloud platform provider Fastly on its agreement to acquire Signal Sciences, one of the fastest growing web application security companies in the world, for approximately $775 million in cash and stock.The acquisition, which is expected to close this year, will expand Fastly's robust security portfolio at a time when security at the ...\n# 【8†Signal Sciences†info.signalsciences.com】\nSignal Sciences next-gen WAF and RASP technology is designed to work quickly and effectively, enabling application developers and operations teams to deliver modern, business-critical web applications and APIs while ensuring they're well protected and running performantly.. There are many vendors claiming to provide effective and scalable offerings to protect applications and APIs, so we ...\n# 【9†Gartner names Signal Sciences a Visionary in the 2020 Magic ... - Fastly†www.fastly.com】\nWe believe Signal Sciences' innovation earned them recognition in the 2019 Gartner Magic Quadrant for WAF, ... Gartner research publications consist of the opinions of Gartner's research organization and should not be construed as statements of fact. Gartner disclaims all warranties, express or implied, with respect to this research ...\n# 【10†Fastly acquires Signal Sciences - 2020-08-27 - Crunchbase†www.crunchbase.com】\nSignal Sciences Signal Sciences is a software as a service platform providing security monitoring and defense for your web applications. Acquiring Organization: Fastly Fastly helps digital businesses keep pace with their customer expectations by delivering secure and online experiences. Announced Date Aug 27, 2020;\n# 【11†Signal Sciences - Funding, Financials, Valuation & Investors - Crunchbase†www.crunchbase.com】\nFunding. Signal Sciences has raised a total of. $61.7M. in funding over 4 rounds. Their latest funding was raised on Feb 5, 2019 from a Series C round. Signal Sciences is funded by 8 investors. CRV and Harrison Metal are the most recent investors. Signal Sciences has invested in Open Raven on Feb 11, 2020. This investment - Seed Round - Open ...\n# 【12†Signal Sciences WAF Review and Alternatives - Comparitech†www.comparitech.com】\nSignal Sciences WAF is completely agentless as it works in the cloud to apply both managed and custom rulesets to your application traffic. Protections extend beyond the OWASP Top 10 and include defense from botnet attacks, account takeovers, credential stuffing, API abuse, and DDoS mitigation. Automatic blocking and scanning are applied to ...\n# 【13†Integrations - Web Application Security - Signal Sciences now part of ...†www.signalsciences.com】\nThe Signal Sciences site extension protects any IIS web application hosted on AAS and provides production visibility necessary to detect and block malicious web requests before they can compromise your business-critical apps, APIs and microservices. ... and efficiently into an organization's existing hybrid, or cloud architecture ...\n# 【14†Fastly Next-Gen WAF professional services | Fastly Products†docs.fastly.com】\nFastly Next-Gen WAF (powered by Signal Sciences) (Next-Gen WAF) professional services provide your organization with training, implementation, and maintenance services for the Next-Gen WAF. Depending on the service offerings you select, our team will provide training and work with you to plan, test, deploy, and maintain a solution to protect ...\n# 【15†Andrew Peterson - Founder & CEO @ Signal Sciences - Crunchbase†www.crunchbase.com】\nAndrew Peterson is the founder and CEO of Signal Sciences, a Software-as-a-Service platform that provides security monitoring and defense solutions for web applications. He was previously the group product manager of international growth at Etsy. Prior to Etsy, Peterson was a health information management officer at the Clinton Foundation\n# 【16†Signal Sciences†info.signalsciences.com】\nThe webinar will cover: The most common attacks targeting distributed web apps. How a web application firewall (WAF) can inspect and decision on both north-south (client-to-app origin) and east-west (service-to-service) web requests. How to detect and prevent malicious client-side activity originating from website supply chain vendors. A live ...\n# 【17†Next-Generation Cloud Web Application Firewall (WAF) - Signal Sciences ...†www.signalsciences.com】\nWith a single change to a DNS record, web requests are routed to the Signal Sciences Cloud WAF to inspect and decision on those requests and block bad requests. Our unique solution provides robust detection and protection capabilities no matter where your organization's applications operate or how rapidly they scale. No complex software ...\n# 【18†How to Configure SAML 2.0 for Signal Sciences - UserDocs†saml-doc.okta.com】\nConfiguration Steps. Log in to Signal Sciences as an Organization Administrator. Navigate to organization > Settings.. Click the Switch to SAML button:. Click the I understand, configure SSO button:. On the Configure SSO with SAML page, enter the following:. Copy and paste the following IDP Login page into the SAML endpoint field:. Sign into the Okta Admin Dashboard to generate this variable.\n# 【19†Powering the best of the internet | Fastly†www.fastly.com】\nEverybody needs speed, reliability, security, savings, and scale – but different industries have different needs. Fastly's powerful network and smarter solutions can be tailored to your organization. We partner with you to guarantee a smooth migration, so you can deliver the best possible user experiences.\nVisible: 0% - 100%", + "pub_date": null, + "extra": null + } + ], + "original_query": null + }, + "command": "search", + "args": ["Signal Sciences organization"], + "status": "finished", + "is_complete": true, + "message_type": null, + "model_slug": "gpt-4", + "parent_id": "5cc5e87b-2b74-485a-b3a7-23cb25f202db", + "request_id": "8594dde3b85705f4-IAD", + "timestamp_": "absolute" + }, + "recipient": "all" + }, + "parent": "5cc5e87b-2b74-485a-b3a7-23cb25f202db", + "children": ["4b3aec6b-5146-4bad-ae8e-204fdb6accda"] + }, + "4b3aec6b-5146-4bad-ae8e-204fdb6accda": { + "id": "4b3aec6b-5146-4bad-ae8e-204fdb6accda", + "message": { + "id": "4b3aec6b-5146-4bad-ae8e-204fdb6accda", + "author": { + "role": "assistant", + "name": null, + "metadata": {} + }, + "create_time": 1708580164.175371, + "update_time": null, + "content": { + "content_type": "text", + "parts": [ + "Signal Sciences is a web application security company that was founded on March 10, 2014, by Andrew Peterson, Nick Galbreath, and Zane Lackey. It operates as a for-profit company with its legal name being Signal Sciences Corp. The company has achieved significant growth and is recognized as the fastest-growing web application security company in the world. Signal Sciences developed a next-gen web application firewall (NGWAF) and runtime application self-protection (RASP) technologies designed to increase security and maintain reliability without compromising the performance of modern web applications distributed across cloud, on-premise, edge, or hybrid environments【3:0†source】【3:1†source】.\n\nIn a major development, Fastly, Inc., a provider of an edge cloud platform, announced the completion of its acquisition of Signal Sciences on October 1, 2020. This acquisition was valued at approximately $775 million in cash and stock. By integrating Signal Sciences' powerful web application and API security solutions with Fastly's edge cloud platform and existing security offerings, they aimed to form a unified suite of security solutions. The merger was aimed at expanding Fastly's security portfolio, particularly at a time when digital security has become paramount for businesses operating online【3:4†source】【3:7†source】." + ] + }, + "status": "finished_successfully", + "end_turn": true, + "weight": 1, + "metadata": { + "finish_details": { + "type": "stop", + "stop_tokens": [100260] + }, + "citations": [ + { + "start_ix": 674, + "end_ix": 686, + "citation_format_type": "tether_og", + "metadata": { + "type": "webpage", + "title": "Signal Sciences - Crunchbase Company Profile & Funding", + "url": "https://www.crunchbase.com/organization/signal-sciences", + "text": "", + "pub_date": null, + "extra": { + "evidence_text": "source", + "cited_message_idx": 3, + "search_result_idx": 0 + } + } + }, + { + "start_ix": 686, + "end_ix": 698, + "citation_format_type": "tether_og", + "metadata": { + "type": "webpage", + "title": "Demand More from Your WAF - Signal Sciences now part of Fastly", + "url": "https://www.signalsciences.com/", + "text": "", + "pub_date": null, + "extra": { + "evidence_text": "source", + "cited_message_idx": 3, + "search_result_idx": 1 + } + } + }, + { + "start_ix": 1306, + "end_ix": 1318, + "citation_format_type": "tether_og", + "metadata": { + "type": "webpage", + "title": "Fastly Completes Acquisition of Signal Sciences | Fastly", + "url": "https://www.fastly.com/press/press-releases/fastly-completes-acquisition-signal-sciences", + "text": "", + "pub_date": null, + "extra": { + "evidence_text": "source", + "cited_message_idx": 3, + "search_result_idx": 4 + } + } + }, + { + "start_ix": 1318, + "end_ix": 1330, + "citation_format_type": "tether_og", + "metadata": { + "type": "webpage", + "title": "Fastly Agrees to Acquire Signal Sciences for $775 Million - Cooley", + "url": "https://www.cooley.com/news/coverage/2020/2020-08-27-fastly-agrees-to-acquire-signal-sciences-for-775-million", + "text": "", + "pub_date": null, + "extra": { + "evidence_text": "source", + "cited_message_idx": 3, + "search_result_idx": 7 + } + } + } + ], + "gizmo_id": null, + "is_complete": true, + "message_type": null, + "model_slug": "gpt-4", + "parent_id": "9e874379-5008-4a2d-aa2e-628d1d705a04", + "request_id": "8594dde3b85705f4-IAD", + "timestamp_": "absolute" + }, + "recipient": "all" + }, + "parent": "9e874379-5008-4a2d-aa2e-628d1d705a04", + "children": ["aaa27ed5-1e1c-4189-ac6c-737608404ec1"] + } + }, + "moderation_results": [], + "current_node": "88a0cf9f-e860-4b34-8e7e-65f8346f4862", + "plugin_ids": null, + "conversation_id": "d6523d1e-7ec3-474f-a363-0e9dffdb3d93", + "conversation_template_id": null, + "gizmo_id": null, + "is_archived": false, + "safe_urls": [ + "https://www.carsales.com.au/volkswagen/transporter/price/2014/tdi340-381631/", + "https://www.drive.com.au/reviews/volkswagen-transporter-review/", + "https://www.ultimatespecs.com/car-specs/volkswagen/118424/2021-volkswagen-transporter-t61-l2h1-20-tdi-110hp.html" + ], + "default_model_slug": null, + "id": "d6523d1e-7ec3-474f-a363-0e9dffdb3d93" + } +] diff --git a/api/server/utils/import/importers.js b/api/server/utils/import/importers.js index e262f1ae120..9c4c0dc5909 100644 --- a/api/server/utils/import/importers.js +++ b/api/server/utils/import/importers.js @@ -277,34 +277,39 @@ function processConversation(conv, importBatchBuilder, requestUserId) { /** * Processes text content of messages authored by an assistant, inserting citation links as required. - * Applies citation metadata to construct regex patterns and replacements for inserting links into the text. + * Uses citation start and end indices to place links at the correct positions. * * @param {ChatGPTMessage} messageData - The message data containing metadata about citations. * @param {string} messageText - The original text of the message which may be altered by inserting citation links. * @returns {string} - The updated message text after processing for citations. */ function processAssistantMessage(messageData, messageText) { - const citations = messageData.metadata.citations ?? []; + if (!messageText) { + return messageText; + } + + const citations = messageData.metadata?.citations ?? []; + + const sortedCitations = [...citations].sort((a, b) => b.start_ix - a.start_ix); - for (const citation of citations) { + let result = messageText; + for (const citation of sortedCitations) { if ( - !citation.metadata || - !citation.metadata.extra || - !citation.metadata.extra.cited_message_idx || - (citation.metadata.type && citation.metadata.type !== 'webpage') + !citation.metadata?.type || + citation.metadata.type !== 'webpage' || + typeof citation.start_ix !== 'number' || + typeof citation.end_ix !== 'number' || + citation.start_ix >= citation.end_ix ) { continue; } - const pattern = new RegExp( - `\\u3010${citation.metadata.extra.cited_message_idx}\\u2020.+?\\u3011`, - 'g', - ); const replacement = ` ([${citation.metadata.title}](${citation.metadata.url}))`; - messageText = messageText.replace(pattern, replacement); + + result = result.slice(0, citation.start_ix) + replacement + result.slice(citation.end_ix); } - return messageText; + return result; } /** @@ -342,4 +347,4 @@ function formatMessageText(messageData) { return messageText; } -module.exports = { getImporter }; +module.exports = { getImporter, processAssistantMessage }; diff --git a/api/server/utils/import/importers.spec.js b/api/server/utils/import/importers.spec.js index fa0b2e9f33b..a68bc3e7f3b 100644 --- a/api/server/utils/import/importers.spec.js +++ b/api/server/utils/import/importers.spec.js @@ -2,10 +2,10 @@ const fs = require('fs'); const path = require('path'); const { EModelEndpoint, Constants, openAISettings } = require('librechat-data-provider'); const { bulkSaveConvos: _bulkSaveConvos } = require('~/models/Conversation'); +const { getImporter, processAssistantMessage } = require('./importers'); const { ImportBatchBuilder } = require('./importBatchBuilder'); const { bulkSaveMessages } = require('~/models/Message'); const getLogStores = require('~/cache/getLogStores'); -const { getImporter } = require('./importers'); jest.mock('~/cache/getLogStores'); const mockedCacheGet = jest.fn(); @@ -404,3 +404,234 @@ describe('getImporter', () => { expect(() => getImporter(jsonData)).toThrow('Unsupported import type'); }); }); + +describe('processAssistantMessage', () => { + const testMessage = 'This is a test citation 【3:0†source】【3:1†source】'; + + const messageData = { + metadata: { + citations: [ + { + start_ix: 23, // Position of first "【3:0†source】" + end_ix: 36, // End of first citation (including closing bracket) + citation_format_type: 'tether_og', + metadata: { + type: 'webpage', + title: 'Signal Sciences - Crunchbase Company Profile & Funding', + url: 'https://www.crunchbase.com/organization/signal-sciences', + text: '', + pub_date: null, + extra: { + evidence_text: 'source', + cited_message_idx: 3, + search_result_idx: 0, + }, + }, + }, + { + start_ix: 36, // Position of second "【3:1†source】" + end_ix: 49, // End of second citation (including closing bracket) + citation_format_type: 'tether_og', + metadata: { + type: 'webpage', + title: 'Demand More from Your WAF - Signal Sciences now part of Fastly', + url: 'https://www.signalsciences.com/', + text: '', + pub_date: null, + extra: { + evidence_text: 'source', + cited_message_idx: 3, + search_result_idx: 1, + }, + }, + }, + ], + }, + }; + + const messageText = testMessage; + const expectedOutput = + 'This is a test citation ([Signal Sciences - Crunchbase Company Profile & Funding](https://www.crunchbase.com/organization/signal-sciences)) ([Demand More from Your WAF - Signal Sciences now part of Fastly](https://www.signalsciences.com/))'; + + test('should correctly process citations and replace them with markdown links', () => { + const result = processAssistantMessage(messageData, messageText); + expect(result).toBe(expectedOutput); + }); + + test('should handle message with no citations', () => { + const messageWithNoCitations = { + metadata: {}, + }; + const result = processAssistantMessage(messageWithNoCitations, messageText); + expect(result).toBe(messageText); + }); + + test('should handle citations with missing metadata', () => { + const messageWithBadCitation = { + metadata: { + citations: [ + { + start_ix: 85, + end_ix: 97, + }, + ], + }, + }; + const result = processAssistantMessage(messageWithBadCitation, messageText); + expect(result).toBe(messageText); + }); + + test('should handle citations with non-webpage type', () => { + const messageWithNonWebpage = { + metadata: { + citations: [ + { + start_ix: 85, + end_ix: 97, + metadata: { + type: 'other', + title: 'Test', + url: 'http://test.com', + }, + }, + ], + }, + }; + const result = processAssistantMessage(messageWithNonWebpage, messageText); + expect(result).toBe(messageText); + }); + + test('should handle empty message text', () => { + const result = processAssistantMessage(messageData, ''); + expect(result).toBe(''); + }); + + test('should handle undefined message text', () => { + const result = processAssistantMessage(messageData, undefined); + expect(result).toBe(undefined); + }); + + test('should handle invalid citation indices', () => { + const messageWithBadIndices = { + metadata: { + citations: [ + { + start_ix: 100, + end_ix: 90, // end before start + metadata: { + type: 'webpage', + title: 'Test', + url: 'http://test.com', + }, + }, + ], + }, + }; + const result = processAssistantMessage(messageWithBadIndices, messageText); + expect(result).toBe(messageText); + }); + + test('should correctly process citations from real ChatGPT data', () => { + const jsonData = JSON.parse( + fs.readFileSync(path.join(__dirname, '__data__', 'chatgpt-citations.json'), 'utf8'), + ); + + // Get the message containing citations from the JSON data + const assistantMessage = jsonData[0].mapping['4b3aec6b-5146-4bad-ae8e-204fdb6accda'].message; + + const messageText = assistantMessage.content.parts[0]; + const citations = assistantMessage.metadata.citations; + + // Expected output should have all citations replaced with markdown links + const expectedOutput = + 'Signal Sciences is a web application security company that was founded on March 10, 2014, by Andrew Peterson, Nick Galbreath, and Zane Lackey. It operates as a for-profit company with its legal name being Signal Sciences Corp. The company has achieved significant growth and is recognized as the fastest-growing web application security company in the world. Signal Sciences developed a next-gen web application firewall (NGWAF) and runtime application self-protection (RASP) technologies designed to increase security and maintain reliability without compromising the performance of modern web applications distributed across cloud, on-premise, edge, or hybrid environments ([Signal Sciences - Crunchbase Company Profile & Funding](https://www.crunchbase.com/organization/signal-sciences)) ([Demand More from Your WAF - Signal Sciences now part of Fastly](https://www.signalsciences.com/)).\n\nIn a major development, Fastly, Inc., a provider of an edge cloud platform, announced the completion of its acquisition of Signal Sciences on October 1, 2020. This acquisition was valued at approximately $775 million in cash and stock. By integrating Signal Sciences\' powerful web application and API security solutions with Fastly\'s edge cloud platform and existing security offerings, they aimed to form a unified suite of security solutions. The merger was aimed at expanding Fastly\'s security portfolio, particularly at a time when digital security has become paramount for businesses operating online ([Fastly Completes Acquisition of Signal Sciences | Fastly](https://www.fastly.com/press/press-releases/fastly-completes-acquisition-signal-sciences)) ([Fastly Agrees to Acquire Signal Sciences for $775 Million - Cooley](https://www.cooley.com/news/coverage/2020/2020-08-27-fastly-agrees-to-acquire-signal-sciences-for-775-million)).'; + + const result = processAssistantMessage(assistantMessage, messageText); + expect(result).toBe(expectedOutput); + + // Additional checks to verify citation processing + citations.forEach((citation) => { + // Verify each citation was replaced + const markdownLink = `([${citation.metadata.title}](${citation.metadata.url}))`; + expect(result).toContain(markdownLink); + + // Verify original citation format is not present + const originalCitation = messageText.slice(citation.start_ix, citation.end_ix); + expect(result).not.toContain(originalCitation); + }); + }); + + test('should handle potential ReDoS attack payloads', () => { + // Test with increasing input sizes to check for exponential behavior + const sizes = [32, 33, 34]; // Adding more sizes would increase test time + const regExp = '(a+)+'; + const results = []; + + sizes.forEach((size) => { + const startTime = process.hrtime(); + + const maliciousMessageData = { + metadata: { + citations: [ + { + start_ix: 0, + end_ix: size, + citation_format_type: 'tether_og', + metadata: { + type: 'webpage', + title: 'Test', + url: 'http://test.com', + extra: { + cited_message_idx: regExp, + }, + }, + }, + ], + }, + }; + + const maliciousText = '【' + 'a'.repeat(size) + '】'; + + processAssistantMessage(maliciousMessageData, maliciousText); + + const endTime = process.hrtime(startTime); + const duration = endTime[0] * 1000 + endTime[1] / 1000000; // Convert to milliseconds + results.push(duration); + }); + + // Check if processing time increases exponentially + // In a ReDoS vulnerability, time would roughly double with each size increase + for (let i = 1; i < results.length; i++) { + const ratio = results[i] / results[i - 1]; + expect(ratio).toBeLessThan(2); // Processing time should not double + console.log(`Size ${sizes[i]} processing time ratio: ${ratio}`); + } + + // Also test with the exact payload from the security report + const maliciousPayload = { + metadata: { + citations: [ + { + metadata: { + extra: { + cited_message_idx: '(a+)+', + }, + type: 'webpage', + title: '1', + url: '2', + }, + }, + ], + }, + }; + + const text = '【' + 'a'.repeat(32); + const startTime = process.hrtime(); + processAssistantMessage(maliciousPayload, text); + const endTime = process.hrtime(startTime); + const duration = endTime[0] * 1000 + endTime[1] / 1000000; + + // The processing should complete quickly (under 100ms) + expect(duration).toBeLessThan(100); + }); +}); diff --git a/api/strategies/ldapStrategy.js b/api/strategies/ldapStrategy.js index 756e1da4227..4d9124bb6ad 100644 --- a/api/strategies/ldapStrategy.js +++ b/api/strategies/ldapStrategy.js @@ -14,6 +14,7 @@ const { LDAP_FULL_NAME, LDAP_ID, LDAP_USERNAME, + LDAP_EMAIL, LDAP_TLS_REJECT_UNAUTHORIZED, } = process.env; @@ -43,6 +44,9 @@ if (LDAP_ID) { if (LDAP_USERNAME) { searchAttributes.push(LDAP_USERNAME); } +if (LDAP_EMAIL) { + searchAttributes.push(LDAP_EMAIL); +} const rejectUnauthorized = isEnabled(LDAP_TLS_REJECT_UNAUTHORIZED); const ldapOptions = { @@ -76,15 +80,6 @@ const ldapLogin = new LdapStrategy(ldapOptions, async (userinfo, done) => { return done(null, false, { message: 'Invalid credentials' }); } - if (!userinfo.mail) { - logger.warn( - '[ldapStrategy]', - 'No email attributes found in userinfo', - JSON.stringify(userinfo, null, 2), - ); - return done(null, false, { message: 'Invalid credentials' }); - } - try { const ldapId = (LDAP_ID && userinfo[LDAP_ID]) || userinfo.uid || userinfo.sAMAccountName || userinfo.mail; @@ -100,12 +95,25 @@ const ldapLogin = new LdapStrategy(ldapOptions, async (userinfo, done) => { const username = (LDAP_USERNAME && userinfo[LDAP_USERNAME]) || userinfo.givenName || userinfo.mail; + const mail = (LDAP_EMAIL && userinfo[LDAP_EMAIL]) || userinfo.mail || username + '@ldap.local'; + + if (!userinfo.mail && !(LDAP_EMAIL && userinfo[LDAP_EMAIL])) { + logger.warn( + '[ldapStrategy]', + `No valid email attribute found in LDAP userinfo. Using fallback email: ${username}@ldap.local`, + `LDAP_EMAIL env var: ${LDAP_EMAIL || 'not set'}`, + `Available userinfo attributes: ${Object.keys(userinfo).join(', ')}`, + 'Full userinfo:', + JSON.stringify(userinfo, null, 2), + ); + } + if (!user) { user = { provider: 'ldap', ldapId, username, - email: userinfo.mail, + email: mail, emailVerified: true, // The ldap server administrator should verify the email name: fullName, }; @@ -116,7 +124,7 @@ const ldapLogin = new LdapStrategy(ldapOptions, async (userinfo, done) => { // so update the user information with the values registered in LDAP user.provider = 'ldap'; user.ldapId = ldapId; - user.email = userinfo.mail; + user.email = mail; user.username = username; user.name = fullName; } diff --git a/api/strategies/localStrategy.js b/api/strategies/localStrategy.js index 9c87a5b3194..bffb4f845f8 100644 --- a/api/strategies/localStrategy.js +++ b/api/strategies/localStrategy.js @@ -48,7 +48,12 @@ async function passportLogin(req, email, password, done) { user.emailVerified = true; } - if (!user.emailVerified && !isEnabled(process.env.ALLOW_UNVERIFIED_EMAIL_LOGIN)) { + const unverifiedAllowed = isEnabled(process.env.ALLOW_UNVERIFIED_EMAIL_LOGIN); + if (user.expiresAt && unverifiedAllowed) { + await updateUser(user._id, {}); + } + + if (!user.emailVerified && !unverifiedAllowed) { logError('Passport Local Strategy - Email not verified', { email }); logger.error(`[Login] [Login failed] [Username: ${email}] [Request-IP: ${req.ip}]`); return done(null, user, { message: 'Email not verified.' }); diff --git a/api/typedefs.js b/api/typedefs.js index b2548acf152..8f0e9fef252 100644 --- a/api/typedefs.js +++ b/api/typedefs.js @@ -646,9 +646,22 @@ * @property {string} [temp_file_id] - The temporary identifier of the file. * @property {string} endpoint - The conversation endpoint origin for the file upload. * @property {string} [assistant_id] - The assistant ID if file upload is in the `knowledge` context. + * @property {string} [tool_resource] - The relevant tool resource for the file upload. * @memberof typedefs */ +/** + * @exports FileObject + * @typedef {{file_id: string, filepath: string, source: string, bytes?: number, width?: number, height?: number}} FileObject + * @memberof typedefs + * + +/** + * @exports ArtifactPromises + * @typedef {Promise[]} ArtifactPromises + * @memberof typedefs + * + /** * @typedef {Object} ImageOnlyMetadata * @property {number} width - The width of the image. @@ -706,6 +719,12 @@ * @memberof typedefs */ +/** + * @exports TAttachment + * @typedef {import('librechat-data-provider').TAttachment} TAttachment + * @memberof typedefs + */ + /** * @exports AssistantCreateParams * @typedef {import('librechat-data-provider').AssistantCreateParams} AssistantCreateParams @@ -1443,7 +1462,19 @@ */ /** - * @typedef {AnthropicStreamUsage} StreamUsage - Stream usage for all providers (currently only Anthropic) + * @exports OpenAIUsageMetadata + * @typedef {Object} OpenAIUsageMetadata - Usage statistics related to the run. This value will be `null` if the run is not in a terminal state (i.e. `in_progress`, `queued`, etc.). + * @property {number} [usage.completion_tokens] - Number of completion tokens used over the course of the run. + * @property {number} [usage.prompt_tokens] - Number of prompt tokens used over the course of the run. + * @property {number} [usage.total_tokens] - Total number of tokens used (prompt + completion). + * @property {number} [usage.reasoning_tokens] - Total number of tokens used for reasoning (OpenAI o1 models). + * @property {Object} [usage.completion_tokens_details] - Further details on the completion tokens used (OpenAI o1 models). + * @property {number} [usage.completion_tokens_details.reasoning_tokens] - Total number of tokens used for reasoning (OpenAI o1 models). + * @memberof typedefs + */ + +/** + * @typedef {AnthropicStreamUsage | OpenAIUsageMetadata | UsageMetadata} StreamUsage - Stream usage for all providers (currently only Anthropic, OpenAI, LangChain) */ /* Native app/client methods */ diff --git a/api/utils/logAxiosError.js b/api/utils/axios.js similarity index 97% rename from api/utils/logAxiosError.js rename to api/utils/axios.js index 17fac85f47d..8b12a5ca993 100644 --- a/api/utils/logAxiosError.js +++ b/api/utils/axios.js @@ -42,4 +42,4 @@ const logAxiosError = ({ message, error }) => { } }; -module.exports = logAxiosError; +module.exports = { logAxiosError }; diff --git a/api/utils/index.js b/api/utils/index.js index 29357f7adb8..62d61586bf0 100644 --- a/api/utils/index.js +++ b/api/utils/index.js @@ -1,17 +1,17 @@ const loadYaml = require('./loadYaml'); +const axiosHelpers = require('./axios'); const tokenHelpers = require('./tokens'); const azureUtils = require('./azureUtils'); const deriveBaseURL = require('./deriveBaseURL'); -const logAxiosError = require('./logAxiosError'); const extractBaseURL = require('./extractBaseURL'); const findMessageContent = require('./findMessageContent'); module.exports = { loadYaml, - ...tokenHelpers, - ...azureUtils, deriveBaseURL, - logAxiosError, extractBaseURL, + ...azureUtils, + ...axiosHelpers, + ...tokenHelpers, findMessageContent, }; diff --git a/api/utils/tokens.js b/api/utils/tokens.js index ec248fe92c5..362b2fc6701 100644 --- a/api/utils/tokens.js +++ b/api/utils/tokens.js @@ -2,6 +2,9 @@ const z = require('zod'); const { EModelEndpoint } = require('librechat-data-provider'); const openAIModels = { + o1: 127500, // -500 from max + 'o1-mini': 127500, // -500 from max + 'o1-preview': 127500, // -500 from max 'gpt-4': 8187, // -5 from max 'gpt-4-0613': 8187, // -5 from max 'gpt-4-32k': 32758, // -10 from max @@ -11,6 +14,7 @@ const openAIModels = { 'gpt-4-0125': 127500, // -500 from max 'gpt-4o': 127500, // -500 from max 'gpt-4o-mini': 127500, // -500 from max + 'gpt-4o-2024-05-13': 127500, // -500 from max 'gpt-4o-2024-08-06': 127500, // -500 from max 'gpt-4-turbo': 127500, // -500 from max 'gpt-4-vision': 127500, // -500 from max @@ -68,6 +72,8 @@ const anthropicModels = { 'claude-3-opus': 200000, 'claude-3-5-sonnet': 200000, 'claude-3.5-sonnet': 200000, + 'claude-3-5-sonnet-latest': 200000, + 'claude-3.5-sonnet-latest': 200000, }; const metaModels = { @@ -113,6 +119,28 @@ const maxTokensMap = { [EModelEndpoint.bedrock]: bedrockModels, }; +const modelMaxOutputs = { + o1: 32268, // -500 from max: 32,768 + 'o1-mini': 65136, // -500 from max: 65,536 + 'o1-preview': 32268, // -500 from max: 32,768 + system_default: 1024, +}; + +const anthropicMaxOutputs = { + 'claude-3-haiku': 4096, + 'claude-3-sonnet': 4096, + 'claude-3-opus': 4096, + 'claude-3.5-sonnet': 8192, + 'claude-3-5-sonnet': 8192, +}; + +const maxOutputTokensMap = { + [EModelEndpoint.anthropic]: anthropicMaxOutputs, + [EModelEndpoint.azureOpenAI]: modelMaxOutputs, + [EModelEndpoint.openAI]: modelMaxOutputs, + [EModelEndpoint.custom]: modelMaxOutputs, +}; + /** * Finds the first matching pattern in the tokens map. * @param {string} modelName @@ -132,27 +160,15 @@ function findMatchingPattern(modelName, tokensMap) { } /** - * Retrieves the maximum tokens for a given model name. If the exact model name isn't found, - * it searches for partial matches within the model name, checking keys in reverse order. + * Retrieves a token value for a given model name from a tokens map. * * @param {string} modelName - The name of the model to look up. - * @param {string} endpoint - The endpoint (default is 'openAI'). - * @param {EndpointTokenConfig} [endpointTokenConfig] - Token Config for current endpoint to use for max tokens lookup - * @returns {number|undefined} The maximum tokens for the given model or undefined if no match is found. - * - * @example - * getModelMaxTokens('gpt-4-32k-0613'); // Returns 32767 - * getModelMaxTokens('gpt-4-32k-unknown'); // Returns 32767 - * getModelMaxTokens('unknown-model'); // Returns undefined + * @param {EndpointTokenConfig | Record} tokensMap - The map of model names to token values. + * @param {string} [key='context'] - The key to look up in the tokens map. + * @returns {number|undefined} The token value for the given model or undefined if no match is found. */ -function getModelMaxTokens(modelName, endpoint = EModelEndpoint.openAI, endpointTokenConfig) { - if (typeof modelName !== 'string') { - return undefined; - } - - /** @type {EndpointTokenConfig | Record} */ - const tokensMap = endpointTokenConfig ?? maxTokensMap[endpoint]; - if (!tokensMap) { +function getModelTokenValue(modelName, tokensMap, key = 'context') { + if (typeof modelName !== 'string' || !tokensMap) { return undefined; } @@ -168,10 +184,36 @@ function getModelMaxTokens(modelName, endpoint = EModelEndpoint.openAI, endpoint if (matchedPattern) { const result = tokensMap[matchedPattern]; - return result?.context ?? result; + return result?.[key] ?? result ?? tokensMap.system_default; } - return undefined; + return tokensMap.system_default; +} + +/** + * Retrieves the maximum tokens for a given model name. + * + * @param {string} modelName - The name of the model to look up. + * @param {string} endpoint - The endpoint (default is 'openAI'). + * @param {EndpointTokenConfig} [endpointTokenConfig] - Token Config for current endpoint to use for max tokens lookup + * @returns {number|undefined} The maximum tokens for the given model or undefined if no match is found. + */ +function getModelMaxTokens(modelName, endpoint = EModelEndpoint.openAI, endpointTokenConfig) { + const tokensMap = endpointTokenConfig ?? maxTokensMap[endpoint]; + return getModelTokenValue(modelName, tokensMap); +} + +/** + * Retrieves the maximum output tokens for a given model name. + * + * @param {string} modelName - The name of the model to look up. + * @param {string} endpoint - The endpoint (default is 'openAI'). + * @param {EndpointTokenConfig} [endpointTokenConfig] - Token Config for current endpoint to use for max tokens lookup + * @returns {number|undefined} The maximum output tokens for the given model or undefined if no match is found. + */ +function getModelMaxOutputTokens(modelName, endpoint = EModelEndpoint.openAI, endpointTokenConfig) { + const tokensMap = endpointTokenConfig ?? maxOutputTokensMap[endpoint]; + return getModelTokenValue(modelName, tokensMap, 'output'); } /** @@ -298,7 +340,8 @@ module.exports = { maxTokensMap, inputSchema, modelSchema, - getModelMaxTokens, matchModelName, processModelData, + getModelMaxTokens, + getModelMaxOutputTokens, }; diff --git a/helmchart/.helmignore b/charts/librechat/.helmignore similarity index 100% rename from helmchart/.helmignore rename to charts/librechat/.helmignore diff --git a/helmchart/Chart.yaml b/charts/librechat/Chart.yaml similarity index 100% rename from helmchart/Chart.yaml rename to charts/librechat/Chart.yaml diff --git a/helmchart/templates/NOTES.txt b/charts/librechat/templates/NOTES.txt similarity index 100% rename from helmchart/templates/NOTES.txt rename to charts/librechat/templates/NOTES.txt diff --git a/helmchart/templates/_helpers.tpl b/charts/librechat/templates/_helpers.tpl similarity index 100% rename from helmchart/templates/_helpers.tpl rename to charts/librechat/templates/_helpers.tpl diff --git a/helmchart/templates/configmap-env.yaml b/charts/librechat/templates/configmap-env.yaml similarity index 100% rename from helmchart/templates/configmap-env.yaml rename to charts/librechat/templates/configmap-env.yaml diff --git a/helmchart/templates/deployment.yaml b/charts/librechat/templates/deployment.yaml similarity index 100% rename from helmchart/templates/deployment.yaml rename to charts/librechat/templates/deployment.yaml diff --git a/helmchart/templates/hpa.yaml b/charts/librechat/templates/hpa.yaml similarity index 100% rename from helmchart/templates/hpa.yaml rename to charts/librechat/templates/hpa.yaml diff --git a/helmchart/templates/ingress.yaml b/charts/librechat/templates/ingress.yaml similarity index 100% rename from helmchart/templates/ingress.yaml rename to charts/librechat/templates/ingress.yaml diff --git a/helmchart/templates/service.yaml b/charts/librechat/templates/service.yaml similarity index 100% rename from helmchart/templates/service.yaml rename to charts/librechat/templates/service.yaml diff --git a/helmchart/templates/serviceaccount.yaml b/charts/librechat/templates/serviceaccount.yaml similarity index 100% rename from helmchart/templates/serviceaccount.yaml rename to charts/librechat/templates/serviceaccount.yaml diff --git a/helmchart/values.yaml b/charts/librechat/values.yaml similarity index 100% rename from helmchart/values.yaml rename to charts/librechat/values.yaml diff --git a/client/index.html b/client/index.html index 50e60585dbe..c72d3951cbf 100644 --- a/client/index.html +++ b/client/index.html @@ -2,47 +2,57 @@ - - - - - QIMA-GPT - - - - - - + + + + + LibreChat + + + + + + + + -
- - +
+
+
+ diff --git a/client/package.json b/client/package.json index 2419d667740..3910f7bedab 100644 --- a/client/package.json +++ b/client/package.json @@ -1,6 +1,6 @@ { "name": "@librechat/frontend", - "version": "v0.7.5-rc2", + "version": "v0.7.5", "description": "", "type": "module", "scripts": { @@ -28,7 +28,7 @@ }, "homepage": "https://librechat.ai", "dependencies": { - "@ariakit/react": "^0.4.8", + "@ariakit/react": "^0.4.11", "@codesandbox/sandpack-react": "^2.18.2", "@dicebear/collection": "^7.0.4", "@dicebear/core": "^7.0.4", @@ -54,7 +54,7 @@ "@tanstack/react-query": "^4.28.0", "@tanstack/react-table": "^8.11.7", "@zattoo/use-double-click": "1.2.0", - "axios": "^1.3.4", + "axios": "^1.7.7", "class-variance-authority": "^0.6.0", "clsx": "^1.2.1", "copy-to-clipboard": "^3.3.3", @@ -63,6 +63,7 @@ "downloadjs": "^1.4.7", "export-from-json": "^1.7.2", "filenamify": "^6.0.0", + "framer-motion": "^11.5.4", "html-to-image": "^1.11.11", "image-blob-reduce": "^4.1.0", "js-cookie": "^3.0.5", @@ -137,6 +138,6 @@ "typescript": "^5.0.4", "vite": "^5.1.1", "vite-plugin-node-polyfills": "^0.17.0", - "vite-plugin-pwa": "^0.19.8" + "vite-plugin-pwa": "^0.20.5" } } diff --git a/client/src/common/agents-types.ts b/client/src/common/agents-types.ts index 07633a68dbd..fd268e8cb77 100644 --- a/client/src/common/agents-types.ts +++ b/client/src/common/agents-types.ts @@ -1,17 +1,16 @@ -import { Capabilities } from 'librechat-data-provider'; +import { AgentCapabilities } from 'librechat-data-provider'; import type { Agent, AgentProvider, AgentModelParameters } from 'librechat-data-provider'; import type { OptionWithIcon, ExtendedFile } from './types'; export type TAgentOption = OptionWithIcon & Agent & { - files?: Array<[string, ExtendedFile]>; + knowledge_files?: Array<[string, ExtendedFile]>; code_files?: Array<[string, ExtendedFile]>; }; -export type AgentCapabilities = { - [Capabilities.code_interpreter]: boolean; - [Capabilities.image_vision]: boolean; - [Capabilities.retrieval]: boolean; +export type TAgentCapabilities = { + [AgentCapabilities.execute_code]: boolean; + [AgentCapabilities.file_search]: boolean; }; export type AgentForm = { @@ -24,4 +23,4 @@ export type AgentForm = { model_parameters: AgentModelParameters; tools?: string[]; provider?: AgentProvider | OptionWithIcon; -} & AgentCapabilities; +} & TAgentCapabilities; diff --git a/client/src/common/types.ts b/client/src/common/types.ts index b969346e70a..3590b279b8d 100644 --- a/client/src/common/types.ts +++ b/client/src/common/types.ts @@ -194,6 +194,7 @@ export type AgentModelPanelProps = { setActivePanel: React.Dispatch>; providers: Option[]; models: Record; + agent_id?: string; }; export type AugmentedColumnDef = ColumnDef & DataColumnMeta; @@ -430,6 +431,8 @@ export type Option = Record & { value: string | number | null; }; +export type StringOption = Option & { value: string | null }; + export type VoiceOption = { value: string; label: string; @@ -540,3 +543,9 @@ export type TVectorStore = { }; export type TThread = { id: string; createdAt: string }; + +declare global { + interface Window { + google_tag_manager?: unknown; + } +} diff --git a/client/src/components/Audio/Voices.tsx b/client/src/components/Audio/Voices.tsx index ff37efaa7d2..ab6b0a86090 100644 --- a/client/src/components/Audio/Voices.tsx +++ b/client/src/components/Audio/Voices.tsx @@ -3,6 +3,7 @@ import { useRecoilState } from 'recoil'; import type { Option } from '~/common'; import DropdownNoState from '~/components/ui/DropdownNoState'; import { useLocalize, useTTSBrowser, useTTSEdge, useTTSExternal } from '~/hooks'; +import { Dropdown } from '~/components/ui'; import { logger } from '~/utils'; import store from '~/store'; @@ -22,13 +23,12 @@ export function EdgeVoiceDropdown() { return (
{localize('com_nav_voice_select')}
-
@@ -51,13 +51,12 @@ export function BrowserVoiceDropdown() { return (
{localize('com_nav_voice_select')}
-
@@ -80,13 +79,12 @@ export function ExternalVoiceDropdown() { return (
{localize('com_nav_voice_select')}
-
diff --git a/client/src/components/Auth/AuthLayout.tsx b/client/src/components/Auth/AuthLayout.tsx index aec61c9375c..0ff99f357cf 100644 --- a/client/src/components/Auth/AuthLayout.tsx +++ b/client/src/components/Auth/AuthLayout.tsx @@ -3,7 +3,8 @@ import { BlinkAnimation } from './BlinkAnimation'; import { TStartupConfig } from 'librechat-data-provider'; import SocialLoginRender from './SocialLoginRender'; import { ThemeSelector } from '~/components/ui'; -import Footer from '../Input/FooterQima'; +import { Banner } from '../Banners'; +import Footer from './Footer'; const ErrorRender = ({ children }: { children: React.ReactNode }) => (
@@ -56,8 +57,9 @@ function AuthLayout({ return (
+ -
+
Logo
diff --git a/client/src/components/Auth/LoginForm.tsx b/client/src/components/Auth/LoginForm.tsx index 3e50e92578a..3404a787293 100644 --- a/client/src/components/Auth/LoginForm.tsx +++ b/client/src/components/Auth/LoginForm.tsx @@ -81,7 +81,7 @@ const LoginForm: React.FC = ({ onSubmit, startupConfig, error, method="POST" onSubmit={handleSubmit((data) => onSubmit(data))} > -
+
= ({ onSubmit, startupConfig, error, }, })} aria-invalid={!!errors.email} - className="webkit-dark-styles peer block w-full appearance-none rounded-md border border-gray-300 bg-transparent px-3.5 pb-3.5 pt-4 text-sm text-gray-900 focus:border-green-500 focus:outline-none focus:ring-0 dark:border-gray-600 dark:text-white dark:focus:border-green-500" + className=" + webkit-dark-styles transition-color peer w-full rounded-2xl border border-border-light + bg-surface-primary px-3.5 pb-2.5 pt-3 text-text-primary duration-200 focus:border-green-500 focus:outline-none + " placeholder=" " />