diff --git a/apps/docs/content/docs/en/workflows/blocks/meta.json b/apps/docs/content/docs/en/workflows/blocks/meta.json index 567a0c3417f..8b5cd58012a 100644 --- a/apps/docs/content/docs/en/workflows/blocks/meta.json +++ b/apps/docs/content/docs/en/workflows/blocks/meta.json @@ -2,7 +2,6 @@ "title": "Core Blocks", "pages": [ "agent", - "pi", "api", "function", "condition", @@ -17,6 +16,7 @@ "human-in-the-loop", "variables", "wait", - "credential" + "credential", + "pi" ] } diff --git a/apps/docs/content/docs/en/workflows/blocks/pi.mdx b/apps/docs/content/docs/en/workflows/blocks/pi.mdx index f98095bdb1a..2bee2901742 100644 --- a/apps/docs/content/docs/en/workflows/blocks/pi.mdx +++ b/apps/docs/content/docs/en/workflows/blocks/pi.mdx @@ -25,7 +25,7 @@ Pick the mode with the **Mode** dropdown. The fields below it change to match. Cloud runs entirely inside a disposable sandbox, so it never touches your machine. It clones the repo, lets the agent work with full read/shell/edit/git, pushes a branch, and opens a PR you review and merge. - Requires sandbox execution to be enabled (the Cloud option only appears when it is). -- Requires **your own provider API key (BYOK)** — the model key is handed to the sandbox, so Sim never injects a hosted key there. +- Requires **your own provider API key (BYOK)** — the model key is handed to the sandbox. - Needs a **GitHub token** with permission to clone, push, and open a PR (see [Setup](#setup-cloud)). - The deliverable is a **pull request** — nothing is committed to your default branch directly. @@ -118,7 +118,7 @@ The one case neither layer can rescue is a *first* prompt that already exceeds t ## Setup -### Cloud +### Cloud [#setup-cloud] Cloud runs in a sandbox image with the Pi CLI and git baked in. @@ -128,7 +128,7 @@ Cloud runs in a sandbox image with the Pi CLI and git baked in. - *Fine-grained:* select the repo, then **Contents: Read and write** + **Pull requests: Read and write**. - *Classic:* the **`repo`** scope. For org repos, authorize the token for SSO. -### Local +### Local [#setup-local] 1. **Enable SSH** on the target machine (on macOS: System Settings → General → Sharing → Remote Login). 2. **Expose it on a public host.** Sim blocks `localhost`/LAN, so use a TCP tunnel — for example `ngrok tcp 22`, which gives a `host:port` to put in **Host** and **Port**. diff --git a/apps/realtime/package.json b/apps/realtime/package.json index 99867ef852d..e8b1e1607be 100644 --- a/apps/realtime/package.json +++ b/apps/realtime/package.json @@ -26,6 +26,7 @@ "@sim/logger": "workspace:*", "@sim/platform-authz": "workspace:*", "@sim/realtime-protocol": "workspace:*", + "@sim/runtime-secrets": "workspace:*", "@sim/security": "workspace:*", "@sim/utils": "workspace:*", "@sim/workflow-persistence": "workspace:*", diff --git a/apps/realtime/src/bootstrap.ts b/apps/realtime/src/bootstrap.ts new file mode 100644 index 00000000000..fe786372052 --- /dev/null +++ b/apps/realtime/src/bootstrap.ts @@ -0,0 +1,9 @@ +/** + * Container entrypoint. Hydrates `process.env` from the runtime secret before + * loading the Socket.IO server, whose modules (`@/env`, DB preflight) read env + * at import time. See `@sim/runtime-secrets`. + */ +import { loadRuntimeSecrets } from '@sim/runtime-secrets' + +await loadRuntimeSecrets() +await import('@/index') diff --git a/apps/sim/app/chat/hooks/use-chat-streaming.ts b/apps/sim/app/chat/hooks/use-chat-streaming.ts index f4c9fcb9b97..dd315dafe73 100644 --- a/apps/sim/app/chat/hooks/use-chat-streaming.ts +++ b/apps/sim/app/chat/hooks/use-chat-streaming.ts @@ -3,6 +3,7 @@ import { useRef, useState } from 'react' import { createLogger } from '@sim/logger' import { generateId } from '@sim/utils/id' +import { readSSEEvents } from '@/lib/core/utils/sse' import { isUserFileWithMetadata } from '@/lib/core/utils/user-file' import type { ChatFile, ChatMessage } from '@/app/chat/components/message/message' import { CHAT_ERROR_MESSAGES } from '@/app/chat/constants' @@ -125,14 +126,12 @@ export function useChatStreaming() { streamingOptions?.voiceSettings?.autoPlayResponses && streamingOptions?.audioStreamHandler - const reader = response.body?.getReader() - if (!reader) { + if (!response.body) { setIsLoading(false) setIsStreamingResponse(false) return } - const decoder = new TextDecoder() let accumulatedText = '' let lastAudioPosition = 0 @@ -192,264 +191,252 @@ export function useChatStreaming() { setIsLoading(false) + let terminated = false + try { - while (true) { - // Check if aborted - if (abortControllerRef.current === null) { - break + await readSSEEvents<{ + blockId?: string + chunk?: string + event?: string + error?: string + data?: { + success: boolean + error?: string | { message?: string } + output?: Record> } - - const { done, value } = await reader.read() - - if (done) { - flushUI() - // Stream any remaining text for TTS - if ( - shouldPlayAudio && - streamingOptions?.audioStreamHandler && - accumulatedText.length > lastAudioPosition - ) { - const remainingText = accumulatedText.substring(lastAudioPosition).trim() - if (remainingText) { - try { - await streamingOptions.audioStreamHandler(remainingText) - } catch (error) { - logger.error('TTS error for remaining text:', error) - } - } + }>(response.body, { + signal: abortControllerRef.current.signal, + onParseError: (_data, parseError) => { + logger.error('Error parsing stream data:', parseError) + }, + onEvent: async (json) => { + const { blockId, chunk: contentChunk, event: eventType } = json + + if (eventType === 'error' || json.event === 'error') { + const errorMessage = json.error || CHAT_ERROR_MESSAGES.GENERIC_ERROR + setMessages((prev) => + prev.map((msg) => + msg.id === messageId + ? { + ...msg, + content: errorMessage, + isStreaming: false, + type: 'assistant' as const, + } + : msg + ) + ) + setIsLoading(false) + terminated = true + return true } - break - } - const chunk = decoder.decode(value, { stream: true }) - const lines = chunk.split('\n\n') + if (eventType === 'final' && json.data) { + flushUI() + const finalData = json.data - for (const line of lines) { - if (line.startsWith('data: ')) { - const data = line.substring(6) + const outputConfigs = streamingOptions?.outputConfigs + const formattedOutputs: string[] = [] + let extractedFiles: ChatFile[] = [] - if (data === '[DONE]') { - continue - } + const formatValue = (value: any): string | null => { + if (value === null || value === undefined) { + return null + } - try { - const json = JSON.parse(data) - const { blockId, chunk: contentChunk, event: eventType } = json - - if (eventType === 'error' || json.event === 'error') { - const errorMessage = json.error || CHAT_ERROR_MESSAGES.GENERIC_ERROR - setMessages((prev) => - prev.map((msg) => - msg.id === messageId - ? { - ...msg, - content: errorMessage, - isStreaming: false, - type: 'assistant' as const, - } - : msg - ) - ) - setIsLoading(false) - return + if (isUserFileWithMetadata(value)) { + return null } - if (eventType === 'final' && json.data) { - flushUI() - const finalData = json.data as { - success: boolean - error?: string | { message?: string } - output?: Record> + if (Array.isArray(value) && value.length === 0) { + return null + } + + if (typeof value === 'string') { + return value + } + + if (typeof value === 'object') { + try { + return `\`\`\`json\n${JSON.stringify(value, null, 2)}\n\`\`\`` + } catch { + return String(value) } + } - const outputConfigs = streamingOptions?.outputConfigs - const formattedOutputs: string[] = [] - let extractedFiles: ChatFile[] = [] + return String(value) + } - const formatValue = (value: any): string | null => { - if (value === null || value === undefined) { - return null - } + const getOutputValue = (blockOutputs: Record, path?: string) => { + if (!path || path === 'content') { + if (blockOutputs.content !== undefined) return blockOutputs.content + if (blockOutputs.result !== undefined) return blockOutputs.result + return blockOutputs + } - if (isUserFileWithMetadata(value)) { - return null - } + if (blockOutputs[path] !== undefined) { + return blockOutputs[path] + } - if (Array.isArray(value) && value.length === 0) { - return null + if (path.includes('.')) { + return path.split('.').reduce((current, segment) => { + if (current && typeof current === 'object' && segment in current) { + return current[segment] } + return undefined + }, blockOutputs) + } - if (typeof value === 'string') { - return value - } + return undefined + } - if (typeof value === 'object') { - try { - return `\`\`\`json\n${JSON.stringify(value, null, 2)}\n\`\`\`` - } catch { - return String(value) - } - } + if (outputConfigs?.length && finalData.output) { + for (const config of outputConfigs) { + const blockOutputs = finalData.output[config.blockId] + if (!blockOutputs) continue + + const value = getOutputValue(blockOutputs, config.path) + + if (isUserFileWithMetadata(value)) { + extractedFiles.push({ + id: value.id, + name: value.name, + url: value.url, + key: value.key, + size: value.size, + type: value.type, + context: value.context, + }) + continue + } - return String(value) + const nestedFiles = extractFilesFromData(value) + if (nestedFiles.length > 0) { + extractedFiles = [...extractedFiles, ...nestedFiles] + continue } - const getOutputValue = (blockOutputs: Record, path?: string) => { - if (!path || path === 'content') { - if (blockOutputs.content !== undefined) return blockOutputs.content - if (blockOutputs.result !== undefined) return blockOutputs.result - return blockOutputs - } + const formatted = formatValue(value) + if (formatted) { + formattedOutputs.push(formatted) + } + } + } - if (blockOutputs[path] !== undefined) { - return blockOutputs[path] - } + let finalContent = accumulatedText - if (path.includes('.')) { - return path.split('.').reduce((current, segment) => { - if (current && typeof current === 'object' && segment in current) { - return current[segment] - } - return undefined - }, blockOutputs) - } + if (formattedOutputs.length > 0) { + const nonEmptyOutputs = formattedOutputs.filter((output) => output.trim()) + if (nonEmptyOutputs.length > 0) { + const combinedOutputs = nonEmptyOutputs.join('\n\n') + finalContent = finalContent + ? `${finalContent.trim()}\n\n${combinedOutputs}` + : combinedOutputs + } + } - return undefined + if (!finalContent && extractedFiles.length === 0) { + if (finalData.error) { + if (typeof finalData.error === 'string') { + finalContent = finalData.error + } else if (typeof finalData.error?.message === 'string') { + finalContent = finalData.error.message } + } else if (finalData.success && finalData.output) { + const fallbackOutput = Object.values(finalData.output) + .map((block) => formatValue(block)?.trim()) + .filter(Boolean)[0] + if (fallbackOutput) { + finalContent = fallbackOutput + } + } + } - if (outputConfigs?.length && finalData.output) { - for (const config of outputConfigs) { - const blockOutputs = finalData.output[config.blockId] - if (!blockOutputs) continue - - const value = getOutputValue(blockOutputs, config.path) - - if (isUserFileWithMetadata(value)) { - extractedFiles.push({ - id: value.id, - name: value.name, - url: value.url, - key: value.key, - size: value.size, - type: value.type, - context: value.context, - }) - continue - } - - const nestedFiles = extractFilesFromData(value) - if (nestedFiles.length > 0) { - extractedFiles = [...extractedFiles, ...nestedFiles] - continue + setMessages((prev) => + prev.map((msg) => + msg.id === messageId + ? { + ...msg, + isStreaming: false, + content: finalContent ?? msg.content, + files: extractedFiles.length > 0 ? extractedFiles : undefined, } + : msg + ) + ) - const formatted = formatValue(value) - if (formatted) { - formattedOutputs.push(formatted) - } - } - } + accumulatedTextRef.current = '' + lastStreamedPositionRef.current = 0 + lastDisplayedPositionRef.current = 0 + audioStreamingActiveRef.current = false - let finalContent = accumulatedText + terminated = true + return true + } - if (formattedOutputs.length > 0) { - const nonEmptyOutputs = formattedOutputs.filter((output) => output.trim()) - if (nonEmptyOutputs.length > 0) { - const combinedOutputs = nonEmptyOutputs.join('\n\n') - finalContent = finalContent - ? `${finalContent.trim()}\n\n${combinedOutputs}` - : combinedOutputs - } - } + if (blockId && contentChunk) { + if (!messageIdMap.has(blockId)) { + messageIdMap.set(blockId, messageId) + } - if (!finalContent && extractedFiles.length === 0) { - if (finalData.error) { - if (typeof finalData.error === 'string') { - finalContent = finalData.error - } else if (typeof finalData.error?.message === 'string') { - finalContent = finalData.error.message - } - } else if (finalData.success && finalData.output) { - const fallbackOutput = Object.values(finalData.output) - .map((block) => formatValue(block)?.trim()) - .filter(Boolean)[0] - if (fallbackOutput) { - finalContent = fallbackOutput - } - } + accumulatedText += contentChunk + accumulatedTextRef.current = accumulatedText + logger.debug('[useChatStreaming] Received chunk', { + blockId, + chunkLength: contentChunk.length, + totalLength: accumulatedText.length, + messageId, + chunk: contentChunk.substring(0, 20), + }) + uiDirty = true + scheduleUIFlush() + + if (shouldPlayAudio && streamingOptions?.audioStreamHandler) { + const newText = accumulatedText.substring(lastAudioPosition) + const sentenceEndings = ['. ', '! ', '? ', '.\n', '!\n', '?\n', '.', '!', '?'] + let sentenceEnd = -1 + + for (const ending of sentenceEndings) { + const index = newText.indexOf(ending) + if (index > 0) { + sentenceEnd = index + ending.length + break } - - setMessages((prev) => - prev.map((msg) => - msg.id === messageId - ? { - ...msg, - isStreaming: false, - content: finalContent ?? msg.content, - files: extractedFiles.length > 0 ? extractedFiles : undefined, - } - : msg - ) - ) - - accumulatedTextRef.current = '' - lastStreamedPositionRef.current = 0 - lastDisplayedPositionRef.current = 0 - audioStreamingActiveRef.current = false - - return } - if (blockId && contentChunk) { - if (!messageIdMap.has(blockId)) { - messageIdMap.set(blockId, messageId) - } - - accumulatedText += contentChunk - accumulatedTextRef.current = accumulatedText - logger.debug('[useChatStreaming] Received chunk', { - blockId, - chunkLength: contentChunk.length, - totalLength: accumulatedText.length, - messageId, - chunk: contentChunk.substring(0, 20), - }) - uiDirty = true - scheduleUIFlush() - - // Real-time TTS for voice mode - if (shouldPlayAudio && streamingOptions?.audioStreamHandler) { - const newText = accumulatedText.substring(lastAudioPosition) - const sentenceEndings = ['. ', '! ', '? ', '.\n', '!\n', '?\n', '.', '!', '?'] - let sentenceEnd = -1 - - for (const ending of sentenceEndings) { - const index = newText.indexOf(ending) - if (index > 0) { - sentenceEnd = index + ending.length - break - } - } - - if (sentenceEnd > 0) { - const sentence = newText.substring(0, sentenceEnd).trim() - if (sentence && sentence.length >= 3) { - try { - await streamingOptions.audioStreamHandler(sentence) - lastAudioPosition += sentenceEnd - } catch (error) { - logger.error('TTS error:', error) - } - } + if (sentenceEnd > 0) { + const sentence = newText.substring(0, sentenceEnd).trim() + if (sentence && sentence.length >= 3) { + try { + await streamingOptions.audioStreamHandler(sentence) + lastAudioPosition += sentenceEnd + } catch (error) { + logger.error('TTS error:', error) } } - } else if (blockId && eventType === 'end') { - setMessages((prev) => - prev.map((msg) => (msg.id === messageId ? { ...msg, isStreaming: false } : msg)) - ) } - } catch (parseError) { - logger.error('Error parsing stream data:', parseError) + } + } else if (blockId && eventType === 'end') { + setMessages((prev) => + prev.map((msg) => (msg.id === messageId ? { ...msg, isStreaming: false } : msg)) + ) + } + }, + }) + + if (!terminated) { + flushUI() + if ( + shouldPlayAudio && + streamingOptions?.audioStreamHandler && + accumulatedText.length > lastAudioPosition + ) { + const remainingText = accumulatedText.substring(lastAudioPosition).trim() + if (remainingText) { + try { + await streamingOptions.audioStreamHandler(remainingText) + } catch (error) { + logger.error('TTS error for remaining text:', error) } } } diff --git a/apps/sim/app/workspace/[workspaceId]/files/page.tsx b/apps/sim/app/workspace/[workspaceId]/files/page.tsx index ab21f2f3b72..2ed876e4ba0 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/page.tsx +++ b/apps/sim/app/workspace/[workspaceId]/files/page.tsx @@ -1,5 +1,8 @@ import { Suspense } from 'react' +import { dehydrate, HydrationBoundary } from '@tanstack/react-query' import type { Metadata } from 'next' +import { getQueryClient } from '@/app/_shell/providers/get-query-client' +import { prefetchFilesBrowser } from '@/app/workspace/[workspaceId]/files/prefetch' import { Files } from './files' import FilesLoading from './loading' @@ -15,10 +18,17 @@ export const metadata: Metadata = { * table headers) so a suspend never shows a blank frame; the route-level * `loading.tsx` covers the navigation/chunk-load transition the same way. */ -export default function FilesPage() { +export default async function FilesPage({ params }: { params: Promise<{ workspaceId: string }> }) { + const { workspaceId } = await params + + const queryClient = getQueryClient() + await prefetchFilesBrowser(queryClient, workspaceId) + return ( - }> - - + + }> + + + ) } diff --git a/apps/sim/app/workspace/[workspaceId]/files/prefetch.ts b/apps/sim/app/workspace/[workspaceId]/files/prefetch.ts new file mode 100644 index 00000000000..8780aa537fa --- /dev/null +++ b/apps/sim/app/workspace/[workspaceId]/files/prefetch.ts @@ -0,0 +1,43 @@ +import type { QueryClient } from '@tanstack/react-query' +import type { WorkspaceFileFolderApi } from '@/lib/api/contracts/workspace-file-folders' +import type { ListWorkspaceFilesResponse } from '@/lib/api/contracts/workspace-files' +import { prefetchInternalJson } from '@/app/workspace/[workspaceId]/lib/prefetch-internal-fetch' +import { workspaceFileFolderKeys } from '@/hooks/queries/workspace-file-folders' +import { workspaceFilesKeys } from '@/hooks/queries/workspace-files' + +/** + * Prefetches the Files browser's two lists — workspace files and file folders — + * under the same query keys their client hooks (`useWorkspaceFiles`, + * `useWorkspaceFileFolders`) use (scope `active`), so the browser paints + * populated on first render. + * + * Both payloads carry `Date` fields, so they go through their routes and cache + * the serialized wire shape — see {@link prefetchInternalJson}. + */ +export async function prefetchFilesBrowser( + queryClient: QueryClient, + workspaceId: string +): Promise { + await Promise.all([ + queryClient.prefetchQuery({ + queryKey: workspaceFilesKeys.list(workspaceId, 'active'), + queryFn: async () => { + const data = await prefetchInternalJson( + `/api/workspaces/${workspaceId}/files?scope=active` + ) + return data.success ? data.files : [] + }, + staleTime: 30 * 1000, + }), + queryClient.prefetchQuery({ + queryKey: workspaceFileFolderKeys.list(workspaceId, 'active'), + queryFn: async () => { + const data = await prefetchInternalJson<{ folders?: WorkspaceFileFolderApi[] }>( + `/api/workspaces/${workspaceId}/files/folders?scope=active` + ) + return data.folders ?? [] + }, + staleTime: 30 * 1000, + }), + ]) +} diff --git a/apps/sim/app/workspace/[workspaceId]/home/hooks/use-chat.ts b/apps/sim/app/workspace/[workspaceId]/home/hooks/use-chat.ts index b1ec30520af..e81ec0603f9 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/hooks/use-chat.ts +++ b/apps/sim/app/workspace/[workspaceId]/home/hooks/use-chat.ts @@ -63,6 +63,7 @@ import { } from '@/lib/copilot/tools/client/run-tool-execution' import { setCurrentChatTraceparent } from '@/lib/copilot/tools/client/trace-context' import { isWorkflowToolName } from '@/lib/copilot/tools/workflow-tools' +import { readSSELines } from '@/lib/core/utils/sse' import { getQueryClient } from '@/app/_shell/providers/get-query-client' import { useFilePreviewController } from '@/app/workspace/[workspaceId]/home/hooks/preview' import { @@ -1934,7 +1935,6 @@ export function useChat( shouldContinue?: () => boolean } ) => { - const decoder = new TextDecoder() const ctx = createStreamLoopContext({ workspaceId, queryClient, @@ -1987,71 +1987,47 @@ export function useChat( return { sawStreamError: false, sawComplete: false } } streamReaderRef.current = reader - let buffer = '' try { - const pendingLines: string[] = [] - - while (true) { - if (pendingLines.length === 0) { - // Don't read another chunk after `complete` has drained. - if (state.sawCompleteEvent) break - const { done, value } = await reader.read() - if (done) break - if (ops.isStale()) continue - - buffer += decoder.decode(value, { stream: true }) - const lines = buffer.split('\n') - buffer = lines.pop() || '' - pendingLines.push(...lines) - if (pendingLines.length === 0) { - continue + await readSSELines(reader, { + onData: (raw) => { + if (state.sawCompleteEvent) return true + if (ops.isStale()) return + + const parsedResult = parsePersistedStreamEventEnvelopeJson(raw) + if (!parsedResult.ok) { + const error = createStreamSchemaValidationError(parsedResult, 'Live SSE event.') + logger.error('Rejected chat SSE event due to client-side schema enforcement', { + reason: parsedResult.reason, + message: parsedResult.message, + errors: parsedResult.errors, + error: error.message, + }) + throw error } - } - - const line = pendingLines.shift() - if (line === undefined) { - continue - } - if (ops.isStale()) { - pendingLines.length = 0 - continue - } - if (!line.startsWith('data: ')) continue - const raw = line.slice(6) - - const parsedResult = parsePersistedStreamEventEnvelopeJson(raw) - if (!parsedResult.ok) { - const error = createStreamSchemaValidationError(parsedResult, 'Live SSE event.') - logger.error('Rejected chat SSE event due to client-side schema enforcement', { - reason: parsedResult.reason, - message: parsedResult.message, - errors: parsedResult.errors, - error: error.message, - }) - throw error - } - const parsed = parsedResult.event + const parsed = parsedResult.event - if (parsed.trace?.requestId && parsed.trace.requestId !== state.streamRequestId) { - state.streamRequestId = parsed.trace.requestId - streamRequestIdRef.current = state.streamRequestId - ops.flush() - } - if (parsed.stream?.streamId) { - streamIdRef.current = parsed.stream.streamId - } - const eventCursor = parsed.stream?.cursor ?? String(parsed.seq) - if (isAlreadyProcessedStreamCursor(eventCursor, lastCursorRef.current)) { - continue - } - if (eventCursor) { - lastCursorRef.current = eventCursor - } + if (parsed.trace?.requestId && parsed.trace.requestId !== state.streamRequestId) { + state.streamRequestId = parsed.trace.requestId + streamRequestIdRef.current = state.streamRequestId + ops.flush() + } + if (parsed.stream?.streamId) { + streamIdRef.current = parsed.stream.streamId + } + const eventCursor = parsed.stream?.cursor ?? String(parsed.seq) + if (isAlreadyProcessedStreamCursor(eventCursor, lastCursorRef.current)) { + return + } + if (eventCursor) { + lastCursorRef.current = eventCursor + } - logger.debug('SSE event received', parsed) - dispatchStreamEvent(ctx, parsed) - } + logger.debug('SSE event received', parsed) + dispatchStreamEvent(ctx, parsed) + if (state.sawCompleteEvent) return true + }, + }) } finally { if (state.sawStreamError && !state.sawCompleteEvent) { applyTurnTerminal(state.model, 'error') diff --git a/apps/sim/app/workspace/[workspaceId]/home/page.tsx b/apps/sim/app/workspace/[workspaceId]/home/page.tsx index e29acc640ee..13595d65398 100644 --- a/apps/sim/app/workspace/[workspaceId]/home/page.tsx +++ b/apps/sim/app/workspace/[workspaceId]/home/page.tsx @@ -1,6 +1,9 @@ import { Suspense } from 'react' +import { dehydrate, HydrationBoundary } from '@tanstack/react-query' import type { Metadata } from 'next' import { getSession } from '@/lib/auth' +import { getQueryClient } from '@/app/_shell/providers/get-query-client' +import { prefetchHomeLists } from '@/app/workspace/[workspaceId]/home/prefetch' import { Home } from './home' import { HomeFallback } from './home-fallback' @@ -8,11 +11,20 @@ export const metadata: Metadata = { title: 'New chat', } -export default async function HomePage() { +export default async function HomePage({ params }: { params: Promise<{ workspaceId: string }> }) { + const { workspaceId } = await params + + const queryClient = getQueryClient() + const listsPrefetch = prefetchHomeLists(queryClient, workspaceId) + const session = await getSession() + await listsPrefetch + return ( - }> - - + + }> + + + ) } diff --git a/apps/sim/app/workspace/[workspaceId]/home/prefetch.ts b/apps/sim/app/workspace/[workspaceId]/home/prefetch.ts new file mode 100644 index 00000000000..956a9e95555 --- /dev/null +++ b/apps/sim/app/workspace/[workspaceId]/home/prefetch.ts @@ -0,0 +1,48 @@ +import type { QueryClient } from '@tanstack/react-query' +import type { FolderApi } from '@/lib/api/contracts' +import type { ListWorkspaceFilesResponse } from '@/lib/api/contracts/workspace-files' +import { prefetchInternalJson } from '@/app/workspace/[workspaceId]/lib/prefetch-internal-fetch' +import { FOLDER_LIST_STALE_TIME, mapFolder } from '@/hooks/queries/folders' +import { folderKeys } from '@/hooks/queries/utils/folder-keys' +import { workspaceFilesKeys } from '@/hooks/queries/workspace-files' + +/** + * Prefetches the home page's secondary lists — folders and workspace files — + * under the same query keys their client hooks (`useFolders`, + * `useWorkspaceFiles`) use, so the home view paints populated on first render. + * + * The workflow list (`workflowKeys.list(ws, 'active')`) is already hydrated by + * the workspace sidebar prefetch and is intentionally not repeated here. + * + * Folders are fetched through the route and mapped with the same `mapFolder` + * the hook applies, matching its cached shape (string dates → `Date`). Files + * carry `Date` fields, so they go through the route and cache the serialized + * wire shape — see {@link prefetchInternalJson}. + */ +export async function prefetchHomeLists( + queryClient: QueryClient, + workspaceId: string +): Promise { + await Promise.all([ + queryClient.prefetchQuery({ + queryKey: folderKeys.list(workspaceId, 'active'), + queryFn: async () => { + const { folders } = await prefetchInternalJson<{ folders?: FolderApi[] }>( + `/api/folders?workspaceId=${workspaceId}&scope=active` + ) + return (folders ?? []).map(mapFolder) + }, + staleTime: FOLDER_LIST_STALE_TIME, + }), + queryClient.prefetchQuery({ + queryKey: workspaceFilesKeys.list(workspaceId, 'active'), + queryFn: async () => { + const data = await prefetchInternalJson( + `/api/workspaces/${workspaceId}/files?scope=active` + ) + return data.success ? data.files : [] + }, + staleTime: 30 * 1000, + }), + ]) +} diff --git a/apps/sim/app/workspace/[workspaceId]/knowledge/page.tsx b/apps/sim/app/workspace/[workspaceId]/knowledge/page.tsx index be3743be659..6243dc42035 100644 --- a/apps/sim/app/workspace/[workspaceId]/knowledge/page.tsx +++ b/apps/sim/app/workspace/[workspaceId]/knowledge/page.tsx @@ -1,8 +1,26 @@ +import { dehydrate, HydrationBoundary } from '@tanstack/react-query' import type { Metadata } from 'next' +import { getQueryClient } from '@/app/_shell/providers/get-query-client' +import { prefetchKnowledgeBases } from '@/app/workspace/[workspaceId]/knowledge/prefetch' import { Knowledge } from './knowledge' export const metadata: Metadata = { title: 'Knowledge Base', } -export default Knowledge +export default async function KnowledgePage({ + params, +}: { + params: Promise<{ workspaceId: string }> +}) { + const { workspaceId } = await params + + const queryClient = getQueryClient() + await prefetchKnowledgeBases(queryClient, workspaceId) + + return ( + + + + ) +} diff --git a/apps/sim/app/workspace/[workspaceId]/knowledge/prefetch.ts b/apps/sim/app/workspace/[workspaceId]/knowledge/prefetch.ts new file mode 100644 index 00000000000..0e8e5578840 --- /dev/null +++ b/apps/sim/app/workspace/[workspaceId]/knowledge/prefetch.ts @@ -0,0 +1,28 @@ +import type { QueryClient } from '@tanstack/react-query' +import type { KnowledgeBaseData } from '@/lib/api/contracts/knowledge' +import { prefetchInternalJson } from '@/app/workspace/[workspaceId]/lib/prefetch-internal-fetch' +import { knowledgeKeys } from '@/hooks/queries/kb/knowledge' + +/** + * Prefetches the workspace's knowledge-bases list under the same query key the + * client `useKnowledgeBasesQuery` hook uses (scope `active`), so the list paints + * populated on first render. + * + * The list carries `Date` fields, so it goes through the `/api/knowledge` route + * and caches the serialized wire shape — see {@link prefetchInternalJson}. + */ +export async function prefetchKnowledgeBases( + queryClient: QueryClient, + workspaceId: string +): Promise { + await queryClient.prefetchQuery({ + queryKey: knowledgeKeys.list(workspaceId, 'active'), + queryFn: async () => { + const result = await prefetchInternalJson<{ data: KnowledgeBaseData[] }>( + `/api/knowledge?workspaceId=${workspaceId}&scope=active` + ) + return result.data + }, + staleTime: 60 * 1000, + }) +} diff --git a/apps/sim/app/workspace/[workspaceId]/lib/prefetch-internal-fetch.ts b/apps/sim/app/workspace/[workspaceId]/lib/prefetch-internal-fetch.ts new file mode 100644 index 00000000000..e48f6064c17 --- /dev/null +++ b/apps/sim/app/workspace/[workspaceId]/lib/prefetch-internal-fetch.ts @@ -0,0 +1,25 @@ +import { headers } from 'next/headers' +import { getInternalApiBaseUrl } from '@/lib/core/utils/urls' + +/** + * Server-side GET against an internal `/api` route, forwarding the incoming + * request's cookie so the route authenticates as the current user. + * + * List prefetches go through the route (rather than the data layer) when the + * payload carries `Date` fields: `NextResponse.json` serializes them to the + * string wire shape the client caches via `requestJson`, so the + * server-hydrated entry byte-matches the client-fetched one through + * dehydration. Calling the data layer directly would cache raw `Date` objects + * and drift from that wire shape. Mirrors the settings/subscription prefetch. + */ +export async function prefetchInternalJson(path: string): Promise { + const cookie = (await headers()).get('cookie') + // boundary-raw-fetch: server-side RSC prefetch forwarding the session cookie to an internal API route; requestJson is client-only and cannot run here + const response = await fetch(`${getInternalApiBaseUrl()}${path}`, { + headers: cookie ? { cookie } : {}, + }) + if (!response.ok) { + throw new Error(`Prefetch failed for ${path}: ${response.status}`) + } + return response.json() as Promise +} diff --git a/apps/sim/app/workspace/[workspaceId]/lib/prefetch.test.ts b/apps/sim/app/workspace/[workspaceId]/lib/prefetch.test.ts new file mode 100644 index 00000000000..d031c0648ef --- /dev/null +++ b/apps/sim/app/workspace/[workspaceId]/lib/prefetch.test.ts @@ -0,0 +1,169 @@ +/** + * @vitest-environment node + */ +import { QueryClient } from '@tanstack/react-query' +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const { mockPrefetchInternalJson } = vi.hoisted(() => ({ + mockPrefetchInternalJson: vi.fn(), +})) + +vi.mock('@/app/workspace/[workspaceId]/lib/prefetch-internal-fetch', () => ({ + prefetchInternalJson: mockPrefetchInternalJson, +})) + +vi.mock('@/components/emcn', () => ({ + toast: { success: vi.fn(), error: vi.fn() }, +})) + +import { prefetchFilesBrowser } from '@/app/workspace/[workspaceId]/files/prefetch' +import { prefetchHomeLists } from '@/app/workspace/[workspaceId]/home/prefetch' +import { prefetchKnowledgeBases } from '@/app/workspace/[workspaceId]/knowledge/prefetch' +import { prefetchTables } from '@/app/workspace/[workspaceId]/tables/prefetch' +import { knowledgeKeys } from '@/hooks/queries/kb/knowledge' +import { tableKeys } from '@/hooks/queries/tables' +import { folderKeys } from '@/hooks/queries/utils/folder-keys' +import { workspaceFileFolderKeys } from '@/hooks/queries/workspace-file-folders' +import { workspaceFilesKeys } from '@/hooks/queries/workspace-files' + +const WORKSPACE_ID = 'ws-123' + +function makeClient() { + return new QueryClient({ defaultOptions: { queries: { retry: false } } }) +} + +describe('workspace list prefetches', () => { + beforeEach(() => { + vi.clearAllMocks() + }) + + describe('prefetchKnowledgeBases', () => { + it('primes the exact key useKnowledgeBasesQuery reads and unwraps data', async () => { + const bases = [{ id: 'kb-1' }] + mockPrefetchInternalJson.mockResolvedValue({ data: bases }) + const client = makeClient() + + await prefetchKnowledgeBases(client, WORKSPACE_ID) + + expect(mockPrefetchInternalJson).toHaveBeenCalledWith( + `/api/knowledge?workspaceId=${WORKSPACE_ID}&scope=active` + ) + expect(client.getQueryData(knowledgeKeys.list(WORKSPACE_ID, 'active'))).toEqual(bases) + }) + }) + + describe('prefetchTables', () => { + it('primes the exact key useTablesList reads and unwraps data.tables', async () => { + const tables = [{ id: 't-1' }] + mockPrefetchInternalJson.mockResolvedValue({ data: { tables } }) + const client = makeClient() + + await prefetchTables(client, WORKSPACE_ID) + + expect(mockPrefetchInternalJson).toHaveBeenCalledWith( + `/api/table?workspaceId=${WORKSPACE_ID}&scope=active` + ) + expect(client.getQueryData(tableKeys.list(WORKSPACE_ID, 'active'))).toEqual(tables) + }) + }) + + describe('prefetchFilesBrowser', () => { + it('primes both file + folder keys the client hooks read', async () => { + const files = [{ id: 'f-1' }] + const folders = [{ id: 'folder-1' }] + mockPrefetchInternalJson.mockImplementation(async (path: string) => + path.includes('/folders') ? { folders } : { success: true, files } + ) + const client = makeClient() + + await prefetchFilesBrowser(client, WORKSPACE_ID) + + expect(mockPrefetchInternalJson).toHaveBeenCalledWith( + `/api/workspaces/${WORKSPACE_ID}/files?scope=active` + ) + expect(mockPrefetchInternalJson).toHaveBeenCalledWith( + `/api/workspaces/${WORKSPACE_ID}/files/folders?scope=active` + ) + expect(client.getQueryData(workspaceFilesKeys.list(WORKSPACE_ID, 'active'))).toEqual(files) + expect(client.getQueryData(workspaceFileFolderKeys.list(WORKSPACE_ID, 'active'))).toEqual( + folders + ) + }) + + it('caches an empty file list when the route reports failure', async () => { + mockPrefetchInternalJson.mockImplementation(async (path: string) => + path.includes('/folders') ? { folders: [] } : { success: false, files: [] } + ) + const client = makeClient() + + await prefetchFilesBrowser(client, WORKSPACE_ID) + + expect(client.getQueryData(workspaceFilesKeys.list(WORKSPACE_ID, 'active'))).toEqual([]) + }) + }) + + describe('prefetchHomeLists', () => { + it('primes folder + file keys, mapping folder rows to the client shape', async () => { + const folderRow = { + id: 'folder-1', + name: 'Docs', + userId: 'u-1', + workspaceId: WORKSPACE_ID, + parentId: null, + color: null, + isExpanded: true, + locked: false, + sortOrder: 0, + createdAt: '2026-01-01T00:00:00.000Z', + updatedAt: '2026-01-02T00:00:00.000Z', + archivedAt: null, + } + const files = [{ id: 'f-1' }] + mockPrefetchInternalJson.mockImplementation(async (path: string) => + path.startsWith('/api/folders') ? { folders: [folderRow] } : { success: true, files } + ) + const client = makeClient() + + await prefetchHomeLists(client, WORKSPACE_ID) + + expect(mockPrefetchInternalJson).toHaveBeenCalledWith( + `/api/folders?workspaceId=${WORKSPACE_ID}&scope=active` + ) + const cachedFolders = client.getQueryData(folderKeys.list(WORKSPACE_ID, 'active')) as Array<{ + id: string + color: string + createdAt: Date + }> + expect(cachedFolders).toHaveLength(1) + expect(cachedFolders[0].color).toBe('#6B7280') + expect(cachedFolders[0].createdAt).toBeInstanceOf(Date) + expect(client.getQueryData(workspaceFilesKeys.list(WORKSPACE_ID, 'active'))).toEqual(files) + }) + }) + + describe('graceful failure', () => { + it.each([ + [ + 'prefetchKnowledgeBases', + prefetchKnowledgeBases, + knowledgeKeys.list(WORKSPACE_ID, 'active'), + ], + ['prefetchTables', prefetchTables, tableKeys.list(WORKSPACE_ID, 'active')], + ['prefetchHomeLists', prefetchHomeLists, folderKeys.list(WORKSPACE_ID, 'active')], + [ + 'prefetchFilesBrowser', + prefetchFilesBrowser, + workspaceFilesKeys.list(WORKSPACE_ID, 'active'), + ], + ] as const)( + '%s does not throw when the fetcher rejects (page still renders, client refetches)', + async (_name, prefetch, queryKey) => { + mockPrefetchInternalJson.mockRejectedValue(new Error('500')) + const client = makeClient() + + await expect(prefetch(client, WORKSPACE_ID)).resolves.toBeUndefined() + expect(client.getQueryData(queryKey)).toBeUndefined() + } + ) + }) +}) diff --git a/apps/sim/app/workspace/[workspaceId]/tables/page.tsx b/apps/sim/app/workspace/[workspaceId]/tables/page.tsx index 15a016a25a2..0e9390a5d95 100644 --- a/apps/sim/app/workspace/[workspaceId]/tables/page.tsx +++ b/apps/sim/app/workspace/[workspaceId]/tables/page.tsx @@ -1,6 +1,9 @@ import { Suspense } from 'react' +import { dehydrate, HydrationBoundary } from '@tanstack/react-query' import type { Metadata } from 'next' +import { getQueryClient } from '@/app/_shell/providers/get-query-client' import TablesLoading from '@/app/workspace/[workspaceId]/tables/loading' +import { prefetchTables } from '@/app/workspace/[workspaceId]/tables/prefetch' import { Tables } from './tables' export const metadata: Metadata = { @@ -13,10 +16,17 @@ export const metadata: Metadata = { * fallback renders the real chrome so a suspend never shows a blank frame; the * route-level `loading.tsx` covers the navigation/chunk-load transition. */ -export default function TablesPage() { +export default async function TablesPage({ params }: { params: Promise<{ workspaceId: string }> }) { + const { workspaceId } = await params + + const queryClient = getQueryClient() + await prefetchTables(queryClient, workspaceId) + return ( - }> - - + + }> + + + ) } diff --git a/apps/sim/app/workspace/[workspaceId]/tables/prefetch.ts b/apps/sim/app/workspace/[workspaceId]/tables/prefetch.ts new file mode 100644 index 00000000000..60d6a79a735 --- /dev/null +++ b/apps/sim/app/workspace/[workspaceId]/tables/prefetch.ts @@ -0,0 +1,26 @@ +import type { QueryClient } from '@tanstack/react-query' +import type { TableDefinition } from '@/lib/table' +import { prefetchInternalJson } from '@/app/workspace/[workspaceId]/lib/prefetch-internal-fetch' +import { tableKeys } from '@/hooks/queries/tables' + +/** + * Prefetches the workspace's tables list under the same query key the client + * `useTablesList` hook uses (scope `active`), so the list paints populated on + * first render. + * + * Table definitions carry `Date` fields, so the list goes through the + * `/api/table` route and caches the serialized wire shape — see + * {@link prefetchInternalJson}. + */ +export async function prefetchTables(queryClient: QueryClient, workspaceId: string): Promise { + await queryClient.prefetchQuery({ + queryKey: tableKeys.list(workspaceId, 'active'), + queryFn: async () => { + const response = await prefetchInternalJson<{ data: { tables: TableDefinition[] } }>( + `/api/table?workspaceId=${workspaceId}&scope=active` + ) + return response.data.tables + }, + staleTime: 30 * 1000, + }) +} diff --git a/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/chat/chat.tsx b/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/chat/chat.tsx index 588ff6da5cc..d4caa19424f 100644 --- a/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/chat/chat.tsx +++ b/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/chat/chat.tsx @@ -25,6 +25,7 @@ import { extractPathFromOutputId, parseOutputContentSafely, } from '@/lib/core/utils/response-format' +import { readSSEEvents } from '@/lib/core/utils/sse' import { CHAT_ACCEPT_ATTRIBUTE } from '@/lib/uploads/utils/validation' import { normalizeInputFormatValue } from '@/lib/workflows/input-format' import { StartBlockPath, TriggerUtils } from '@/lib/workflows/triggers/triggers' @@ -520,12 +521,10 @@ export function Chat() { * @param responseMessageId - ID of the message to update with streamed content */ const processStreamingResponse = useCallback( - async (stream: ReadableStream, responseMessageId: string) => { + async (stream: ReadableStream, responseMessageId: string) => { const reader = stream.getReader() streamReaderRef.current = reader - const decoder = new TextDecoder() let accumulatedContent = '' - let buffer = '' const BATCH_MAX_MS = 50 let pendingChunks = '' @@ -563,63 +562,34 @@ export function Chat() { } try { - while (true) { - const { done, value } = await reader.read() - if (done) { - flushChunks() - finalizeMessageStream(responseMessageId) - break - } - - const chunk = decoder.decode(value, { stream: true }) - buffer += chunk - - const separatorIndex = buffer.lastIndexOf('\n\n') - if (separatorIndex === -1) { - continue - } - - const processable = buffer.slice(0, separatorIndex) - buffer = buffer.slice(separatorIndex + 2) - - const lines = processable.split('\n\n') - - for (const line of lines) { - if (!line.startsWith('data: ')) continue - - const data = line.substring(6) - if (data === '[DONE]') continue - - try { - const json = JSON.parse(data) - const { event, data: eventData, chunk: contentChunk } = json - - if (event === 'final' && eventData) { - const result = eventData as ExecutionResult - - if ('success' in result && !result.success) { - const errorMessage = result.error || 'Workflow execution failed' - flushChunks() - appendMessageContent( - responseMessageId, - `${accumulatedContent ? '\n\n' : ''}Error: ${errorMessage}` - ) - finalizeMessageStream(responseMessageId) - return - } + await readSSEEvents<{ event?: string; data?: ExecutionResult; chunk?: string }>(reader, { + onParseError: (_data, e) => { + logger.error('Error parsing stream data:', e) + }, + onEvent: (json) => { + const { event, data: eventData, chunk: contentChunk } = json + if (event === 'final' && eventData) { + if ('success' in eventData && !eventData.success) { + const errorMessage = eventData.error || 'Workflow execution failed' flushChunks() - finalizeMessageStream(responseMessageId) - } else if (contentChunk) { - accumulatedContent += contentChunk - pendingChunks += contentChunk - scheduleFlush() + appendMessageContent( + responseMessageId, + `${accumulatedContent ? '\n\n' : ''}Error: ${errorMessage}` + ) } - } catch (e) { - logger.error('Error parsing stream data:', e) + return true } - } - } + + if (contentChunk) { + accumulatedContent += contentChunk + pendingChunks += contentChunk + scheduleFlush() + } + }, + }) + flushChunks() + finalizeMessageStream(responseMessageId) } catch (error) { if ((error as Error)?.name !== 'AbortError') { logger.error('Error processing stream:', error) diff --git a/apps/sim/blocks/blocks/gitlab.ts b/apps/sim/blocks/blocks/gitlab.ts index 59de55f8c26..1cfe24a9e25 100644 --- a/apps/sim/blocks/blocks/gitlab.ts +++ b/apps/sim/blocks/blocks/gitlab.ts @@ -57,6 +57,15 @@ export const GitLabBlock: BlockConfig = { password: true, required: true, }, + // Self-managed GitLab host (defaults to gitlab.com) + { + id: 'host', + title: 'GitLab Host', + type: 'short-input', + placeholder: 'gitlab.com', + mode: 'advanced', + description: 'Self-managed GitLab host. Leave blank for gitlab.com.', + }, // Project ID (required for most operations) { id: 'projectId', @@ -474,6 +483,7 @@ Return ONLY the commit message - no explanations, no extra text.`, params: (params) => { const baseParams: Record = { accessToken: params.accessToken, + host: params.host?.trim() || undefined, } switch (params.operation) { @@ -709,6 +719,7 @@ Return ONLY the commit message - no explanations, no extra text.`, inputs: { operation: { type: 'string', description: 'Operation to perform' }, credential: { type: 'string', description: 'GitLab access token' }, + host: { type: 'string', description: 'Self-managed GitLab host (defaults to gitlab.com)' }, projectId: { type: 'string', description: 'Project ID or URL-encoded path' }, issueIid: { type: 'number', description: 'Issue internal ID' }, mergeRequestIid: { type: 'number', description: 'Merge request internal ID' }, diff --git a/apps/sim/bootstrap.ts b/apps/sim/bootstrap.ts new file mode 100644 index 00000000000..bc2e92b882c --- /dev/null +++ b/apps/sim/bootstrap.ts @@ -0,0 +1,13 @@ +/** + * Container entrypoint. Hydrates `process.env` from the runtime secret before + * loading the Next.js standalone server, so application modules that read env at + * import time see the full configuration. See `@sim/runtime-secrets`. + */ +import { loadRuntimeSecrets } from '@sim/runtime-secrets' + +await loadRuntimeSecrets() +// `server.js` is the Next standalone build artifact, a sibling of this file in +// the image; it does not exist at type-check time, so the specifier is held in a +// variable to keep it out of static module resolution. +const standaloneServer = './server.js' +await import(standaloneServer) diff --git a/apps/sim/connectors/gitlab/gitlab.ts b/apps/sim/connectors/gitlab/gitlab.ts index 18247214006..99586321f48 100644 --- a/apps/sim/connectors/gitlab/gitlab.ts +++ b/apps/sim/connectors/gitlab/gitlab.ts @@ -14,10 +14,10 @@ import { parseTagDate, sizeLimitSkipReason, } from '@/connectors/utils' +import { normalizeGitLabHost, UnsafeGitLabHostError } from '@/tools/gitlab/utils' const logger = createLogger('GitLabConnector') -const DEFAULT_HOST = 'gitlab.com' const PAGE_SIZE = 100 /** Max repository file size to index. Larger blobs are skipped. */ const MAX_FILE_SIZE = CONNECTOR_MAX_FILE_BYTES @@ -175,16 +175,16 @@ interface GitLabProject { } /** - * Normalizes the host config value: trims whitespace, strips any protocol - * prefix and trailing slashes, and falls back to gitlab.com when empty. + * Normalizes the host config value via the shared GitLab host normalizer: + * trims, strips any protocol prefix and trailing slashes, rejects structurally + * unsafe hosts (userinfo, whitespace, embedded path), and falls back to + * gitlab.com when empty. Shared with the GitLab tools and webhook provider so + * every surface resolves and validates hosts identically. + * + * @throws {UnsafeGitLabHostError} when a non-empty host is structurally unsafe. */ function normalizeHost(rawHost: unknown): string { - const host = typeof rawHost === 'string' ? rawHost.trim() : '' - if (!host) return DEFAULT_HOST - return host - .replace(/^https?:\/\//i, '') - .replace(/\/+$/, '') - .trim() + return normalizeGitLabHost(rawHost) } /** @@ -941,7 +941,18 @@ export const gitlabConnector: ConnectorConfig = { return { valid: false, error: 'Max items must be a positive number' } } - const host = normalizeHost(sourceConfig.host) + let host: string + try { + host = normalizeHost(sourceConfig.host) + } catch (error) { + if (error instanceof UnsafeGitLabHostError) { + return { + valid: false, + error: 'Host must be a valid GitLab domain (e.g. gitlab.example.com)', + } + } + throw error + } const apiBase = buildApiBase(host) const encodedProject = encodeProjectId(project) const choice = getContentTypeChoice(sourceConfig) diff --git a/apps/sim/hooks/queries/folders.ts b/apps/sim/hooks/queries/folders.ts index f0d8913cd05..34b695374fd 100644 --- a/apps/sim/hooks/queries/folders.ts +++ b/apps/sim/hooks/queries/folders.ts @@ -25,7 +25,15 @@ import type { WorkflowFolder } from '@/stores/folders/types' const logger = createLogger('FolderQueries') -function mapFolder(folder: FolderApi): WorkflowFolder { +export const FOLDER_LIST_STALE_TIME = 60 * 1000 + +/** + * Maps a wire folder row to the client `WorkflowFolder` shape (string dates → + * `Date`, color default). Exported so the server-side home prefetch produces + * the exact cached value `useFolders` stores, keeping the hydrated entry in + * sync with a client fetch. + */ +export function mapFolder(folder: FolderApi): WorkflowFolder { return { id: folder.id, name: folder.name, diff --git a/apps/sim/hooks/use-execution-stream.test.ts b/apps/sim/hooks/use-execution-stream.test.ts index da52635ff99..f38f028c805 100644 --- a/apps/sim/hooks/use-execution-stream.test.ts +++ b/apps/sim/hooks/use-execution-stream.test.ts @@ -84,4 +84,46 @@ describe('processSSEStream', () => { expect(onEventId).not.toHaveBeenCalled() }) + + it('releases the reader lock after the stream completes', async () => { + const stream = streamEvents([]) + const reader = stream.getReader() + expect(stream.locked).toBe(true) + + await processSSEStream(reader, {}, 'test') + + expect(stream.locked).toBe(false) + }) + + it('releases the reader lock even when a handler throws', async () => { + const event: ExecutionEvent = { + type: 'block:started', + eventId: 7, + timestamp: new Date().toISOString(), + executionId: 'exec-1', + workflowId: 'wf-1', + data: { + blockId: 'block-1', + blockName: 'Block 1', + blockType: 'function', + executionOrder: 1, + }, + } + const stream = streamEvents([event]) + const reader = stream.getReader() + + await expect( + processSSEStream( + reader, + { + onBlockStarted: () => { + throw new Error('boom') + }, + }, + 'test' + ) + ).rejects.toThrow('boom') + + expect(stream.locked).toBe(false) + }) }) diff --git a/apps/sim/hooks/use-execution-stream.ts b/apps/sim/hooks/use-execution-stream.ts index 24a6e0cad4a..ffe862c4710 100644 --- a/apps/sim/hooks/use-execution-stream.ts +++ b/apps/sim/hooks/use-execution-stream.ts @@ -1,6 +1,7 @@ import { useCallback } from 'react' import { createLogger } from '@sim/logger' import { getErrorMessage } from '@sim/utils/errors' +import { readSSEEvents } from '@/lib/core/utils/sse' import type { BlockChildWorkflowStartedData, BlockCompletedData, @@ -82,36 +83,12 @@ export async function processSSEStream( callbacks: ExecutionStreamCallbacks, logPrefix: string ): Promise { - const decoder = new TextDecoder() - let buffer = '' - try { - while (true) { - const { done, value } = await reader.read() - - if (done) break - - buffer += decoder.decode(value, { stream: true }) - const lines = buffer.split('\n\n') - buffer = lines.pop() || '' - - for (const line of lines) { - if (!line.trim() || !line.startsWith('data: ')) continue - - const data = line.substring(6).trim() - if (data === '[DONE]') { - logger.info(`${logPrefix} stream completed`) - continue - } - - let event: ExecutionEvent - try { - event = JSON.parse(data) as ExecutionEvent - } catch (error) { - logger.error('Failed to parse SSE event:', error, { data }) - continue - } - + await readSSEEvents(reader, { + onParseError: (data, error) => { + logger.error('Failed to parse SSE event:', error, { data }) + }, + onEvent: async (event) => { try { switch (event.type) { case 'execution:started': @@ -168,8 +145,9 @@ export async function processSSEStream( error ) } - } - } + }, + }) + logger.debug(`${logPrefix} stream completed`) } finally { reader.releaseLock() } diff --git a/apps/sim/hooks/use-undo-redo.ts b/apps/sim/hooks/use-undo-redo.ts index 025c087de0e..22dfdaa6e0e 100644 --- a/apps/sim/hooks/use-undo-redo.ts +++ b/apps/sim/hooks/use-undo-redo.ts @@ -41,6 +41,7 @@ import { type UpdateParentOperation, useUndoRedoStore, } from '@/stores/undo-redo' +import { deriveDiffFlags } from '@/stores/workflow-diff/types' import { useWorkflowRegistry } from '@/stores/workflows/registry/store' import { useSubBlockStore } from '@/stores/workflows/subblock/store' import { useWorkflowStore } from '@/stores/workflows/workflow/store' @@ -1234,9 +1235,7 @@ export function useUndoRedo() { // Restore diff state with baseline (local UI only) diffStore._batchedStateUpdate({ - hasActiveDiff: true, - isShowingDiff: true, - isDiffReady: true, + ...deriveDiffFlags('showing'), baselineWorkflow: originalBaseline || null, baselineWorkflowId: activeWorkflowId, diffAnalysis: diffAnalysis, @@ -1285,9 +1284,7 @@ export function useUndoRedo() { // Restore diff state with baseline (local UI only) const diffStore = useWorkflowDiffStore.getState() diffStore._batchedStateUpdate({ - hasActiveDiff: true, - isShowingDiff: true, - isDiffReady: true, + ...deriveDiffFlags('showing'), baselineWorkflow: baselineSnapshot || null, baselineWorkflowId: activeWorkflowId, diffAnalysis: diffAnalysis, @@ -1805,9 +1802,7 @@ export function useUndoRedo() { // Restore diff state with original baseline (local UI only) diffStore._batchedStateUpdate({ - hasActiveDiff: true, - isShowingDiff: true, - isDiffReady: true, + ...deriveDiffFlags('showing'), baselineWorkflow: baselineSnapshot, baselineWorkflowId: activeWorkflowId, diffAnalysis: diffAnalysis, @@ -1834,9 +1829,7 @@ export function useUndoRedo() { // Clear diff state FIRST to prevent flash of colors (local UI only) // Use setState directly to ensure synchronous clearing useWorkflowDiffStore.setState({ - hasActiveDiff: false, - isShowingDiff: false, - isDiffReady: false, + ...deriveDiffFlags('none'), baselineWorkflow: null, baselineWorkflowId: null, diffAnalysis: null, @@ -1886,9 +1879,7 @@ export function useUndoRedo() { // Clear diff state FIRST to prevent flash of colors (local UI only) // Use setState directly to ensure synchronous clearing useWorkflowDiffStore.setState({ - hasActiveDiff: false, - isShowingDiff: false, - isDiffReady: false, + ...deriveDiffFlags('none'), baselineWorkflow: null, baselineWorkflowId: null, diffAnalysis: null, diff --git a/apps/sim/lib/core/utils/sse.test.ts b/apps/sim/lib/core/utils/sse.test.ts index 524c00b83d3..579e23fd607 100644 --- a/apps/sim/lib/core/utils/sse.test.ts +++ b/apps/sim/lib/core/utils/sse.test.ts @@ -2,7 +2,13 @@ * @vitest-environment node */ import { describe, expect, it, vi } from 'vitest' -import { encodeSSE, readSSEStream, SSE_HEADERS } from '@/lib/core/utils/sse' +import { + encodeSSE, + readSSEEvents, + readSSELines, + readSSEStream, + SSE_HEADERS, +} from '@/lib/core/utils/sse' function createStreamFromChunks(chunks: Uint8Array[]): ReadableStream { let index = 0 @@ -311,3 +317,316 @@ describe('readSSEStream', () => { }) }) }) + +function streamFromStringChunks(chunks: string[]): ReadableStream { + const encoder = new TextEncoder() + return createStreamFromChunks(chunks.map((c) => encoder.encode(c))) +} + +describe('readSSEEvents', () => { + it('parses `\\n\\n`-framed events', async () => { + const stream = streamFromStringChunks([ + 'data: {"n":1}\n\n', + 'data: {"n":2}\n\n', + 'data: {"n":3}\n\n', + ]) + const events: number[] = [] + await readSSEEvents<{ n: number }>(stream, { + onEvent: (e) => { + events.push(e.n) + }, + }) + expect(events).toEqual([1, 2, 3]) + }) + + it('parses `\\n`-framed events', async () => { + const stream = streamFromStringChunks(['data: {"n":1}\ndata: {"n":2}\ndata: {"n":3}\n']) + const events: number[] = [] + await readSSEEvents<{ n: number }>(stream, { + onEvent: (e) => { + events.push(e.n) + }, + }) + expect(events).toEqual([1, 2, 3]) + }) + + it('reassembles events split across chunk boundaries', async () => { + const stream = streamFromStringChunks(['data: {"ms', 'g":"hel', 'lo"}\n\n']) + const events: Array<{ msg: string }> = [] + await readSSEEvents<{ msg: string }>(stream, { + onEvent: (e) => { + events.push(e) + }, + }) + expect(events).toEqual([{ msg: 'hello' }]) + }) + + it('skips the [DONE] sentinel', async () => { + const stream = streamFromStringChunks(['data: {"n":1}\n\n', 'data: [DONE]\n\n']) + const events: number[] = [] + await readSSEEvents<{ n: number }>(stream, { + onEvent: (e) => { + events.push(e.n) + }, + }) + expect(events).toEqual([1]) + }) + + it('accepts `data:` with and without a leading space', async () => { + const stream = streamFromStringChunks(['data:{"n":1}\n\n', 'data: {"n":2}\n\n']) + const events: number[] = [] + await readSSEEvents<{ n: number }>(stream, { + onEvent: (e) => { + events.push(e.n) + }, + }) + expect(events).toEqual([1, 2]) + }) + + it('strips trailing carriage returns (\\r\\n framing)', async () => { + const stream = streamFromStringChunks(['data: {"n":1}\r\n\r\n', 'data: {"n":2}\r\n\r\n']) + const events: number[] = [] + await readSSEEvents<{ n: number }>(stream, { + onEvent: (e) => { + events.push(e.n) + }, + }) + expect(events).toEqual([1, 2]) + }) + + it('routes unparseable payloads to onParseError and continues', async () => { + const stream = streamFromStringChunks(['data: not-json\n\n', 'data: {"n":2}\n\n']) + const events: number[] = [] + const onParseError = vi.fn() + await readSSEEvents<{ n: number }>(stream, { + onEvent: (e) => { + events.push(e.n) + }, + onParseError, + }) + expect(events).toEqual([2]) + expect(onParseError).toHaveBeenCalledTimes(1) + expect(onParseError).toHaveBeenCalledWith('not-json', expect.any(Error)) + }) + + it('stops early when onEvent returns true', async () => { + const stream = streamFromStringChunks([ + 'data: {"n":1}\n\n', + 'data: {"n":2}\n\n', + 'data: {"n":3}\n\n', + ]) + const events: number[] = [] + await readSSEEvents<{ n: number }>(stream, { + onEvent: (e) => { + events.push(e.n) + return e.n === 2 + }, + }) + expect(events).toEqual([1, 2]) + }) + + it('does not process events once the signal is aborted', async () => { + const controller = new AbortController() + const stream = streamFromStringChunks(['data: {"n":1}\n\n', 'data: {"n":2}\n\n']) + const events: number[] = [] + await readSSEEvents<{ n: number }>(stream, { + signal: controller.signal, + onEvent: (e) => { + events.push(e.n) + controller.abort() + }, + }) + expect(events).toEqual([1]) + }) + + it('returns immediately when the signal is already aborted', async () => { + const controller = new AbortController() + controller.abort() + const stream = streamFromStringChunks(['data: {"n":1}\n\n']) + const onEvent = vi.fn() + await readSSEEvents(stream, { signal: controller.signal, onEvent }) + expect(onEvent).not.toHaveBeenCalled() + }) + + it('releases the reader lock for a stream source', async () => { + const stream = streamFromStringChunks(['data: {"n":1}\n\n']) + await readSSEEvents<{ n: number }>(stream, { onEvent: () => {} }) + expect(() => stream.getReader()).not.toThrow() + }) + + it('does not release the lock for a reader source', async () => { + const stream = streamFromStringChunks(['data: {"n":1}\n\n']) + const reader = stream.getReader() + await readSSEEvents<{ n: number }>(reader, { onEvent: () => {} }) + expect(() => stream.getReader()).toThrow() + reader.releaseLock() + }) + + it('accepts a Response source', async () => { + const response = new Response(streamFromStringChunks(['data: {"n":7}\n\n'])) + const events: number[] = [] + await readSSEEvents<{ n: number }>(response, { + onEvent: (e) => { + events.push(e.n) + }, + }) + expect(events).toEqual([7]) + }) + + it('silently skips unparseable payloads when no onParseError is provided', async () => { + const stream = streamFromStringChunks(['data: not-json\n\n', 'data: {"n":2}\n\n']) + const events: number[] = [] + await expect( + readSSEEvents<{ n: number }>(stream, { + onEvent: (e) => { + events.push(e.n) + }, + }) + ).resolves.toBeUndefined() + expect(events).toEqual([2]) + }) + + it('surfaces a fatal parse error when onParseError throws', async () => { + const stream = streamFromStringChunks(['data: not-json\n\n', 'data: {"n":2}\n\n']) + const events: number[] = [] + await expect( + readSSEEvents<{ n: number }>(stream, { + onEvent: (e) => { + events.push(e.n) + }, + onParseError: () => { + throw new Error('boom') + }, + }) + ).rejects.toThrow('boom') + expect(events).toEqual([]) + }) + + it('stops early when onEvent resolves true asynchronously', async () => { + const stream = streamFromStringChunks([ + 'data: {"n":1}\n\n', + 'data: {"n":2}\n\n', + 'data: {"n":3}\n\n', + ]) + const events: number[] = [] + await readSSEEvents<{ n: number }>(stream, { + onEvent: async (e) => { + events.push(e.n) + return e.n === 2 + }, + }) + expect(events).toEqual([1, 2]) + }) + + it('throws "No response body" for a Response without a body', async () => { + const response = new Response(null) + await expect(readSSEEvents(response, { onEvent: () => {} })).rejects.toThrow('No response body') + }) +}) + +describe('readSSELines', () => { + it('delivers raw (un-parsed) data payloads', async () => { + const stream = streamFromStringChunks(['data: raw-one\n\n', 'data: {"keep":"asString"}\n\n']) + const lines: string[] = [] + await readSSELines(stream, { + onData: (raw) => { + lines.push(raw) + }, + }) + expect(lines).toEqual(['raw-one', '{"keep":"asString"}']) + }) + + it('skips [DONE] and blank separator lines', async () => { + const stream = streamFromStringChunks(['data: a\n\ndata: b\n\ndata: [DONE]\n\n']) + const lines: string[] = [] + await readSSELines(stream, { + onData: (raw) => { + lines.push(raw) + }, + }) + expect(lines).toEqual(['a', 'b']) + }) + + it('preserves the raw payload verbatim (no JSON parsing)', async () => { + const stream = streamFromStringChunks(['data: {"unterminated\n\n', 'data:no-space\n\n']) + const lines: string[] = [] + await readSSELines(stream, { + onData: (raw) => { + lines.push(raw) + }, + }) + expect(lines).toEqual(['{"unterminated', 'no-space']) + }) + + it('strips a trailing carriage return from each line', async () => { + const stream = streamFromStringChunks(['data: one\r\n\r\ndata: two\r\n\r\n']) + const lines: string[] = [] + await readSSELines(stream, { + onData: (raw) => { + lines.push(raw) + }, + }) + expect(lines).toEqual(['one', 'two']) + }) + + it('stops early when onData returns true', async () => { + const stream = streamFromStringChunks(['data: a\n\ndata: b\n\ndata: c\n\n']) + const lines: string[] = [] + await readSSELines(stream, { + onData: (raw) => { + lines.push(raw) + return raw === 'b' + }, + }) + expect(lines).toEqual(['a', 'b']) + }) + + it('does not deliver any line when the signal is already aborted', async () => { + const controller = new AbortController() + controller.abort() + const stream = streamFromStringChunks(['data: a\n\n']) + const onData = vi.fn() + await readSSELines(stream, { signal: controller.signal, onData }) + expect(onData).not.toHaveBeenCalled() + }) + + it('stops between events in the same chunk once aborted mid-stream', async () => { + const controller = new AbortController() + const stream = streamFromStringChunks(['data: a\n\ndata: b\n\ndata: c\n\n']) + const lines: string[] = [] + await readSSELines(stream, { + signal: controller.signal, + onData: (raw) => { + lines.push(raw) + if (raw === 'a') controller.abort() + }, + }) + expect(lines).toEqual(['a']) + }) + + it('releases the lock for a stream source', async () => { + const stream = streamFromStringChunks(['data: a\n\n']) + await readSSELines(stream, { onData: () => {} }) + expect(() => stream.getReader()).not.toThrow() + }) + + it('does not release the lock for a reader source', async () => { + const stream = streamFromStringChunks(['data: a\n\n']) + const reader = stream.getReader() + await readSSELines(reader, { onData: () => {} }) + expect(() => stream.getReader()).toThrow() + reader.releaseLock() + }) + + it('releases the lock for a stream source even when onData throws', async () => { + const stream = streamFromStringChunks(['data: a\n\n']) + await expect( + readSSELines(stream, { + onData: () => { + throw new Error('handler failed') + }, + }) + ).rejects.toThrow('handler failed') + expect(() => stream.getReader()).not.toThrow() + }) +}) diff --git a/apps/sim/lib/core/utils/sse.ts b/apps/sim/lib/core/utils/sse.ts index 9d9d3f785a5..50c758f0013 100644 --- a/apps/sim/lib/core/utils/sse.ts +++ b/apps/sim/lib/core/utils/sse.ts @@ -20,6 +20,181 @@ export function encodeSSE(data: any): Uint8Array { return new TextEncoder().encode(`data: ${JSON.stringify(data)}\n\n`) } +/** + * The sentinel value servers emit to signal end-of-stream. Lines carrying this + * payload are skipped before reaching the consumer's `onEvent` callback. + */ +const DONE_SENTINEL = '[DONE]' + +/** + * A source the SSE reader can consume: a fetch `Response`, its `ReadableStream` + * body, or an already-acquired reader. Passing a `Response`/stream lets the + * primitive own `getReader()` and the reader lifecycle (lock release); passing a + * reader is supported for callers that must acquire it first (e.g. to stash it + * for external cancellation). + */ +export type SSESource = + | Response + | ReadableStream + | ReadableStreamDefaultReader + +/** + * The result of an SSE event/line callback. Only the literal `true` (returned + * synchronously or resolved from a `Promise`) stops processing and returns + * early — useful for terminal events. Any other value (including the + * `undefined` a handler that returns nothing produces) keeps processing. + * + * Typed as `unknown` rather than `boolean | void | Promise` so + * both sync and `async` handlers — including `async` handlers that return + * nothing (`Promise`) — stay assignable, without the confusing + * `void`-inside-a-`Promise` union that the precise type would require. + */ +export type SSEStopSignal = unknown + +/** + * Options for {@link readSSELines} — the low-level line engine that delivers the + * raw `data:` payload string (no JSON parsing). + */ +export interface ReadSSELinesOptions { + /** Invoked once per SSE `data:` line with the raw (un-parsed) payload string. */ + onData: (rawData: string) => SSEStopSignal + /** Aborts the read; checked before each chunk and between events. */ + signal?: AbortSignal +} + +/** + * Options for {@link readSSEEvents} — the JSON convenience layer over + * {@link readSSELines}. + */ +export interface ReadSSEEventsOptions { + /** + * Invoked once per parsed SSE `data:` event with the JSON-parsed payload. + * Return (or resolve) `true` to stop processing and return early. Callers + * narrow the typed payload. + */ + onEvent: (event: T) => SSEStopSignal + /** + * Invoked for a `data:` line whose payload is not valid JSON. Defaults to + * silently skipping the line. Throw from here to surface a fatal parse error. + */ + onParseError?: (rawData: string, error: unknown) => void + /** Aborts the read; checked before each chunk and between events. */ + signal?: AbortSignal +} + +/** + * Resolves an {@link SSESource} to a reader, reporting whether this call + * acquired the lock (and is therefore responsible for releasing it). + */ +function toReader(source: SSESource): { + reader: ReadableStreamDefaultReader + ownsLock: boolean +} { + if (source instanceof ReadableStream) { + return { reader: source.getReader(), ownsLock: true } + } + if (source instanceof Response) { + if (!source.body) throw new Error('No response body') + return { reader: source.body.getReader(), ownsLock: true } + } + return { reader: source, ownsLock: false } +} + +/** + * Strips an optional trailing carriage return from a single SSE line, so both + * `\n`- and `\r\n`-terminated framings parse identically. + */ +function stripCarriageReturn(line: string): string { + return line.endsWith('\r') ? line.slice(0, -1) : line +} + +/** + * The single client-side SSE decode engine. Reads a byte stream, decodes it + * incrementally, splits it into lines, and invokes `onData` once per `data:` + * line with its raw (un-parsed) payload string. + * + * It splits on `\n` and processes each `data:` line individually, which makes it + * tolerant of BOTH `\n`- and `\n\n`-separated framings (the blank separator + * lines between events are simply ignored). Trailing `\r` is stripped, a single + * optional space after `data:` is consumed, and the `[DONE]` sentinel is + * skipped. The reader's lock is always released on completion, abort, or error + * (only when this function acquired it). + * + * This is the low-level engine. Most callers want {@link readSSEEvents}, which + * adds JSON parsing on top. Reach for `readSSELines` only when the payload needs + * custom parsing (e.g. schema-validated decoding). + * + * @param source - A `Response`, `ReadableStream`, or stream reader. + * @param options - The `onData` callback plus an optional `signal`. + */ +export async function readSSELines(source: SSESource, options: ReadSSELinesOptions): Promise { + const { onData, signal } = options + const { reader, ownsLock } = toReader(source) + const decoder = new TextDecoder() + let buffer = '' + + try { + while (true) { + if (signal?.aborted) break + + const { done, value } = await reader.read() + if (done) break + + buffer += decoder.decode(value, { stream: true }) + const lines = buffer.split('\n') + buffer = lines.pop() ?? '' + + for (const rawLine of lines) { + if (signal?.aborted) return + + const line = stripCarriageReturn(rawLine) + if (!line.startsWith('data:')) continue + + let data = line.slice(5) + if (data.startsWith(' ')) data = data.slice(1) + if (data === DONE_SENTINEL) continue + + if ((await onData(data)) === true) return + } + } + } finally { + if (ownsLock) reader.releaseLock() + } +} + +/** + * The JSON convenience layer over {@link readSSELines}: invokes `onEvent` once + * per `data:` event with its JSON-parsed payload. Unparseable lines are passed + * to `onParseError` (default: silently skipped). All framing, `\r`, `[DONE]`, + * abort, and reader-lifecycle behavior is inherited from `readSSELines`. + * + * Higher-level concerns — UI batching, reconnect, error classification, event + * dispatch — belong in the caller's `onEvent`, not here. + * + * @typeParam T - The parsed event type the caller expects (defaults to `unknown`). + * @param source - A `Response`, `ReadableStream`, or stream reader. + * @param options - The `onEvent` callback plus optional `signal`/`onParseError`. + */ +export async function readSSEEvents( + source: SSESource, + options: ReadSSEEventsOptions +): Promise { + const { onEvent, onParseError, signal } = options + await readSSELines(source, { + signal, + onData: (data) => { + let parsed: T + try { + parsed = JSON.parse(data) as T + } catch (error) { + onParseError?.(data, error) + return + } + return onEvent(parsed) + }, + }) +} + /** * Options for reading SSE stream */ diff --git a/apps/sim/lib/oauth/oauth.ts b/apps/sim/lib/oauth/oauth.ts index a9a71270042..8f6b17100fe 100644 --- a/apps/sim/lib/oauth/oauth.ts +++ b/apps/sim/lib/oauth/oauth.ts @@ -711,7 +711,7 @@ export const OAUTH_PROVIDERS: Record = { 'groups:write', 'chat:write', 'chat:write.public', - 'assistant:write', + // TODO: Add 'assistant:write' once Slack app review is approved 'im:write', 'im:read', 'users:read', diff --git a/apps/sim/lib/webhooks/providers/gitlab.ts b/apps/sim/lib/webhooks/providers/gitlab.ts index 3f6ffcbf12e..d812ee19d92 100644 --- a/apps/sim/lib/webhooks/providers/gitlab.ts +++ b/apps/sim/lib/webhooks/providers/gitlab.ts @@ -2,6 +2,7 @@ import { createLogger } from '@sim/logger' import { safeCompare } from '@sim/security/compare' import { generateId } from '@sim/utils/id' import { NextResponse } from 'next/server' +import { secureFetchWithValidation } from '@/lib/core/security/input-validation.server' import { getNotificationUrl, getProviderConfig } from '@/lib/webhooks/provider-subscription-utils' import type { AuthContext, @@ -13,17 +14,16 @@ import type { SubscriptionResult, WebhookProviderHandler, } from '@/lib/webhooks/providers/types' +import { getGitLabApiBase, UnsafeGitLabHostError } from '@/tools/gitlab/utils' const logger = createLogger('WebhookProvider:GitLab') -const GITLAB_API_BASE = 'https://gitlab.com/api/v4' - function asRecord(value: unknown): Record { return (value as Record) || {} } -function gitlabProjectHooksUrl(projectId: string): string { - return `${GITLAB_API_BASE}/projects/${encodeURIComponent(projectId)}/hooks` +function gitlabProjectHooksUrl(projectId: string, host: unknown): string { + return `${getGitLabApiBase(host)}/projects/${encodeURIComponent(projectId)}/hooks` } /** @@ -33,9 +33,10 @@ function gitlabProjectHooksUrl(projectId: string): string { async function cleanupGitLabHookByUrl( projectId: string, accessToken: string, - url: string + url: string, + host: unknown ): Promise { - const res = await fetch(gitlabProjectHooksUrl(projectId), { + const res = await secureFetchWithValidation(gitlabProjectHooksUrl(projectId, host), { headers: { 'PRIVATE-TOKEN': accessToken }, }).catch(() => null) if (!res || !res.ok) return @@ -47,7 +48,7 @@ async function cleanupGitLabHookByUrl( hooks .filter((hook) => hook.url === url && hook.id != null) .map((hook) => - fetch(`${gitlabProjectHooksUrl(projectId)}/${hook.id}`, { + secureFetchWithValidation(`${gitlabProjectHooksUrl(projectId, host)}/${hook.id}`, { method: 'DELETE', headers: { 'PRIVATE-TOKEN': accessToken }, }).catch(() => null) @@ -113,14 +114,28 @@ export const gitlabHandler: WebhookProviderHandler = { const accessToken = config.accessToken as string | undefined const projectId = config.projectId as string | undefined const triggerId = config.triggerId as string | undefined + const host = config.host as string | undefined if (!accessToken) throw new Error('GitLab Personal Access Token is required to create the webhook.') if (!projectId) throw new Error('GitLab Project ID is required to create the webhook.') + // Validate the optional self-managed host up front so a structurally unsafe + // value surfaces as a clear error instead of an unhandled UnsafeGitLabHostError. + try { + getGitLabApiBase(host) + } catch (error) { + if (error instanceof UnsafeGitLabHostError) { + throw new Error( + 'GitLab host is invalid. Provide a domain like gitlab.example.com (no protocol, path, or credentials).' + ) + } + throw error + } + const { getGitLabEventFlags } = await import('@/triggers/gitlab/utils') const secretToken = generateId() - const res = await fetch(gitlabProjectHooksUrl(projectId), { + const res = await secureFetchWithValidation(gitlabProjectHooksUrl(projectId, host), { method: 'POST', headers: { 'PRIVATE-TOKEN': accessToken, 'Content-Type': 'application/json' }, body: JSON.stringify({ @@ -150,7 +165,7 @@ export const gitlabHandler: WebhookProviderHandler = { if (created.id === undefined || created.id === null) { // The hook was created but we can't read its id — delete it by URL so it // is not orphaned in GitLab. - await cleanupGitLabHookByUrl(projectId, accessToken, getNotificationUrl(ctx.webhook)) + await cleanupGitLabHookByUrl(projectId, accessToken, getNotificationUrl(ctx.webhook), host) throw new Error('GitLab webhook created but no hook ID was returned.') } @@ -163,6 +178,7 @@ export const gitlabHandler: WebhookProviderHandler = { const accessToken = config.accessToken as string | undefined const projectId = config.projectId as string | undefined const externalId = config.externalId as string | undefined + const host = config.host as string | undefined if (!accessToken || !projectId || !externalId) { if (ctx.strict) throw new Error('Missing GitLab credentials or hook ID for webhook deletion.') @@ -172,10 +188,30 @@ export const gitlabHandler: WebhookProviderHandler = { return } - const res = await fetch(`${gitlabProjectHooksUrl(projectId)}/${externalId}`, { - method: 'DELETE', - headers: { 'PRIVATE-TOKEN': accessToken }, - }) + // A structurally unsafe host must not abort cleanup in non-strict mode — mirror + // the graceful skip used for missing credentials above. + try { + getGitLabApiBase(host) + } catch (error) { + if (error instanceof UnsafeGitLabHostError) { + if (ctx.strict) { + throw new Error('Cannot delete GitLab webhook: the configured host is invalid.') + } + logger.warn( + `[${ctx.requestId}] Skipping GitLab webhook cleanup — configured host is invalid` + ) + return + } + throw error + } + + const res = await secureFetchWithValidation( + `${gitlabProjectHooksUrl(projectId, host)}/${externalId}`, + { + method: 'DELETE', + headers: { 'PRIVATE-TOKEN': accessToken }, + } + ) if (!res.ok && res.status !== 404) { if (ctx.strict) throw new Error(`Failed to delete GitLab webhook: ${res.status}`) diff --git a/apps/sim/package.json b/apps/sim/package.json index 88e9575836d..dcb9c2bc649 100644 --- a/apps/sim/package.json +++ b/apps/sim/package.json @@ -99,6 +99,7 @@ "@sim/logger": "workspace:*", "@sim/platform-authz": "workspace:*", "@sim/realtime-protocol": "workspace:*", + "@sim/runtime-secrets": "workspace:*", "@sim/security": "workspace:*", "@sim/utils": "workspace:*", "@sim/workflow-persistence": "workspace:*", diff --git a/apps/sim/stores/execution/store.test.ts b/apps/sim/stores/execution/store.test.ts index 6888894b80b..c0b11fc87ce 100644 --- a/apps/sim/stores/execution/store.test.ts +++ b/apps/sim/stores/execution/store.test.ts @@ -1,4 +1,6 @@ /** + * @vitest-environment node + * * Tests for the per-workflow execution store. * * These tests cover: @@ -7,12 +9,19 @@ * - Execution lifecycle (start/stop clears run path) * - Block and edge run status tracking * - Active block management - * - Debug state management + * - The {@link ExecutionStatus} enum and its derived `isExecuting` / + * `isDebugging` booleans (exhaustive status → flag mapping + transitions) * - Execution snapshot management * - Store reset * - Immutability guarantees * * @remarks + * The store under test transitively imports the workflow registry store, + * which drags in the block registry and emcn icon CSS. To keep this a true + * unit test that loads under the node environment, the registry store is + * mocked to a minimal stub (the store actions never touch it — only the + * convenience hooks do, which are not exercised here). + * * Most tests use `it.concurrent` with unique workflow IDs per test. * Because the store isolates state by workflow ID, concurrent tests * do not interfere with each other. The `reset` and `immutability` @@ -21,17 +30,30 @@ import { beforeEach, describe, expect, it, vi } from 'vitest' +vi.mock('@/stores/workflows/registry/store', () => ({ + useWorkflowRegistry: Object.assign( + vi.fn(() => null), + { getState: vi.fn(() => ({ activeWorkflowId: null })) } + ), +})) + vi.unmock('@/stores/execution/store') vi.unmock('@/stores/execution/types') import { useExecutionStore } from '@/stores/execution/store' -import { defaultWorkflowExecutionState, initialState } from '@/stores/execution/types' +import { + defaultWorkflowExecutionState, + deriveExecutionFlags, + type ExecutionStatus, + initialState, +} from '@/stores/execution/types' describe('useExecutionStore', () => { describe('getWorkflowExecution', () => { it.concurrent('should return default state for an unknown workflow', () => { const state = useExecutionStore.getState().getWorkflowExecution('wf-get-default') + expect(state.status).toBe('idle') expect(state.isExecuting).toBe(false) expect(state.isDebugging).toBe(false) expect(state.activeBlockIds.size).toBe(0) @@ -63,22 +85,35 @@ describe('useExecutionStore', () => { }) }) + describe('deriveExecutionFlags', () => { + it.concurrent('maps every status to the documented legacy booleans', () => { + const cases: Array<[ExecutionStatus, boolean, boolean]> = [ + ['idle', false, false], + ['running', true, false], + ['debugging', true, true], + ] + for (const [status, isExecuting, isDebugging] of cases) { + expect(deriveExecutionFlags(status)).toEqual({ isExecuting, isDebugging }) + } + }) + }) + describe('setIsExecuting', () => { - it.concurrent('should set isExecuting to true', () => { + it.concurrent('should set isExecuting to true (status running)', () => { useExecutionStore.getState().setIsExecuting('wf-exec-true', true) - expect(useExecutionStore.getState().getWorkflowExecution('wf-exec-true').isExecuting).toBe( - true - ) + const state = useExecutionStore.getState().getWorkflowExecution('wf-exec-true') + expect(state.isExecuting).toBe(true) + expect(state.status).toBe('running') }) - it.concurrent('should set isExecuting to false', () => { + it.concurrent('should set isExecuting to false (status idle)', () => { useExecutionStore.getState().setIsExecuting('wf-exec-false', true) useExecutionStore.getState().setIsExecuting('wf-exec-false', false) - expect(useExecutionStore.getState().getWorkflowExecution('wf-exec-false').isExecuting).toBe( - false - ) + const state = useExecutionStore.getState().getWorkflowExecution('wf-exec-false') + expect(state.isExecuting).toBe(false) + expect(state.status).toBe('idle') }) it.concurrent('should clear lastRunPath and lastRunEdges when starting execution', () => { @@ -107,6 +142,131 @@ describe('useExecutionStore', () => { expect(state.isExecuting).toBe(false) expect(state.lastRunPath.get('block-1')).toBe('success') }) + + it.concurrent('starting a debug run then setIsExecuting(true) clears the run path', () => { + const wf = 'wf-exec-debug-start-clears' + useExecutionStore.getState().setIsExecuting(wf, true) + useExecutionStore.getState().setIsDebugging(wf, true) + useExecutionStore.getState().setBlockRunStatus(wf, 'block-1', 'success') + + useExecutionStore.getState().setIsExecuting(wf, true) + + const state = useExecutionStore.getState().getWorkflowExecution(wf) + expect(state.status).toBe('debugging') + expect(state.isExecuting).toBe(true) + expect(state.isDebugging).toBe(true) + expect(state.lastRunPath.size).toBe(0) + expect(state.lastRunEdges.size).toBe(0) + }) + }) + + describe('setIsDebugging', () => { + it.concurrent('should toggle debug mode', () => { + const wf = 'wf-debug-toggle' + useExecutionStore.getState().setIsDebugging(wf, true) + + expect(useExecutionStore.getState().getWorkflowExecution(wf).isDebugging).toBe(true) + expect(useExecutionStore.getState().getWorkflowExecution(wf).isExecuting).toBe(true) + expect(useExecutionStore.getState().getWorkflowExecution(wf).status).toBe('debugging') + + useExecutionStore.getState().setIsDebugging(wf, false) + expect(useExecutionStore.getState().getWorkflowExecution(wf).isDebugging).toBe(false) + expect(useExecutionStore.getState().getWorkflowExecution(wf).isExecuting).toBe(true) + expect(useExecutionStore.getState().getWorkflowExecution(wf).status).toBe('running') + }) + + it.concurrent('setIsDebugging(false) while idle is a no-op (stays idle)', () => { + const wf = 'wf-debug-false-idle' + useExecutionStore.getState().setIsDebugging(wf, false) + expect(useExecutionStore.getState().getWorkflowExecution(wf).status).toBe('idle') + expect(useExecutionStore.getState().getWorkflowExecution(wf).isExecuting).toBe(false) + }) + + it.concurrent('setIsDebugging(false) while running keeps running', () => { + const wf = 'wf-debug-false-running' + useExecutionStore.getState().setIsExecuting(wf, true) + useExecutionStore.getState().setIsDebugging(wf, false) + expect(useExecutionStore.getState().getWorkflowExecution(wf).status).toBe('running') + expect(useExecutionStore.getState().getWorkflowExecution(wf).isExecuting).toBe(true) + }) + + it.concurrent('does not clear the run path when entering debug mode', () => { + const wf = 'wf-debug-keeps-path' + useExecutionStore.getState().setBlockRunStatus(wf, 'block-1', 'success') + useExecutionStore.getState().setIsDebugging(wf, true) + expect(useExecutionStore.getState().getWorkflowExecution(wf).lastRunPath.get('block-1')).toBe( + 'success' + ) + }) + }) + + describe('status enum', () => { + it.concurrent('idle derives both flags false', () => { + const wf = 'wf-status-idle' + const state = useExecutionStore.getState().getWorkflowExecution(wf) + expect(state.status).toBe('idle') + expect(state.isExecuting).toBe(false) + expect(state.isDebugging).toBe(false) + }) + + it.concurrent('running derives isExecuting only', () => { + const wf = 'wf-status-running' + useExecutionStore.getState().setStatus(wf, 'running') + const state = useExecutionStore.getState().getWorkflowExecution(wf) + expect(state.status).toBe('running') + expect(state.isExecuting).toBe(true) + expect(state.isDebugging).toBe(false) + }) + + it.concurrent('debugging derives both flags true', () => { + const wf = 'wf-status-debugging' + useExecutionStore.getState().setStatus(wf, 'debugging') + const state = useExecutionStore.getState().getWorkflowExecution(wf) + expect(state.status).toBe('debugging') + expect(state.isExecuting).toBe(true) + expect(state.isDebugging).toBe(true) + }) + + it.concurrent('setStatus preserves the run path unless clearRunPath is passed', () => { + const wf = 'wf-status-path-rules' + useExecutionStore.getState().setStatus(wf, 'debugging') + useExecutionStore.getState().setBlockRunStatus(wf, 'block-1', 'success') + expect(useExecutionStore.getState().getWorkflowExecution(wf).lastRunPath.size).toBe(1) + + useExecutionStore.getState().setStatus(wf, 'running') + expect(useExecutionStore.getState().getWorkflowExecution(wf).lastRunPath.size).toBe(1) + + useExecutionStore.getState().setStatus(wf, 'running', { clearRunPath: true }) + expect(useExecutionStore.getState().getWorkflowExecution(wf).lastRunPath.size).toBe(0) + }) + + it.concurrent('the derived booleans always agree with the stored status', () => { + const wf = 'wf-status-no-drift' + for (const status of ['idle', 'running', 'debugging', 'idle'] as const) { + useExecutionStore.getState().setStatus(wf, status) + const state = useExecutionStore.getState().getWorkflowExecution(wf) + expect({ isExecuting: state.isExecuting, isDebugging: state.isDebugging }).toEqual( + deriveExecutionFlags(status) + ) + } + }) + + it.concurrent('setIsExecuting(true) preserves an active debug session', () => { + const wf = 'wf-status-debug-preserve' + useExecutionStore.getState().setStatus(wf, 'debugging') + useExecutionStore.getState().setIsExecuting(wf, true) + expect(useExecutionStore.getState().getWorkflowExecution(wf).status).toBe('debugging') + }) + + it.concurrent('setIsExecuting(false) returns to idle from any mode', () => { + const wf = 'wf-status-stop' + useExecutionStore.getState().setStatus(wf, 'debugging') + useExecutionStore.getState().setIsExecuting(wf, false) + const state = useExecutionStore.getState().getWorkflowExecution(wf) + expect(state.status).toBe('idle') + expect(state.isExecuting).toBe(false) + expect(state.isDebugging).toBe(false) + }) }) describe('setActiveBlocks', () => { @@ -151,18 +311,6 @@ describe('useExecutionStore', () => { }) }) - describe('setIsDebugging', () => { - it.concurrent('should toggle debug mode', () => { - const wf = 'wf-debug-toggle' - useExecutionStore.getState().setIsDebugging(wf, true) - - expect(useExecutionStore.getState().getWorkflowExecution(wf).isDebugging).toBe(true) - - useExecutionStore.getState().setIsDebugging(wf, false) - expect(useExecutionStore.getState().getWorkflowExecution(wf).isDebugging).toBe(false) - }) - }) - describe('setExecutor', () => { it.concurrent('should store and clear executor', () => { const wf = 'wf-executor' diff --git a/apps/sim/stores/execution/store.ts b/apps/sim/stores/execution/store.ts index 8d9e2827f34..5efa313f021 100644 --- a/apps/sim/stores/execution/store.ts +++ b/apps/sim/stores/execution/store.ts @@ -3,9 +3,11 @@ import { useWorkflowRegistry } from '@/stores/workflows/registry/store' import { type BlockRunStatus, defaultWorkflowExecutionState, + deriveExecutionFlags, type EdgeRunStatus, type ExecutionActions, type ExecutionState, + type ExecutionStatus, initialState, type WorkflowExecutionState, } from './types' @@ -78,9 +80,12 @@ export const useExecutionStore = create()((se }) }, - setIsExecuting: (workflowId, isExecuting) => { - const patch: Partial = { isExecuting } - if (isExecuting) { + setStatus: (workflowId, status, options) => { + const patch: Partial = { + status, + ...deriveExecutionFlags(status), + } + if (options?.clearRunPath) { patch.lastRunPath = new Map() patch.lastRunEdges = new Map() } @@ -89,10 +94,24 @@ export const useExecutionStore = create()((se }) }, + setIsExecuting: (workflowId, isExecuting) => { + const current = getOrCreate(get().workflowExecutions, workflowId) + const nextStatus: ExecutionStatus = isExecuting + ? current.status === 'debugging' + ? 'debugging' + : 'running' + : 'idle' + get().setStatus(workflowId, nextStatus, { clearRunPath: isExecuting }) + }, + setIsDebugging: (workflowId, isDebugging) => { - set({ - workflowExecutions: updatedMap(get().workflowExecutions, workflowId, { isDebugging }), - }) + const current = getOrCreate(get().workflowExecutions, workflowId) + const nextStatus: ExecutionStatus = isDebugging + ? 'debugging' + : current.status === 'debugging' + ? 'running' + : current.status + get().setStatus(workflowId, nextStatus) }, setExecutor: (workflowId, executor) => { diff --git a/apps/sim/stores/execution/types.ts b/apps/sim/stores/execution/types.ts index b36ea43a190..bb48c9d45a2 100644 --- a/apps/sim/stores/execution/types.ts +++ b/apps/sim/stores/execution/types.ts @@ -12,6 +12,22 @@ export type BlockRunStatus = 'success' | 'error' */ export type EdgeRunStatus = 'success' | 'error' +/** + * The mutually-exclusive execution mode of a single workflow. + * + * @remarks + * This is the single source of truth for whether a workflow is running. + * The legacy `isExecuting` / `isDebugging` booleans are derived from it + * via {@link deriveExecutionFlags} so illegal combinations — such as + * "debugging while not executing" — are unrepresentable. + * + * - `idle` — not running. + * - `running` — executing normally (derives `isExecuting`). + * - `debugging` — executing in step-by-step debug mode (derives both + * `isExecuting` and `isDebugging`). + */ +export type ExecutionStatus = 'idle' | 'running' | 'debugging' + /** * Execution state scoped to a single workflow. * @@ -19,9 +35,11 @@ export type EdgeRunStatus = 'success' | 'error' * do not interfere with one another. */ export interface WorkflowExecutionState { - /** Whether this workflow is currently executing */ + /** Mutually-exclusive execution mode; the source of truth for run state */ + status: ExecutionStatus + /** Derived from {@link status}: whether this workflow is currently executing */ isExecuting: boolean - /** Whether this workflow is in step-by-step debug mode */ + /** Derived from {@link status}: whether this workflow is in step-by-step debug mode */ isDebugging: boolean /** Block IDs that are currently running (pulsing in the UI) */ activeBlockIds: Set @@ -39,6 +57,24 @@ export interface WorkflowExecutionState { currentExecutionId: string | null } +/** + * Computes the legacy `isExecuting` / `isDebugging` booleans from a status. + * + * @remarks + * Keeping the derived booleans on the stored state object lets existing + * consumers keep reading `state.isExecuting` / `state.isDebugging` + * unchanged while {@link ExecutionStatus} remains the single source of truth. + */ +export function deriveExecutionFlags(status: ExecutionStatus): { + isExecuting: boolean + isDebugging: boolean +} { + return { + isExecuting: status !== 'idle', + isDebugging: status === 'debugging', + } +} + /** * Default values for a workflow that has never been executed. * @@ -48,8 +84,8 @@ export interface WorkflowExecutionState { * re-renders in Zustand selectors that use `Object.is` equality. */ export const defaultWorkflowExecutionState: WorkflowExecutionState = { - isExecuting: false, - isDebugging: false, + status: 'idle', + ...deriveExecutionFlags('idle'), activeBlockIds: new Set(), pendingBlocks: [], executor: null, @@ -83,9 +119,38 @@ export interface ExecutionActions { getWorkflowExecution: (workflowId: string) => WorkflowExecutionState /** Replaces the set of currently-executing block IDs for a workflow */ setActiveBlocks: (workflowId: string, blockIds: Set) => void - /** Marks a workflow as executing or idle. Starting clears the run path */ + /** + * Sets the {@link ExecutionStatus} for a workflow. + * + * @remarks + * Pass `{ clearRunPath: true }` to also reset `lastRunPath` / `lastRunEdges`. + * Run-path clearing is opt-in: it is owned by + * {@link ExecutionActions.setIsExecuting} (which clears on start), matching + * the legacy behavior where only starting execution wiped the run history. + */ + setStatus: ( + workflowId: string, + status: ExecutionStatus, + options?: { clearRunPath?: boolean } + ) => void + /** + * Marks a workflow as executing or idle. Starting (`true`) clears the run path. + * + * @remarks + * Translates to {@link ExecutionActions.setStatus}: `true` preserves an + * active debug session (`debugging`) and otherwise enters `running`, and + * always clears the run path; `false` returns to `idle` and preserves it. + */ setIsExecuting: (workflowId: string, isExecuting: boolean) => void - /** Toggles debug mode for a workflow */ + /** + * Toggles step-by-step debug mode for a workflow. + * + * @remarks + * Translates to {@link ExecutionActions.setStatus}: `true` enters + * `debugging` (which implies executing); `false` returns to `running` only + * when currently `debugging`, otherwise the status is preserved (e.g. calling + * it while `idle` is a no-op). + */ setIsDebugging: (workflowId: string, isDebugging: boolean) => void /** Sets the list of blocks pending execution during debug stepping */ setPendingBlocks: (workflowId: string, blockIds: string[]) => void diff --git a/apps/sim/stores/workflow-diff/store.test.ts b/apps/sim/stores/workflow-diff/store.test.ts new file mode 100644 index 00000000000..c9bca038048 --- /dev/null +++ b/apps/sim/stores/workflow-diff/store.test.ts @@ -0,0 +1,228 @@ +/** + * @vitest-environment node + * + * Tests for the workflow-diff store's status modeling. + * + * Focus: the {@link WorkflowDiffStatus} enum is the single source of truth and + * the legacy `hasActiveDiff` / `isShowingDiff` / `isDiffReady` booleans are + * derived from it, so contradictory combinations are unrepresentable. We assert + * the exhaustive status → boolean mapping and the status transitions driven by + * the tractable actions (`toggleDiffView`, `clearDiff`, `_batchedStateUpdate`). + * + * @remarks + * The store transitively imports the diff engine, serializer, socket + * operations, and the workflow/registry stores, all of which drag in the block + * registry and emcn icon CSS. Every such dependency is mocked so the suite + * loads under the node environment and exercises only the store + its types. + */ + +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const { applyWorkflowStateToStores } = vi.hoisted(() => ({ + applyWorkflowStateToStores: vi.fn(), +})) + +vi.mock('@sim/logger', () => ({ + createLogger: () => ({ info: vi.fn(), warn: vi.fn(), error: vi.fn(), debug: vi.fn() }), +})) + +vi.mock('@/lib/workflows/diff', () => ({ + WorkflowDiffEngine: class { + clearDiff = vi.fn() + createDiffFromWorkflowState = vi.fn() + }, + stripWorkflowDiffMarkers: vi.fn((s) => s), +})) + +vi.mock('@/lib/workflows/operations/socket-operations', () => ({ + enqueueReplaceWorkflowState: vi.fn().mockResolvedValue(undefined), +})) + +vi.mock('@/lib/workflows/sanitization/validation', () => ({ + validateWorkflowState: vi.fn(() => ({ valid: true, errors: [], sanitizedState: null })), +})) + +vi.mock('@/serializer', () => ({ + Serializer: class { + serializeWorkflow = vi.fn() + deserializeWorkflow = vi.fn() + }, +})) + +vi.mock('@/stores/workflows/registry/store', () => ({ + useWorkflowRegistry: { getState: vi.fn(() => ({ activeWorkflowId: null })) }, +})) + +vi.mock('@/stores/workflows/utils', () => ({ + mergeSubblockState: vi.fn((blocks) => blocks), +})) + +vi.mock('@/stores/workflows/workflow/store', () => ({ + useWorkflowStore: { + getState: vi.fn(() => ({ + getWorkflowState: vi.fn(() => ({ blocks: {}, edges: [], loops: {}, parallels: {} })), + blocks: {}, + lastSaved: 0, + })), + setState: vi.fn(), + }, +})) + +vi.mock('@/stores/workflow-diff/utils', () => ({ + applyWorkflowStateToStores, + captureBaselineSnapshot: vi.fn(), + cloneWorkflowState: vi.fn((s) => s), + createBatchedUpdater: + (set: (u: Record) => void) => (updates: Record) => + set(updates), + getLatestUserMessageId: vi.fn().mockResolvedValue(null), + persistWorkflowStateToServer: vi.fn().mockResolvedValue(true), + WORKFLOW_DIFF_SETTLED_EVENT: 'workflow-diff-settled', +})) + +import { RESET_DIFF_STATE, useWorkflowDiffStore } from '@/stores/workflow-diff/store' +import { + deriveDiffFlags, + type WorkflowDiffState, + type WorkflowDiffStatus, +} from '@/stores/workflow-diff/types' + +function seedStatus(status: WorkflowDiffStatus): void { + useWorkflowDiffStore.setState(deriveDiffFlags(status)) +} + +describe('useWorkflowDiffStore status modeling', () => { + beforeEach(() => { + vi.clearAllMocks() + useWorkflowDiffStore.setState({ + ...RESET_DIFF_STATE, + pendingExternalUpdates: {}, + remoteUpdateVersions: {}, + reconcilingWorkflows: {}, + reconciliationErrors: {}, + } as Partial) + }) + + describe('deriveDiffFlags', () => { + it('maps every status to the documented legacy booleans', () => { + expect(deriveDiffFlags('none')).toEqual({ + status: 'none', + hasActiveDiff: false, + isShowingDiff: false, + isDiffReady: false, + }) + expect(deriveDiffFlags('staged')).toEqual({ + status: 'staged', + hasActiveDiff: true, + isShowingDiff: false, + isDiffReady: true, + }) + expect(deriveDiffFlags('showing')).toEqual({ + status: 'showing', + hasActiveDiff: true, + isShowingDiff: true, + isDiffReady: true, + }) + }) + + it('keeps hasActiveDiff and isDiffReady in lockstep (legacy invariant)', () => { + for (const status of ['none', 'staged', 'showing'] as const) { + const flags = deriveDiffFlags(status) + expect(flags.hasActiveDiff).toBe(flags.isDiffReady) + } + }) + }) + + describe('initial / reset state', () => { + it('starts in the none-derived state', () => { + const state = useWorkflowDiffStore.getState() + expect(state.status).toBe('none') + expect(state.hasActiveDiff).toBe(false) + expect(state.isShowingDiff).toBe(false) + expect(state.isDiffReady).toBe(false) + }) + + it('RESET_DIFF_STATE carries the none-derived flags and clears diff payload', () => { + expect(RESET_DIFF_STATE.status).toBe('none') + expect(RESET_DIFF_STATE.hasActiveDiff).toBe(false) + expect(RESET_DIFF_STATE.isShowingDiff).toBe(false) + expect(RESET_DIFF_STATE.isDiffReady).toBe(false) + expect(RESET_DIFF_STATE.baselineWorkflow).toBeNull() + expect(RESET_DIFF_STATE.diffAnalysis).toBeNull() + }) + }) + + describe('toggleDiffView', () => { + it('is a guarded no-op when there is no active diff', () => { + seedStatus('none') + useWorkflowDiffStore.getState().toggleDiffView() + expect(useWorkflowDiffStore.getState().status).toBe('none') + }) + + it('toggles showing → staged (hides the proposed changes)', () => { + seedStatus('showing') + useWorkflowDiffStore.getState().toggleDiffView() + + const state = useWorkflowDiffStore.getState() + expect(state.status).toBe('staged') + expect(state.hasActiveDiff).toBe(true) + expect(state.isDiffReady).toBe(true) + expect(state.isShowingDiff).toBe(false) + }) + + it('toggles staged → showing (reveals the proposed changes)', () => { + seedStatus('staged') + useWorkflowDiffStore.getState().toggleDiffView() + + const state = useWorkflowDiffStore.getState() + expect(state.status).toBe('showing') + expect(state.isShowingDiff).toBe(true) + }) + }) + + describe('clearDiff', () => { + it('returns the store to the none status', () => { + seedStatus('showing') + useWorkflowDiffStore.getState().clearDiff({ restoreBaseline: false }) + + const state = useWorkflowDiffStore.getState() + expect(state.status).toBe('none') + expect(state.hasActiveDiff).toBe(false) + expect(state.isShowingDiff).toBe(false) + expect(state.isDiffReady).toBe(false) + }) + }) + + describe('_batchedStateUpdate (undo/redo writer)', () => { + it('restores the showing status via deriveDiffFlags', () => { + seedStatus('none') + useWorkflowDiffStore.getState()._batchedStateUpdate({ + ...deriveDiffFlags('showing'), + baselineWorkflow: null, + baselineWorkflowId: 'wf-1', + }) + + const state = useWorkflowDiffStore.getState() + expect(state.status).toBe('showing') + expect(state.hasActiveDiff).toBe(true) + expect(state.isShowingDiff).toBe(true) + expect(state.isDiffReady).toBe(true) + }) + + it('the derived booleans always agree with the stored status', () => { + for (const status of ['none', 'staged', 'showing', 'none'] as const) { + seedStatus(status) + const state = useWorkflowDiffStore.getState() + expect({ + hasActiveDiff: state.hasActiveDiff, + isShowingDiff: state.isShowingDiff, + isDiffReady: state.isDiffReady, + }).toEqual({ + hasActiveDiff: deriveDiffFlags(status).hasActiveDiff, + isShowingDiff: deriveDiffFlags(status).isShowingDiff, + isDiffReady: deriveDiffFlags(status).isDiffReady, + }) + } + }) + }) +}) diff --git a/apps/sim/stores/workflow-diff/store.ts b/apps/sim/stores/workflow-diff/store.ts index 38d08ad3b97..ec46c2192a3 100644 --- a/apps/sim/stores/workflow-diff/store.ts +++ b/apps/sim/stores/workflow-diff/store.ts @@ -8,7 +8,7 @@ import { Serializer } from '@/serializer' import { useWorkflowRegistry } from '../workflows/registry/store' import { mergeSubblockState } from '../workflows/utils' import { useWorkflowStore } from '../workflows/workflow/store' -import type { WorkflowDiffActions, WorkflowDiffState } from './types' +import { deriveDiffFlags, type WorkflowDiffActions, type WorkflowDiffState } from './types' import { applyWorkflowStateToStores, captureBaselineSnapshot, @@ -21,17 +21,20 @@ import { const logger = createLogger('WorkflowDiffStore') const diffEngine = new WorkflowDiffEngine() -const RESET_DIFF_STATE = { - hasActiveDiff: false, - isShowingDiff: false, - isDiffReady: false, + +/** + * Canonical state patch that clears the diff overlay back to `none`: the + * none-derived flags plus a wipe of all diff payload fields. + */ +export const RESET_DIFF_STATE = { + ...deriveDiffFlags('none'), baselineWorkflow: null, baselineWorkflowId: null, diffAnalysis: null, diffMetadata: null, diffError: null, _triggerMessageId: null, -} +} as const /** * Detects when a diff contains no meaningful changes. @@ -70,9 +73,7 @@ export const useWorkflowDiffStore = create { - const { hasActiveDiff, isDiffReady, isShowingDiff } = get() - if (!hasActiveDiff) { - logger.warn('Cannot toggle diff view without active diff') - return - } - if (!isDiffReady) { - logger.warn('Cannot toggle diff view before diff is ready') + const { status } = get() + if (status === 'none') { + logger.warn('Cannot toggle diff view without an active, ready diff') return } - batchedUpdate({ isShowingDiff: !isShowingDiff }) + batchedUpdate(deriveDiffFlags(status === 'showing' ? 'staged' : 'showing')) }, acceptChanges: async (options) => { diff --git a/apps/sim/stores/workflow-diff/types.ts b/apps/sim/stores/workflow-diff/types.ts index 6c3ea3990a1..d4e05e68760 100644 --- a/apps/sim/stores/workflow-diff/types.ts +++ b/apps/sim/stores/workflow-diff/types.ts @@ -1,9 +1,31 @@ import type { DiffAnalysis, WorkflowDiff } from '@/lib/workflows/diff' import type { WorkflowState } from '../workflows/workflow/types' +/** + * The lifecycle stage of the workflow diff overlay. + * + * @remarks + * This is the single source of truth for the diff overlay. The legacy + * `hasActiveDiff` / `isShowingDiff` / `isDiffReady` booleans are derived from + * it via {@link deriveDiffFlags}, which makes contradictory combinations — + * such as "showing a diff that has no active diff" — unrepresentable. + * + * - `none` — no diff staged; the canvas shows the live workflow. + * - `staged` — a diff is staged and ready, but the canvas is showing the + * baseline (proposed changes hidden). + * - `showing` — a diff is staged and ready, and the canvas is showing the + * proposed changes with diff markers. + */ +export type WorkflowDiffStatus = 'none' | 'staged' | 'showing' + export interface WorkflowDiffState { + /** Lifecycle stage of the diff overlay; the source of truth for diff flags */ + status: WorkflowDiffStatus + /** Derived from {@link status}: a diff is staged (`staged` or `showing`) */ hasActiveDiff: boolean + /** Derived from {@link status}: the canvas is rendering the proposed changes */ isShowingDiff: boolean + /** Derived from {@link status}: a staged diff is ready to view/toggle */ isDiffReady: boolean baselineWorkflow: WorkflowState | null baselineWorkflowId: string | null @@ -48,3 +70,31 @@ export interface WorkflowDiffActions { setWorkflowReconciliationError: (workflowId: string, error: string | null) => void _batchedStateUpdate: (updates: Partial) => void } + +/** + * The {@link WorkflowDiffStatus} fields shared by `status` and its derived + * booleans. Spread this into a state patch so the source of truth and the + * legacy flags never drift apart. + */ +export type DiffStatusFlags = Pick< + WorkflowDiffState, + 'status' | 'hasActiveDiff' | 'isShowingDiff' | 'isDiffReady' +> + +/** + * Computes the legacy `hasActiveDiff` / `isShowingDiff` / `isDiffReady` + * booleans (plus the `status` itself) from a {@link WorkflowDiffStatus}. + * + * @remarks + * Keeping the derived booleans on the stored state lets existing consumers + * keep reading `state.hasActiveDiff` etc. unchanged while + * {@link WorkflowDiffStatus} remains the single source of truth. + */ +export function deriveDiffFlags(status: WorkflowDiffStatus): DiffStatusFlags { + return { + status, + hasActiveDiff: status !== 'none', + isShowingDiff: status === 'showing', + isDiffReady: status !== 'none', + } +} diff --git a/apps/sim/tools/gitlab/cancel_pipeline.ts b/apps/sim/tools/gitlab/cancel_pipeline.ts index 62b9e096b95..9707f758a0e 100644 --- a/apps/sim/tools/gitlab/cancel_pipeline.ts +++ b/apps/sim/tools/gitlab/cancel_pipeline.ts @@ -1,4 +1,5 @@ import type { GitLabCancelPipelineParams, GitLabCancelPipelineResponse } from '@/tools/gitlab/types' +import { getGitLabApiBase } from '@/tools/gitlab/utils' import type { ToolConfig } from '@/tools/types' export const gitlabCancelPipelineTool: ToolConfig< @@ -17,6 +18,12 @@ export const gitlabCancelPipelineTool: ToolConfig< visibility: 'user-only', description: 'GitLab Personal Access Token', }, + host: { + type: 'string', + required: false, + visibility: 'user-only', + description: 'Self-managed GitLab host (e.g. gitlab.example.com). Defaults to gitlab.com.', + }, projectId: { type: 'string', required: true, @@ -34,7 +41,7 @@ export const gitlabCancelPipelineTool: ToolConfig< request: { url: (params) => { const encodedId = encodeURIComponent(String(params.projectId)) - return `https://gitlab.com/api/v4/projects/${encodedId}/pipelines/${params.pipelineId}/cancel` + return `${getGitLabApiBase(params.host)}/projects/${encodedId}/pipelines/${params.pipelineId}/cancel` }, method: 'POST', headers: (params) => ({ diff --git a/apps/sim/tools/gitlab/create_issue.ts b/apps/sim/tools/gitlab/create_issue.ts index 6a03e9ff970..cc4475831da 100644 --- a/apps/sim/tools/gitlab/create_issue.ts +++ b/apps/sim/tools/gitlab/create_issue.ts @@ -1,4 +1,5 @@ import type { GitLabCreateIssueParams, GitLabCreateIssueResponse } from '@/tools/gitlab/types' +import { getGitLabApiBase } from '@/tools/gitlab/utils' import type { ToolConfig } from '@/tools/types' export const gitlabCreateIssueTool: ToolConfig = @@ -15,6 +16,12 @@ export const gitlabCreateIssueTool: ToolConfig { const encodedId = encodeURIComponent(String(params.projectId)) - return `https://gitlab.com/api/v4/projects/${encodedId}/issues` + return `${getGitLabApiBase(params.host)}/projects/${encodedId}/issues` }, method: 'POST', headers: (params) => ({ diff --git a/apps/sim/tools/gitlab/create_issue_note.ts b/apps/sim/tools/gitlab/create_issue_note.ts index 150bf729d7e..0ad5c218bee 100644 --- a/apps/sim/tools/gitlab/create_issue_note.ts +++ b/apps/sim/tools/gitlab/create_issue_note.ts @@ -1,4 +1,5 @@ import type { GitLabCreateIssueNoteParams, GitLabCreateNoteResponse } from '@/tools/gitlab/types' +import { getGitLabApiBase } from '@/tools/gitlab/utils' import type { ToolConfig } from '@/tools/types' export const gitlabCreateIssueNoteTool: ToolConfig< @@ -17,6 +18,12 @@ export const gitlabCreateIssueNoteTool: ToolConfig< visibility: 'user-only', description: 'GitLab Personal Access Token', }, + host: { + type: 'string', + required: false, + visibility: 'user-only', + description: 'Self-managed GitLab host (e.g. gitlab.example.com). Defaults to gitlab.com.', + }, projectId: { type: 'string', required: true, @@ -40,7 +47,7 @@ export const gitlabCreateIssueNoteTool: ToolConfig< request: { url: (params) => { const encodedId = encodeURIComponent(String(params.projectId)) - return `https://gitlab.com/api/v4/projects/${encodedId}/issues/${params.issueIid}/notes` + return `${getGitLabApiBase(params.host)}/projects/${encodedId}/issues/${params.issueIid}/notes` }, method: 'POST', headers: (params) => ({ diff --git a/apps/sim/tools/gitlab/create_merge_request.ts b/apps/sim/tools/gitlab/create_merge_request.ts index fdaebbf842a..2c02c2dd0f4 100644 --- a/apps/sim/tools/gitlab/create_merge_request.ts +++ b/apps/sim/tools/gitlab/create_merge_request.ts @@ -2,6 +2,7 @@ import type { GitLabCreateMergeRequestParams, GitLabCreateMergeRequestResponse, } from '@/tools/gitlab/types' +import { getGitLabApiBase } from '@/tools/gitlab/utils' import type { ToolConfig } from '@/tools/types' export const gitlabCreateMergeRequestTool: ToolConfig< @@ -20,6 +21,12 @@ export const gitlabCreateMergeRequestTool: ToolConfig< visibility: 'user-only', description: 'GitLab Personal Access Token', }, + host: { + type: 'string', + required: false, + visibility: 'user-only', + description: 'Self-managed GitLab host (e.g. gitlab.example.com). Defaults to gitlab.com.', + }, projectId: { type: 'string', required: true, @@ -91,7 +98,7 @@ export const gitlabCreateMergeRequestTool: ToolConfig< request: { url: (params) => { const encodedId = encodeURIComponent(String(params.projectId)) - return `https://gitlab.com/api/v4/projects/${encodedId}/merge_requests` + return `${getGitLabApiBase(params.host)}/projects/${encodedId}/merge_requests` }, method: 'POST', headers: (params) => ({ diff --git a/apps/sim/tools/gitlab/create_merge_request_note.ts b/apps/sim/tools/gitlab/create_merge_request_note.ts index 599ec569719..f02f5fa35fb 100644 --- a/apps/sim/tools/gitlab/create_merge_request_note.ts +++ b/apps/sim/tools/gitlab/create_merge_request_note.ts @@ -2,6 +2,7 @@ import type { GitLabCreateMergeRequestNoteParams, GitLabCreateNoteResponse, } from '@/tools/gitlab/types' +import { getGitLabApiBase } from '@/tools/gitlab/utils' import type { ToolConfig } from '@/tools/types' export const gitlabCreateMergeRequestNoteTool: ToolConfig< @@ -20,6 +21,12 @@ export const gitlabCreateMergeRequestNoteTool: ToolConfig< visibility: 'user-only', description: 'GitLab Personal Access Token', }, + host: { + type: 'string', + required: false, + visibility: 'user-only', + description: 'Self-managed GitLab host (e.g. gitlab.example.com). Defaults to gitlab.com.', + }, projectId: { type: 'string', required: true, @@ -43,7 +50,7 @@ export const gitlabCreateMergeRequestNoteTool: ToolConfig< request: { url: (params) => { const encodedId = encodeURIComponent(String(params.projectId)) - return `https://gitlab.com/api/v4/projects/${encodedId}/merge_requests/${params.mergeRequestIid}/notes` + return `${getGitLabApiBase(params.host)}/projects/${encodedId}/merge_requests/${params.mergeRequestIid}/notes` }, method: 'POST', headers: (params) => ({ diff --git a/apps/sim/tools/gitlab/create_pipeline.ts b/apps/sim/tools/gitlab/create_pipeline.ts index 38a15df006b..a27ed7ba372 100644 --- a/apps/sim/tools/gitlab/create_pipeline.ts +++ b/apps/sim/tools/gitlab/create_pipeline.ts @@ -1,4 +1,5 @@ import type { GitLabCreatePipelineParams, GitLabCreatePipelineResponse } from '@/tools/gitlab/types' +import { getGitLabApiBase } from '@/tools/gitlab/utils' import type { ToolConfig } from '@/tools/types' export const gitlabCreatePipelineTool: ToolConfig< @@ -17,6 +18,12 @@ export const gitlabCreatePipelineTool: ToolConfig< visibility: 'user-only', description: 'GitLab Personal Access Token', }, + host: { + type: 'string', + required: false, + visibility: 'user-only', + description: 'Self-managed GitLab host (e.g. gitlab.example.com). Defaults to gitlab.com.', + }, projectId: { type: 'string', required: true, @@ -41,7 +48,7 @@ export const gitlabCreatePipelineTool: ToolConfig< request: { url: (params) => { const encodedId = encodeURIComponent(String(params.projectId)) - return `https://gitlab.com/api/v4/projects/${encodedId}/pipeline` + return `${getGitLabApiBase(params.host)}/projects/${encodedId}/pipeline` }, method: 'POST', headers: (params) => ({ diff --git a/apps/sim/tools/gitlab/delete_issue.ts b/apps/sim/tools/gitlab/delete_issue.ts index 64fbe881e53..475e52d77a8 100644 --- a/apps/sim/tools/gitlab/delete_issue.ts +++ b/apps/sim/tools/gitlab/delete_issue.ts @@ -1,4 +1,5 @@ import type { GitLabDeleteIssueParams, GitLabDeleteIssueResponse } from '@/tools/gitlab/types' +import { getGitLabApiBase } from '@/tools/gitlab/utils' import type { ToolConfig } from '@/tools/types' export const gitlabDeleteIssueTool: ToolConfig = @@ -15,6 +16,12 @@ export const gitlabDeleteIssueTool: ToolConfig { const encodedId = encodeURIComponent(String(params.projectId)) - return `https://gitlab.com/api/v4/projects/${encodedId}/issues/${params.issueIid}` + return `${getGitLabApiBase(params.host)}/projects/${encodedId}/issues/${params.issueIid}` }, method: 'DELETE', headers: (params) => ({ diff --git a/apps/sim/tools/gitlab/get_issue.ts b/apps/sim/tools/gitlab/get_issue.ts index 9a3c5821edb..aa87136e552 100644 --- a/apps/sim/tools/gitlab/get_issue.ts +++ b/apps/sim/tools/gitlab/get_issue.ts @@ -1,4 +1,5 @@ import type { GitLabGetIssueParams, GitLabGetIssueResponse } from '@/tools/gitlab/types' +import { getGitLabApiBase } from '@/tools/gitlab/utils' import type { ToolConfig } from '@/tools/types' export const gitlabGetIssueTool: ToolConfig = { @@ -14,6 +15,12 @@ export const gitlabGetIssueTool: ToolConfig { const encodedId = encodeURIComponent(String(params.projectId)) - return `https://gitlab.com/api/v4/projects/${encodedId}/issues/${params.issueIid}` + return `${getGitLabApiBase(params.host)}/projects/${encodedId}/issues/${params.issueIid}` }, method: 'GET', headers: (params) => ({ diff --git a/apps/sim/tools/gitlab/get_merge_request.ts b/apps/sim/tools/gitlab/get_merge_request.ts index 6e99d3cfbd3..f228cfba2eb 100644 --- a/apps/sim/tools/gitlab/get_merge_request.ts +++ b/apps/sim/tools/gitlab/get_merge_request.ts @@ -2,6 +2,7 @@ import type { GitLabGetMergeRequestParams, GitLabGetMergeRequestResponse, } from '@/tools/gitlab/types' +import { getGitLabApiBase } from '@/tools/gitlab/utils' import type { ToolConfig } from '@/tools/types' export const gitlabGetMergeRequestTool: ToolConfig< @@ -20,6 +21,12 @@ export const gitlabGetMergeRequestTool: ToolConfig< visibility: 'user-only', description: 'GitLab Personal Access Token', }, + host: { + type: 'string', + required: false, + visibility: 'user-only', + description: 'Self-managed GitLab host (e.g. gitlab.example.com). Defaults to gitlab.com.', + }, projectId: { type: 'string', required: true, @@ -37,7 +44,7 @@ export const gitlabGetMergeRequestTool: ToolConfig< request: { url: (params) => { const encodedId = encodeURIComponent(String(params.projectId)) - return `https://gitlab.com/api/v4/projects/${encodedId}/merge_requests/${params.mergeRequestIid}` + return `${getGitLabApiBase(params.host)}/projects/${encodedId}/merge_requests/${params.mergeRequestIid}` }, method: 'GET', headers: (params) => ({ diff --git a/apps/sim/tools/gitlab/get_pipeline.ts b/apps/sim/tools/gitlab/get_pipeline.ts index 5f4f25a0eaa..1494e65e4bb 100644 --- a/apps/sim/tools/gitlab/get_pipeline.ts +++ b/apps/sim/tools/gitlab/get_pipeline.ts @@ -1,4 +1,5 @@ import type { GitLabGetPipelineParams, GitLabGetPipelineResponse } from '@/tools/gitlab/types' +import { getGitLabApiBase } from '@/tools/gitlab/utils' import type { ToolConfig } from '@/tools/types' export const gitlabGetPipelineTool: ToolConfig = @@ -15,6 +16,12 @@ export const gitlabGetPipelineTool: ToolConfig { const encodedId = encodeURIComponent(String(params.projectId)) - return `https://gitlab.com/api/v4/projects/${encodedId}/pipelines/${params.pipelineId}` + return `${getGitLabApiBase(params.host)}/projects/${encodedId}/pipelines/${params.pipelineId}` }, method: 'GET', headers: (params) => ({ diff --git a/apps/sim/tools/gitlab/get_project.ts b/apps/sim/tools/gitlab/get_project.ts index c49369084f5..5ea42920584 100644 --- a/apps/sim/tools/gitlab/get_project.ts +++ b/apps/sim/tools/gitlab/get_project.ts @@ -1,4 +1,5 @@ import type { GitLabGetProjectParams, GitLabGetProjectResponse } from '@/tools/gitlab/types' +import { getGitLabApiBase } from '@/tools/gitlab/utils' import type { ToolConfig } from '@/tools/types' export const gitlabGetProjectTool: ToolConfig = { @@ -14,6 +15,12 @@ export const gitlabGetProjectTool: ToolConfig { const encodedId = encodeURIComponent(String(params.projectId)) - return `https://gitlab.com/api/v4/projects/${encodedId}` + return `${getGitLabApiBase(params.host)}/projects/${encodedId}` }, method: 'GET', headers: (params) => ({ diff --git a/apps/sim/tools/gitlab/list_issues.ts b/apps/sim/tools/gitlab/list_issues.ts index 1607920571e..40a016f3b34 100644 --- a/apps/sim/tools/gitlab/list_issues.ts +++ b/apps/sim/tools/gitlab/list_issues.ts @@ -1,4 +1,5 @@ import type { GitLabListIssuesParams, GitLabListIssuesResponse } from '@/tools/gitlab/types' +import { getGitLabApiBase } from '@/tools/gitlab/utils' import type { ToolConfig } from '@/tools/types' export const gitlabListIssuesTool: ToolConfig = { @@ -14,6 +15,12 @@ export const gitlabListIssuesTool: ToolConfig ({ diff --git a/apps/sim/tools/gitlab/list_merge_requests.ts b/apps/sim/tools/gitlab/list_merge_requests.ts index 0296bc3a24f..2cdae3301c4 100644 --- a/apps/sim/tools/gitlab/list_merge_requests.ts +++ b/apps/sim/tools/gitlab/list_merge_requests.ts @@ -2,6 +2,7 @@ import type { GitLabListMergeRequestsParams, GitLabListMergeRequestsResponse, } from '@/tools/gitlab/types' +import { getGitLabApiBase } from '@/tools/gitlab/utils' import type { ToolConfig } from '@/tools/types' export const gitlabListMergeRequestsTool: ToolConfig< @@ -20,6 +21,12 @@ export const gitlabListMergeRequestsTool: ToolConfig< visibility: 'user-only', description: 'GitLab Personal Access Token', }, + host: { + type: 'string', + required: false, + visibility: 'user-only', + description: 'Self-managed GitLab host (e.g. gitlab.example.com). Defaults to gitlab.com.', + }, projectId: { type: 'string', required: true, @@ -91,7 +98,7 @@ export const gitlabListMergeRequestsTool: ToolConfig< if (params.page) queryParams.append('page', String(params.page)) const query = queryParams.toString() - return `https://gitlab.com/api/v4/projects/${encodedId}/merge_requests${query ? `?${query}` : ''}` + return `${getGitLabApiBase(params.host)}/projects/${encodedId}/merge_requests${query ? `?${query}` : ''}` }, method: 'GET', headers: (params) => ({ diff --git a/apps/sim/tools/gitlab/list_pipelines.ts b/apps/sim/tools/gitlab/list_pipelines.ts index d4aed464736..80294e85f73 100644 --- a/apps/sim/tools/gitlab/list_pipelines.ts +++ b/apps/sim/tools/gitlab/list_pipelines.ts @@ -1,4 +1,5 @@ import type { GitLabListPipelinesParams, GitLabListPipelinesResponse } from '@/tools/gitlab/types' +import { getGitLabApiBase } from '@/tools/gitlab/utils' import type { ToolConfig } from '@/tools/types' export const gitlabListPipelinesTool: ToolConfig< @@ -17,6 +18,12 @@ export const gitlabListPipelinesTool: ToolConfig< visibility: 'user-only', description: 'GitLab Personal Access Token', }, + host: { + type: 'string', + required: false, + visibility: 'user-only', + description: 'Self-managed GitLab host (e.g. gitlab.example.com). Defaults to gitlab.com.', + }, projectId: { type: 'string', required: true, @@ -75,7 +82,7 @@ export const gitlabListPipelinesTool: ToolConfig< if (params.page) queryParams.append('page', String(params.page)) const query = queryParams.toString() - return `https://gitlab.com/api/v4/projects/${encodedId}/pipelines${query ? `?${query}` : ''}` + return `${getGitLabApiBase(params.host)}/projects/${encodedId}/pipelines${query ? `?${query}` : ''}` }, method: 'GET', headers: (params) => ({ diff --git a/apps/sim/tools/gitlab/list_projects.ts b/apps/sim/tools/gitlab/list_projects.ts index ec8018215a4..b6d6dd4c4a7 100644 --- a/apps/sim/tools/gitlab/list_projects.ts +++ b/apps/sim/tools/gitlab/list_projects.ts @@ -1,4 +1,5 @@ import type { GitLabListProjectsParams, GitLabListProjectsResponse } from '@/tools/gitlab/types' +import { getGitLabApiBase } from '@/tools/gitlab/utils' import type { ToolConfig } from '@/tools/types' export const gitlabListProjectsTool: ToolConfig< @@ -17,6 +18,12 @@ export const gitlabListProjectsTool: ToolConfig< visibility: 'user-only', description: 'GitLab Personal Access Token', }, + host: { + type: 'string', + required: false, + visibility: 'user-only', + description: 'Self-managed GitLab host (e.g. gitlab.example.com). Defaults to gitlab.com.', + }, owned: { type: 'boolean', required: false, @@ -80,7 +87,7 @@ export const gitlabListProjectsTool: ToolConfig< if (params.page) queryParams.append('page', String(params.page)) const query = queryParams.toString() - return `https://gitlab.com/api/v4/projects${query ? `?${query}` : ''}` + return `${getGitLabApiBase(params.host)}/projects${query ? `?${query}` : ''}` }, method: 'GET', headers: (params) => ({ diff --git a/apps/sim/tools/gitlab/merge_merge_request.ts b/apps/sim/tools/gitlab/merge_merge_request.ts index d63686a16f4..500e6ebfd07 100644 --- a/apps/sim/tools/gitlab/merge_merge_request.ts +++ b/apps/sim/tools/gitlab/merge_merge_request.ts @@ -2,6 +2,7 @@ import type { GitLabMergeMergeRequestParams, GitLabMergeMergeRequestResponse, } from '@/tools/gitlab/types' +import { getGitLabApiBase } from '@/tools/gitlab/utils' import type { ToolConfig } from '@/tools/types' export const gitlabMergeMergeRequestTool: ToolConfig< @@ -20,6 +21,12 @@ export const gitlabMergeMergeRequestTool: ToolConfig< visibility: 'user-only', description: 'GitLab Personal Access Token', }, + host: { + type: 'string', + required: false, + visibility: 'user-only', + description: 'Self-managed GitLab host (e.g. gitlab.example.com). Defaults to gitlab.com.', + }, projectId: { type: 'string', required: true, @@ -67,7 +74,7 @@ export const gitlabMergeMergeRequestTool: ToolConfig< request: { url: (params) => { const encodedId = encodeURIComponent(String(params.projectId)) - return `https://gitlab.com/api/v4/projects/${encodedId}/merge_requests/${params.mergeRequestIid}/merge` + return `${getGitLabApiBase(params.host)}/projects/${encodedId}/merge_requests/${params.mergeRequestIid}/merge` }, method: 'PUT', headers: (params) => ({ diff --git a/apps/sim/tools/gitlab/retry_pipeline.ts b/apps/sim/tools/gitlab/retry_pipeline.ts index 3c0fe6f2b4d..48143109c97 100644 --- a/apps/sim/tools/gitlab/retry_pipeline.ts +++ b/apps/sim/tools/gitlab/retry_pipeline.ts @@ -1,4 +1,5 @@ import type { GitLabRetryPipelineParams, GitLabRetryPipelineResponse } from '@/tools/gitlab/types' +import { getGitLabApiBase } from '@/tools/gitlab/utils' import type { ToolConfig } from '@/tools/types' export const gitlabRetryPipelineTool: ToolConfig< @@ -17,6 +18,12 @@ export const gitlabRetryPipelineTool: ToolConfig< visibility: 'user-only', description: 'GitLab Personal Access Token', }, + host: { + type: 'string', + required: false, + visibility: 'user-only', + description: 'Self-managed GitLab host (e.g. gitlab.example.com). Defaults to gitlab.com.', + }, projectId: { type: 'string', required: true, @@ -34,7 +41,7 @@ export const gitlabRetryPipelineTool: ToolConfig< request: { url: (params) => { const encodedId = encodeURIComponent(String(params.projectId)) - return `https://gitlab.com/api/v4/projects/${encodedId}/pipelines/${params.pipelineId}/retry` + return `${getGitLabApiBase(params.host)}/projects/${encodedId}/pipelines/${params.pipelineId}/retry` }, method: 'POST', headers: (params) => ({ diff --git a/apps/sim/tools/gitlab/types.ts b/apps/sim/tools/gitlab/types.ts index 9722c16422b..af865ed3ef8 100644 --- a/apps/sim/tools/gitlab/types.ts +++ b/apps/sim/tools/gitlab/types.ts @@ -194,6 +194,11 @@ interface GitLabMilestone { interface GitLabBaseParams { accessToken: string + /** + * Self-managed GitLab host (e.g. `gitlab.example.com`). Optional — defaults to + * `gitlab.com` so existing workflows keep working. + */ + host?: string } // ===== Project Parameters ===== diff --git a/apps/sim/tools/gitlab/update_issue.ts b/apps/sim/tools/gitlab/update_issue.ts index 27c1fb70164..acf7ca25402 100644 --- a/apps/sim/tools/gitlab/update_issue.ts +++ b/apps/sim/tools/gitlab/update_issue.ts @@ -1,4 +1,5 @@ import type { GitLabUpdateIssueParams, GitLabUpdateIssueResponse } from '@/tools/gitlab/types' +import { getGitLabApiBase } from '@/tools/gitlab/utils' import type { ToolConfig } from '@/tools/types' export const gitlabUpdateIssueTool: ToolConfig = @@ -15,6 +16,12 @@ export const gitlabUpdateIssueTool: ToolConfig { const encodedId = encodeURIComponent(String(params.projectId)) - return `https://gitlab.com/api/v4/projects/${encodedId}/issues/${params.issueIid}` + return `${getGitLabApiBase(params.host)}/projects/${encodedId}/issues/${params.issueIid}` }, method: 'PUT', headers: (params) => ({ diff --git a/apps/sim/tools/gitlab/update_merge_request.ts b/apps/sim/tools/gitlab/update_merge_request.ts index c02d4f13b08..69632a637d5 100644 --- a/apps/sim/tools/gitlab/update_merge_request.ts +++ b/apps/sim/tools/gitlab/update_merge_request.ts @@ -2,6 +2,7 @@ import type { GitLabUpdateMergeRequestParams, GitLabUpdateMergeRequestResponse, } from '@/tools/gitlab/types' +import { getGitLabApiBase } from '@/tools/gitlab/utils' import type { ToolConfig } from '@/tools/types' export const gitlabUpdateMergeRequestTool: ToolConfig< @@ -20,6 +21,12 @@ export const gitlabUpdateMergeRequestTool: ToolConfig< visibility: 'user-only', description: 'GitLab Personal Access Token', }, + host: { + type: 'string', + required: false, + visibility: 'user-only', + description: 'Self-managed GitLab host (e.g. gitlab.example.com). Defaults to gitlab.com.', + }, projectId: { type: 'string', required: true, @@ -97,7 +104,7 @@ export const gitlabUpdateMergeRequestTool: ToolConfig< request: { url: (params) => { const encodedId = encodeURIComponent(String(params.projectId)) - return `https://gitlab.com/api/v4/projects/${encodedId}/merge_requests/${params.mergeRequestIid}` + return `${getGitLabApiBase(params.host)}/projects/${encodedId}/merge_requests/${params.mergeRequestIid}` }, method: 'PUT', headers: (params) => ({ diff --git a/apps/sim/tools/gitlab/utils.test.ts b/apps/sim/tools/gitlab/utils.test.ts new file mode 100644 index 00000000000..f7eca36aef8 --- /dev/null +++ b/apps/sim/tools/gitlab/utils.test.ts @@ -0,0 +1,71 @@ +/** + * @vitest-environment node + */ +import { describe, expect, it } from 'vitest' +import { getGitLabApiBase, normalizeGitLabHost, UnsafeGitLabHostError } from '@/tools/gitlab/utils' + +describe('normalizeGitLabHost', () => { + it('defaults to gitlab.com when the host is empty, blank, or not a string', () => { + expect(normalizeGitLabHost(undefined)).toBe('gitlab.com') + expect(normalizeGitLabHost(null)).toBe('gitlab.com') + expect(normalizeGitLabHost('')).toBe('gitlab.com') + expect(normalizeGitLabHost(' ')).toBe('gitlab.com') + expect(normalizeGitLabHost(42)).toBe('gitlab.com') + }) + + it('strips protocol and trailing slashes from a self-managed host', () => { + expect(normalizeGitLabHost('gitlab.example.com')).toBe('gitlab.example.com') + expect(normalizeGitLabHost('https://gitlab.example.com')).toBe('gitlab.example.com') + expect(normalizeGitLabHost('http://gitlab.example.com/')).toBe('gitlab.example.com') + expect(normalizeGitLabHost(' https://gitlab.example.com// ')).toBe('gitlab.example.com') + }) + + it('preserves an explicit port and IDN punycode labels', () => { + expect(normalizeGitLabHost('gitlab.example.com:8443')).toBe('gitlab.example.com:8443') + expect(normalizeGitLabHost('xn--80ak6aa92e.com')).toBe('xn--80ak6aa92e.com') + }) + + it('rejects hosts that could redirect the request authority (SSRF / token exfiltration)', () => { + const unsafe = [ + 'legit.com@evil.com', + 'user:pass@evil.com', + 'gitlab.com#@evil.com', + 'gitlab.com /api', + 'line\nbreak.com', + 'evil.com/path', + 'evil.com?x=1', + '[::1]', + 'a..b.com', + '.gitlab.com', + 'gitlab.com.', + ] + for (const host of unsafe) { + expect(() => normalizeGitLabHost(host), host).toThrow(UnsafeGitLabHostError) + } + }) + + it('accepts bare IP literals at the STRUCTURAL layer by design (private/metadata IPs are rejected later by the fetch-layer DNS guard)', () => { + // This guard is structural only — it prevents authority confusion (userinfo, + // path, whitespace). SSRF to private/loopback/metadata addresses is the + // responsibility of validateUrlWithDNS / secureFetchWithValidation at fetch + // time, the single SSRF chokepoint shared by tools, webhooks, and connectors. + // These hosts are therefore structurally valid here, then blocked at fetch. + expect(normalizeGitLabHost('127.0.0.1')).toBe('127.0.0.1') + expect(normalizeGitLabHost('169.254.169.254')).toBe('169.254.169.254') + expect(normalizeGitLabHost('localhost')).toBe('localhost') + }) +}) + +describe('getGitLabApiBase', () => { + it('builds the v4 REST base for the default and self-managed hosts', () => { + expect(getGitLabApiBase(undefined)).toBe('https://gitlab.com/api/v4') + expect(getGitLabApiBase('gitlab.example.com')).toBe('https://gitlab.example.com/api/v4') + expect(getGitLabApiBase('https://gitlab.example.com:8443/')).toBe( + 'https://gitlab.example.com:8443/api/v4' + ) + }) + + it('propagates rejection of unsafe hosts', () => { + expect(() => getGitLabApiBase('legit.com@evil.com')).toThrow(UnsafeGitLabHostError) + }) +}) diff --git a/apps/sim/tools/gitlab/utils.ts b/apps/sim/tools/gitlab/utils.ts new file mode 100644 index 00000000000..6334a7030ee --- /dev/null +++ b/apps/sim/tools/gitlab/utils.ts @@ -0,0 +1,68 @@ +const DEFAULT_GITLAB_HOST = 'gitlab.com' + +/** + * Error thrown when a user-supplied GitLab host is structurally unsafe to use + * as the target of a server-side request that carries the user's access token. + */ +export class UnsafeGitLabHostError extends Error { + constructor(rawHost: string) { + super(`Invalid GitLab host: ${rawHost}`) + this.name = 'UnsafeGitLabHostError' + } +} + +/** + * Rejects a host that is structurally unsafe to fetch with the caller's token. + * + * The host is later interpolated into `https:///api/v4`, so anything that + * could change the request's authority (userinfo `@`, an embedded path/query/ + * fragment, whitespace, or control characters) must be rejected to prevent the + * `PRIVATE-TOKEN` header from being sent to an attacker-controlled origin. The + * allowed alphabet is hostname labels plus an optional `:port`, so self-managed + * hosts such as `gitlab.example.com` or `gitlab.example.com:8443` keep working. + * This is a structural guard only; DNS-based private-IP/SSRF checks remain the + * responsibility of the fetch layer. + */ +function assertSafeGitLabHostString(host: string, rawHost: string): void { + const hostnameWithoutPort = host.replace(/:\d+$/, '') + const allowedHostChars = /^[A-Za-z0-9.-]+$/ + if (!allowedHostChars.test(hostnameWithoutPort)) { + throw new UnsafeGitLabHostError(rawHost) + } + if (hostnameWithoutPort.startsWith('.') || hostnameWithoutPort.endsWith('.')) { + throw new UnsafeGitLabHostError(rawHost) + } + if (hostnameWithoutPort.split('.').some((label) => label.length === 0)) { + throw new UnsafeGitLabHostError(rawHost) + } +} + +/** + * Normalizes a GitLab host value: trims whitespace, strips any protocol prefix + * and trailing slashes, validates that the result is a bare host (optionally + * with a port), and falls back to gitlab.com when empty. Mirrors the GitLab + * connector so tools, triggers, and connectors resolve hosts identically. + * + * @throws {UnsafeGitLabHostError} when a non-empty host is structurally unsafe. + */ +export function normalizeGitLabHost(rawHost: unknown): string { + const raw = typeof rawHost === 'string' ? rawHost.trim() : '' + if (!raw) return DEFAULT_GITLAB_HOST + const host = raw + .replace(/^https?:\/\//i, '') + .replace(/\/+$/, '') + .trim() + if (!host) return DEFAULT_GITLAB_HOST + assertSafeGitLabHostString(host, String(rawHost)) + return host +} + +/** + * Builds the REST API v4 base URL for the configured host. Defaults to + * gitlab.com so existing workflows that never set a host keep working. + * + * @throws {UnsafeGitLabHostError} when a non-empty host is structurally unsafe. + */ +export function getGitLabApiBase(rawHost: unknown): string { + return `https://${normalizeGitLabHost(rawHost)}/api/v4` +} diff --git a/apps/sim/tools/mistral/parser.ts b/apps/sim/tools/mistral/parser.ts index c0e7ee5e11d..948e2753f92 100644 --- a/apps/sim/tools/mistral/parser.ts +++ b/apps/sim/tools/mistral/parser.ts @@ -12,6 +12,19 @@ import type { ToolConfig } from '@/tools/types' const logger = createLogger('MistralParserTool') +/** + * Mistral OCR 4 standard pricing, in USD per page ($4 per 1,000 pages). + * + * This tool calls the synchronous `/v1/ocr` endpoint with the `mistral-ocr-latest` + * alias, which Mistral repointed to OCR 4 (`mistral-ocr-4-0`) on 2026-06-23, so the + * standard (non-batch) OCR 4 rate applies. Document AI / annotation pages are priced + * separately, but this tool does not submit annotation requests. + * + * @see https://mistral.ai/news/ocr-4/ + * @see https://docs.mistral.ai/getting-started/changelog + */ +const MISTRAL_OCR_COST_PER_PAGE = 0.004 + const MISTRAL_OCR_HOSTING = { envKeyPrefix: 'MISTRAL_API_KEY', apiKeyParam: 'apiKey', @@ -19,9 +32,6 @@ const MISTRAL_OCR_HOSTING = { pricing: { type: 'custom' as const, getCost: (_params: unknown, output: Record) => { - // Mistral OCR 3 standard pricing: $2 per 1,000 pages ($0.002/page). - // Annotated pages are priced separately at $3 per 1,000 annotated pages, but this tool does - // not submit annotation requests. Source: https://docs.mistral.ai/models/ocr-3-25-12 const rawUsageInfo = output.usage_info as { pages_processed?: number } | undefined const transformedUsageInfo = ( output.metadata as { usageInfo?: { pagesProcessed?: number } } | undefined @@ -33,7 +43,7 @@ const MISTRAL_OCR_HOSTING = { 'Mistral OCR response missing pages_processed in usage_info or metadata.usageInfo.pagesProcessed' ) } - const cost = pagesProcessed * 0.002 + const cost = pagesProcessed * MISTRAL_OCR_COST_PER_PAGE return { cost, metadata: { pagesProcessed } } }, }, diff --git a/apps/sim/triggers/gitlab/utils.ts b/apps/sim/triggers/gitlab/utils.ts index 6f7848ad2fe..a25d09e3447 100644 --- a/apps/sim/triggers/gitlab/utils.ts +++ b/apps/sim/triggers/gitlab/utils.ts @@ -103,6 +103,15 @@ export function buildGitLabExtraFields(triggerId: string): SubBlockConfig[] { mode: 'trigger', condition: { field: 'selectedTriggerId', value: triggerId }, }, + { + id: 'host', + title: 'GitLab Host', + type: 'short-input', + placeholder: 'gitlab.com', + description: 'Self-managed GitLab host. Leave blank for gitlab.com.', + mode: 'trigger', + condition: { field: 'selectedTriggerId', value: triggerId }, + }, ] } diff --git a/apps/sim/triggers/slack/capabilities.ts b/apps/sim/triggers/slack/capabilities.ts index 1a532e4ebf2..317c240621f 100644 --- a/apps/sim/triggers/slack/capabilities.ts +++ b/apps/sim/triggers/slack/capabilities.ts @@ -105,16 +105,7 @@ export const SLACK_CAPABILITIES: readonly SlackCapability[] = [ scopes: ['channels:history', 'groups:history', 'im:history', 'mpim:history'], events: [], }, - { - id: 'action_assistant', - label: 'Manage assistant threads', - description: - "Let the bot set the status indicator (the 'is thinking…' shimmer), title, and suggested prompts on AI app threads.", - defaultChecked: true, - group: 'action', - scopes: ['assistant:write'], - events: [], - }, + // TODO: Restore the 'action_assistant' capability (scope 'assistant:write') once Slack app review is approved { id: 'action_read_files', label: 'Read file attachments', diff --git a/apps/sim/vitest.setup.ts b/apps/sim/vitest.setup.ts index 92e945dcca0..b9a9ae7d14a 100644 --- a/apps/sim/vitest.setup.ts +++ b/apps/sim/vitest.setup.ts @@ -41,6 +41,7 @@ vi.mock('@/stores/execution/store', () => ({ useExecutionStore: { getState: vi.fn().mockReturnValue({ getWorkflowExecution: vi.fn().mockReturnValue({ + status: 'idle', isExecuting: false, isDebugging: false, activeBlockIds: new Set(), @@ -50,6 +51,7 @@ vi.mock('@/stores/execution/store', () => ({ lastRunPath: new Map(), lastRunEdges: new Map(), }), + setStatus: vi.fn(), setIsExecuting: vi.fn(), setIsDebugging: vi.fn(), setPendingBlocks: vi.fn(), @@ -61,6 +63,7 @@ vi.mock('@/stores/execution/store', () => ({ }), }, useCurrentWorkflowExecution: vi.fn().mockReturnValue({ + status: 'idle', isExecuting: false, isDebugging: false, activeBlockIds: new Set(), diff --git a/bun.lock b/bun.lock index e9bb4f978b3..c20f6bfa669 100644 --- a/bun.lock +++ b/bun.lock @@ -1,5 +1,6 @@ { "lockfileVersion": 1, + "configVersion": 0, "workspaces": { "": { "name": "simstudio", @@ -67,6 +68,7 @@ "@sim/logger": "workspace:*", "@sim/platform-authz": "workspace:*", "@sim/realtime-protocol": "workspace:*", + "@sim/runtime-secrets": "workspace:*", "@sim/security": "workspace:*", "@sim/utils": "workspace:*", "@sim/workflow-persistence": "workspace:*", @@ -158,6 +160,7 @@ "@sim/logger": "workspace:*", "@sim/platform-authz": "workspace:*", "@sim/realtime-protocol": "workspace:*", + "@sim/runtime-secrets": "workspace:*", "@sim/security": "workspace:*", "@sim/utils": "workspace:*", "@sim/workflow-persistence": "workspace:*", @@ -400,6 +403,21 @@ "typescript": "^5.7.3", }, }, + "packages/runtime-secrets": { + "name": "@sim/runtime-secrets", + "version": "0.1.0", + "dependencies": { + "@aws-sdk/client-secrets-manager": "3.1032.0", + "@sim/logger": "workspace:*", + "@sim/utils": "workspace:*", + }, + "devDependencies": { + "@sim/tsconfig": "workspace:*", + "@types/node": "24.2.1", + "typescript": "^5.7.3", + "vitest": "^4.1.0", + }, + }, "packages/security": { "name": "@sim/security", "version": "0.1.0", @@ -1469,6 +1487,8 @@ "@sim/realtime-protocol": ["@sim/realtime-protocol@workspace:packages/realtime-protocol"], + "@sim/runtime-secrets": ["@sim/runtime-secrets@workspace:packages/runtime-secrets"], + "@sim/security": ["@sim/security@workspace:packages/security"], "@sim/testing": ["@sim/testing@workspace:packages/testing"], @@ -4237,6 +4257,8 @@ "@sim/realtime/@types/node": ["@types/node@24.2.1", "", { "dependencies": { "undici-types": "~7.10.0" } }, "sha512-DRh5K+ka5eJic8CjH7td8QpYEV6Zo10gfRkjHCO3weqZHWDtAaSTFtl4+VMqOJ4N5jcuhZ9/l+yy8rVgw7BQeQ=="], + "@sim/runtime-secrets/@types/node": ["@types/node@24.2.1", "", { "dependencies": { "undici-types": "~7.10.0" } }, "sha512-DRh5K+ka5eJic8CjH7td8QpYEV6Zo10gfRkjHCO3weqZHWDtAaSTFtl4+VMqOJ4N5jcuhZ9/l+yy8rVgw7BQeQ=="], + "@sim/security/@types/node": ["@types/node@24.2.1", "", { "dependencies": { "undici-types": "~7.10.0" } }, "sha512-DRh5K+ka5eJic8CjH7td8QpYEV6Zo10gfRkjHCO3weqZHWDtAaSTFtl4+VMqOJ4N5jcuhZ9/l+yy8rVgw7BQeQ=="], "@smithy/middleware-compression/fflate": ["fflate@0.8.1", "", {}, "sha512-/exOvEuc+/iaUm105QIiOt4LpBdMTWsXxqR0HDF35vx3fmaKzw7354gTilCh5rkzEt8WYyG//ku3h3nRmd7CHQ=="], @@ -4749,6 +4771,8 @@ "@sim/realtime/@types/node/undici-types": ["undici-types@7.10.0", "", {}, "sha512-t5Fy/nfn+14LuOc2KNYg75vZqClpAiqscVvMygNnlsHBFpSXdJaYtXMcdNLpl/Qvc3P2cB3s6lOV51nqsFq4ag=="], + "@sim/runtime-secrets/@types/node/undici-types": ["undici-types@7.10.0", "", {}, "sha512-t5Fy/nfn+14LuOc2KNYg75vZqClpAiqscVvMygNnlsHBFpSXdJaYtXMcdNLpl/Qvc3P2cB3s6lOV51nqsFq4ag=="], + "@sim/security/@types/node/undici-types": ["undici-types@7.10.0", "", {}, "sha512-t5Fy/nfn+14LuOc2KNYg75vZqClpAiqscVvMygNnlsHBFpSXdJaYtXMcdNLpl/Qvc3P2cB3s6lOV51nqsFq4ag=="], "@trigger.dev/core/@opentelemetry/api-logs/@opentelemetry/api": ["@opentelemetry/api@1.9.1", "", {}, "sha512-gLyJlPHPZYdAk1JENA9LeHejZe1Ti77/pTeFm/nMXmQH/HFZlcS/O2XJB+L8fkbrNSqhdtlvjBVjxwUYanNH5Q=="], diff --git a/docker/app.Dockerfile b/docker/app.Dockerfile index ff0ea1ccc28..e6e7f22bb53 100644 --- a/docker/app.Dockerfile +++ b/docker/app.Dockerfile @@ -81,6 +81,13 @@ RUN --mount=type=cache,id=next-cache-${TARGETPLATFORM},target=/app/apps/sim/.nex --mount=type=cache,id=turbo-cache-${TARGETPLATFORM},target=/app/.turbo \ bun run build +# Bundle the secrets-loading bootstrap into a self-contained entrypoint. It runs +# before (and outside) the Next standalone server, so its dependencies +# (@sim/runtime-secrets, AWS SDK) are inlined here rather than resolved from the +# pruned standalone node_modules. The dynamic import of ./server.js stays a +# runtime import. +RUN bun build apps/sim/bootstrap.ts --target=bun --outfile=apps/sim/bootstrap.js + # ======================================== # Runner Stage: Run the actual app # ======================================== @@ -100,6 +107,10 @@ COPY --from=builder --chown=nextjs:nodejs /app/apps/sim/public ./apps/sim/public COPY --from=builder --chown=nextjs:nodejs /app/apps/sim/.next/standalone ./ COPY --from=builder --chown=nextjs:nodejs /app/apps/sim/.next/static ./apps/sim/.next/static +# Self-contained secrets-loading bootstrap (bundled in the builder stage). Runs +# before the standalone server.js to hydrate process.env from the runtime secret. +COPY --from=builder --chown=nextjs:nodejs /app/apps/sim/bootstrap.js ./apps/sim/bootstrap.js + # Copy blog/author content for runtime filesystem reads (not part of the JS bundle) COPY --from=builder --chown=nextjs:nodejs /app/apps/sim/content ./apps/sim/content @@ -128,4 +139,4 @@ EXPOSE 3000 ENV PORT=3000 \ HOSTNAME="0.0.0.0" -CMD ["bun", "apps/sim/server.js"] +CMD ["bun", "apps/sim/bootstrap.js"] diff --git a/docker/realtime.Dockerfile b/docker/realtime.Dockerfile index 16f3cd1c32f..d403c906462 100644 --- a/docker/realtime.Dockerfile +++ b/docker/realtime.Dockerfile @@ -49,4 +49,4 @@ USER nextjs EXPOSE 3002 -CMD ["bun", "apps/realtime/src/index.ts"] +CMD ["bun", "apps/realtime/src/bootstrap.ts"] diff --git a/packages/runtime-secrets/package.json b/packages/runtime-secrets/package.json new file mode 100644 index 00000000000..ee57201f3d1 --- /dev/null +++ b/packages/runtime-secrets/package.json @@ -0,0 +1,38 @@ +{ + "name": "@sim/runtime-secrets", + "version": "0.1.0", + "private": true, + "sideEffects": false, + "type": "module", + "license": "Apache-2.0", + "engines": { + "bun": ">=1.2.13", + "node": ">=20.0.0" + }, + "exports": { + ".": { + "types": "./src/index.ts", + "default": "./src/index.ts" + } + }, + "scripts": { + "type-check": "tsc --noEmit", + "lint": "biome check --write --unsafe .", + "lint:check": "biome check .", + "format": "biome format --write .", + "format:check": "biome format .", + "test": "vitest run", + "test:watch": "vitest" + }, + "dependencies": { + "@aws-sdk/client-secrets-manager": "3.1032.0", + "@sim/logger": "workspace:*", + "@sim/utils": "workspace:*" + }, + "devDependencies": { + "@sim/tsconfig": "workspace:*", + "@types/node": "24.2.1", + "typescript": "^5.7.3", + "vitest": "^4.1.0" + } +} diff --git a/packages/runtime-secrets/src/index.test.ts b/packages/runtime-secrets/src/index.test.ts new file mode 100644 index 00000000000..dfd5cec1f4b --- /dev/null +++ b/packages/runtime-secrets/src/index.test.ts @@ -0,0 +1,91 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' + +const { mockSend } = vi.hoisted(() => ({ mockSend: vi.fn() })) + +vi.mock('@aws-sdk/client-secrets-manager', () => ({ + SecretsManagerClient: class SecretsManagerClient { + send = mockSend + }, + GetSecretValueCommand: class GetSecretValueCommand { + constructor(public input: unknown) {} + }, +})) + +vi.mock('@sim/logger', () => ({ + createLogger: () => ({ info: vi.fn(), warn: vi.fn(), error: vi.fn() }), +})) + +vi.mock('@sim/utils/helpers', () => ({ + sleep: vi.fn().mockResolvedValue(undefined), +})) + +import { loadRuntimeSecrets } from './index' + +const TOUCHED = ['SIM_ENV_SECRET_ID', 'FOO', 'BAZ'] as const + +describe('loadRuntimeSecrets', () => { + beforeEach(() => { + vi.clearAllMocks() + for (const key of TOUCHED) delete process.env[key] + }) + + afterEach(() => { + for (const key of TOUCHED) delete process.env[key] + }) + + it('no-ops when SIM_ENV_SECRET_ID is unset', async () => { + await loadRuntimeSecrets() + expect(mockSend).not.toHaveBeenCalled() + }) + + it('hydrates process.env from the parsed secret JSON', async () => { + process.env.SIM_ENV_SECRET_ID = '/test/sim/env-vars' + mockSend.mockResolvedValue({ SecretString: JSON.stringify({ FOO: 'bar', BAZ: 'qux' }) }) + + await loadRuntimeSecrets() + + expect(process.env.FOO).toBe('bar') + expect(process.env.BAZ).toBe('qux') + }) + + it('never overwrites an already-set env var', async () => { + process.env.SIM_ENV_SECRET_ID = '/test/sim/env-vars' + process.env.FOO = 'existing' + mockSend.mockResolvedValue({ SecretString: JSON.stringify({ FOO: 'new', BAZ: 'qux' }) }) + + await loadRuntimeSecrets() + + expect(process.env.FOO).toBe('existing') + expect(process.env.BAZ).toBe('qux') + }) + + it('throws when the secret is not valid JSON', async () => { + process.env.SIM_ENV_SECRET_ID = '/test/sim/env-vars' + mockSend.mockResolvedValue({ SecretString: 'not json' }) + + await expect(loadRuntimeSecrets()).rejects.toThrow(/not valid JSON/) + }) + + it('throws when the secret JSON is not an object', async () => { + process.env.SIM_ENV_SECRET_ID = '/test/sim/env-vars' + mockSend.mockResolvedValue({ SecretString: JSON.stringify(['a', 'b']) }) + + await expect(loadRuntimeSecrets()).rejects.toThrow(/must be a JSON object/) + }) + + it('throws immediately on a binary secret (no SecretString), without retrying', async () => { + process.env.SIM_ENV_SECRET_ID = '/test/sim/env-vars' + mockSend.mockResolvedValue({}) + + await expect(loadRuntimeSecrets()).rejects.toThrow(/binary secrets/) + expect(mockSend).toHaveBeenCalledTimes(1) + }) + + it('retries then throws when the fetch keeps failing', async () => { + process.env.SIM_ENV_SECRET_ID = '/test/sim/env-vars' + mockSend.mockRejectedValue(new Error('boom')) + + await expect(loadRuntimeSecrets()).rejects.toThrow(/Failed to fetch runtime secrets/) + expect(mockSend).toHaveBeenCalledTimes(3) + }) +}) diff --git a/packages/runtime-secrets/src/index.ts b/packages/runtime-secrets/src/index.ts new file mode 100644 index 00000000000..86c79e7952d --- /dev/null +++ b/packages/runtime-secrets/src/index.ts @@ -0,0 +1,102 @@ +import type { GetSecretValueCommandOutput } from '@aws-sdk/client-secrets-manager' +import { GetSecretValueCommand, SecretsManagerClient } from '@aws-sdk/client-secrets-manager' +import { createLogger } from '@sim/logger' +import { getErrorMessage } from '@sim/utils/errors' +import { sleep } from '@sim/utils/helpers' +import { backoffWithJitter } from '@sim/utils/retry' + +const logger = createLogger('RuntimeSecrets') + +/** Plaintext env var (set in the ECS task definition) naming the secret to ingest. */ +const SECRET_ID_ENV = 'SIM_ENV_SECRET_ID' + +const MAX_ATTEMPTS = 3 + +/** Bounds each Secrets Manager request so a stalled response can't hang boot. */ +const REQUEST_TIMEOUT_MS = 5000 + +/** + * Fetches the combined `/{env}/sim/env-vars` secret once at container boot and + * hydrates `process.env`, so secrets no longer have to be fanned out into the + * ECS task definition (which is approaching the 64 KB rendered-document limit). + * + * Must run before any application module that reads env at import time. No-ops + * when {@link SECRET_ID_ENV} is unset (local dev / self-hosted keep using their + * own env). Existing `process.env` keys are never overwritten, so explicit + * task-definition `environment` entries win. Throws on any fetch/parse failure + * so a misconfigured container crashes instead of booting without its config. + */ +export async function loadRuntimeSecrets(): Promise { + const secretId = process.env[SECRET_ID_ENV] + if (!secretId) { + logger.info(`${SECRET_ID_ENV} not set; skipping runtime secret ingestion`) + return + } + + const client = new SecretsManagerClient( + process.env.AWS_REGION ? { region: process.env.AWS_REGION } : {} + ) + + const secretString = await fetchSecretString(client, secretId) + const entries = parseSecretJson(secretString) + + let loaded = 0 + let skipped = 0 + for (const [key, value] of Object.entries(entries)) { + if (key in process.env) { + skipped++ + continue + } + process.env[key] = typeof value === 'string' ? value : JSON.stringify(value) + loaded++ + } + + logger.info('Runtime secrets ingested', { secretId, loaded, skipped }) +} + +async function fetchSecretString(client: SecretsManagerClient, secretId: string): Promise { + const response = await sendWithRetry(client, secretId) + if (!response.SecretString) { + // Non-retriable: a binary secret will never become a string between attempts. + throw new Error('Secret has no SecretString (binary secrets are not supported)') + } + return response.SecretString +} + +async function sendWithRetry( + client: SecretsManagerClient, + secretId: string +): Promise { + let lastError: unknown + for (let attempt = 1; attempt <= MAX_ATTEMPTS; attempt++) { + try { + return await client.send(new GetSecretValueCommand({ SecretId: secretId }), { + abortSignal: AbortSignal.timeout(REQUEST_TIMEOUT_MS), + }) + } catch (error) { + lastError = error + if (attempt < MAX_ATTEMPTS) { + const delay = backoffWithJitter(attempt, null, { baseMs: 200, maxMs: 2000 }) + logger.warn( + `Failed to fetch runtime secrets (attempt ${attempt}/${MAX_ATTEMPTS}), retrying`, + { error: getErrorMessage(error) } + ) + await sleep(delay) + } + } + } + throw new Error(`Failed to fetch runtime secrets from ${secretId}: ${getErrorMessage(lastError)}`) +} + +function parseSecretJson(secretString: string): Record { + let parsed: unknown + try { + parsed = JSON.parse(secretString) + } catch (error) { + throw new Error(`Runtime secret is not valid JSON: ${getErrorMessage(error)}`) + } + if (typeof parsed !== 'object' || parsed === null || Array.isArray(parsed)) { + throw new Error('Runtime secret must be a JSON object of key/value pairs') + } + return parsed as Record +} diff --git a/packages/runtime-secrets/tsconfig.json b/packages/runtime-secrets/tsconfig.json new file mode 100644 index 00000000000..1ffa3d2e844 --- /dev/null +++ b/packages/runtime-secrets/tsconfig.json @@ -0,0 +1,5 @@ +{ + "extends": "@sim/tsconfig/library.json", + "include": ["src/**/*"], + "exclude": ["node_modules", "dist"] +} diff --git a/packages/runtime-secrets/vitest.config.ts b/packages/runtime-secrets/vitest.config.ts new file mode 100644 index 00000000000..2b1c323fe22 --- /dev/null +++ b/packages/runtime-secrets/vitest.config.ts @@ -0,0 +1,7 @@ +import { defineConfig } from 'vitest/config' + +export default defineConfig({ + test: { + environment: 'node', + }, +})