Skip to content

Commit

Permalink
fix: fix some bug about scheme uri
Browse files Browse the repository at this point in the history
  • Loading branch information
2214962083 committed Jan 17, 2025
1 parent be6fc89 commit a74858c
Show file tree
Hide file tree
Showing 33 changed files with 860 additions and 160 deletions.
44 changes: 43 additions & 1 deletion src/extension/actions/git-project-actions.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,9 @@
import { gitUtils } from '@extension/file-utils/git'
import {
traverseFileOrFolders,
type FileInfo,
type FolderInfo
} from '@extension/file-utils/traverse-fs'
import { vfs } from '@extension/file-utils/vfs'
import { gitProjectSchemeHandler } from '@extension/file-utils/vfs/schemes/git-project-scheme'
import { gitProjectDB } from '@extension/lowdb/git-project-db'
Expand Down Expand Up @@ -166,7 +171,6 @@ export class GitProjectActionsCollection extends ServerActionCollection {
// Remove project directories
await settledPromiseResults(
projects.map(async project => {
if (!project) return
const schemeUri = gitProjectSchemeHandler.createSchemeUri({
name: project.name,
type: project.type,
Expand Down Expand Up @@ -218,4 +222,42 @@ export class GitProjectActionsCollection extends ServerActionCollection {

return project
}

async getGitProjectFilesAndFolders(
context: ActionContext<{ projectIds?: string[] }>
): Promise<Record<string, (FileInfo | FolderInfo)[]>> {
const { actionParams } = context
const projectIds =
actionParams.projectIds ?? (await gitProjectDB.getAll()).map(p => p.id)
const projects = await gitProjectDB.getAll()
const result: Record<string, (FileInfo | FolderInfo)[]> = {}

// Get files and folders for each project
await settledPromiseResults(
projectIds.map(async projectId => {
const project = projects.find(p => p.id === projectId)
if (!project) {
result[projectId] = []
return
}

const gitProjectSchemeUri = gitProjectSchemeHandler.createSchemeUri({
name: project.name,
type: project.type,
relativePath: './'
})

const items = await traverseFileOrFolders({
type: 'fileOrFolder',
schemeUris: [gitProjectSchemeUri],
isGetFileContent: false,
itemCallback: item => item
})

result[projectId] = items
})
)

return result
}
}
44 changes: 44 additions & 0 deletions src/extension/actions/project-actions.ts
Original file line number Diff line number Diff line change
@@ -1,7 +1,14 @@
import {
traverseFileOrFolders,
type FileInfo,
type FolderInfo
} from '@extension/file-utils/traverse-fs'
import { projectSchemeHandler } from '@extension/file-utils/vfs/schemes/project-scheme'
import { projectDB } from '@extension/lowdb/project-db'
import { ServerActionCollection } from '@shared/actions/server-action-collection'
import type { ActionContext } from '@shared/actions/types'
import type { Project } from '@shared/entities'
import { settledPromiseResults } from '@shared/utils/common'
import { z } from 'zod'

// Create schema for validation
Expand Down Expand Up @@ -115,4 +122,41 @@ export class ProjectActionsCollection extends ServerActionCollection {
project.path.toLowerCase().includes(query.toLowerCase())
)
}

async getProjectFilesAndFolders(
context: ActionContext<{ projectIds?: string[] }>
): Promise<Record<string, (FileInfo | FolderInfo)[]>> {
const { actionParams } = context
const projectIds =
actionParams.projectIds ?? (await projectDB.getAll()).map(p => p.id)
const projects = await projectDB.getAll()
const result: Record<string, (FileInfo | FolderInfo)[]> = {}

// Get files and folders for each project
await settledPromiseResults(
projectIds.map(async projectId => {
const project = projects.find(p => p.id === projectId)
if (!project) {
result[projectId] = []
return
}

const projectSchemeUri = projectSchemeHandler.createSchemeUri({
name: project.name,
relativePath: './'
})

const items = await traverseFileOrFolders({
type: 'fileOrFolder',
schemeUris: [projectSchemeUri],
isGetFileContent: false,
itemCallback: item => item
})

result[projectId] = items
})
)

return result
}
}
58 changes: 58 additions & 0 deletions src/extension/actions/prompt-snippet-actions.ts
Original file line number Diff line number Diff line change
Expand Up @@ -9,14 +9,66 @@ import type { ActionContext } from '@shared/actions/types'
import type { PromptSnippet, SettingsSaveType } from '@shared/entities'
import { settledPromiseResults } from '@shared/utils/common'
import { v4 as uuidv4 } from 'uuid'
import { z } from 'zod'

export type PromptSnippetWithSaveType = PromptSnippet & {
saveType: SettingsSaveType
}

// Add schema validation
const promptSnippetSchema = z.object({
title: z
.string()
.min(1, 'Title is required')
.refine(
async title => {
const globalSnippets = await promptSnippetsGlobalDB.getAll()
const workspaceSnippets = await promptSnippetsWorkspaceDB.getAll()
const allSnippets = [...globalSnippets, ...workspaceSnippets]
return !allSnippets.some(s => s.title === title)
},
{
message: 'Title must be unique'
}
),
contents: z.array(
z.object({
type: z.literal('text'),
text: z.string()
})
),
richText: z.string().optional(),
mentions: z.array(z.any()).optional()
})

export class PromptSnippetActionsCollection extends ServerActionCollection {
readonly categoryName = 'promptSnippet'

// Add validation method
private async validateSnippet(
data: Partial<PromptSnippet>,
excludeId?: string
): Promise<void> {
const globalSnippets = await promptSnippetsGlobalDB.getAll()
const workspaceSnippets = await promptSnippetsWorkspaceDB.getAll()
const allSnippets = [...globalSnippets, ...workspaceSnippets]

const schema = promptSnippetSchema.extend({
title: z
.string()
.min(1, 'Title is required')
.refine(
async title =>
!allSnippets.some(s => s.title === title && s.id !== excludeId),
{
message: 'Title must be unique'
}
)
})

await schema.parseAsync(data)
}

async refreshSnippet(
context: ActionContext<{ snippet: PromptSnippet }>
): Promise<PromptSnippet> {
Expand Down Expand Up @@ -171,6 +223,9 @@ export class PromptSnippetActionsCollection extends ServerActionCollection {
const { saveType, snippet, isRefresh } = actionParams

try {
// Add validation
await this.validateSnippet(snippet)

const now = Date.now()
let updatedSnippet = {
...snippet,
Expand Down Expand Up @@ -211,6 +266,9 @@ export class PromptSnippetActionsCollection extends ServerActionCollection {
const { actionParams } = context
const { id, updates, isRefresh } = actionParams
try {
// Add validation
await this.validateSnippet(updates, id)

const originalSnippet = await this.getSnippet({
...context,
actionParams: { id, isRefresh }
Expand Down
3 changes: 2 additions & 1 deletion src/extension/chat/utils/doc-crawler.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import url from 'url'
import { aidePaths, getSemanticHashName } from '@extension/file-utils/paths'
import { vfs } from '@extension/file-utils/vfs'
import { logger } from '@extension/logger'
import { settledPromiseResults } from '@shared/utils/common'
import * as cheerio from 'cheerio'
import type { Element } from 'domhandler'
import TurndownService from 'turndown'
Expand Down Expand Up @@ -162,7 +163,7 @@ export class DocCrawler {
while (this.queue.length > 0 && this.visited.size < this.options.maxPages) {
const batch = this.queue.splice(0, this.options.concurrency)
const promises = batch.map(item => this.crawlPage(item.url, item.depth))
await Promise.allSettled(promises)
await settledPromiseResults(promises)
await new Promise(resolve => setTimeout(resolve, this.options.delay))
this.progressReporter.setProcessedItems(this.visited.size)
}
Expand Down
5 changes: 3 additions & 2 deletions src/extension/chat/vectordb/base-indexer.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ import type { BaseEmbeddings } from '@extension/ai/embeddings/types'
import { getFileHash } from '@extension/file-utils/get-file-hash'
import { vfs } from '@extension/file-utils/vfs'
import { logger } from '@extension/logger'
import { settledPromiseResults } from '@shared/utils/common'
import {
Field,
FixedSizeList,
Expand Down Expand Up @@ -218,7 +219,7 @@ export abstract class BaseIndexer<T extends IndexRow> {
}
})

await Promise.allSettled(tasksPromises)
await settledPromiseResults(tasksPromises)

this.totalFiles = fileSchemeUrisNeedReindex.length
this.progressReporter.setTotalItems(this.totalFiles)
Expand Down Expand Up @@ -246,7 +247,7 @@ export abstract class BaseIndexer<T extends IndexRow> {
)

try {
await Promise.allSettled(processingPromises)
await settledPromiseResults(processingPromises)
} catch (error) {
logger.error(`Error indexing files:`, error)
} finally {
Expand Down
2 changes: 1 addition & 1 deletion src/extension/commands/batch-processor/command.ts
Original file line number Diff line number Diff line change
Expand Up @@ -103,7 +103,7 @@ export class BatchProcessorCommand extends BaseCommand {
)
)

await Promise.allSettled(promises)
await settledPromiseResults(promises)

hideProcessLoading()

Expand Down
52 changes: 38 additions & 14 deletions src/extension/file-utils/ignore-patterns.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
import path from 'path'
import { getConfigKey } from '@extension/config'
import { logger } from '@extension/logger'
import { toUnixPath } from '@shared/utils/common'
import { settledPromiseResults, toUnixPath } from '@shared/utils/common'
import { SchemeUriHelper } from '@shared/utils/scheme-uri-helper'
import { glob } from 'glob'
import ignore from 'ignore'
Expand All @@ -20,6 +21,7 @@ export const createShouldIgnore = async (
) => {
const ignorePatterns = await getConfigKey('ignorePatterns')
const respectGitIgnore = await getConfigKey('respectGitIgnore')
const fullDirPath = await vfs.resolveFullPathProAsync(dirSchemeUri, false)

if (customIgnorePatterns) {
ignorePatterns.push(...customIgnorePatterns)
Expand Down Expand Up @@ -58,20 +60,32 @@ export const createShouldIgnore = async (
* @returns A boolean indicating whether the file should be ignored.
*/
const shouldIgnore = (schemeUriOrFileFullPath: string) => {
const relativePath = vfs.resolveRelativePathProSync(schemeUriOrFileFullPath)
const unixRelativePath = toUnixPath(relativePath)
try {
let relativePath

if (!unixRelativePath) return false
if (vfs.isSchemeUri(schemeUriOrFileFullPath)) {
relativePath = vfs.resolveRelativePathProSync(schemeUriOrFileFullPath)
} else {
relativePath = path.relative(fullDirPath, schemeUriOrFileFullPath)
}

if (['.', './', '..', '../', '/'].includes(unixRelativePath)) {
return false
}
const unixRelativePath = toUnixPath(relativePath)

if (ig && ig.ignores(unixRelativePath)) {
return true
}
if (!unixRelativePath) return false

return mms.some(mm => mm.match(unixRelativePath))
if (['.', './', '..', '../', '/'].includes(unixRelativePath)) {
return false
}

if (ig && ig.ignores(unixRelativePath)) {
return true
}

return mms.some(mm => mm.match(unixRelativePath))
} catch (error) {
logger.warn('shouldIgnore error', error)
return false
}
}

return shouldIgnore
Expand All @@ -96,11 +110,16 @@ export const getAllValidFiles = async (
nodir: true,
absolute: true,
follow: false,
posix: true,
dot: true,
fs: vfs,
ignore: {
ignored(p) {
return shouldIgnore(p.fullpath())
try {
return shouldIgnore(p.fullpath())
} catch {
return false
}
},
childrenIgnored(p) {
try {
Expand Down Expand Up @@ -147,10 +166,15 @@ export const getAllValidFolders = async (
absolute: true,
follow: false,
dot: true,
posix: true,
fs: vfs,
ignore: {
ignored(p) {
return shouldIgnore(p.fullpath())
try {
return shouldIgnore(p.fullpath())
} catch {
return false
}
},
childrenIgnored(p) {
try {
Expand All @@ -170,7 +194,7 @@ export const getAllValidFolders = async (
}
})

await Promise.allSettled(promises)
await settledPromiseResults(promises)

if (!vfs.isSchemeUri(dirSchemeUri)) {
return folders
Expand Down
6 changes: 3 additions & 3 deletions src/extension/file-utils/traverse-fs.ts
Original file line number Diff line number Diff line change
Expand Up @@ -113,12 +113,12 @@ const traverseOneProjectFs = async <T, Type extends FsType>(
return await getAllValidFiles(schemeUri, shouldIgnore)
}
// For 'fileOrFolder' type, get both files and folders
const files = await getAllValidFiles(schemeUri, shouldIgnore)
const folders = await getAllValidFolders(schemeUri, shouldIgnore)
const files = await getAllValidFiles(schemeUri, shouldIgnore)
return [...files, ...folders]
}

await Promise.allSettled(
await settledPromiseResults(
schemeUris.map(async schemeUri => {
const stat = await vfs.promises.stat(schemeUri)

Expand All @@ -130,7 +130,7 @@ const traverseOneProjectFs = async <T, Type extends FsType>(
const allItemSchemeUris =
await getAllValidItemsWithCustomIgnore(schemeUri)

await Promise.allSettled(
await settledPromiseResults(
allItemSchemeUris.map(async itemSchemeUri => {
const itemStat = await vfs.promises.stat(itemSchemeUri)

Expand Down
Loading

0 comments on commit a74858c

Please sign in to comment.