Skip to content
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
529 changes: 52 additions & 477 deletions packages/core/src/filesystem.ts

Large diffs are not rendered by default.

62 changes: 44 additions & 18 deletions packages/core/src/filesystem/search.ts
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,11 @@ export interface FileInput {
readonly kind?: "file" | "directory" | "all"
}

export interface FileResult {
readonly path: string
readonly type: "file" | "directory"
}

export interface GlobInput {
readonly cwd: string
readonly pattern: string
Expand Down Expand Up @@ -61,7 +66,7 @@ export interface Interface {
readonly files: Ripgrep.Interface["files"]
readonly tree: Ripgrep.Interface["tree"]
readonly search: (input: Ripgrep.SearchInput) => Effect.Effect<Result, SearchError>
readonly file: (input: FileInput) => Effect.Effect<string[] | undefined, SearchError>
readonly file: (input: FileInput) => Effect.Effect<readonly FileResult[], SearchError>
readonly glob: (input: GlobInput) => Effect.Effect<{ files: string[]; truncated: boolean }, SearchError>
readonly open: (input: { cwd?: string; file: string }) => Effect.Effect<void, SearchError>
readonly warm: (cwd: string) => Effect.Effect<void>
Expand Down Expand Up @@ -131,46 +136,67 @@ function item(hit: Fff.Hit): Item {
}
}

function collectPaths<T>(items: T[], scores: Array<{ total: number }>, toPath: (item: T) => string): string[] {
const rows = items.flatMap((item, index): Array<{ text: string; score: number }> => {
const text = toPath(item)
if (!text) return []
return [{ text, score: scores[index]?.total ?? 0 }]
function collectPaths<T>(
items: T[],
scores: Array<{ total: number }>,
toResult: (item: T) => FileResult,
): FileResult[] {
const rows = items.flatMap((item, index): Array<FileResult & { score: number }> => {
const result = toResult(item)
if (!result.path) return []
return [{ ...result, score: scores[index]?.total ?? 0 }]
})
rows.sort(
(a, b) => b.score - a.score || a.text.length - b.text.length || (a.text < b.text ? -1 : a.text > b.text ? 1 : 0),
(a, b) =>
b.score - a.score ||
a.path.length - b.path.length ||
(a.path < b.path ? -1 : a.path > b.path ? 1 : 0),
)

return Array.from(new Set(rows.map((item) => item.text)))
const seen = new Set<string>()
return rows.flatMap((item) => {
if (seen.has(item.path)) return []
seen.add(item.path)
return [{ path: item.path, type: item.type }]
})
}

function searchFff(
pick: Fff.Picker,
kind: "file" | "directory" | "all",
query: string,
opts: { currentFile?: string; pageIndex?: number; pageSize?: number },
): Fff.Result<string[]> {
): Fff.Result<FileResult[]> {
if (kind === "directory") {
const out = pick.directorySearch(query, opts)
if (!out.ok) return out
return {
ok: true,
value: collectPaths(out.value.items, out.value.scores, (entry) => normalize(entry.relativePath)),
value: collectPaths(out.value.items, out.value.scores, (entry) => ({
path: normalize(entry.relativePath),
type: "directory",
})),
}
}
if (kind === "all") {
const out = pick.mixedSearch(query, opts)
if (!out.ok) return out
return {
ok: true,
value: collectPaths(out.value.items, out.value.scores, (entry) => normalize(entry.item.relativePath)),
value: collectPaths(out.value.items, out.value.scores, (entry) => ({
path: normalize(entry.item.relativePath),
type: entry.type,
})),
}
}
const out = pick.fileSearch(query, opts)
if (!out.ok) return out
return {
ok: true,
value: collectPaths(out.value.items, out.value.scores, (entry) => normalize(entry.relativePath)),
value: collectPaths(out.value.items, out.value.scores, (entry) => ({
path: normalize(entry.relativePath),
type: "file",
})),
}
}

Expand Down Expand Up @@ -341,8 +367,9 @@ export const layer: Layer.Layer<Service, never, FSUtil.Service | Ripgrep.Service
const query = input.query.trim()
const kind = input.kind ?? "file"

const entry = yield* acquire(input.cwd).pipe(Effect.catch(() => Effect.succeed<Picker | undefined>(undefined)))
if (!entry) return undefined
const entry = yield* acquire(input.cwd)
if (!entry) return yield* Effect.fail(new Error("fff is unavailable"))
yield* entry.ready
const dir = FSUtil.resolve(input.cwd)
const limit = input.limit ?? 100
const fffResult = yield* fffSync(`${kind} search`, () =>
Expand All @@ -354,22 +381,21 @@ export const layer: Layer.Layer<Service, never, FSUtil.Service | Ripgrep.Service
).pipe(
Effect.catch((error) =>
Effect.logWarning(`fff ${kind} search failed`, { dir, query, error }).pipe(
Effect.as<Fff.Result<string[]> | undefined>(undefined),
Effect.andThen(Effect.fail(error)),
),
),
)
if (!fffResult) return undefined
if (!fffResult.ok) {
yield* Effect.logWarning(`fff ${kind} search failed`, { dir, query, error: fffResult.error })
return undefined
return yield* Effect.fail(new Error(fffResult.error))
}

const rows = fffResult.value
remember(
state,
dir,
query,
rows.map((row) => path.join(dir, row)),
rows.map((row) => path.join(dir, row.path)),
)
return rows.slice(0, limit)
})
Expand Down
2 changes: 2 additions & 0 deletions packages/core/src/location-layer.ts
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ import { PermissionV2 } from "./permission"
import { PermissionSaved } from "./permission/saved"
import { FileSystem } from "./filesystem"
import { Watcher } from "./filesystem/watcher"
import { Search } from "./filesystem/search"
import { LocationMutation } from "./location-mutation"
import { LocationSearch } from "./location-search"
import { FileMutation } from "./file-mutation"
Expand Down Expand Up @@ -124,5 +125,6 @@ export class LocationServiceMap extends LayerMap.Service<LocationServiceMap>()("
FetchHttpClient.layer,
ToolOutputStore.defaultCleanupLayer,
ApplicationTools.layer,
Search.defaultLayer,
],
}) {}
43 changes: 36 additions & 7 deletions packages/core/src/location-search.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,10 +2,12 @@ export * as LocationSearch from "./location-search"

import path from "path"
import { Context, Effect, Layer, Option, Schema } from "effect"
import { FileSystem } from "./filesystem"
import { FSUtil } from "./fs-util"
import { Global } from "./global"
import { Location } from "./location"
import { Ripgrep } from "./ripgrep"
import { NonNegativeInt, PositiveInt, RelativePath } from "./schema"
import { ToolOutputStore } from "./tool-output-store"

/**
* Location-scoped raw search substrate. Search authority is selected only by
Expand All @@ -25,15 +27,15 @@ export const ResultLimit = PositiveInt.check(Schema.isLessThanOrEqualTo(MAX_RESU

export const FilesInput = Schema.Struct({
pattern: Schema.String,
...FileSystem.ListInput.fields,
path: Schema.String.pipe(Schema.optional),
limit: ResultLimit.pipe(Schema.optional),
})
export type FilesInput = typeof FilesInput.Type & { readonly signal?: AbortSignal }

export const GrepInput = Schema.Struct({
pattern: Schema.String,
include: Schema.String.pipe(Schema.optional),
...FileSystem.ListInput.fields,
path: Schema.String.pipe(Schema.optional),
limit: ResultLimit.pipe(Schema.optional),
})
export type GrepInput = typeof GrepInput.Type & { readonly signal?: AbortSignal }
Expand Down Expand Up @@ -89,10 +91,37 @@ export const layer = Layer.effect(
Service,
Effect.gen(function* () {
const fs = yield* FSUtil.Service
const filesystem = yield* FileSystem.Service
const location = yield* Location.Service
const global = yield* Effect.serviceOption(Global.Service)
const ripgrep = yield* Ripgrep.Service

const candidate = Effect.fnUntraced(function* (root: FileSystem.RootTarget, cwd: string, value: string) {
const resolve = Effect.fnUntraced(function* (input?: string) {
const directory = input && path.isAbsolute(input) ? path.dirname(input) : location.directory
const absolute = path.resolve(location.directory, input ?? ".")
if (!path.isAbsolute(input ?? "") && !FSUtil.contains(location.directory, absolute))
return yield* Effect.die(new globalThis.Error("Path escapes the location"))
if (path.isAbsolute(input ?? "")) {
const managed = path.join(
Option.match(global, { onNone: () => Global.Path.data, onSome: (value) => value.data }),
ToolOutputStore.MANAGED_DIRECTORY,
)
if (directory !== managed || !path.basename(absolute).startsWith("tool_"))
return yield* Effect.die(new globalThis.Error("Absolute path is not managed tool output"))
}
const real = yield* fs.realPath(absolute).pipe(Effect.orDie)
const root = yield* fs.realPath(directory).pipe(Effect.orDie)
if (!FSUtil.contains(root, real)) return yield* Effect.die(new globalThis.Error("Path escapes the search root"))
const info = yield* fs.stat(real).pipe(Effect.orDie)
const type = info.type === "File" ? ("file" as const) : info.type === "Directory" ? ("directory" as const) : undefined
if (!type) return yield* Effect.die(new globalThis.Error("Search root is not a file or directory"))
return { real, root, resource: slash(path.relative(root, real)) || ".", type }
})

const candidate = Effect.fnUntraced(function* (
root: { readonly real: string; readonly root: string; readonly type: "file" | "directory" },
cwd: string,
value: string,
) {
const absolute = path.resolve(cwd, value)
const lexicallyContained =
root.type === "directory" ? FSUtil.contains(root.real, absolute) : absolute === root.real
Expand All @@ -115,7 +144,7 @@ export const layer = Layer.effect(

return Service.of({
files: Effect.fn("LocationSearch.files")(function* (input) {
const root = yield* filesystem.resolveRoot(input)
const root = yield* resolve(input.path)
if (root.type !== "directory")
return yield* Effect.die(new globalThis.Error("Files search path must be a directory"))
const result = yield* ripgrep.files({
Expand All @@ -137,7 +166,7 @@ export const layer = Layer.effect(
})
}),
grep: Effect.fn("LocationSearch.grep")(function* (input) {
const root = yield* filesystem.resolveRoot(input)
const root = yield* resolve(input.path)
const cwd = root.type === "directory" ? root.real : path.dirname(root.real)
const result = yield* ripgrep.grep({
cwd,
Expand Down
3 changes: 2 additions & 1 deletion packages/core/src/tool/builtins.ts
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ import { GlobTool } from "./glob"
import { GrepTool } from "./grep"
import { QuestionTool } from "./question"
import { ReadTool } from "./read"
import { ReadToolFileSystem } from "./read-filesystem"
import { SkillTool } from "./skill"
import { TodoWriteTool } from "./todowrite"
import { WebFetchTool } from "./webfetch"
Expand All @@ -34,7 +35,7 @@ export const locationLayer = Layer.mergeAll(
GlobTool.layer,
GrepTool.layer,
QuestionTool.layer,
ReadTool.layer,
ReadTool.layer.pipe(Layer.provide(ReadToolFileSystem.layer)),
SkillTool.layer,
TodoWriteTool.layer,
WebFetchTool.layer,
Expand Down
5 changes: 1 addition & 4 deletions packages/core/src/tool/glob.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,6 @@ export * as GlobTool from "./glob"

import { ToolFailure, toolText } from "@opencode-ai/llm"
import { Effect, Layer, Schema } from "effect"
import { FileSystem } from "../filesystem"
import { LocationSearch } from "../location-search"
import { PermissionV2 } from "../permission"
import { Tool } from "./tool"
Expand Down Expand Up @@ -42,7 +41,6 @@ export const toModelOutput = (output: ModelOutput) => {
export const layer = Layer.effectDiscard(
Effect.gen(function* () {
const tools = yield* Tools.Service
const filesystem = yield* FileSystem.Service
const search = yield* LocationSearch.Service
const permission = yield* PermissionV2.Service

Expand All @@ -56,13 +54,12 @@ export const layer = Layer.effectDiscard(
toModelOutput: ({ output }) => [toolText({ type: "text", text: toModelOutput(output) })],
execute: (input, context) =>
Effect.gen(function* () {
const root = yield* filesystem.resolveRoot({ path: input.path })
yield* permission.assert({
action: name,
resources: [input.pattern],
save: ["*"],
metadata: {
root: root.resource,
root: input.path ?? ".",
path: input.path,
limit: input.limit,
},
Expand Down
5 changes: 1 addition & 4 deletions packages/core/src/tool/grep.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,6 @@ export * as GrepTool from "./grep"

import { ToolFailure, toolText } from "@opencode-ai/llm"
import { Effect, Layer, Schema } from "effect"
import { FileSystem } from "../filesystem"
import { LocationSearch } from "../location-search"
import { Ripgrep } from "../ripgrep"
import { PermissionV2 } from "../permission"
Expand Down Expand Up @@ -57,7 +56,6 @@ export const toModelOutput = (output: Output) => {
export const layer = Layer.effectDiscard(
Effect.gen(function* () {
const tools = yield* Tools.Service
const filesystem = yield* FileSystem.Service
const search = yield* LocationSearch.Service
const permission = yield* PermissionV2.Service

Expand All @@ -71,13 +69,12 @@ export const layer = Layer.effectDiscard(
toModelOutput: ({ output }) => [toolText({ type: "text", text: toModelOutput(output) })],
execute: (input, context) =>
Effect.gen(function* () {
const root = yield* filesystem.resolveRoot(input)
yield* permission.assert({
action: name,
resources: [input.pattern],
save: ["*"],
metadata: {
root: root.resource,
root: input.path ?? ".",
path: input.path,
include: input.include,
limit: input.limit,
Expand Down
Loading
Loading