refactor(ripgrep): use embedded wasm backend (#21703)

This commit is contained in:
Shoubhit Dash
2026-04-14 11:56:23 +05:30
committed by GitHub
parent 9b2648dd57
commit d6840868d4
15 changed files with 868 additions and 577 deletions
+3
View File
@@ -396,6 +396,7 @@
"opentui-spinner": "0.0.6", "opentui-spinner": "0.0.6",
"partial-json": "0.1.7", "partial-json": "0.1.7",
"remeda": "catalog:", "remeda": "catalog:",
"ripgrep": "0.3.1",
"semver": "^7.6.3", "semver": "^7.6.3",
"solid-js": "catalog:", "solid-js": "catalog:",
"strip-ansi": "7.1.2", "strip-ansi": "7.1.2",
@@ -4345,6 +4346,8 @@
"rimraf": ["rimraf@2.6.3", "", { "dependencies": { "glob": "^7.1.3" }, "bin": { "rimraf": "./bin.js" } }, "sha512-mwqeW5XsA2qAejG46gYdENaxXjx9onRNCfn7L0duuP4hCuTIi/QO7PDK07KJfp1d+izWPrzEJDcSqBa0OZQriA=="], "rimraf": ["rimraf@2.6.3", "", { "dependencies": { "glob": "^7.1.3" }, "bin": { "rimraf": "./bin.js" } }, "sha512-mwqeW5XsA2qAejG46gYdENaxXjx9onRNCfn7L0duuP4hCuTIi/QO7PDK07KJfp1d+izWPrzEJDcSqBa0OZQriA=="],
"ripgrep": ["ripgrep@0.3.1", "", { "bin": { "rg": "lib/rg.mjs", "ripgrep": "lib/rg.mjs" } }, "sha512-6bDtNIBh1qPviVIU685/4uv0Ap5t8eS4wiJhy/tR2LdIeIey9CVasENlGS+ul3HnTmGANIp7AjnfsztsRmALfQ=="],
"roarr": ["roarr@2.15.4", "", { "dependencies": { "boolean": "^3.0.1", "detect-node": "^2.0.4", "globalthis": "^1.0.1", "json-stringify-safe": "^5.0.1", "semver-compare": "^1.0.0", "sprintf-js": "^1.1.2" } }, "sha512-CHhPh+UNHD2GTXNYhPWLnU8ONHdI+5DI+4EYIAOaiD63rHeYlZvyh8P+in5999TTSFgUYuKUAjzRI4mdh/p+2A=="], "roarr": ["roarr@2.15.4", "", { "dependencies": { "boolean": "^3.0.1", "detect-node": "^2.0.4", "globalthis": "^1.0.1", "json-stringify-safe": "^5.0.1", "semver-compare": "^1.0.0", "sprintf-js": "^1.1.2" } }, "sha512-CHhPh+UNHD2GTXNYhPWLnU8ONHdI+5DI+4EYIAOaiD63rHeYlZvyh8P+in5999TTSFgUYuKUAjzRI4mdh/p+2A=="],
"rollup": ["rollup@4.60.1", "", { "dependencies": { "@types/estree": "1.0.8" }, "optionalDependencies": { "@rollup/rollup-android-arm-eabi": "4.60.1", "@rollup/rollup-android-arm64": "4.60.1", "@rollup/rollup-darwin-arm64": "4.60.1", "@rollup/rollup-darwin-x64": "4.60.1", "@rollup/rollup-freebsd-arm64": "4.60.1", "@rollup/rollup-freebsd-x64": "4.60.1", "@rollup/rollup-linux-arm-gnueabihf": "4.60.1", "@rollup/rollup-linux-arm-musleabihf": "4.60.1", "@rollup/rollup-linux-arm64-gnu": "4.60.1", "@rollup/rollup-linux-arm64-musl": "4.60.1", "@rollup/rollup-linux-loong64-gnu": "4.60.1", "@rollup/rollup-linux-loong64-musl": "4.60.1", "@rollup/rollup-linux-ppc64-gnu": "4.60.1", "@rollup/rollup-linux-ppc64-musl": "4.60.1", "@rollup/rollup-linux-riscv64-gnu": "4.60.1", "@rollup/rollup-linux-riscv64-musl": "4.60.1", "@rollup/rollup-linux-s390x-gnu": "4.60.1", "@rollup/rollup-linux-x64-gnu": "4.60.1", "@rollup/rollup-linux-x64-musl": "4.60.1", "@rollup/rollup-openbsd-x64": "4.60.1", "@rollup/rollup-openharmony-arm64": "4.60.1", "@rollup/rollup-win32-arm64-msvc": "4.60.1", "@rollup/rollup-win32-ia32-msvc": "4.60.1", "@rollup/rollup-win32-x64-gnu": "4.60.1", "@rollup/rollup-win32-x64-msvc": "4.60.1", "fsevents": "~2.3.2" }, "bin": { "rollup": "dist/bin/rollup" } }, "sha512-VmtB2rFU/GroZ4oL8+ZqXgSA38O6GR8KSIvWmEFv63pQ0G6KaBH9s07PO8XTXP4vI+3UJUEypOfjkGfmSBBR0w=="], "rollup": ["rollup@4.60.1", "", { "dependencies": { "@types/estree": "1.0.8" }, "optionalDependencies": { "@rollup/rollup-android-arm-eabi": "4.60.1", "@rollup/rollup-android-arm64": "4.60.1", "@rollup/rollup-darwin-arm64": "4.60.1", "@rollup/rollup-darwin-x64": "4.60.1", "@rollup/rollup-freebsd-arm64": "4.60.1", "@rollup/rollup-freebsd-x64": "4.60.1", "@rollup/rollup-linux-arm-gnueabihf": "4.60.1", "@rollup/rollup-linux-arm-musleabihf": "4.60.1", "@rollup/rollup-linux-arm64-gnu": "4.60.1", "@rollup/rollup-linux-arm64-musl": "4.60.1", "@rollup/rollup-linux-loong64-gnu": "4.60.1", "@rollup/rollup-linux-loong64-musl": "4.60.1", "@rollup/rollup-linux-ppc64-gnu": "4.60.1", "@rollup/rollup-linux-ppc64-musl": "4.60.1", "@rollup/rollup-linux-riscv64-gnu": "4.60.1", "@rollup/rollup-linux-riscv64-musl": "4.60.1", "@rollup/rollup-linux-s390x-gnu": "4.60.1", "@rollup/rollup-linux-x64-gnu": "4.60.1", "@rollup/rollup-linux-x64-musl": "4.60.1", "@rollup/rollup-openbsd-x64": "4.60.1", "@rollup/rollup-openharmony-arm64": "4.60.1", "@rollup/rollup-win32-arm64-msvc": "4.60.1", "@rollup/rollup-win32-ia32-msvc": "4.60.1", "@rollup/rollup-win32-x64-gnu": "4.60.1", "@rollup/rollup-win32-x64-msvc": "4.60.1", "fsevents": "~2.3.2" }, "bin": { "rollup": "dist/bin/rollup" } }, "sha512-VmtB2rFU/GroZ4oL8+ZqXgSA38O6GR8KSIvWmEFv63pQ0G6KaBH9s07PO8XTXP4vI+3UJUEypOfjkGfmSBBR0w=="],
+1
View File
@@ -153,6 +153,7 @@
"opentui-spinner": "0.0.6", "opentui-spinner": "0.0.6",
"partial-json": "0.1.7", "partial-json": "0.1.7",
"remeda": "catalog:", "remeda": "catalog:",
"ripgrep": "0.3.1",
"semver": "^7.6.3", "semver": "^7.6.3",
"solid-js": "catalog:", "solid-js": "catalog:",
"strip-ansi": "7.1.2", "strip-ansi": "7.1.2",
@@ -46,7 +46,7 @@ const FilesCommand = cmd({
async handler(args) { async handler(args) {
await bootstrap(process.cwd(), async () => { await bootstrap(process.cwd(), async () => {
const files: string[] = [] const files: string[] = []
for await (const file of Ripgrep.files({ for await (const file of await Ripgrep.files({
cwd: Instance.directory, cwd: Instance.directory,
glob: args.glob ? [args.glob] : undefined, glob: args.glob ? [args.glob] : undefined,
})) { })) {
+34 -2
View File
@@ -1,8 +1,10 @@
import { BusEvent } from "@/bus/bus-event" import { BusEvent } from "@/bus/bus-event"
import { InstanceState } from "@/effect/instance-state" import { InstanceState } from "@/effect/instance-state"
import { makeRuntime } from "@/effect/run-service"
import { AppFileSystem } from "@/filesystem" import { AppFileSystem } from "@/filesystem"
import { Git } from "@/git" import { Git } from "@/git"
import { Effect, Layer, Context } from "effect" import { Effect, Layer, Context } from "effect"
import * as Stream from "effect/Stream"
import { formatPatch, structuredPatch } from "diff" import { formatPatch, structuredPatch } from "diff"
import fuzzysort from "fuzzysort" import fuzzysort from "fuzzysort"
import ignore from "ignore" import ignore from "ignore"
@@ -342,6 +344,7 @@ export namespace File {
Service, Service,
Effect.gen(function* () { Effect.gen(function* () {
const appFs = yield* AppFileSystem.Service const appFs = yield* AppFileSystem.Service
const rg = yield* Ripgrep.Service
const git = yield* Git.Service const git = yield* Git.Service
const state = yield* InstanceState.make<State>( const state = yield* InstanceState.make<State>(
@@ -381,7 +384,10 @@ export namespace File {
next.dirs = Array.from(dirs).toSorted() next.dirs = Array.from(dirs).toSorted()
} else { } else {
const files = yield* Effect.promise(() => Array.fromAsync(Ripgrep.files({ cwd: Instance.directory }))) const files = yield* rg.files({ cwd: Instance.directory }).pipe(
Stream.runCollect,
Effect.map((chunk) => [...chunk]),
)
const seen = new Set<string>() const seen = new Set<string>()
for (const file of files) { for (const file of files) {
next.files.push(file) next.files.push(file)
@@ -642,5 +648,31 @@ export namespace File {
}), }),
) )
export const defaultLayer = layer.pipe(Layer.provide(AppFileSystem.defaultLayer), Layer.provide(Git.defaultLayer)) export const defaultLayer = layer.pipe(
Layer.provide(Ripgrep.defaultLayer),
Layer.provide(AppFileSystem.defaultLayer),
Layer.provide(Git.defaultLayer),
)
const { runPromise } = makeRuntime(Service, defaultLayer)
export function init() {
return runPromise((svc) => svc.init())
}
export async function status() {
return runPromise((svc) => svc.status())
}
export async function read(file: string): Promise<Content> {
return runPromise((svc) => svc.read(file))
}
export async function list(dir?: string) {
return runPromise((svc) => svc.list(dir))
}
export async function search(input: { query: string; limit?: number; dirs?: boolean; type?: "file" | "directory" }) {
return runPromise((svc) => svc.search(input))
}
} }
+441 -387
View File
@@ -1,28 +1,16 @@
// Ripgrep utility functions
import path from "path"
import { Global } from "../global"
import fs from "fs/promises" import fs from "fs/promises"
import path from "path"
import { fileURLToPath } from "url"
import z from "zod" import z from "zod"
import { Effect, Layer, Context, Schema } from "effect" import { Cause, Context, Effect, Layer, Queue, Stream } from "effect"
import * as Stream from "effect/Stream" import { ripgrep } from "ripgrep"
import { ChildProcess } from "effect/unstable/process" import { makeRuntime } from "@/effect/run-service"
import { ChildProcessSpawner } from "effect/unstable/process/ChildProcessSpawner" import { Filesystem } from "@/util/filesystem"
import * as CrossSpawnSpawner from "@/effect/cross-spawn-spawner"
import type { PlatformError } from "effect/PlatformError"
import { NamedError } from "@opencode-ai/util/error"
import { lazy } from "../util/lazy"
import { Filesystem } from "../util/filesystem"
import { AppFileSystem } from "../filesystem"
import { Process } from "../util/process"
import { which } from "../util/which"
import { text } from "node:stream/consumers"
import { ZipReader, BlobReader, BlobWriter } from "@zip.js/zip.js"
import { Log } from "@/util/log" import { Log } from "@/util/log"
export namespace Ripgrep { export namespace Ripgrep {
const log = Log.create({ service: "ripgrep" }) const log = Log.create({ service: "ripgrep" })
const Stats = z.object({ const Stats = z.object({
elapsed: z.object({ elapsed: z.object({
secs: z.number(), secs: z.number(),
@@ -94,437 +82,503 @@ export namespace Ripgrep {
const Result = z.union([Begin, Match, End, Summary]) const Result = z.union([Begin, Match, End, Summary])
const Hit = Schema.Struct({
type: Schema.Literal("match"),
data: Schema.Struct({
path: Schema.Struct({
text: Schema.String,
}),
lines: Schema.Struct({
text: Schema.String,
}),
line_number: Schema.Number,
absolute_offset: Schema.Number,
submatches: Schema.mutable(
Schema.Array(
Schema.Struct({
match: Schema.Struct({
text: Schema.String,
}),
start: Schema.Number,
end: Schema.Number,
}),
),
),
}),
})
const Row = Schema.Union([
Schema.Struct({ type: Schema.Literal("begin"), data: Schema.Unknown }),
Hit,
Schema.Struct({ type: Schema.Literal("end"), data: Schema.Unknown }),
Schema.Struct({ type: Schema.Literal("summary"), data: Schema.Unknown }),
])
const decode = Schema.decodeUnknownEffect(Schema.fromJsonString(Row))
export type Result = z.infer<typeof Result> export type Result = z.infer<typeof Result>
export type Match = z.infer<typeof Match> export type Match = z.infer<typeof Match>
export type Item = Match["data"] export type Item = Match["data"]
export type Begin = z.infer<typeof Begin> export type Begin = z.infer<typeof Begin>
export type End = z.infer<typeof End> export type End = z.infer<typeof End>
export type Summary = z.infer<typeof Summary> export type Summary = z.infer<typeof Summary>
const PLATFORM = { export type Row = Match["data"]
"arm64-darwin": { platform: "aarch64-apple-darwin", extension: "tar.gz" },
"arm64-linux": {
platform: "aarch64-unknown-linux-gnu",
extension: "tar.gz",
},
"x64-darwin": { platform: "x86_64-apple-darwin", extension: "tar.gz" },
"x64-linux": { platform: "x86_64-unknown-linux-musl", extension: "tar.gz" },
"arm64-win32": { platform: "aarch64-pc-windows-msvc", extension: "zip" },
"x64-win32": { platform: "x86_64-pc-windows-msvc", extension: "zip" },
} as const
export const ExtractionFailedError = NamedError.create( export interface SearchResult {
"RipgrepExtractionFailedError", items: Item[]
z.object({ partial: boolean
filepath: z.string(),
stderr: z.string(),
}),
)
export const UnsupportedPlatformError = NamedError.create(
"RipgrepUnsupportedPlatformError",
z.object({
platform: z.string(),
}),
)
export const DownloadFailedError = NamedError.create(
"RipgrepDownloadFailedError",
z.object({
url: z.string(),
status: z.number(),
}),
)
const state = lazy(async () => {
const system = which("rg")
if (system) {
const stat = await fs.stat(system).catch(() => undefined)
if (stat?.isFile()) return { filepath: system }
log.warn("bun.which returned invalid rg path", { filepath: system })
}
const filepath = path.join(Global.Path.bin, "rg" + (process.platform === "win32" ? ".exe" : ""))
if (!(await Filesystem.exists(filepath))) {
const platformKey = `${process.arch}-${process.platform}` as keyof typeof PLATFORM
const config = PLATFORM[platformKey]
if (!config) throw new UnsupportedPlatformError({ platform: platformKey })
const version = "14.1.1"
const filename = `ripgrep-${version}-${config.platform}.${config.extension}`
const url = `https://github.com/BurntSushi/ripgrep/releases/download/${version}/${filename}`
const response = await fetch(url)
if (!response.ok) throw new DownloadFailedError({ url, status: response.status })
const arrayBuffer = await response.arrayBuffer()
const archivePath = path.join(Global.Path.bin, filename)
await Filesystem.write(archivePath, Buffer.from(arrayBuffer))
if (config.extension === "tar.gz") {
const args = ["tar", "-xzf", archivePath, "--strip-components=1"]
if (platformKey.endsWith("-darwin")) args.push("--include=*/rg")
if (platformKey.endsWith("-linux")) args.push("--wildcards", "*/rg")
const proc = Process.spawn(args, {
cwd: Global.Path.bin,
stderr: "pipe",
stdout: "pipe",
})
const exit = await proc.exited
if (exit !== 0) {
const stderr = proc.stderr ? await text(proc.stderr) : ""
throw new ExtractionFailedError({
filepath,
stderr,
})
}
}
if (config.extension === "zip") {
const zipFileReader = new ZipReader(new BlobReader(new Blob([arrayBuffer])))
const entries = await zipFileReader.getEntries()
let rgEntry: any
for (const entry of entries) {
if (entry.filename.endsWith("rg.exe")) {
rgEntry = entry
break
}
} }
if (!rgEntry) { export interface FilesInput {
throw new ExtractionFailedError({
filepath: archivePath,
stderr: "rg.exe not found in zip archive",
})
}
const rgBlob = await rgEntry.getData(new BlobWriter())
if (!rgBlob) {
throw new ExtractionFailedError({
filepath: archivePath,
stderr: "Failed to extract rg.exe from zip archive",
})
}
await Filesystem.write(filepath, Buffer.from(await rgBlob.arrayBuffer()))
await zipFileReader.close()
}
await fs.unlink(archivePath)
if (!platformKey.endsWith("-win32")) await fs.chmod(filepath, 0o755)
}
return {
filepath,
}
})
export async function filepath() {
const { filepath } = await state()
return filepath
}
export async function* files(input: {
cwd: string cwd: string
glob?: string[] glob?: string[]
hidden?: boolean hidden?: boolean
follow?: boolean follow?: boolean
maxDepth?: number maxDepth?: number
signal?: AbortSignal signal?: AbortSignal
}) {
input.signal?.throwIfAborted()
const args = [await filepath(), "--files", "--glob=!.git/*"]
if (input.follow) args.push("--follow")
if (input.hidden !== false) args.push("--hidden")
if (input.maxDepth !== undefined) args.push(`--max-depth=${input.maxDepth}`)
if (input.glob) {
for (const g of input.glob) {
args.push(`--glob=${g}`)
}
} }
// Guard against invalid cwd to provide a consistent ENOENT error. export interface SearchInput {
if (!(await fs.stat(input.cwd).catch(() => undefined))?.isDirectory()) {
throw Object.assign(new Error(`No such file or directory: '${input.cwd}'`), {
code: "ENOENT",
errno: -2,
path: input.cwd,
})
}
const proc = Process.spawn(args, {
cwd: input.cwd,
stdout: "pipe",
stderr: "ignore",
abort: input.signal,
})
if (!proc.stdout) {
throw new Error("Process output not available")
}
let buffer = ""
const stream = proc.stdout as AsyncIterable<Buffer | string>
for await (const chunk of stream) {
input.signal?.throwIfAborted()
buffer += typeof chunk === "string" ? chunk : chunk.toString()
// Handle both Unix (\n) and Windows (\r\n) line endings
const lines = buffer.split(/\r?\n/)
buffer = lines.pop() || ""
for (const line of lines) {
if (line) yield line
}
}
if (buffer) yield buffer
await proc.exited
input.signal?.throwIfAborted()
}
export interface Interface {
readonly files: (input: {
cwd: string
glob?: string[]
hidden?: boolean
follow?: boolean
maxDepth?: number
}) => Stream.Stream<string, PlatformError>
readonly search: (input: {
cwd: string cwd: string
pattern: string pattern: string
glob?: string[] glob?: string[]
limit?: number limit?: number
follow?: boolean follow?: boolean
file?: string[] file?: string[]
}) => Effect.Effect<{ items: Item[]; partial: boolean }, PlatformError | Error> signal?: AbortSignal
}
export interface TreeInput {
cwd: string
limit?: number
signal?: AbortSignal
}
export interface Interface {
readonly files: (input: FilesInput) => Stream.Stream<string, Error>
readonly tree: (input: TreeInput) => Effect.Effect<string, Error>
readonly search: (input: SearchInput) => Effect.Effect<SearchResult, Error>
} }
export class Service extends Context.Service<Service, Interface>()("@opencode/Ripgrep") {} export class Service extends Context.Service<Service, Interface>()("@opencode/Ripgrep") {}
export const layer: Layer.Layer<Service, never, ChildProcessSpawner | AppFileSystem.Service> = Layer.effect( type Run = { kind: "files" | "search"; cwd: string; args: string[] }
type WorkerResult = {
type: "result"
code: number
stdout: string
stderr: string
}
type WorkerLine = {
type: "line"
line: string
}
type WorkerDone = {
type: "done"
code: number
stderr: string
}
type WorkerError = {
type: "error"
error: {
message: string
name?: string
stack?: string
}
}
function env() {
const env = Object.fromEntries(
Object.entries(process.env).filter((item): item is [string, string] => item[1] !== undefined),
)
delete env.RIPGREP_CONFIG_PATH
return env
}
function text(input: unknown) {
if (typeof input === "string") return input
if (input instanceof ArrayBuffer) return Buffer.from(input).toString()
if (ArrayBuffer.isView(input)) return Buffer.from(input.buffer, input.byteOffset, input.byteLength).toString()
return String(input)
}
function toError(input: unknown) {
if (input instanceof Error) return input
if (typeof input === "string") return new Error(input)
return new Error(String(input))
}
function abort(signal?: AbortSignal) {
const err = signal?.reason
if (err instanceof Error) return err
const out = new Error("Aborted")
out.name = "AbortError"
return out
}
function error(stderr: string, code: number) {
const err = new Error(stderr.trim() || `ripgrep failed with code ${code}`)
err.name = "RipgrepError"
return err
}
function clean(file: string) {
return path.normalize(file.replace(/^\.[\\/]/, ""))
}
function row(data: Row): Row {
return {
...data,
path: {
...data.path,
text: clean(data.path.text),
},
}
}
function opts(cwd: string) {
return {
env: env(),
preopens: { ".": cwd },
}
}
function check(cwd: string) {
return Effect.tryPromise({
try: () => fs.stat(cwd).catch(() => undefined),
catch: toError,
}).pipe(
Effect.flatMap((stat) =>
stat?.isDirectory()
? Effect.void
: Effect.fail(
Object.assign(new Error(`No such file or directory: '${cwd}'`), {
code: "ENOENT",
errno: -2,
path: cwd,
}),
),
),
)
}
function filesArgs(input: FilesInput) {
const args = ["--files", "--glob=!.git/*"]
if (input.follow) args.push("--follow")
if (input.hidden !== false) args.push("--hidden")
if (input.maxDepth !== undefined) args.push(`--max-depth=${input.maxDepth}`)
if (input.glob) {
for (const glob of input.glob) {
args.push(`--glob=${glob}`)
}
}
args.push(".")
return args
}
function searchArgs(input: SearchInput) {
const args = ["--json", "--hidden", "--glob=!.git/*", "--no-messages"]
if (input.follow) args.push("--follow")
if (input.glob) {
for (const glob of input.glob) {
args.push(`--glob=${glob}`)
}
}
if (input.limit) args.push(`--max-count=${input.limit}`)
args.push("--", input.pattern, ...(input.file ?? ["."]))
return args
}
function parse(stdout: string) {
return stdout
.trim()
.split(/\r?\n/)
.filter(Boolean)
.map((line) => Result.parse(JSON.parse(line)))
.flatMap((item) => (item.type === "match" ? [row(item.data)] : []))
}
function target() {
const js = new URL("./ripgrep.worker.js", import.meta.url)
return Effect.tryPromise({
try: () => Filesystem.exists(fileURLToPath(js)),
catch: toError,
}).pipe(Effect.map((exists) => (exists ? js : new URL("./ripgrep.worker.ts", import.meta.url))))
}
function worker() {
return target().pipe(Effect.flatMap((file) => Effect.sync(() => new Worker(file, { env: env() }))))
}
function drain(buf: string, chunk: unknown, push: (line: string) => void) {
const lines = (buf + text(chunk)).split(/\r?\n/)
buf = lines.pop() || ""
for (const line of lines) {
if (line) push(line)
}
return buf
}
function fail(queue: Queue.Queue<string, Error | Cause.Done>, err: Error) {
Queue.failCauseUnsafe(queue, Cause.fail(err))
}
function searchDirect(input: SearchInput) {
return Effect.tryPromise({
try: () =>
ripgrep(searchArgs(input), {
buffer: true,
...opts(input.cwd),
}),
catch: toError,
}).pipe(
Effect.flatMap((ret) => {
const out = ret.stdout ?? ""
if (ret.code !== 0 && ret.code !== 1 && ret.code !== 2) {
return Effect.fail(error(ret.stderr ?? "", ret.code ?? 1))
}
return Effect.sync(() => ({
items: ret.code === 1 ? [] : parse(out),
partial: ret.code === 2,
}))
}),
)
}
function searchWorker(input: SearchInput) {
if (input.signal?.aborted) return Effect.fail(abort(input.signal))
return Effect.acquireUseRelease(
worker(),
(w) =>
Effect.callback<SearchResult, Error>((resume, signal) => {
let open = true
const done = (effect: Effect.Effect<SearchResult, Error>) => {
if (!open) return
open = false
resume(effect)
}
const onabort = () => done(Effect.fail(abort(input.signal)))
w.onerror = (evt) => {
done(Effect.fail(toError(evt.error ?? evt.message)))
}
w.onmessage = (evt: MessageEvent<WorkerResult | WorkerError>) => {
const msg = evt.data
if (msg.type === "error") {
done(Effect.fail(Object.assign(new Error(msg.error.message), msg.error)))
return
}
if (msg.code === 1) {
done(Effect.succeed({ items: [], partial: false }))
return
}
if (msg.code !== 0 && msg.code !== 1 && msg.code !== 2) {
done(Effect.fail(error(msg.stderr, msg.code)))
return
}
done(
Effect.sync(() => ({
items: parse(msg.stdout),
partial: msg.code === 2,
})),
)
}
input.signal?.addEventListener("abort", onabort, { once: true })
signal.addEventListener("abort", onabort, { once: true })
w.postMessage({
kind: "search",
cwd: input.cwd,
args: searchArgs(input),
} satisfies Run)
return Effect.sync(() => {
input.signal?.removeEventListener("abort", onabort)
signal.removeEventListener("abort", onabort)
w.onerror = null
w.onmessage = null
})
}),
(w) => Effect.sync(() => w.terminate()),
)
}
function filesDirect(input: FilesInput) {
return Stream.callback<string, Error>(
Effect.fnUntraced(function* (queue: Queue.Queue<string, Error | Cause.Done>) {
let buf = ""
let err = ""
const out = {
write(chunk: unknown) {
buf = drain(buf, chunk, (line) => {
Queue.offerUnsafe(queue, clean(line))
})
},
}
const stderr = {
write(chunk: unknown) {
err += text(chunk)
},
}
yield* Effect.forkScoped(
Effect.gen(function* () {
yield* check(input.cwd)
const ret = yield* Effect.tryPromise({
try: () =>
ripgrep(filesArgs(input), {
stdout: out,
stderr,
...opts(input.cwd),
}),
catch: toError,
})
if (buf) Queue.offerUnsafe(queue, clean(buf))
if (ret.code === 0 || ret.code === 1) {
Queue.endUnsafe(queue)
return
}
fail(queue, error(err, ret.code ?? 1))
}).pipe(
Effect.catch((err) =>
Effect.sync(() => {
fail(queue, err)
}),
),
),
)
}),
)
}
function filesWorker(input: FilesInput) {
return Stream.callback<string, Error>(
Effect.fnUntraced(function* (queue: Queue.Queue<string, Error | Cause.Done>) {
if (input.signal?.aborted) {
fail(queue, abort(input.signal))
return
}
const w = yield* Effect.acquireRelease(worker(), (w) => Effect.sync(() => w.terminate()))
let open = true
const close = () => {
if (!open) return false
open = false
return true
}
const onabort = () => {
if (!close()) return
fail(queue, abort(input.signal))
}
w.onerror = (evt) => {
if (!close()) return
fail(queue, toError(evt.error ?? evt.message))
}
w.onmessage = (evt: MessageEvent<WorkerLine | WorkerDone | WorkerError>) => {
const msg = evt.data
if (msg.type === "line") {
if (open) Queue.offerUnsafe(queue, msg.line)
return
}
if (!close()) return
if (msg.type === "error") {
fail(queue, Object.assign(new Error(msg.error.message), msg.error))
return
}
if (msg.code === 0 || msg.code === 1) {
Queue.endUnsafe(queue)
return
}
fail(queue, error(msg.stderr, msg.code))
}
yield* Effect.acquireRelease(
Effect.sync(() => {
input.signal?.addEventListener("abort", onabort, { once: true })
w.postMessage({
kind: "files",
cwd: input.cwd,
args: filesArgs(input),
} satisfies Run)
}),
() =>
Effect.sync(() => {
input.signal?.removeEventListener("abort", onabort)
w.onerror = null
w.onmessage = null
}),
)
}),
)
}
export const layer = Layer.effect(
Service, Service,
Effect.gen(function* () { Effect.gen(function* () {
const spawner = yield* ChildProcessSpawner const source = (input: FilesInput) => {
const afs = yield* AppFileSystem.Service const useWorker = !!input.signal && typeof Worker !== "undefined"
const bin = Effect.fn("Ripgrep.path")(function* () { if (!useWorker && input.signal) {
return yield* Effect.promise(() => filepath()) log.warn("worker unavailable, ripgrep abort disabled")
})
const args = Effect.fn("Ripgrep.args")(function* (input: {
mode: "files" | "search"
glob?: string[]
hidden?: boolean
follow?: boolean
maxDepth?: number
limit?: number
pattern?: string
file?: string[]
}) {
const out = [yield* bin(), input.mode === "search" ? "--json" : "--files", "--glob=!.git/*"]
if (input.follow) out.push("--follow")
if (input.hidden !== false) out.push("--hidden")
if (input.maxDepth !== undefined) out.push(`--max-depth=${input.maxDepth}`)
if (input.glob) {
for (const g of input.glob) {
out.push(`--glob=${g}`)
} }
} return useWorker ? filesWorker(input) : filesDirect(input)
if (input.limit) out.push(`--max-count=${input.limit}`)
if (input.mode === "search") out.push("--no-messages")
if (input.pattern) out.push("--", input.pattern, ...(input.file ?? []))
return out
})
const files = Effect.fn("Ripgrep.files")(function* (input: {
cwd: string
glob?: string[]
hidden?: boolean
follow?: boolean
maxDepth?: number
}) {
const rgPath = yield* bin()
const isDir = yield* afs.isDir(input.cwd)
if (!isDir) {
return yield* Effect.die(
Object.assign(new Error(`No such file or directory: '${input.cwd}'`), {
code: "ENOENT" as const,
errno: -2,
path: input.cwd,
}),
)
} }
const cmd = yield* args({ const files: Interface["files"] = (input) => source(input)
mode: "files",
glob: input.glob,
hidden: input.hidden,
follow: input.follow,
maxDepth: input.maxDepth,
})
return spawner const tree: Interface["tree"] = Effect.fn("Ripgrep.tree")(function* (input: TreeInput) {
.streamLines(ChildProcess.make(cmd[0], cmd.slice(1), { cwd: input.cwd }))
.pipe(Stream.filter((line: string) => line.length > 0))
})
const search = Effect.fn("Ripgrep.search")(function* (input: {
cwd: string
pattern: string
glob?: string[]
limit?: number
follow?: boolean
file?: string[]
}) {
return yield* Effect.scoped(
Effect.gen(function* () {
const cmd = yield* args({
mode: "search",
glob: input.glob,
follow: input.follow,
limit: input.limit,
pattern: input.pattern,
file: input.file,
})
const handle = yield* spawner.spawn(
ChildProcess.make(cmd[0], cmd.slice(1), {
cwd: input.cwd,
stdin: "ignore",
}),
)
const [items, stderr, code] = yield* Effect.all(
[
Stream.decodeText(handle.stdout).pipe(
Stream.splitLines,
Stream.filter((line) => line.length > 0),
Stream.mapEffect((line) =>
decode(line).pipe(Effect.mapError((cause) => new Error("invalid ripgrep output", { cause }))),
),
Stream.filter((row): row is Schema.Schema.Type<typeof Hit> => row.type === "match"),
Stream.map((row): Item => row.data),
Stream.runCollect,
Effect.map((chunk) => [...chunk]),
),
Stream.mkString(Stream.decodeText(handle.stderr)),
handle.exitCode,
],
{ concurrency: "unbounded" },
)
if (code !== 0 && code !== 1 && code !== 2) {
return yield* Effect.fail(new Error(`ripgrep failed: ${stderr}`))
}
return {
items,
partial: code === 2,
}
}),
)
})
return Service.of({
files: (input) => Stream.unwrap(files(input)),
search,
})
}),
)
export const defaultLayer = layer.pipe(
Layer.provide(AppFileSystem.defaultLayer),
Layer.provide(CrossSpawnSpawner.defaultLayer),
)
export async function tree(input: { cwd: string; limit?: number; signal?: AbortSignal }) {
log.info("tree", input) log.info("tree", input)
const files = await Array.fromAsync(Ripgrep.files({ cwd: input.cwd, signal: input.signal })) const list = Array.from(yield* source({ cwd: input.cwd, signal: input.signal }).pipe(Stream.runCollect))
interface Node { interface Node {
name: string name: string
children: Map<string, Node> children: Map<string, Node>
} }
function dir(node: Node, name: string) { function child(node: Node, name: string) {
const existing = node.children.get(name) const item = node.children.get(name)
if (existing) return existing if (item) return item
const next = { name, children: new Map() } const next = { name, children: new Map() }
node.children.set(name, next) node.children.set(name, next)
return next return next
} }
function count(node: Node): number {
return Array.from(node.children.values()).reduce((sum, child) => sum + 1 + count(child), 0)
}
const root: Node = { name: "", children: new Map() } const root: Node = { name: "", children: new Map() }
for (const file of files) { for (const file of list) {
if (file.includes(".opencode")) continue if (file.includes(".opencode")) continue
const parts = file.split(path.sep) const parts = file.split(path.sep)
if (parts.length < 2) continue if (parts.length < 2) continue
let node = root let node = root
for (const part of parts.slice(0, -1)) { for (const part of parts.slice(0, -1)) {
node = dir(node, part) node = child(node, part)
} }
} }
function count(node: Node): number {
let total = 0
for (const child of node.children.values()) {
total += 1 + count(child)
}
return total
}
const total = count(root) const total = count(root)
const limit = input.limit ?? total const limit = input.limit ?? total
const lines: string[] = [] const lines: string[] = []
const queue: { node: Node; path: string }[] = [] const queue: Array<{ node: Node; path: string }> = Array.from(root.children.values())
for (const child of Array.from(root.children.values()).sort((a, b) => a.name.localeCompare(b.name))) { .sort((a, b) => a.name.localeCompare(b.name))
queue.push({ node: child, path: child.name }) .map((node) => ({ node, path: node.name }))
}
let used = 0 let used = 0
for (let i = 0; i < queue.length && used < limit; i++) { for (let i = 0; i < queue.length && used < limit; i++) {
const { node, path } = queue[i] const item = queue[i]
lines.push(path) lines.push(item.path)
used++ used++
for (const child of Array.from(node.children.values()).sort((a, b) => a.name.localeCompare(b.name))) { queue.push(
queue.push({ node: child, path: `${path}/${child.name}` }) ...Array.from(item.node.children.values())
} .sort((a, b) => a.name.localeCompare(b.name))
.map((node) => ({ node, path: `${item.path}/${node.name}` })),
)
} }
if (total > used) lines.push(`[${total - used} truncated]`) if (total > used) lines.push(`[${total - used} truncated]`)
return lines.join("\n") return lines.join("\n")
})
const search: Interface["search"] = Effect.fn("Ripgrep.search")(function* (input: SearchInput) {
const useWorker = !!input.signal && typeof Worker !== "undefined"
if (!useWorker && input.signal) {
log.warn("worker unavailable, ripgrep abort disabled")
}
return yield* useWorker ? searchWorker(input) : searchDirect(input)
})
return Service.of({ files, tree, search })
}),
)
export const defaultLayer = layer
const { runPromise } = makeRuntime(Service, defaultLayer)
export function files(input: FilesInput) {
return runPromise((svc) => Stream.toAsyncIterableEffect(svc.files(input)))
}
export function tree(input: TreeInput) {
return runPromise((svc) => svc.tree(input))
}
export function search(input: SearchInput) {
return runPromise((svc) => svc.search(input))
} }
} }
@@ -0,0 +1,103 @@
import { ripgrep } from "ripgrep"
function env() {
const env = Object.fromEntries(
Object.entries(process.env).filter((item): item is [string, string] => item[1] !== undefined),
)
delete env.RIPGREP_CONFIG_PATH
return env
}
function opts(cwd: string) {
return {
env: env(),
preopens: { ".": cwd },
}
}
type Run = {
kind: "files" | "search"
cwd: string
args: string[]
}
function text(input: unknown) {
if (typeof input === "string") return input
if (input instanceof ArrayBuffer) return Buffer.from(input).toString()
if (ArrayBuffer.isView(input)) return Buffer.from(input.buffer, input.byteOffset, input.byteLength).toString()
return String(input)
}
function error(input: unknown) {
if (input instanceof Error) {
return {
message: input.message,
name: input.name,
stack: input.stack,
}
}
return {
message: String(input),
}
}
function clean(file: string) {
return file.replace(/^\.[\\/]/, "")
}
onmessage = async (evt: MessageEvent<Run>) => {
const msg = evt.data
try {
if (msg.kind === "search") {
const ret = await ripgrep(msg.args, {
buffer: true,
...opts(msg.cwd),
})
postMessage({
type: "result",
code: ret.code ?? 0,
stdout: ret.stdout ?? "",
stderr: ret.stderr ?? "",
})
return
}
let buf = ""
let err = ""
const out = {
write(chunk: unknown) {
buf += text(chunk)
const lines = buf.split(/\r?\n/)
buf = lines.pop() || ""
for (const line of lines) {
if (line) postMessage({ type: "line", line: clean(line) })
}
},
}
const stderr = {
write(chunk: unknown) {
err += text(chunk)
},
}
const ret = await ripgrep(msg.args, {
stdout: out,
stderr,
...opts(msg.cwd),
})
if (buf) postMessage({ type: "line", line: clean(buf) })
postMessage({
type: "done",
code: ret.code ?? 0,
stderr: err,
})
} catch (err) {
postMessage({
type: "error",
error: error(err),
})
}
}
+4 -3
View File
@@ -46,7 +46,7 @@ import { Process } from "@/util/process"
import { Cause, Effect, Exit, Layer, Option, Scope, Context } from "effect" import { Cause, Effect, Exit, Layer, Option, Scope, Context } from "effect"
import { EffectLogger } from "@/effect/logger" import { EffectLogger } from "@/effect/logger"
import { InstanceState } from "@/effect/instance-state" import { InstanceState } from "@/effect/instance-state"
import { makeRuntime } from "@/effect/run-service" import { attach, makeRuntime } from "@/effect/run-service"
import { TaskTool, type TaskPromptOps } from "@/tool/task" import { TaskTool, type TaskPromptOps } from "@/tool/task"
import { SessionRunState } from "./run-state" import { SessionRunState } from "./run-state"
@@ -108,8 +108,9 @@ export namespace SessionPrompt {
const run = { const run = {
promise: <A, E>(effect: Effect.Effect<A, E>) => promise: <A, E>(effect: Effect.Effect<A, E>) =>
Effect.runPromise(effect.pipe(Effect.provide(EffectLogger.layer))), Effect.runPromise(attach(effect).pipe(Effect.provide(EffectLogger.layer))),
fork: <A, E>(effect: Effect.Effect<A, E>) => Effect.runFork(effect.pipe(Effect.provide(EffectLogger.layer))), fork: <A, E>(effect: Effect.Effect<A, E>) =>
Effect.runFork(attach(effect).pipe(Effect.provide(EffectLogger.layer))),
} }
const cancel = Effect.fn("SessionPrompt.cancel")(function* (sessionID: SessionID) { const cancel = Effect.fn("SessionPrompt.cancel")(function* (sessionID: SessionID) {
@@ -1,6 +1,7 @@
import path from "path" import path from "path"
import { Effect } from "effect" import { Effect } from "effect"
import { EffectLogger } from "@/effect/logger" import { EffectLogger } from "@/effect/logger"
import { InstanceState } from "@/effect/instance-state"
import type { Tool } from "./tool" import type { Tool } from "./tool"
import { Instance } from "../project/instance" import { Instance } from "../project/instance"
import { AppFileSystem } from "../filesystem" import { AppFileSystem } from "../filesystem"
@@ -21,8 +22,9 @@ export const assertExternalDirectoryEffect = Effect.fn("Tool.assertExternalDirec
if (options?.bypass) return if (options?.bypass) return
const ins = yield* InstanceState.context
const full = process.platform === "win32" ? AppFileSystem.normalizePath(target) : target const full = process.platform === "win32" ? AppFileSystem.normalizePath(target) : target
if (Instance.containsPath(full)) return if (Instance.containsPath(full, ins)) return
const kind = options?.kind ?? "file" const kind = options?.kind ?? "file"
const dir = kind === "directory" ? full : path.dirname(full) const dir = kind === "directory" ? full : path.dirname(full)
+13 -12
View File
@@ -1,13 +1,13 @@
import z from "zod"
import path from "path" import path from "path"
import z from "zod"
import { Effect, Option } from "effect" import { Effect, Option } from "effect"
import * as Stream from "effect/Stream" import * as Stream from "effect/Stream"
import { Tool } from "./tool" import { InstanceState } from "@/effect/instance-state"
import DESCRIPTION from "./glob.txt"
import { Ripgrep } from "../file/ripgrep"
import { Instance } from "../project/instance"
import { assertExternalDirectoryEffect } from "./external-directory"
import { AppFileSystem } from "../filesystem" import { AppFileSystem } from "../filesystem"
import { Ripgrep } from "../file/ripgrep"
import { assertExternalDirectoryEffect } from "./external-directory"
import DESCRIPTION from "./glob.txt"
import { Tool } from "./tool"
export const GlobTool = Tool.define( export const GlobTool = Tool.define(
"glob", "glob",
@@ -28,6 +28,7 @@ export const GlobTool = Tool.define(
}), }),
execute: (params: { pattern: string; path?: string }, ctx: Tool.Context) => execute: (params: { pattern: string; path?: string }, ctx: Tool.Context) =>
Effect.gen(function* () { Effect.gen(function* () {
const ins = yield* InstanceState.context
yield* ctx.ask({ yield* ctx.ask({
permission: "glob", permission: "glob",
patterns: [params.pattern], patterns: [params.pattern],
@@ -38,8 +39,8 @@ export const GlobTool = Tool.define(
}, },
}) })
let search = params.path ?? Instance.directory let search = params.path ?? ins.directory
search = path.isAbsolute(search) ? search : path.resolve(Instance.directory, search) search = path.isAbsolute(search) ? search : path.resolve(ins.directory, search)
const info = yield* fs.stat(search).pipe(Effect.catch(() => Effect.succeed(undefined))) const info = yield* fs.stat(search).pipe(Effect.catch(() => Effect.succeed(undefined)))
if (info?.type === "File") { if (info?.type === "File") {
throw new Error(`glob path must be a directory: ${search}`) throw new Error(`glob path must be a directory: ${search}`)
@@ -48,14 +49,14 @@ export const GlobTool = Tool.define(
const limit = 100 const limit = 100
let truncated = false let truncated = false
const files = yield* rg.files({ cwd: search, glob: [params.pattern] }).pipe( const files = yield* rg.files({ cwd: search, glob: [params.pattern], signal: ctx.abort }).pipe(
Stream.mapEffect((file) => Stream.mapEffect((file) =>
Effect.gen(function* () { Effect.gen(function* () {
const full = path.resolve(search, file) const full = path.resolve(search, file)
const info = yield* fs.stat(full).pipe(Effect.catch(() => Effect.succeed(undefined))) const info = yield* fs.stat(full).pipe(Effect.catch(() => Effect.succeed(undefined)))
const mtime = const mtime =
info?.mtime.pipe( info?.mtime.pipe(
Option.map((d) => d.getTime()), Option.map((date) => date.getTime()),
Option.getOrElse(() => 0), Option.getOrElse(() => 0),
) ?? 0 ) ?? 0
return { path: full, mtime } return { path: full, mtime }
@@ -75,7 +76,7 @@ export const GlobTool = Tool.define(
const output = [] const output = []
if (files.length === 0) output.push("No files found") if (files.length === 0) output.push("No files found")
if (files.length > 0) { if (files.length > 0) {
output.push(...files.map((f) => f.path)) output.push(...files.map((file) => file.path))
if (truncated) { if (truncated) {
output.push("") output.push("")
output.push( output.push(
@@ -85,7 +86,7 @@ export const GlobTool = Tool.define(
} }
return { return {
title: path.relative(Instance.worktree, search), title: path.relative(ins.worktree, search),
metadata: { metadata: {
count: files.length, count: files.length,
truncated, truncated,
+34 -37
View File
@@ -1,13 +1,12 @@
import path from "path"
import z from "zod" import z from "zod"
import { Effect, Option } from "effect" import { Effect, Option } from "effect"
import { Tool } from "./tool" import { InstanceState } from "@/effect/instance-state"
import { Ripgrep } from "../file/ripgrep"
import { AppFileSystem } from "../filesystem" import { AppFileSystem } from "../filesystem"
import { Ripgrep } from "../file/ripgrep"
import DESCRIPTION from "./grep.txt"
import { Instance } from "../project/instance"
import path from "path"
import { assertExternalDirectoryEffect } from "./external-directory" import { assertExternalDirectoryEffect } from "./external-directory"
import DESCRIPTION from "./grep.txt"
import { Tool } from "./tool"
const MAX_LINE_LENGTH = 2000 const MAX_LINE_LENGTH = 2000
@@ -46,15 +45,16 @@ export const GrepTool = Tool.define(
}, },
}) })
const searchPath = AppFileSystem.resolve( const ins = yield* InstanceState.context
path.isAbsolute(params.path ?? Instance.directory) const search = AppFileSystem.resolve(
? (params.path ?? Instance.directory) path.isAbsolute(params.path ?? ins.directory)
: path.join(Instance.directory, params.path ?? "."), ? (params.path ?? ins.directory)
: path.join(ins.directory, params.path ?? "."),
) )
const info = yield* fs.stat(searchPath).pipe(Effect.catch(() => Effect.succeed(undefined))) const info = yield* fs.stat(search).pipe(Effect.catch(() => Effect.succeed(undefined)))
const cwd = info?.type === "Directory" ? searchPath : path.dirname(searchPath) const cwd = info?.type === "Directory" ? search : path.dirname(search)
const file = info?.type === "Directory" ? undefined : [searchPath] const file = info?.type === "Directory" ? undefined : [path.relative(cwd, search)]
yield* assertExternalDirectoryEffect(ctx, searchPath, { yield* assertExternalDirectoryEffect(ctx, search, {
kind: info?.type === "Directory" ? "directory" : "file", kind: info?.type === "Directory" ? "directory" : "file",
}) })
@@ -63,8 +63,8 @@ export const GrepTool = Tool.define(
pattern: params.pattern, pattern: params.pattern,
glob: params.include ? [params.include] : undefined, glob: params.include ? [params.include] : undefined,
file, file,
signal: ctx.abort,
}) })
if (result.items.length === 0) return empty if (result.items.length === 0) return empty
const rows = result.items.map((item) => ({ const rows = result.items.map((item) => ({
@@ -101,46 +101,43 @@ export const GrepTool = Tool.define(
const limit = 100 const limit = 100
const truncated = matches.length > limit const truncated = matches.length > limit
const finalMatches = truncated ? matches.slice(0, limit) : matches const final = truncated ? matches.slice(0, limit) : matches
if (final.length === 0) return empty
if (finalMatches.length === 0) return empty const total = matches.length
const output = [`Found ${total} matches${truncated ? ` (showing first ${limit})` : ""}`]
const totalMatches = matches.length let current = ""
const outputLines = [`Found ${totalMatches} matches${truncated ? ` (showing first ${limit})` : ""}`] for (const match of final) {
if (current !== match.path) {
let currentFile = "" if (current !== "") output.push("")
for (const match of finalMatches) { current = match.path
if (currentFile !== match.path) { output.push(`${match.path}:`)
if (currentFile !== "") {
outputLines.push("")
} }
currentFile = match.path const text =
outputLines.push(`${match.path}:`)
}
const truncatedLineText =
match.text.length > MAX_LINE_LENGTH ? match.text.substring(0, MAX_LINE_LENGTH) + "..." : match.text match.text.length > MAX_LINE_LENGTH ? match.text.substring(0, MAX_LINE_LENGTH) + "..." : match.text
outputLines.push(` Line ${match.line}: ${truncatedLineText}`) output.push(` Line ${match.line}: ${text}`)
} }
if (truncated) { if (truncated) {
outputLines.push("") output.push("")
outputLines.push( output.push(
`(Results truncated: showing ${limit} of ${totalMatches} matches (${totalMatches - limit} hidden). Consider using a more specific path or pattern.)`, `(Results truncated: showing ${limit} of ${total} matches (${total - limit} hidden). Consider using a more specific path or pattern.)`,
) )
} }
if (result.partial) { if (result.partial) {
outputLines.push("") output.push("")
outputLines.push("(Some paths were inaccessible and skipped)") output.push("(Some paths were inaccessible and skipped)")
} }
return { return {
title: params.pattern, title: params.pattern,
metadata: { metadata: {
matches: totalMatches, matches: total,
truncated, truncated,
}, },
output: outputLines.join("\n"), output: output.join("\n"),
} }
}).pipe(Effect.orDie), }).pipe(Effect.orDie),
} }
+32 -44
View File
@@ -1,12 +1,12 @@
import * as path from "path"
import z from "zod" import z from "zod"
import { Effect } from "effect" import { Effect } from "effect"
import * as Stream from "effect/Stream" import * as Stream from "effect/Stream"
import { Tool } from "./tool" import { InstanceState } from "@/effect/instance-state"
import * as path from "path"
import DESCRIPTION from "./ls.txt"
import { Instance } from "../project/instance"
import { Ripgrep } from "../file/ripgrep" import { Ripgrep } from "../file/ripgrep"
import { assertExternalDirectoryEffect } from "./external-directory" import { assertExternalDirectoryEffect } from "./external-directory"
import DESCRIPTION from "./ls.txt"
import { Tool } from "./tool"
export const IGNORE_PATTERNS = [ export const IGNORE_PATTERNS = [
"node_modules/", "node_modules/",
@@ -53,80 +53,68 @@ export const ListTool = Tool.define(
}), }),
execute: (params: { path?: string; ignore?: string[] }, ctx: Tool.Context) => execute: (params: { path?: string; ignore?: string[] }, ctx: Tool.Context) =>
Effect.gen(function* () { Effect.gen(function* () {
const searchPath = path.resolve(Instance.directory, params.path || ".") const ins = yield* InstanceState.context
yield* assertExternalDirectoryEffect(ctx, searchPath, { kind: "directory" }) const search = path.resolve(ins.directory, params.path || ".")
yield* assertExternalDirectoryEffect(ctx, search, { kind: "directory" })
yield* ctx.ask({ yield* ctx.ask({
permission: "list", permission: "list",
patterns: [searchPath], patterns: [search],
always: ["*"], always: ["*"],
metadata: { metadata: {
path: searchPath, path: search,
}, },
}) })
const ignoreGlobs = IGNORE_PATTERNS.map((p) => `!${p}*`).concat(params.ignore?.map((p) => `!${p}`) || []) const glob = IGNORE_PATTERNS.map((item) => `!${item}*`).concat(params.ignore?.map((item) => `!${item}`) || [])
const files = yield* rg.files({ cwd: searchPath, glob: ignoreGlobs }).pipe( const files = yield* rg.files({ cwd: search, glob, signal: ctx.abort }).pipe(
Stream.take(LIMIT), Stream.take(LIMIT + 1),
Stream.runCollect, Stream.runCollect,
Effect.map((chunk) => [...chunk]), Effect.map((chunk) => [...chunk]),
) )
// Build directory structure const truncated = files.length > LIMIT
const dirs = new Set<string>() if (truncated) files.length = LIMIT
const filesByDir = new Map<string, string[]>()
const dirs = new Set<string>()
const map = new Map<string, string[]>()
for (const file of files) { for (const file of files) {
const dir = path.dirname(file) const dir = path.dirname(file)
const parts = dir === "." ? [] : dir.split("/") const parts = dir === "." ? [] : dir.split("/")
// Add all parent directories
for (let i = 0; i <= parts.length; i++) { for (let i = 0; i <= parts.length; i++) {
const dirPath = i === 0 ? "." : parts.slice(0, i).join("/") dirs.add(i === 0 ? "." : parts.slice(0, i).join("/"))
dirs.add(dirPath) }
if (!map.has(dir)) map.set(dir, [])
map.get(dir)!.push(path.basename(file))
} }
// Add file to its directory function render(dir: string, depth: number): string {
if (!filesByDir.has(dir)) filesByDir.set(dir, [])
filesByDir.get(dir)!.push(path.basename(file))
}
function renderDir(dirPath: string, depth: number): string {
const indent = " ".repeat(depth) const indent = " ".repeat(depth)
let output = "" let output = ""
if (depth > 0) output += `${indent}${path.basename(dir)}/\n`
if (depth > 0) { const child = " ".repeat(depth + 1)
output += `${indent}${path.basename(dirPath)}/\n` const dirs2 = Array.from(dirs)
} .filter((item) => path.dirname(item) === dir && item !== dir)
const childIndent = " ".repeat(depth + 1)
const children = Array.from(dirs)
.filter((d) => path.dirname(d) === dirPath && d !== dirPath)
.sort() .sort()
for (const item of dirs2) {
// Render subdirectories first output += render(item, depth + 1)
for (const child of children) {
output += renderDir(child, depth + 1)
} }
// Render files const files = map.get(dir) || []
const files = filesByDir.get(dirPath) || []
for (const file of files.sort()) { for (const file of files.sort()) {
output += `${childIndent}${file}\n` output += `${child}${file}\n`
} }
return output return output
} }
const output = `${searchPath}/\n` + renderDir(".", 0)
return { return {
title: path.relative(Instance.worktree, searchPath), title: path.relative(ins.worktree, search),
metadata: { metadata: {
count: files.length, count: files.length,
truncated: files.length >= LIMIT, truncated,
}, },
output, output: `${search}/\n` + render(".", 0),
} }
}).pipe(Effect.orDie), }).pipe(Effect.orDie),
} }
+6 -7
View File
@@ -2,11 +2,11 @@ import path from "path"
import { pathToFileURL } from "url" import { pathToFileURL } from "url"
import z from "zod" import z from "zod"
import { Effect } from "effect" import { Effect } from "effect"
import { EffectLogger } from "@/effect/logger"
import * as Stream from "effect/Stream" import * as Stream from "effect/Stream"
import { Tool } from "./tool" import { EffectLogger } from "@/effect/logger"
import { Skill } from "../skill"
import { Ripgrep } from "../file/ripgrep" import { Ripgrep } from "../file/ripgrep"
import { Skill } from "../skill"
import { Tool } from "./tool"
const Parameters = z.object({ const Parameters = z.object({
name: z.string().describe("The name of the skill from available_skills"), name: z.string().describe("The name of the skill from available_skills"),
@@ -17,6 +17,7 @@ export const SkillTool = Tool.define(
Effect.gen(function* () { Effect.gen(function* () {
const skill = yield* Skill.Service const skill = yield* Skill.Service
const rg = yield* Ripgrep.Service const rg = yield* Ripgrep.Service
return () => return () =>
Effect.gen(function* () { Effect.gen(function* () {
const list = yield* skill.available().pipe(Effect.provide(EffectLogger.layer)) const list = yield* skill.available().pipe(Effect.provide(EffectLogger.layer))
@@ -45,10 +46,9 @@ export const SkillTool = Tool.define(
execute: (params: z.infer<typeof Parameters>, ctx: Tool.Context) => execute: (params: z.infer<typeof Parameters>, ctx: Tool.Context) =>
Effect.gen(function* () { Effect.gen(function* () {
const info = yield* skill.get(params.name) const info = yield* skill.get(params.name)
if (!info) { if (!info) {
const all = yield* skill.all() const all = yield* skill.all()
const available = all.map((s) => s.name).join(", ") const available = all.map((item) => item.name).join(", ")
throw new Error(`Skill "${params.name}" not found. Available skills: ${available || "none"}`) throw new Error(`Skill "${params.name}" not found. Available skills: ${available || "none"}`)
} }
@@ -61,9 +61,8 @@ export const SkillTool = Tool.define(
const dir = path.dirname(info.location) const dir = path.dirname(info.location)
const base = pathToFileURL(dir).href const base = pathToFileURL(dir).href
const limit = 10 const limit = 10
const files = yield* rg.files({ cwd: dir, follow: false, hidden: true }).pipe( const files = yield* rg.files({ cwd: dir, follow: false, hidden: true, signal: ctx.abort }).pipe(
Stream.filter((file) => !file.includes("SKILL.md")), Stream.filter((file) => !file.includes("SKILL.md")),
Stream.map((file) => path.resolve(dir, file)), Stream.map((file) => path.resolve(dir, file)),
Stream.take(limit), Stream.take(limit),
+130 -17
View File
@@ -6,6 +6,21 @@ import path from "path"
import { tmpdir } from "../fixture/fixture" import { tmpdir } from "../fixture/fixture"
import { Ripgrep } from "../../src/file/ripgrep" import { Ripgrep } from "../../src/file/ripgrep"
async function seed(dir: string, count: number, size = 16) {
const txt = "a".repeat(size)
await Promise.all(Array.from({ length: count }, (_, i) => Bun.write(path.join(dir, `file-${i}.txt`), `${txt}${i}\n`)))
}
function env(name: string, value: string | undefined) {
const prev = process.env[name]
if (value === undefined) delete process.env[name]
else process.env[name] = value
return () => {
if (prev === undefined) delete process.env[name]
else process.env[name] = prev
}
}
describe("file.ripgrep", () => { describe("file.ripgrep", () => {
test("defaults to include hidden", async () => { test("defaults to include hidden", async () => {
await using tmp = await tmpdir({ await using tmp = await tmpdir({
@@ -16,11 +31,9 @@ describe("file.ripgrep", () => {
}, },
}) })
const files = await Array.fromAsync(Ripgrep.files({ cwd: tmp.path })) const files = await Array.fromAsync(await Ripgrep.files({ cwd: tmp.path }))
const hasVisible = files.includes("visible.txt") expect(files.includes("visible.txt")).toBe(true)
const hasHidden = files.includes(path.join(".opencode", "thing.json")) expect(files.includes(path.join(".opencode", "thing.json"))).toBe(true)
expect(hasVisible).toBe(true)
expect(hasHidden).toBe(true)
}) })
test("hidden false excludes hidden", async () => { test("hidden false excludes hidden", async () => {
@@ -32,15 +45,11 @@ describe("file.ripgrep", () => {
}, },
}) })
const files = await Array.fromAsync(Ripgrep.files({ cwd: tmp.path, hidden: false })) const files = await Array.fromAsync(await Ripgrep.files({ cwd: tmp.path, hidden: false }))
const hasVisible = files.includes("visible.txt") expect(files.includes("visible.txt")).toBe(true)
const hasHidden = files.includes(path.join(".opencode", "thing.json")) expect(files.includes(path.join(".opencode", "thing.json"))).toBe(false)
expect(hasVisible).toBe(true)
expect(hasHidden).toBe(false)
}) })
})
describe("Ripgrep.Service", () => {
test("search returns empty when nothing matches", async () => { test("search returns empty when nothing matches", async () => {
await using tmp = await tmpdir({ await using tmp = await tmpdir({
init: async (dir) => { init: async (dir) => {
@@ -48,15 +57,119 @@ describe("Ripgrep.Service", () => {
}, },
}) })
const result = await Effect.gen(function* () { const result = await Ripgrep.search({ cwd: tmp.path, pattern: "needle" })
const rg = yield* Ripgrep.Service
return yield* rg.search({ cwd: tmp.path, pattern: "needle" })
}).pipe(Effect.provide(Ripgrep.defaultLayer), Effect.runPromise)
expect(result.partial).toBe(false) expect(result.partial).toBe(false)
expect(result.items).toEqual([]) expect(result.items).toEqual([])
}) })
test("search returns match metadata with normalized path", async () => {
await using tmp = await tmpdir({
init: async (dir) => {
await fs.mkdir(path.join(dir, "src"), { recursive: true })
await Bun.write(path.join(dir, "src", "match.ts"), "const needle = 1\n")
},
})
const result = await Ripgrep.search({ cwd: tmp.path, pattern: "needle" })
expect(result.partial).toBe(false)
expect(result.items).toHaveLength(1)
expect(result.items[0]?.path.text).toBe(path.join("src", "match.ts"))
expect(result.items[0]?.line_number).toBe(1)
expect(result.items[0]?.lines.text).toContain("needle")
})
test("files returns empty when glob matches no files in worker mode", async () => {
await using tmp = await tmpdir({
init: async (dir) => {
await fs.mkdir(path.join(dir, "packages", "console"), { recursive: true })
await Bun.write(path.join(dir, "packages", "console", "package.json"), "{}")
},
})
const ctl = new AbortController()
const files = await Array.fromAsync(
await Ripgrep.files({
cwd: tmp.path,
glob: ["packages/*"],
signal: ctl.signal,
}),
)
expect(files).toEqual([])
})
test("ignores RIPGREP_CONFIG_PATH in direct mode", async () => {
await using tmp = await tmpdir({
init: async (dir) => {
await Bun.write(path.join(dir, "match.ts"), "const needle = 1\n")
},
})
const restore = env("RIPGREP_CONFIG_PATH", path.join(tmp.path, "missing-ripgreprc"))
try {
const result = await Ripgrep.search({ cwd: tmp.path, pattern: "needle" })
expect(result.items).toHaveLength(1)
} finally {
restore()
}
})
test("ignores RIPGREP_CONFIG_PATH in worker mode", async () => {
await using tmp = await tmpdir({
init: async (dir) => {
await Bun.write(path.join(dir, "match.ts"), "const needle = 1\n")
},
})
const restore = env("RIPGREP_CONFIG_PATH", path.join(tmp.path, "missing-ripgreprc"))
try {
const ctl = new AbortController()
const result = await Ripgrep.search({
cwd: tmp.path,
pattern: "needle",
signal: ctl.signal,
})
expect(result.items).toHaveLength(1)
} finally {
restore()
}
})
test("aborts files scan in worker mode", async () => {
await using tmp = await tmpdir({
init: async (dir) => {
await seed(dir, 4000)
},
})
const ctl = new AbortController()
const iter = await Ripgrep.files({ cwd: tmp.path, signal: ctl.signal })
const pending = Array.fromAsync(iter)
setTimeout(() => ctl.abort(), 0)
const err = await pending.catch((err) => err)
expect(err).toBeInstanceOf(Error)
expect(err.name).toBe("AbortError")
}, 15_000)
test("aborts search in worker mode", async () => {
await using tmp = await tmpdir({
init: async (dir) => {
await seed(dir, 512, 64 * 1024)
},
})
const ctl = new AbortController()
const pending = Ripgrep.search({ cwd: tmp.path, pattern: "needle", signal: ctl.signal })
setTimeout(() => ctl.abort(), 0)
const err = await pending.catch((err) => err)
expect(err).toBeInstanceOf(Error)
expect(err.name).toBe("AbortError")
}, 15_000)
})
describe("Ripgrep.Service", () => {
test("search returns matched rows", async () => { test("search returns matched rows", async () => {
await using tmp = await tmpdir({ await using tmp = await tmpdir({
init: async (dir) => { init: async (dir) => {
+3 -3
View File
@@ -32,18 +32,18 @@ const ctx = {
ask: () => Effect.void, ask: () => Effect.void,
} }
const projectRoot = path.join(__dirname, "../..") const root = path.join(__dirname, "../..")
describe("tool.grep", () => { describe("tool.grep", () => {
it.live("basic search", () => it.live("basic search", () =>
Effect.gen(function* () { Effect.gen(function* () {
const info = yield* GrepTool const info = yield* GrepTool
const grep = yield* info.init() const grep = yield* info.init()
const result = yield* provideInstance(projectRoot)( const result = yield* provideInstance(root)(
grep.execute( grep.execute(
{ {
pattern: "export", pattern: "export",
path: path.join(projectRoot, "src/tool"), path: path.join(root, "src/tool"),
include: "*.ts", include: "*.ts",
}, },
ctx, ctx,
+36 -39
View File
@@ -1,10 +1,6 @@
import { Effect, Layer, ManagedRuntime } from "effect"
import { Agent } from "../../src/agent/agent"
import { Skill } from "../../src/skill"
import * as CrossSpawnSpawner from "../../src/effect/cross-spawn-spawner" import * as CrossSpawnSpawner from "../../src/effect/cross-spawn-spawner"
import { Ripgrep } from "../../src/file/ripgrep" import { Effect, Layer } from "effect"
import { Truncate } from "../../src/tool/truncate" import { afterEach, describe, expect } from "bun:test"
import { afterEach, describe, expect, test } from "bun:test"
import path from "path" import path from "path"
import { pathToFileURL } from "url" import { pathToFileURL } from "url"
import type { Permission } from "../../src/permission" import type { Permission } from "../../src/permission"
@@ -12,7 +8,7 @@ import type { Tool } from "../../src/tool/tool"
import { Instance } from "../../src/project/instance" import { Instance } from "../../src/project/instance"
import { SkillTool } from "../../src/tool/skill" import { SkillTool } from "../../src/tool/skill"
import { ToolRegistry } from "../../src/tool/registry" import { ToolRegistry } from "../../src/tool/registry"
import { provideTmpdirInstance, tmpdir } from "../fixture/fixture" import { provideTmpdirInstance } from "../fixture/fixture"
import { SessionID, MessageID } from "../../src/session/schema" import { SessionID, MessageID } from "../../src/session/schema"
import { testEffect } from "../lib/effect" import { testEffect } from "../lib/effect"
@@ -131,13 +127,14 @@ description: ${description}
), ),
) )
test("execute returns skill content block with files", async () => { it.live("execute returns skill content block with files", () =>
await using tmp = await tmpdir({ provideTmpdirInstance(
git: true, (dir) =>
init: async (dir) => { Effect.gen(function* () {
const skillDir = path.join(dir, ".opencode", "skill", "tool-skill") const skill = path.join(dir, ".opencode", "skill", "tool-skill")
await Bun.write( yield* Effect.promise(() =>
path.join(skillDir, "SKILL.md"), Bun.write(
path.join(skill, "SKILL.md"),
`--- `---
name: tool-skill name: tool-skill
description: Skill for tool tests. description: Skill for tool tests.
@@ -147,23 +144,27 @@ description: Skill for tool tests.
Use this skill. Use this skill.
`, `,
),
) )
await Bun.write(path.join(skillDir, "scripts", "demo.txt"), "demo") yield* Effect.promise(() => Bun.write(path.join(skill, "scripts", "demo.txt"), "demo"))
},
})
const home = process.env.OPENCODE_TEST_HOME const home = process.env.OPENCODE_TEST_HOME
process.env.OPENCODE_TEST_HOME = tmp.path process.env.OPENCODE_TEST_HOME = dir
yield* Effect.addFinalizer(() =>
try { Effect.sync(() => {
await Instance.provide({ process.env.OPENCODE_TEST_HOME = home
directory: tmp.path, }),
fn: async () => {
const runtime = ManagedRuntime.make(
Layer.mergeAll(Skill.defaultLayer, Ripgrep.defaultLayer, Truncate.defaultLayer, Agent.defaultLayer),
) )
const info = await runtime.runPromise(SkillTool)
const tool = await runtime.runPromise(info.init()) const registry = yield* ToolRegistry.Service
const agent = { name: "build", mode: "primary" as const, permission: [], options: {} }
const tool = (yield* registry.tools({
providerID: "opencode" as any,
modelID: "gpt-5" as any,
agent,
})).find((tool) => tool.id === SkillTool.id)
if (!tool) throw new Error("Skill tool not found")
const requests: Array<Omit<Permission.Request, "id" | "sessionID" | "tool">> = [] const requests: Array<Omit<Permission.Request, "id" | "sessionID" | "tool">> = []
const ctx: Tool.Context = { const ctx: Tool.Context = {
...baseCtx, ...baseCtx,
@@ -173,23 +174,19 @@ Use this skill.
}), }),
} }
const result = await runtime.runPromise(tool.execute({ name: "tool-skill" }, ctx)) const result = yield* tool.execute({ name: "tool-skill" }, ctx)
const dir = path.join(tmp.path, ".opencode", "skill", "tool-skill") const file = path.resolve(skill, "scripts", "demo.txt")
const file = path.resolve(dir, "scripts", "demo.txt")
expect(requests.length).toBe(1) expect(requests.length).toBe(1)
expect(requests[0].permission).toBe("skill") expect(requests[0].permission).toBe("skill")
expect(requests[0].patterns).toContain("tool-skill") expect(requests[0].patterns).toContain("tool-skill")
expect(requests[0].always).toContain("tool-skill") expect(requests[0].always).toContain("tool-skill")
expect(result.metadata.dir).toBe(skill)
expect(result.metadata.dir).toBe(dir)
expect(result.output).toContain(`<skill_content name="tool-skill">`) expect(result.output).toContain(`<skill_content name="tool-skill">`)
expect(result.output).toContain(`Base directory for this skill: ${pathToFileURL(dir).href}`) expect(result.output).toContain(`Base directory for this skill: ${pathToFileURL(skill).href}`)
expect(result.output).toContain(`<file>${file}</file>`) expect(result.output).toContain(`<file>${file}</file>`)
}, }),
}) { git: true },
} finally { ),
process.env.OPENCODE_TEST_HOME = home )
}
})
}) })