feat(scout): materialize configured reference repos (#26692)
This commit is contained in:
Shoubhit Dash
228
packages/opencode/src/reference/reference.ts
Normal file
228
packages/opencode/src/reference/reference.ts
Normal file
@@ -0,0 +1,228 @@
|
||||
import path from "path"
|
||||
import { Effect, Context, Layer, Scope } from "effect"
|
||||
import { AppFileSystem } from "@opencode-ai/core/filesystem"
|
||||
import { Flag } from "@opencode-ai/core/flag/flag"
|
||||
import { Global } from "@opencode-ai/core/global"
|
||||
import { Config } from "@/config/config"
|
||||
import { InstanceState } from "@/effect/instance-state"
|
||||
import { Git } from "@/git"
|
||||
import { parseRepositoryReference, repositoryCachePath, type Reference as RepositoryReference } from "@/util/repository"
|
||||
import { RepositoryCache } from "./repository-cache"
|
||||
|
||||
type ReferenceEntry = NonNullable<Config.Info["reference"]>[string]
|
||||
|
||||
export type Resolved =
|
||||
| {
|
||||
name: string
|
||||
kind: "local"
|
||||
path: string
|
||||
}
|
||||
| {
|
||||
name: string
|
||||
kind: "git"
|
||||
repository: string
|
||||
reference: RepositoryReference
|
||||
path: string
|
||||
branch?: string
|
||||
}
|
||||
| {
|
||||
name: string
|
||||
kind: "invalid"
|
||||
repository: string
|
||||
message: string
|
||||
}
|
||||
|
||||
type State = {
|
||||
references: Resolved[]
|
||||
materializeAll: Effect.Effect<void>
|
||||
materializeByPath: { path: string; run: Effect.Effect<void> }[]
|
||||
}
|
||||
|
||||
export interface Interface {
|
||||
readonly init: () => Effect.Effect<void>
|
||||
readonly list: () => Effect.Effect<Resolved[]>
|
||||
readonly get: (name: string) => Effect.Effect<Resolved | undefined>
|
||||
readonly ensure: (target?: string) => Effect.Effect<void>
|
||||
readonly contains: (target?: string) => Effect.Effect<boolean>
|
||||
}
|
||||
|
||||
export class Service extends Context.Service<Service, Interface>()("@opencode/Reference") {}
|
||||
|
||||
export function referencePath(input: { directory: string; worktree: string; value: string }) {
|
||||
if (input.value.startsWith("~/")) return path.join(Global.Path.home, input.value.slice(2))
|
||||
return path.isAbsolute(input.value)
|
||||
? input.value
|
||||
: path.resolve(input.worktree === "/" ? input.directory : input.worktree, input.value)
|
||||
}
|
||||
|
||||
function resolveGit(
|
||||
input: { name: string; repository: string } | { name: string; repository: string; branch: string | undefined },
|
||||
): Resolved {
|
||||
const parsed = parseRepositoryReference(input.repository)
|
||||
if (!parsed || parsed.protocol === "file:") {
|
||||
return {
|
||||
name: input.name,
|
||||
kind: "invalid",
|
||||
repository: input.repository,
|
||||
message: "Repository must be a git URL, host/path reference, or GitHub owner/repo shorthand",
|
||||
}
|
||||
}
|
||||
return {
|
||||
name: input.name,
|
||||
kind: "git",
|
||||
repository: input.repository,
|
||||
reference: parsed,
|
||||
path: repositoryCachePath(parsed),
|
||||
...("branch" in input ? { branch: input.branch } : {}),
|
||||
}
|
||||
}
|
||||
|
||||
function branchLabel(branch: string | undefined) {
|
||||
return branch ?? "default branch"
|
||||
}
|
||||
|
||||
function normalizedTarget(target?: string) {
|
||||
if (!target) return
|
||||
return process.platform === "win32" ? AppFileSystem.normalizePath(target) : target
|
||||
}
|
||||
|
||||
function containsReferencePath(referencePath: string, target: string) {
|
||||
return AppFileSystem.contains(normalizedTarget(referencePath) ?? referencePath, target)
|
||||
}
|
||||
|
||||
export function resolve(input: { name: string; reference: ReferenceEntry; directory: string; worktree: string }): Resolved {
|
||||
if (typeof input.reference === "string") {
|
||||
if (input.reference.startsWith(".") || input.reference.startsWith("/") || input.reference.startsWith("~")) {
|
||||
return { name: input.name, kind: "local", path: referencePath({ ...input, value: input.reference }) }
|
||||
}
|
||||
return resolveGit({ name: input.name, repository: input.reference })
|
||||
}
|
||||
|
||||
if ("path" in input.reference) {
|
||||
return { name: input.name, kind: "local", path: referencePath({ ...input, value: input.reference.path }) }
|
||||
}
|
||||
|
||||
return resolveGit({ name: input.name, repository: input.reference.repository, branch: input.reference.branch })
|
||||
}
|
||||
|
||||
export function resolveAll(input: {
|
||||
references: NonNullable<Config.Info["reference"]>
|
||||
directory: string
|
||||
worktree: string
|
||||
}) {
|
||||
const seen = new Map<string, { name: string; branch?: string }>()
|
||||
return Object.entries(input.references).map(([name, reference]) => {
|
||||
const resolved = resolve({ name, reference, directory: input.directory, worktree: input.worktree })
|
||||
if (resolved.kind !== "git") return resolved
|
||||
|
||||
const existing = seen.get(resolved.path)
|
||||
if (!existing) {
|
||||
seen.set(resolved.path, { name, branch: resolved.branch })
|
||||
return resolved
|
||||
}
|
||||
if (existing.branch === resolved.branch) return resolved
|
||||
|
||||
return {
|
||||
name,
|
||||
kind: "invalid" as const,
|
||||
repository: resolved.repository,
|
||||
message: `Reference conflicts with @${existing.name}: both use ${resolved.path}, but @${existing.name} requests ${branchLabel(existing.branch)} and @${name} requests ${branchLabel(resolved.branch)}`,
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
export const layer = Layer.effect(
|
||||
Service,
|
||||
Effect.gen(function* () {
|
||||
const config = yield* Config.Service
|
||||
const fs = yield* AppFileSystem.Service
|
||||
const git = yield* Git.Service
|
||||
const scope = yield* Scope.Scope
|
||||
|
||||
const state = yield* InstanceState.make<State>(
|
||||
Effect.fn("Reference.state")(function* (ctx) {
|
||||
const cfg = yield* config.get()
|
||||
const references = resolveAll({ references: cfg.reference ?? {}, directory: ctx.directory, worktree: ctx.worktree })
|
||||
const seenPath = new Set<string>()
|
||||
const gitReferences = references.filter((reference): reference is Extract<Resolved, { kind: "git" }> => {
|
||||
if (reference.kind !== "git") return false
|
||||
if (seenPath.has(reference.path)) return false
|
||||
seenPath.add(reference.path)
|
||||
return true
|
||||
})
|
||||
const materializeByPath = yield* Effect.forEach(
|
||||
gitReferences,
|
||||
Effect.fnUntraced(function* (reference) {
|
||||
const run = yield* Effect.cached(
|
||||
RepositoryCache.ensure(
|
||||
{ reference: reference.reference, branch: reference.branch, refresh: true },
|
||||
{ fs, git },
|
||||
).pipe(
|
||||
Effect.asVoid,
|
||||
Effect.catchCause((cause) =>
|
||||
Effect.logWarning("failed to materialize reference repository", { name: reference.name, cause }),
|
||||
),
|
||||
),
|
||||
)
|
||||
return { path: reference.path, run }
|
||||
}),
|
||||
{ concurrency: "unbounded" },
|
||||
)
|
||||
|
||||
const materializeAll = yield* Effect.cached(
|
||||
Flag.OPENCODE_EXPERIMENTAL_SCOUT
|
||||
? Effect.gen(function* () {
|
||||
yield* Effect.forEach(
|
||||
materializeByPath,
|
||||
Effect.fnUntraced(function* (item) {
|
||||
yield* item.run
|
||||
}),
|
||||
{ concurrency: 4, discard: true },
|
||||
)
|
||||
})
|
||||
: Effect.void,
|
||||
)
|
||||
|
||||
return { references, materializeAll, materializeByPath }
|
||||
}),
|
||||
)
|
||||
|
||||
return Service.of({
|
||||
init: Effect.fn("Reference.init")(function* () {
|
||||
if (!Flag.OPENCODE_EXPERIMENTAL_SCOUT) return
|
||||
yield* InstanceState.useEffect(state, (s) => s.materializeAll).pipe(Effect.forkIn(scope), Effect.asVoid)
|
||||
}),
|
||||
list: Effect.fn("Reference.list")(function* () {
|
||||
return yield* InstanceState.use(state, (s) => s.references)
|
||||
}),
|
||||
get: Effect.fn("Reference.get")(function* (name: string) {
|
||||
return yield* InstanceState.use(state, (s) => s.references.find((reference) => reference.name === name))
|
||||
}),
|
||||
ensure: Effect.fn("Reference.ensure")(function* (target?: string) {
|
||||
if (!Flag.OPENCODE_EXPERIMENTAL_SCOUT) return
|
||||
const full = normalizedTarget(target)
|
||||
if (!full) return yield* InstanceState.useEffect(state, (s) => s.materializeAll)
|
||||
return yield* InstanceState.useEffect(
|
||||
state,
|
||||
(s) => s.materializeByPath.find((item) => containsReferencePath(item.path, full))?.run ?? Effect.void,
|
||||
)
|
||||
}),
|
||||
contains: Effect.fn("Reference.contains")(function* (target?: string) {
|
||||
if (!Flag.OPENCODE_EXPERIMENTAL_SCOUT) return false
|
||||
const full = normalizedTarget(target)
|
||||
if (!full) return false
|
||||
return yield* InstanceState.use(state, (s) =>
|
||||
s.references.some((reference) => reference.kind === "git" && containsReferencePath(reference.path, full)),
|
||||
)
|
||||
}),
|
||||
})
|
||||
}),
|
||||
)
|
||||
|
||||
export const defaultLayer = layer.pipe(
|
||||
Layer.provide(Config.defaultLayer),
|
||||
Layer.provide(AppFileSystem.defaultLayer),
|
||||
Layer.provide(Git.defaultLayer),
|
||||
)
|
||||
|
||||
export * as Reference from "./reference"
|
||||
155
packages/opencode/src/reference/repository-cache.ts
Normal file
155
packages/opencode/src/reference/repository-cache.ts
Normal file
@@ -0,0 +1,155 @@
|
||||
import path from "path"
|
||||
import { Effect } from "effect"
|
||||
import { AppFileSystem } from "@opencode-ai/core/filesystem"
|
||||
import { Flock } from "@opencode-ai/core/util/flock"
|
||||
import { Git } from "@/git"
|
||||
import {
|
||||
repositoryCachePath,
|
||||
sameRepositoryReference,
|
||||
parseRepositoryReference,
|
||||
validateRepositoryBranch,
|
||||
type Reference as RepositoryReference,
|
||||
} from "@/util/repository"
|
||||
|
||||
export type Result = {
|
||||
repository: string
|
||||
host: string
|
||||
remote: string
|
||||
localPath: string
|
||||
status: "cached" | "cloned" | "refreshed"
|
||||
head?: string
|
||||
branch?: string
|
||||
}
|
||||
|
||||
function statusForRepository(input: { reuse: boolean; refresh?: boolean; branchMatches?: boolean }) {
|
||||
if (!input.reuse) return "cloned" as const
|
||||
if (input.branchMatches === false) return "refreshed" as const
|
||||
if (input.refresh) return "refreshed" as const
|
||||
return "cached" as const
|
||||
}
|
||||
|
||||
function resetTarget(input: {
|
||||
requestedBranch?: string
|
||||
remoteHead: { code: number; stdout: string }
|
||||
branch: { code: number; stdout: string }
|
||||
}) {
|
||||
if (input.requestedBranch) return `origin/${input.requestedBranch}`
|
||||
if (input.remoteHead.code === 0 && input.remoteHead.stdout) {
|
||||
return input.remoteHead.stdout.replace(/^refs\/remotes\//, "")
|
||||
}
|
||||
if (input.branch.code === 0 && input.branch.stdout) {
|
||||
return `origin/${input.branch.stdout}`
|
||||
}
|
||||
return "HEAD"
|
||||
}
|
||||
|
||||
export const ensure = Effect.fn("RepositoryCache.ensure")(function* (
|
||||
input: {
|
||||
reference: RepositoryReference
|
||||
refresh?: boolean
|
||||
branch?: string
|
||||
},
|
||||
services: {
|
||||
fs: AppFileSystem.Interface
|
||||
git: Git.Interface
|
||||
},
|
||||
) {
|
||||
if (input.branch) validateRepositoryBranch(input.branch)
|
||||
|
||||
const repository = input.reference.label
|
||||
const remote = input.reference.remote
|
||||
const localPath = repositoryCachePath(input.reference)
|
||||
const cloneTarget = parseRepositoryReference(remote) ?? input.reference
|
||||
|
||||
return yield* Effect.acquireUseRelease(
|
||||
Effect.promise((signal) => Flock.acquire(`repo-clone:${localPath}`, { signal })),
|
||||
() =>
|
||||
Effect.gen(function* () {
|
||||
yield* services.fs.ensureDir(path.dirname(localPath)).pipe(Effect.orDie)
|
||||
|
||||
const exists = yield* services.fs.existsSafe(localPath)
|
||||
const hasGitDir = yield* services.fs.existsSafe(path.join(localPath, ".git"))
|
||||
const origin = hasGitDir
|
||||
? yield* services.git.run(["config", "--get", "remote.origin.url"], { cwd: localPath })
|
||||
: undefined
|
||||
const originReference = origin?.exitCode === 0 ? parseRepositoryReference(origin.text().trim()) : undefined
|
||||
const reuse = hasGitDir && Boolean(originReference && sameRepositoryReference(originReference, cloneTarget))
|
||||
if (exists && !reuse) {
|
||||
yield* services.fs.remove(localPath, { recursive: true }).pipe(Effect.orDie)
|
||||
}
|
||||
|
||||
const currentBranch = hasGitDir ? yield* services.git.branch(localPath) : undefined
|
||||
const status = statusForRepository({
|
||||
reuse,
|
||||
refresh: input.refresh,
|
||||
branchMatches: input.branch ? currentBranch === input.branch : undefined,
|
||||
})
|
||||
|
||||
if (status === "cloned") {
|
||||
const clone = yield* services.git.run(
|
||||
[
|
||||
"clone",
|
||||
"--depth",
|
||||
"100",
|
||||
...(input.branch ? ["--branch", input.branch] : []),
|
||||
"--",
|
||||
remote,
|
||||
localPath,
|
||||
],
|
||||
{ cwd: path.dirname(localPath) },
|
||||
)
|
||||
if (clone.exitCode !== 0) {
|
||||
throw new Error(clone.stderr.toString().trim() || clone.text().trim() || `Failed to clone ${repository}`)
|
||||
}
|
||||
}
|
||||
|
||||
if (status === "refreshed") {
|
||||
const fetch = yield* services.git.run(["fetch", "--all", "--prune"], { cwd: localPath })
|
||||
if (fetch.exitCode !== 0) {
|
||||
throw new Error(fetch.stderr.toString().trim() || fetch.text().trim() || `Failed to refresh ${repository}`)
|
||||
}
|
||||
|
||||
if (input.branch) {
|
||||
const checkout = yield* services.git.run(["checkout", "-B", input.branch, `origin/${input.branch}`], {
|
||||
cwd: localPath,
|
||||
})
|
||||
if (checkout.exitCode !== 0) {
|
||||
throw new Error(
|
||||
checkout.stderr.toString().trim() || checkout.text().trim() || `Failed to checkout ${input.branch}`,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
const remoteHead = yield* services.git.run(["symbolic-ref", "refs/remotes/origin/HEAD"], { cwd: localPath })
|
||||
const branch = yield* services.git.run(["symbolic-ref", "--quiet", "--short", "HEAD"], { cwd: localPath })
|
||||
const target = resetTarget({
|
||||
requestedBranch: input.branch,
|
||||
remoteHead: { code: remoteHead.exitCode, stdout: remoteHead.text().trim() },
|
||||
branch: { code: branch.exitCode, stdout: branch.text().trim() },
|
||||
})
|
||||
|
||||
const reset = yield* services.git.run(["reset", "--hard", target], { cwd: localPath })
|
||||
if (reset.exitCode !== 0) {
|
||||
throw new Error(reset.stderr.toString().trim() || reset.text().trim() || `Failed to reset ${repository}`)
|
||||
}
|
||||
}
|
||||
|
||||
const head = yield* services.git.run(["rev-parse", "HEAD"], { cwd: localPath })
|
||||
const branch = yield* services.git.branch(localPath)
|
||||
const headText = head.exitCode === 0 ? head.text().trim() : undefined
|
||||
|
||||
return {
|
||||
repository,
|
||||
host: input.reference.host,
|
||||
remote,
|
||||
localPath,
|
||||
status,
|
||||
head: headText,
|
||||
branch,
|
||||
} satisfies Result
|
||||
}),
|
||||
(lock) => Effect.promise(() => lock.release()).pipe(Effect.ignore),
|
||||
)
|
||||
})
|
||||
|
||||
export * as RepositoryCache from "./repository-cache"
|
||||
Reference in New Issue
Block a user