fix: use htmlrewriter2 instead of HTMLRewriter for node compat (#26309)

This commit is contained in:
Brendan Allan
2026-05-13 11:28:08 +08:00
committed by GitHub
parent 46daede10c
commit ad6a8a1850
4 changed files with 41 additions and 28 deletions

View File

@@ -1,5 +1,6 @@
import { Effect, Schema } from "effect"
import { HttpClient, HttpClientRequest } from "effect/unstable/http"
import { Parser } from "htmlparser2"
import * as Tool from "./tool"
import TurndownService from "turndown"
import DESCRIPTION from "./webfetch.txt"
@@ -139,8 +140,7 @@ export const WebFetchTool = Tool.define(
case "text":
if (contentType.includes("text/html")) {
const text = yield* Effect.promise(() => extractTextFromHTML(content))
return { output: text, title, metadata: {} }
return { output: extractTextFromHTML(content), title, metadata: {} }
}
return { output: content, title, metadata: {} }
@@ -155,35 +155,27 @@ export const WebFetchTool = Tool.define(
}),
)
async function extractTextFromHTML(html: string) {
function extractTextFromHTML(html: string) {
let text = ""
let skipContent = false
let skipDepth = 0
const rewriter = new HTMLRewriter()
.on("script, style, noscript, iframe, object, embed", {
element() {
skipContent = true
},
text() {
// Skip text content inside these elements
},
})
.on("*", {
element(element) {
// Reset skip flag when entering other elements
if (!["script", "style", "noscript", "iframe", "object", "embed"].includes(element.tagName)) {
skipContent = false
}
},
text(input) {
if (!skipContent) {
text += input.text
}
},
})
.transform(new Response(html))
const parser = new Parser({
onopentag(name) {
if (skipDepth > 0 || ["script", "style", "noscript", "iframe", "object", "embed"].includes(name)) {
skipDepth++
}
},
ontext(input) {
if (skipDepth === 0) text += input
},
onclosetag() {
if (skipDepth > 0) skipDepth--
},
})
parser.write(html)
parser.end()
await rewriter.text()
return text.trim()
}