Initial sanitized code sync
This commit is contained in:
zqq61
279
pkg/httpclient/client.go
Normal file
279
pkg/httpclient/client.go
Normal file
@@ -0,0 +1,279 @@
|
||||
package httpclient
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
fhttp "github.com/bogdanfinn/fhttp"
|
||||
tls_client "github.com/bogdanfinn/tls-client"
|
||||
"github.com/bogdanfinn/tls-client/profiles"
|
||||
)
|
||||
|
||||
const defaultUserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/145.0.0.0 Safari/537.36"
|
||||
|
||||
// Client wraps tls-client with Chrome TLS + HTTP/2 fingerprint and proxy support.
|
||||
type Client struct {
|
||||
tlsClient tls_client.HttpClient
|
||||
jar tls_client.CookieJar
|
||||
}
|
||||
|
||||
// NewClient creates a new Client with Chrome TLS + HTTP/2 fingerprint.
|
||||
func NewClient(proxyURL string) (*Client, error) {
|
||||
jar := tls_client.NewCookieJar()
|
||||
|
||||
options := []tls_client.HttpClientOption{
|
||||
tls_client.WithClientProfile(profiles.Chrome_146),
|
||||
tls_client.WithCookieJar(jar),
|
||||
tls_client.WithTimeoutSeconds(30),
|
||||
// Follow redirects by default (needed for OAuth flow)
|
||||
}
|
||||
|
||||
if proxyURL != "" {
|
||||
options = append(options, tls_client.WithProxyUrl(proxyURL))
|
||||
}
|
||||
|
||||
client, err := tls_client.NewHttpClient(nil, options...)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("create tls client: %w", err)
|
||||
}
|
||||
|
||||
return &Client{
|
||||
tlsClient: client,
|
||||
jar: jar,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// GetCookieJar returns a wrapper that implements http.CookieJar.
|
||||
func (c *Client) GetCookieJar() http.CookieJar {
|
||||
return &cookieJarWrapper{jar: c.jar}
|
||||
}
|
||||
|
||||
// ResetCookies creates a fresh cookie jar, clearing all existing cookies.
|
||||
func (c *Client) ResetCookies() {
|
||||
jar := tls_client.NewCookieJar()
|
||||
c.jar = jar
|
||||
c.tlsClient.SetCookieJar(jar)
|
||||
}
|
||||
|
||||
// Do executes a standard net/http request by converting to fhttp.
|
||||
func (c *Client) Do(req *http.Request) (*http.Response, error) {
|
||||
if req.Header.Get("User-Agent") == "" {
|
||||
req.Header.Set("User-Agent", defaultUserAgent)
|
||||
}
|
||||
fReq, err := convertRequest(req)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
fResp, err := c.tlsClient.Do(fReq)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return convertResponse(fResp), nil
|
||||
}
|
||||
|
||||
// Get performs a GET request with optional headers.
|
||||
func (c *Client) Get(rawURL string, headers map[string]string) (*http.Response, error) {
|
||||
req, err := http.NewRequest(http.MethodGet, rawURL, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for k, v := range headers {
|
||||
req.Header.Set(k, v)
|
||||
}
|
||||
return c.Do(req)
|
||||
}
|
||||
|
||||
// PostJSON performs a POST request with a JSON body and optional headers.
|
||||
func (c *Client) PostJSON(rawURL string, body interface{}, headers map[string]string) (*http.Response, error) {
|
||||
data, err := json.Marshal(body)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("marshal json body: %w", err)
|
||||
}
|
||||
req, err := http.NewRequest(http.MethodPost, rawURL, bytes.NewReader(data))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
for k, v := range headers {
|
||||
req.Header.Set(k, v)
|
||||
}
|
||||
return c.Do(req)
|
||||
}
|
||||
|
||||
// DoNoRedirect executes an HTTP request without following redirects.
|
||||
func (c *Client) DoNoRedirect(req *http.Request) (*http.Response, error) {
|
||||
if req.Header.Get("User-Agent") == "" {
|
||||
req.Header.Set("User-Agent", defaultUserAgent)
|
||||
}
|
||||
fReq, err := convertRequest(req)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// Set redirect func to stop following
|
||||
c.tlsClient.SetFollowRedirect(false)
|
||||
fResp, err := c.tlsClient.Do(fReq)
|
||||
c.tlsClient.SetFollowRedirect(true) // restore
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return convertResponse(fResp), nil
|
||||
}
|
||||
|
||||
// PostForm performs a POST request with form-encoded body and optional headers.
|
||||
func (c *Client) PostForm(rawURL string, values url.Values, headers map[string]string) (*http.Response, error) {
|
||||
req, err := http.NewRequest(http.MethodPost, rawURL, strings.NewReader(values.Encode()))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
|
||||
for k, v := range headers {
|
||||
req.Header.Set(k, v)
|
||||
}
|
||||
return c.Do(req)
|
||||
}
|
||||
|
||||
// DoWithRetry executes a request-building function with retry on 403/network errors.
|
||||
func (c *Client) DoWithRetry(ctx context.Context, maxRetries int, buildReq func() (*http.Request, error)) (*http.Response, error) {
|
||||
var lastErr error
|
||||
for attempt := 1; attempt <= maxRetries; attempt++ {
|
||||
req, err := buildReq()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
resp, err := c.Do(req)
|
||||
if err != nil {
|
||||
lastErr = fmt.Errorf("attempt %d: %w", attempt, err)
|
||||
log.Printf("[http] attempt %d/%d failed: %v", attempt, maxRetries, err)
|
||||
} else if resp.StatusCode == 403 {
|
||||
body, _ := ReadBody(resp)
|
||||
bodyStr := string(body)
|
||||
|
||||
// Don't retry API-level rejections
|
||||
if strings.Contains(bodyStr, "unsupported_country") ||
|
||||
strings.Contains(bodyStr, "request_forbidden") {
|
||||
return nil, fmt.Errorf("HTTP 403: %s", bodyStr)
|
||||
}
|
||||
|
||||
lastErr = fmt.Errorf("attempt %d: HTTP 403 (len=%d)", attempt, len(body))
|
||||
log.Printf("[http] attempt %d/%d got 403 from %s: %s", attempt, maxRetries, req.URL.Host, bodyStr)
|
||||
} else {
|
||||
return resp, nil
|
||||
}
|
||||
|
||||
if attempt < maxRetries {
|
||||
wait := time.Duration(3*attempt) * time.Second
|
||||
log.Printf("[http] waiting %v before retry...", wait)
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return nil, ctx.Err()
|
||||
case <-time.After(wait):
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil, fmt.Errorf("failed after %d attempts: %w", maxRetries, lastErr)
|
||||
}
|
||||
|
||||
// ReadBody reads and closes the response body fully.
|
||||
func ReadBody(resp *http.Response) ([]byte, error) {
|
||||
defer resp.Body.Close()
|
||||
return io.ReadAll(resp.Body)
|
||||
}
|
||||
|
||||
// ─── Type conversion helpers (net/http <-> fhttp) ───
|
||||
|
||||
// convertRequest converts a standard *http.Request to *fhttp.Request.
|
||||
func convertRequest(req *http.Request) (*fhttp.Request, error) {
|
||||
// Read the body fully so fhttp gets a fresh bytes.Reader with known length.
|
||||
// This avoids ContentLength mismatches when the original body is an
|
||||
// io.NopCloser(*strings.Reader) that fhttp cannot introspect.
|
||||
var body io.Reader
|
||||
if req.Body != nil {
|
||||
bodyBytes, err := io.ReadAll(req.Body)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("read request body for conversion: %w", err)
|
||||
}
|
||||
body = bytes.NewReader(bodyBytes)
|
||||
}
|
||||
|
||||
fReq, err := fhttp.NewRequest(req.Method, req.URL.String(), body)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// Copy headers
|
||||
for k, vs := range req.Header {
|
||||
for _, v := range vs {
|
||||
fReq.Header.Add(k, v)
|
||||
}
|
||||
}
|
||||
return fReq, nil
|
||||
}
|
||||
|
||||
// convertResponse converts an *fhttp.Response to a standard *http.Response.
|
||||
func convertResponse(fResp *fhttp.Response) *http.Response {
|
||||
resp := &http.Response{
|
||||
Status: fResp.Status,
|
||||
StatusCode: fResp.StatusCode,
|
||||
Proto: fResp.Proto,
|
||||
ProtoMajor: fResp.ProtoMajor,
|
||||
ProtoMinor: fResp.ProtoMinor,
|
||||
Body: fResp.Body,
|
||||
ContentLength: fResp.ContentLength,
|
||||
Header: http.Header{},
|
||||
Request: &http.Request{URL: fResp.Request.URL},
|
||||
}
|
||||
for k, vs := range fResp.Header {
|
||||
for _, v := range vs {
|
||||
resp.Header.Add(k, v)
|
||||
}
|
||||
}
|
||||
return resp
|
||||
}
|
||||
|
||||
// ─── Cookie jar wrapper (tls_client.CookieJar -> http.CookieJar) ───
|
||||
|
||||
type cookieJarWrapper struct {
|
||||
jar tls_client.CookieJar
|
||||
}
|
||||
|
||||
func (w *cookieJarWrapper) SetCookies(u *url.URL, cookies []*http.Cookie) {
|
||||
fCookies := make([]*fhttp.Cookie, len(cookies))
|
||||
for i, c := range cookies {
|
||||
fCookies[i] = &fhttp.Cookie{
|
||||
Name: c.Name,
|
||||
Value: c.Value,
|
||||
Path: c.Path,
|
||||
Domain: c.Domain,
|
||||
Expires: c.Expires,
|
||||
MaxAge: c.MaxAge,
|
||||
Secure: c.Secure,
|
||||
HttpOnly: c.HttpOnly,
|
||||
}
|
||||
}
|
||||
w.jar.SetCookies(u, fCookies)
|
||||
}
|
||||
|
||||
func (w *cookieJarWrapper) Cookies(u *url.URL) []*http.Cookie {
|
||||
fCookies := w.jar.Cookies(u)
|
||||
cookies := make([]*http.Cookie, len(fCookies))
|
||||
for i, c := range fCookies {
|
||||
cookies[i] = &http.Cookie{
|
||||
Name: c.Name,
|
||||
Value: c.Value,
|
||||
Path: c.Path,
|
||||
Domain: c.Domain,
|
||||
Expires: c.Expires,
|
||||
MaxAge: c.MaxAge,
|
||||
Secure: c.Secure,
|
||||
HttpOnly: c.HttpOnly,
|
||||
}
|
||||
}
|
||||
return cookies
|
||||
}
|
||||
Reference in New Issue
Block a user