From d133b45575d7928ae564d17ce516175f6732ebed Mon Sep 17 00:00:00 2001 From: ngn Date: Sat, 3 May 2025 21:40:49 +0300 Subject: [PATCH] save flaresolverr solution to use it again Signed-off-by: ngn --- .dockerignore | 10 +++ src/routes/image.go | 6 +- src/routes/question.go | 17 +++-- src/utils/request.go | 152 +++++++++++++++++++++++++++++++---------- 4 files changed, 141 insertions(+), 44 deletions(-) create mode 100644 .dockerignore diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..473ec1d --- /dev/null +++ b/.dockerignore @@ -0,0 +1,10 @@ +.git + +ups.json +renovate.json + +README.md +LICENSE.txt + +docker-compose.example.yml +Dockerfile diff --git a/src/routes/image.go b/src/routes/image.go index eb2023b..d2a669b 100644 --- a/src/routes/image.go +++ b/src/routes/image.go @@ -51,15 +51,15 @@ func GetImage(c *gin.Context) { } // download the image - body, _, headers, err := utils.GET(claims.ImageURL) + res, err := utils.GET(claims.ImageURL) if err != nil { c.AbortWithStatus(500) return } // set the content type - c.Header("Content-Type", headers.Get("Content-Type")) + c.Header("Content-Type", res.Header().Get("Content-Type")) // write the image to the response - c.Writer.Write(body) + c.Writer.Write(res.Body()) } diff --git a/src/routes/question.go b/src/routes/question.go index f5791fb..2b41d25 100644 --- a/src/routes/question.go +++ b/src/routes/question.go @@ -53,18 +53,25 @@ func ViewQuestion(c *gin.Context) { } soLink := fmt.Sprintf("https://%s/questions/%s/%s?answertab=%s", domain, questionId, params.QuestionTitle, params.SoSortValue) - body, code, _, err := utils.GET(soLink) + res, err := utils.GET(soLink) + + if err != nil { + fmt.Printf("failed to get %s: %s", soLink, err.Error()) - if code != 200 { utils.Render(c, 500, "home", gin.H{ - "errorMessage": fmt.Sprintf("Received a non-OK status code %d", code), + "errorMessage": fmt.Sprintf("Request to server failed"), }) return } - respBodyReader := bytes.NewReader(body) + if res.StatusCode() != 200 { + utils.Render(c, 500, "home", gin.H{ + "errorMessage": fmt.Sprintf("Received a non-OK status code: %d", res.StatusCode()), + }) + return + } - doc, err := goquery.NewDocumentFromReader(respBodyReader) + doc, err := goquery.NewDocumentFromReader(bytes.NewReader(res.Body())) if err != nil { utils.Render(c, 500, "home", gin.H{ "errorMessage": "Unable to parse question data", diff --git a/src/utils/request.go b/src/utils/request.go index d5fa7f1..f8518de 100644 --- a/src/utils/request.go +++ b/src/utils/request.go @@ -1,70 +1,150 @@ package utils import ( + "encoding/json" "fmt" "net/http" "net/url" "os" + "time" "github.com/go-resty/resty/v2" ) // https://github.com/FlareSolverr/FlareSolverr#-requestget -type request struct { - Cmd string `json:"cmd"` - Url string `json:"url"` - MaxTimeout int `json:"maxTimeout"` +type Request struct { + Cmd string `json:"cmd"` + Url string `json:"url"` + MaxTimeout int `json:"maxTimeout"` + OnlyCookies bool `json:"returnOnlyCookies"` } -type solution struct { - Status int `json:"status"` - Response []byte `json:"response"` - Headers map[string]string `json:"headers"` +type Cookie struct { + Name string `json:"name"` + Value string `json:"value"` + Domain string `json:"domain"` + Path string `json:"path"` + Expires time.Time `json:"expires"` + Size uint64 `json:"size"` + HttpOnly bool `json:"httpOnly"` + Secure bool `json:"secure"` + Session bool `json:"session"` + SameSite string `json:"sameSite"` } -type response struct { - Solution solution `json:"solution"` +type Solution struct { + Status int `json:"status"` + Cookies []Cookie `json:"cookies"` + Agent string `json:"userAgent"` } -func GET(target string) ([]byte, int, http.Header, error) { - var ( - client *resty.Client = resty.New() - frurl string = "" - ) +type Response struct { + Solution Solution `json:"solution"` +} - if frurl = os.Getenv("FLARESOLVER"); frurl == "" { - if res, err := client.R().Get(target); err != nil { - return nil, 0, nil, err - } else { - return res.Body(), res.StatusCode(), res.Header(), nil - } +func (c *Cookie) ToCookie() *http.Cookie { + ss := http.SameSiteNoneMode + + switch c.SameSite { + case "Lax": + ss = http.SameSiteLaxMode + + case "Strict": + ss = http.SameSiteStrictMode + + case "Default": + ss = http.SameSiteDefaultMode } - frurl, _ = url.JoinPath(frurl, "/v1") + return &http.Cookie{ + Name: c.Name, + Value: c.Value, + Domain: c.Domain, + Path: c.Path, + Expires: c.Expires, + HttpOnly: c.HttpOnly, + Secure: c.Secure, + SameSite: ss, + } +} + +func (s *Solution) HttpCookies() []*http.Cookie { + cookies := []*http.Cookie{} + + for i := range s.Cookies { + cookies = append(cookies, s.Cookies[i].ToCookie()) + } + + return cookies +} + +const USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/134.0.0.0 Safari/537.3" + +var solution *Solution = nil + +func Solve(target string) error { + fsurl := os.Getenv("FLARESOLVER") + + if fsurl == "" { + return fmt.Errorf("flaresolver is not configured") + } + + fsurl, _ = url.JoinPath(fsurl, "/v1") + response := Response{} + client := resty.New() res, err := client.R(). - SetBody(request{ - Cmd: "request.get", - Url: target, - MaxTimeout: 40_000, + SetBody(Request{ + Cmd: "request.get", + Url: target, + MaxTimeout: 40_000, + OnlyCookies: true, }). - SetResult(&response{}). - Post(frurl) + Post(fsurl) if err != nil { - return nil, 0, nil, err + return fmt.Errorf("request failed: %s", err.Error()) } if res.StatusCode() != 200 { - return nil, 0, nil, fmt.Errorf("flaresolver failure") + return fmt.Errorf("bad status code: %d", res.StatusCode()) } - response := res.Result().(*response) - headers := http.Header{} - - for k, v := range response.Solution.Headers { - headers.Add(k, v) + if err := json.Unmarshal(res.Body(), &response); err != nil { + return fmt.Errorf("failed to parse body: %s", err.Error()) } - return response.Solution.Response, response.Solution.Status, headers, nil + solution = &response.Solution + return nil +} + +func GET(target string, no_retry ...bool) (*resty.Response, error) { + client := resty.New() + req := client.R() + + if solution != nil { + req.SetCookies(solution.HttpCookies()) + req.SetHeader("User-Agent", solution.Agent) + } else { + req.SetHeader("User-Agent", USER_AGENT) + } + + res, err := req.Get(target) + if err != nil { + return nil, err + } + + if res.StatusCode() != http.StatusForbidden { + return res, err + } + + if err = Solve(target); err != nil { + return nil, err + } + + if len(no_retry) == 0 { + return GET(target) + } + + return nil, fmt.Errorf("solution did not work") }