egor3f/rssalchemy
1
0
Fork 0
Code Actions Packages Releases Wiki Activity

cookies fix

Browse Source
This commit is contained in:
Egor Aristov 2025-01-24 18:17:58 +03:00
parent fed51635f5
commit ef7433a594
Signed by: egor3f
GPG Key ID: 40482A264AAEC85F
3 changed files with 15 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
cmd/extractor/extractor.go
View File

@ -16,9 +16,9 @@ func main() {
// todo: rewrite not to use hardcoded tasks // todo: rewrite not to use hardcoded tasks
task := models.Task{ task := models.Task{
URL: "https://2ip.ru", URL: "https://vombat.su",
SelectorPost: "div.ip", SelectorPost: "div.post-body",
SelectorTitle: "span", SelectorTitle: "h1 a",
SelectorLink: "h1 a", SelectorLink: "h1 a",
SelectorDescription: "div.post-content-block p", SelectorDescription: "div.post-content-block p",
SelectorAuthor: "a:has(> span.post-author)", SelectorAuthor: "a:has(> span.post-author)",

8
internal/extractors/pwextractor/pwextractor.go
View File

@ -10,6 +10,7 @@ import (
"github.com/markusmobius/go-dateparser" "github.com/markusmobius/go-dateparser"
"github.com/playwright-community/playwright-go" "github.com/playwright-community/playwright-go"
"maps" "maps"
"strings"
) )
// Timeouts // Timeouts
@ -96,12 +97,13 @@ func (e *PwExtractor) visitPage(task models.Task, cb func(page playwright.Page)
} }
var pwCookies []playwright.OptionalCookie var pwCookies []playwright.OptionalCookie
for k, v := range cookies { for _, cook := range cookies {
pwCookies = append(pwCookies, playwright.OptionalCookie{ pwCookies = append(pwCookies, playwright.OptionalCookie{
Name: k, Name: cook[0],
Value: v, Value: cook[1],
Domain: playwright.String(fmt.Sprintf(".%s", baseDomain)), Domain: playwright.String(fmt.Sprintf(".%s", baseDomain)),
Path: playwright.String("/"), Path: playwright.String("/"),
Secure: playwright.Bool(strings.HasPrefix(cook[0], "__Secure")),
}) })
} }

15
internal/extractors/pwextractor/utils.go
View File

@ -63,21 +63,20 @@ func parseBaseDomain(urlStr string) (string, error) {
return fmt.Sprintf("%s.%s", domainParts[1], domainParts[0]), nil return fmt.Sprintf("%s.%s", domainParts[1], domainParts[0]), nil
} }
func parseCookieString(cookieStr string) (map[string]string, error) { func parseCookieString(cookieStr string) ([][2]string, error) {
result := make(map[string]string) var result [][2]string
failed := fmt.Errorf("failed to parse cookies")
for _, cook := range strings.Split(cookieStr, ";") { for _, cook := range strings.Split(cookieStr, ";") {
kv := strings.Split(cook, "=") kv := strings.Split(cook, "=")
if len(kv) != 2 { if len(kv) < 2 {
return nil, failed return nil, fmt.Errorf("failed to parse cookies: split by =: count<2")
} }
k, err1 := url.QueryUnescape(kv[0]) k, err1 := url.QueryUnescape(kv[0])
v, err2 := url.QueryUnescape(kv[1]) v, err2 := url.QueryUnescape(strings.Join(kv[1:], "="))
if err1 != nil || err2 != nil { if err1 != nil || err2 != nil {
return nil, failed return nil, fmt.Errorf("failed to parse cookies: unescape k=%w v=%w", err1, err2)
} }
result[k] = v result = append(result, [2]string{strings.TrimSpace(k), strings.TrimSpace(v)})
} }
return result, nil return result, nil