egor3f/rssalchemy
1
0
Fork 0
Code Actions Packages Releases Wiki Activity

retries

Browse Source
This commit is contained in:
Egor Aristov 2025-03-20 22:27:36 +03:00
parent 8da176ffba
commit a0876409c5
Signed by: egor3f
GPG Key ID: 40482A264AAEC85F
1 changed files with 10 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
internal/extractors/pwextractor/pwextractor.go
View File

@ -2,6 +2,7 @@ package pwextractor
import ( import (
"context" "context"
"errors"
"fmt" "fmt"
"github.com/egor3f/rssalchemy/internal/limiter" "github.com/egor3f/rssalchemy/internal/limiter"
"github.com/egor3f/rssalchemy/internal/models" "github.com/egor3f/rssalchemy/internal/models"
@ -94,6 +95,8 @@ func (e *PwExtractor) Stop() error {
return nil return nil
} }
const MAX_RETRIES = 3 // todo: config
func (e *PwExtractor) visitPage(task models.Task, cb func(page playwright.Page) error) (errRet error) { func (e *PwExtractor) visitPage(task models.Task, cb func(page playwright.Page) error) (errRet error) {
baseDomain, scheme, err := parseBaseDomain(task.URL) baseDomain, scheme, err := parseBaseDomain(task.URL)
@ -214,7 +217,13 @@ func (e *PwExtractor) visitPage(task models.Task, cb func(page playwright.Page)
} }
} }
if _, err := page.Goto(task.URL, playwright.PageGotoOptions{Timeout: pwDuration("10s")}); err != nil { for retry := 0; retry < MAX_RETRIES; retry++ {
_, err = page.Goto(task.URL, playwright.PageGotoOptions{Timeout: pwDuration("10s")})
if !errors.Is(err, playwright.ErrTimeout) {
break
}
}
if err != nil {
return fmt.Errorf("goto page: %w", err) return fmt.Errorf("goto page: %w", err)
} }
log.Debugf("Url %s visited, starting cb", task.URL) log.Debugf("Url %s visited, starting cb", task.URL)