refactor: add more fields when retrying

ngshiheng · Jun 9, 2024 · cd391f0 · cd391f0
1 parent fd60ca7
commit cd391f0
Showing 1 changed file with 10 additions and 6 deletions.
diff --git a/pkg/crawler/crawler.go b/pkg/crawler/crawler.go
@@ -233,10 +233,12 @@ func (a *App) Crawl() {
 			a.clearCache(r.Request)
 			log.WithFields(
 				log.Fields{
-					"attempt": attempt,
-					"url":     r.Request.URL,
+					"attempt":     attempt,
+					"error":       err,
+					"status_code": r.StatusCode,
+					"url":         r.Request.URL,
 				},
-			).Warnf("delay for %d seconds before next request", delay)
+			).Warnf("delay for %v before next request", delay)
 			r.Ctx.Put("attempt", attempt+1)
 			time.Sleep(delay)
 			r.Request.Retry()
@@ -270,10 +272,12 @@ func (a *App) Crawl() {
 			a.clearCache(r.Request)
 			log.WithFields(
 				log.Fields{
-					"attempt": attempt,
-					"url":     r.Request.URL,
+					"attempt":     attempt,
+					"error":       err,
+					"status_code": r.StatusCode,
+					"url":         r.Request.URL,
 				},
-			).Warnf("delay for %d seconds before next request", delay)
+			).Warnf("delay for %v before next request", delay)
 			r.Ctx.Put("attempt", attempt+1)
 			time.Sleep(delay)
 			r.Request.Retry()