fix: add a timeout to scraping
This commit is contained in:
parent
e80d0148b0
commit
757658c48c
2
main.go
2
main.go
|
@ -5,6 +5,7 @@ import (
|
|||
"os"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/gocolly/colly"
|
||||
log "github.com/sirupsen/logrus"
|
||||
|
@ -28,6 +29,7 @@ func main() {
|
|||
|
||||
// When receiving HTML:
|
||||
collector.OnHTML("html", func(page *colly.HTMLElement) {
|
||||
time.Sleep(time.Second)
|
||||
// Define the URL for brevity
|
||||
url := page.Request.URL.String()
|
||||
if strings.HasSuffix(url, "/groups") {
|
||||
|
|
Loading…
Reference in New Issue