1
Fork 0

fix: add a timeout to scraping

This commit is contained in:
Bauke 2019-08-14 12:41:06 +02:00
parent e80d0148b0
commit 757658c48c
Signed by: Bauke
GPG Key ID: C1C0F29952BCF558
1 changed files with 2 additions and 0 deletions

View File

@ -5,6 +5,7 @@ import (
"os"
"sort"
"strings"
"time"
"github.com/gocolly/colly"
log "github.com/sirupsen/logrus"
@ -28,6 +29,7 @@ func main() {
// When receiving HTML:
collector.OnHTML("html", func(page *colly.HTMLElement) {
time.Sleep(time.Second)
// Define the URL for brevity
url := page.Request.URL.String()
if strings.HasSuffix(url, "/groups") {