Bauke/tildes-wiki-sitemap
Bauke
/
tildes-wiki-sitemap
1
Fork 0
Code Releases 2 Activity

fix: add a timeout to scraping

This commit is contained in:
Bauke 2019-08-14 12:41:06 +02:00
parent e80d0148b0
commit 757658c48c
Signed by: Bauke
GPG Key ID: C1C0F29952BCF558
1 changed files with 2 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
main.go
View File

@ -5,6 +5,7 @@ import (
"os" "os"
"sort" "sort"
"strings" "strings"
"time"
"github.com/gocolly/colly" "github.com/gocolly/colly"
log "github.com/sirupsen/logrus" log "github.com/sirupsen/logrus"
@ -28,6 +29,7 @@ func main() {
// When receiving HTML: // When receiving HTML:
collector.OnHTML("html", func(page *colly.HTMLElement) { collector.OnHTML("html", func(page *colly.HTMLElement) {
time.Sleep(time.Second)
// Define the URL for brevity // Define the URL for brevity
url := page.Request.URL.String() url := page.Request.URL.String()
if strings.HasSuffix(url, "/groups") { if strings.HasSuffix(url, "/groups") {