aboutsummaryrefslogtreecommitdiffstats
path: root/crawler
diff options
context:
space:
mode:
authorAdam Mathes <adam@trenchant.org>2017-02-25 17:50:38 -0800
committerAdam Mathes <adam@trenchant.org>2017-02-25 17:50:38 -0800
commit8359198e5edb272311e10c77e6bf4cb662abd5ec (patch)
tree064589554fa36d5bf6c17436d39b68964034b1c2 /crawler
parent09af7d2f740eaa639fdcc2a7aa8c2ee1e3c3dd4d (diff)
downloadneko-8359198e5edb272311e10c77e6bf4cb662abd5ec.tar.gz
neko-8359198e5edb272311e10c77e6bf4cb662abd5ec.tar.bz2
neko-8359198e5edb272311e10c77e6bf4cb662abd5ec.zip
silent by default, verbose option added. -feeds cmd line option added
Diffstat (limited to 'crawler')
-rw-r--r--crawler/crawler.go15
1 files changed, 8 insertions, 7 deletions
diff --git a/crawler/crawler.go b/crawler/crawler.go
index 63e9486..ea9f694 100644
--- a/crawler/crawler.go
+++ b/crawler/crawler.go
@@ -7,6 +7,7 @@ import (
"net/http"
"time"
"github.com/mmcdole/gofeed"
+ "adammathes.com/neko/vlog"
)
@@ -19,12 +20,12 @@ func Crawl() {
log.Fatal(err)
}
for _, f := range feeds {
- log.Printf("crawling %s", f.Url)
+ vlog.Printf("crawling %s\n", f.Url)
go CrawlFeed(f, ch)
}
for i := 0; i < len(feeds); i++ {
- log.Println(<-ch)
+ vlog.Println(<-ch)
}
}
@@ -42,8 +43,8 @@ func CrawlFeed(f *feed.Feed, ch chan<- string) {
feed, err := fp.ParseURL(f.Url)
if err != nil {
- log.Print(err)
- ch <- "failed to fetch and parse for " + f.Url
+ vlog.Println(err)
+ ch <- "failed to fetch and parse for " + f.Url + "\n"
return
}
@@ -51,7 +52,7 @@ func CrawlFeed(f *feed.Feed, ch chan<- string) {
f.Update()
for _, i := range feed.Items {
- log.Printf("storing item: %s", i.Title)
+ vlog.Printf("storing item: %s", i.Link)
var item item.Item
item.Title = i.Title
item.Url = i.Link
@@ -81,8 +82,8 @@ func CrawlFeed(f *feed.Feed, ch chan<- string) {
item.FeedId = f.Id
err := item.Create()
if err != nil {
- log.Println(err)
+ vlog.Println(err)
}
}
- ch <- "successfully crawled " + f.Url
+ ch <- "successfully crawled " + f.Url + "\n"
}