From 741cd94da338e00e2eee0e12d572e04c3be46530 Mon Sep 17 00:00:00 2001 From: Adam Mathes Date: Sun, 19 Nov 2017 21:23:04 -0700 Subject: enable full_text and header_image --- crawler/crawler.go | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) (limited to 'crawler') diff --git a/crawler/crawler.go b/crawler/crawler.go index ea9f694..e84e219 100644 --- a/crawler/crawler.go +++ b/crawler/crawler.go @@ -1,21 +1,20 @@ package crawler import ( - "log" "adammathes.com/neko/models/feed" "adammathes.com/neko/models/item" + "adammathes.com/neko/vlog" + "github.com/mmcdole/gofeed" + "log" "net/http" "time" - "github.com/mmcdole/gofeed" - "adammathes.com/neko/vlog" ) - func Crawl() { ch := make(chan string) - feeds,err := feed.All() + feeds, err := feed.All() if err != nil { log.Fatal(err) } @@ -64,7 +63,7 @@ func CrawlFeed(f *feed.Feed, ch chan<- string) { // a lot of RSS2.0 generated by wordpress and others // uses - e,ok := i.Extensions["content"]["encoded"] + e, ok := i.Extensions["content"]["encoded"] var encoded = "" if ok { encoded = e[0].Value @@ -73,16 +72,18 @@ func CrawlFeed(f *feed.Feed, ch chan<- string) { item.Description = encoded } - if(i.PublishedParsed != nil) { + if i.PublishedParsed != nil { item.PublishDate = i.PublishedParsed.Format("2006-01-02 15:04:05") } else { item.PublishDate = time.Now().Format("2006-01-02 15:04:05") } - + item.FeedId = f.Id err := item.Create() if err != nil { vlog.Println(err) + } else { + item.GetFullContent() } } ch <- "successfully crawled " + f.Url + "\n" -- cgit v1.2.3