diff options
Diffstat (limited to 'crawler/crawler.go')
| -rw-r--r-- | crawler/crawler.go | 17 |
1 files changed, 9 insertions, 8 deletions
diff --git a/crawler/crawler.go b/crawler/crawler.go index ea9f694..e84e219 100644 --- a/crawler/crawler.go +++ b/crawler/crawler.go @@ -1,21 +1,20 @@ package crawler import ( - "log" "adammathes.com/neko/models/feed" "adammathes.com/neko/models/item" + "adammathes.com/neko/vlog" + "github.com/mmcdole/gofeed" + "log" "net/http" "time" - "github.com/mmcdole/gofeed" - "adammathes.com/neko/vlog" ) - func Crawl() { ch := make(chan string) - feeds,err := feed.All() + feeds, err := feed.All() if err != nil { log.Fatal(err) } @@ -64,7 +63,7 @@ func CrawlFeed(f *feed.Feed, ch chan<- string) { // a lot of RSS2.0 generated by wordpress and others // uses <content:encoded> - e,ok := i.Extensions["content"]["encoded"] + e, ok := i.Extensions["content"]["encoded"] var encoded = "" if ok { encoded = e[0].Value @@ -73,16 +72,18 @@ func CrawlFeed(f *feed.Feed, ch chan<- string) { item.Description = encoded } - if(i.PublishedParsed != nil) { + if i.PublishedParsed != nil { item.PublishDate = i.PublishedParsed.Format("2006-01-02 15:04:05") } else { item.PublishDate = time.Now().Format("2006-01-02 15:04:05") } - + item.FeedId = f.Id err := item.Create() if err != nil { vlog.Println(err) + } else { + item.GetFullContent() } } ch <- "successfully crawled " + f.Url + "\n" |