aboutsummaryrefslogtreecommitdiffstats
path: root/crawler
diff options
context:
space:
mode:
Diffstat (limited to 'crawler')
-rw-r--r--crawler/crawler.go17
1 files changed, 9 insertions, 8 deletions
diff --git a/crawler/crawler.go b/crawler/crawler.go
index ea9f694..e84e219 100644
--- a/crawler/crawler.go
+++ b/crawler/crawler.go
@@ -1,21 +1,20 @@
package crawler
import (
- "log"
"adammathes.com/neko/models/feed"
"adammathes.com/neko/models/item"
+ "adammathes.com/neko/vlog"
+ "github.com/mmcdole/gofeed"
+ "log"
"net/http"
"time"
- "github.com/mmcdole/gofeed"
- "adammathes.com/neko/vlog"
)
-
func Crawl() {
ch := make(chan string)
- feeds,err := feed.All()
+ feeds, err := feed.All()
if err != nil {
log.Fatal(err)
}
@@ -64,7 +63,7 @@ func CrawlFeed(f *feed.Feed, ch chan<- string) {
// a lot of RSS2.0 generated by wordpress and others
// uses <content:encoded>
- e,ok := i.Extensions["content"]["encoded"]
+ e, ok := i.Extensions["content"]["encoded"]
var encoded = ""
if ok {
encoded = e[0].Value
@@ -73,16 +72,18 @@ func CrawlFeed(f *feed.Feed, ch chan<- string) {
item.Description = encoded
}
- if(i.PublishedParsed != nil) {
+ if i.PublishedParsed != nil {
item.PublishDate = i.PublishedParsed.Format("2006-01-02 15:04:05")
} else {
item.PublishDate = time.Now().Format("2006-01-02 15:04:05")
}
-
+
item.FeedId = f.Id
err := item.Create()
if err != nil {
vlog.Println(err)
+ } else {
+ item.GetFullContent()
}
}
ch <- "successfully crawled " + f.Url + "\n"