diff options
Diffstat (limited to 'crawler/crawler.go')
| -rw-r--r-- | crawler/crawler.go | 29 |
1 files changed, 24 insertions, 5 deletions
diff --git a/crawler/crawler.go b/crawler/crawler.go index faf0b70..4d91241 100644 --- a/crawler/crawler.go +++ b/crawler/crawler.go @@ -6,7 +6,7 @@ import ( "neko/models/item" "net/http" "time" - "github.com/SlyMarbo/rss" + "github.com/mmcdole/gofeed" ) @@ -37,7 +37,10 @@ func CrawlFeed(f *feed.Feed, ch chan<- string) { Timeout: 5 * time.Second, } - feed, err := rss.FetchByClient(f.Url, c) + fp := gofeed.NewParser() + fp.Client = c + + feed, err := fp.ParseURL(f.Url) if err != nil { log.Print(err) ch <- "failed to fetch and parse for " + f.Url @@ -52,10 +55,26 @@ func CrawlFeed(f *feed.Feed, ch chan<- string) { var item item.Item item.Title = i.Title item.Url = i.Link - item.Description = i.Content - if item.Description == "" { - item.Description = i.Summary + + item.Description = i.Description + if len(i.Content) > len(item.Description) { + item.Description = i.Content + } + + // a lot of RSS2.0 generated by wordpress and others + // uses <content:encoded> + + e,ok := i.Extensions["content"]["encoded"] + var encoded = "" + if ok { + encoded = e[0].Value + } + + if len(encoded) > len(item.Description) { + item.Description = encoded } + + item.FeedId = f.Id err := item.Create() if err != nil { |