diff options
| -rw-r--r-- | crawler/crawler.go | 15 | ||||
| -rw-r--r-- | main.go | 35 | ||||
| -rw-r--r-- | models/item/item.go | 9 | ||||
| -rw-r--r-- | vlog/vlog.go | 25 |
4 files changed, 65 insertions, 19 deletions
diff --git a/crawler/crawler.go b/crawler/crawler.go index 63e9486..ea9f694 100644 --- a/crawler/crawler.go +++ b/crawler/crawler.go @@ -7,6 +7,7 @@ import ( "net/http" "time" "github.com/mmcdole/gofeed" + "adammathes.com/neko/vlog" ) @@ -19,12 +20,12 @@ func Crawl() { log.Fatal(err) } for _, f := range feeds { - log.Printf("crawling %s", f.Url) + vlog.Printf("crawling %s\n", f.Url) go CrawlFeed(f, ch) } for i := 0; i < len(feeds); i++ { - log.Println(<-ch) + vlog.Println(<-ch) } } @@ -42,8 +43,8 @@ func CrawlFeed(f *feed.Feed, ch chan<- string) { feed, err := fp.ParseURL(f.Url) if err != nil { - log.Print(err) - ch <- "failed to fetch and parse for " + f.Url + vlog.Println(err) + ch <- "failed to fetch and parse for " + f.Url + "\n" return } @@ -51,7 +52,7 @@ func CrawlFeed(f *feed.Feed, ch chan<- string) { f.Update() for _, i := range feed.Items { - log.Printf("storing item: %s", i.Title) + vlog.Printf("storing item: %s", i.Link) var item item.Item item.Title = i.Title item.Url = i.Link @@ -81,8 +82,8 @@ func CrawlFeed(f *feed.Feed, ch chan<- string) { item.FeedId = f.Id err := item.Create() if err != nil { - log.Println(err) + vlog.Println(err) } } - ch <- "successfully crawled " + f.Url + ch <- "successfully crawled " + f.Url + "\n" } @@ -7,36 +7,49 @@ import ( "adammathes.com/neko/models" "adammathes.com/neko/models/feed" "adammathes.com/neko/web" - "log" + "adammathes.com/neko/vlog" + "fmt" ) func main() { - var serve, update bool + var serve, update, verbose, printFeeds bool var configFile, newFeed string flag.StringVar(&configFile, "c", "config.json", "`configuration` file") flag.BoolVar(&update, "update", false, "update items by fetching feeds") flag.BoolVar(&serve, "serve", false, "run http server") - + flag.BoolVar(&verbose, "verbose", false, "verbose output") + flag.BoolVar(&printFeeds, "feeds", false, "list all currently crawled feeds") flag.StringVar(&newFeed, "add", "", "add feed `http://example.com/rss.xml`") flag.Parse() - if !update && !serve && newFeed == ""{ + // no command + if !update && !serve && !printFeeds && newFeed == ""{ flag.Usage() return } config.Read(configFile) models.InitDB(config.Config.DBServer) - + vlog.VERBOSE = verbose + + if update { + crawler.Crawl() + } + if serve { + vlog.Printf("starting web server at %s", config.Config.WebServer) + web.Serve() + } if newFeed != "" { feed.NewFeed(newFeed) } - if serve { - log.Printf("starting web server at %s", config.Config.WebServer) - web.Serve() + if printFeeds { + feeds,err := feed.All() + if err != nil { + panic(err) + } + for _,f := range(feeds) { + fmt.Printf("%s\n", f.Url) + } } - if update { - crawler.Crawl() - } } diff --git a/models/item/item.go b/models/item/item.go index 9e22aa8..5a4c274 100644 --- a/models/item/item.go +++ b/models/item/item.go @@ -115,8 +115,15 @@ func Filter(max_id int64, feed_id int64, unread_only bool, starred_only bool) ([ log.Println(err) return nil, err } + + // sanitize all fields from external input + // should do this at ingest time, probably, for efficiency + // but still may need to adjust rules + i.Title = p.Sanitize(i.Title) i.Description = p.Sanitize(i.Description) - // TODO: sanitize other fields + i.Url = p.Sanitize(i.Url) + i.FeedTitle = p.Sanitize(i.FeedTitle) + i.FeedUrl = p.Sanitize(i.FeedUrl) items = append(items, i) } if err = rows.Err(); err != nil { diff --git a/vlog/vlog.go b/vlog/vlog.go new file mode 100644 index 0000000..ab48478 --- /dev/null +++ b/vlog/vlog.go @@ -0,0 +1,25 @@ +// vlog -- verbose logger -- wraps log functions and only performs them if "verbose" +package vlog + +import ( + "fmt" +) + +var VERBOSE bool + +func init() { + VERBOSE=false +} + +func Printf(format string, v ...interface{}) { + if VERBOSE { + fmt.Printf(format, v...) + } +} + +func Println(v ...interface{}) { + if VERBOSE { + fmt.Println(v...) + } +} + |