aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
-rw-r--r--crawler/crawler.go15
-rw-r--r--main.go35
-rw-r--r--models/item/item.go9
-rw-r--r--vlog/vlog.go25
4 files changed, 65 insertions, 19 deletions
diff --git a/crawler/crawler.go b/crawler/crawler.go
index 63e9486..ea9f694 100644
--- a/crawler/crawler.go
+++ b/crawler/crawler.go
@@ -7,6 +7,7 @@ import (
"net/http"
"time"
"github.com/mmcdole/gofeed"
+ "adammathes.com/neko/vlog"
)
@@ -19,12 +20,12 @@ func Crawl() {
log.Fatal(err)
}
for _, f := range feeds {
- log.Printf("crawling %s", f.Url)
+ vlog.Printf("crawling %s\n", f.Url)
go CrawlFeed(f, ch)
}
for i := 0; i < len(feeds); i++ {
- log.Println(<-ch)
+ vlog.Println(<-ch)
}
}
@@ -42,8 +43,8 @@ func CrawlFeed(f *feed.Feed, ch chan<- string) {
feed, err := fp.ParseURL(f.Url)
if err != nil {
- log.Print(err)
- ch <- "failed to fetch and parse for " + f.Url
+ vlog.Println(err)
+ ch <- "failed to fetch and parse for " + f.Url + "\n"
return
}
@@ -51,7 +52,7 @@ func CrawlFeed(f *feed.Feed, ch chan<- string) {
f.Update()
for _, i := range feed.Items {
- log.Printf("storing item: %s", i.Title)
+ vlog.Printf("storing item: %s", i.Link)
var item item.Item
item.Title = i.Title
item.Url = i.Link
@@ -81,8 +82,8 @@ func CrawlFeed(f *feed.Feed, ch chan<- string) {
item.FeedId = f.Id
err := item.Create()
if err != nil {
- log.Println(err)
+ vlog.Println(err)
}
}
- ch <- "successfully crawled " + f.Url
+ ch <- "successfully crawled " + f.Url + "\n"
}
diff --git a/main.go b/main.go
index 26dd1fc..d13efe7 100644
--- a/main.go
+++ b/main.go
@@ -7,36 +7,49 @@ import (
"adammathes.com/neko/models"
"adammathes.com/neko/models/feed"
"adammathes.com/neko/web"
- "log"
+ "adammathes.com/neko/vlog"
+ "fmt"
)
func main() {
- var serve, update bool
+ var serve, update, verbose, printFeeds bool
var configFile, newFeed string
flag.StringVar(&configFile, "c", "config.json", "`configuration` file")
flag.BoolVar(&update, "update", false, "update items by fetching feeds")
flag.BoolVar(&serve, "serve", false, "run http server")
-
+ flag.BoolVar(&verbose, "verbose", false, "verbose output")
+ flag.BoolVar(&printFeeds, "feeds", false, "list all currently crawled feeds")
flag.StringVar(&newFeed, "add", "", "add feed `http://example.com/rss.xml`")
flag.Parse()
- if !update && !serve && newFeed == ""{
+ // no command
+ if !update && !serve && !printFeeds && newFeed == ""{
flag.Usage()
return
}
config.Read(configFile)
models.InitDB(config.Config.DBServer)
-
+ vlog.VERBOSE = verbose
+
+ if update {
+ crawler.Crawl()
+ }
+ if serve {
+ vlog.Printf("starting web server at %s", config.Config.WebServer)
+ web.Serve()
+ }
if newFeed != "" {
feed.NewFeed(newFeed)
}
- if serve {
- log.Printf("starting web server at %s", config.Config.WebServer)
- web.Serve()
+ if printFeeds {
+ feeds,err := feed.All()
+ if err != nil {
+ panic(err)
+ }
+ for _,f := range(feeds) {
+ fmt.Printf("%s\n", f.Url)
+ }
}
- if update {
- crawler.Crawl()
- }
}
diff --git a/models/item/item.go b/models/item/item.go
index 9e22aa8..5a4c274 100644
--- a/models/item/item.go
+++ b/models/item/item.go
@@ -115,8 +115,15 @@ func Filter(max_id int64, feed_id int64, unread_only bool, starred_only bool) ([
log.Println(err)
return nil, err
}
+
+ // sanitize all fields from external input
+ // should do this at ingest time, probably, for efficiency
+ // but still may need to adjust rules
+ i.Title = p.Sanitize(i.Title)
i.Description = p.Sanitize(i.Description)
- // TODO: sanitize other fields
+ i.Url = p.Sanitize(i.Url)
+ i.FeedTitle = p.Sanitize(i.FeedTitle)
+ i.FeedUrl = p.Sanitize(i.FeedUrl)
items = append(items, i)
}
if err = rows.Err(); err != nil {
diff --git a/vlog/vlog.go b/vlog/vlog.go
new file mode 100644
index 0000000..ab48478
--- /dev/null
+++ b/vlog/vlog.go
@@ -0,0 +1,25 @@
+// vlog -- verbose logger -- wraps log functions and only performs them if "verbose"
+package vlog
+
+import (
+ "fmt"
+)
+
+var VERBOSE bool
+
+func init() {
+ VERBOSE=false
+}
+
+func Printf(format string, v ...interface{}) {
+ if VERBOSE {
+ fmt.Printf(format, v...)
+ }
+}
+
+func Println(v ...interface{}) {
+ if VERBOSE {
+ fmt.Println(v...)
+ }
+}
+