1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
|
package main
import (
"fmt"
"os"
"time"
"adammathes.com/neko/config"
"adammathes.com/neko/crawler"
"adammathes.com/neko/exporter"
"adammathes.com/neko/models"
"adammathes.com/neko/models/feed"
"flag"
"adammathes.com/neko/vlog"
"adammathes.com/neko/web"
)
var Version, Build string
func main() {
if err := Run(os.Args[1:]); err != nil {
fmt.Fprintf(os.Stderr, "%v\n", err)
os.Exit(1)
}
}
func Run(args []string) error {
var help, update, verbose, proxyImages bool
var configFile, dbfile, newFeed, export, password string
var port, minutes int
f := flag.NewFlagSet("neko", flag.ContinueOnError)
// config file
f.StringVar(&configFile, "config", "", "read configuration from file")
f.StringVar(&configFile, "c", "", "read configuration from file (short)")
// commands
f.BoolVar(&help, "help", false, "display help")
f.BoolVar(&help, "h", false, "display help (short)")
f.BoolVar(&update, "update", false, "fetch feeds and store new items")
f.BoolVar(&update, "u", false, "fetch feeds and store new items (short)")
f.StringVar(&newFeed, "add", "", "add the feed at URL")
f.StringVar(&newFeed, "a", "", "add the feed at URL (short)")
f.StringVar(&export, "export", "", "export feed: text, opml, html, json")
f.StringVar(&export, "x", "", "export feed (short)")
// options
f.StringVar(&dbfile, "database", "", "sqlite database file")
f.StringVar(&dbfile, "d", "", "sqlite database file (short)")
f.IntVar(&port, "http", 0, "HTTP port to serve on")
f.IntVar(&port, "s", 0, "HTTP port to serve on (short)")
f.IntVar(&minutes, "minutes", 0, "minutes between crawling feeds")
f.IntVar(&minutes, "m", 0, "minutes between crawling feeds (short)")
f.BoolVar(&proxyImages, "imageproxy", false, "rewrite and proxy all image requests")
f.BoolVar(&proxyImages, "i", false, "rewrite and proxy all image requests (short)")
f.BoolVar(&verbose, "verbose", false, "verbose output")
f.BoolVar(&verbose, "v", false, "verbose output (short)")
f.StringVar(&password, "password", "", "password for web interface")
f.StringVar(&password, "p", "", "password for web interface (short)")
f.Usage = func() {
fmt.Fprintf(os.Stderr, "Usage of %s:\n", os.Args[0])
f.PrintDefaults()
}
if err := f.Parse(args); err != nil {
return err
}
if help {
fmt.Printf("neko v%s | build %s\n", Version, Build)
f.Usage()
return nil
}
// reads config if present and sets defaults
if err := config.Init(configFile); err != nil {
return fmt.Errorf("config error: %v", err)
}
// override config file with flags if present
vlog.VERBOSE = verbose
if dbfile != "" {
config.Config.DBFile = dbfile
}
if port != 0 {
config.Config.Port = port
}
if password != "" {
config.Config.DigestPassword = password
}
if minutes != 0 {
config.Config.CrawlMinutes = minutes
}
if proxyImages != false {
config.Config.ProxyImages = proxyImages
}
models.InitDB()
if update {
vlog.Printf("starting crawl\n")
crawler.Crawl()
return nil
}
if newFeed != "" {
vlog.Printf("creating new feed\n")
feed.NewFeed(newFeed)
return nil
}
if export != "" {
vlog.Printf("exporting feeds in format %s\n", export)
fmt.Printf("%s", exporter.ExportFeeds(export))
return nil
}
// For testing, we might want to avoid starting a web server
if config.Config.Port == -1 {
return nil
}
go backgroundCrawl(config.Config.CrawlMinutes)
vlog.Printf("starting web server at 127.0.0.1:%d\n",
config.Config.Port)
web.Serve()
return nil
}
func backgroundCrawl(minutes int) {
if minutes < 1 {
return
}
vlog.Printf("starting background crawl every %d minutes\n", minutes)
for {
time.Sleep(time.Minute * time.Duration(minutes))
crawler.Crawl()
}
}
|
