diff options
| -rw-r--r-- | .thicket/tickets.jsonl | 1 | ||||
| -rw-r--r-- | internal/crawler/crawler.go | 11 | ||||
| -rw-r--r-- | internal/safehttp/safehttp.go | 110 | ||||
| -rw-r--r-- | internal/safehttp/safehttp_test.go | 53 | ||||
| -rw-r--r-- | models/feed/feed.go | 8 | ||||
| -rw-r--r-- | web/web.go | 5 |
6 files changed, 171 insertions, 17 deletions
diff --git a/.thicket/tickets.jsonl b/.thicket/tickets.jsonl index 4984dec..a91d077 100644 --- a/.thicket/tickets.jsonl +++ b/.thicket/tickets.jsonl @@ -66,6 +66,7 @@ {"id":"NK-qwef98","title":"UI Styling: Controls \u0026 Header","description":"","type":"","status":"closed","priority":2,"labels":null,"assignee":"","created":"2026-02-13T18:05:18.450759919Z","updated":"2026-02-13T18:11:46.291830432Z"} {"id":"NK-r6nhj0","title":"import/export","description":"Import/Export has only ever been partially implemented. Let's finish it up across OPML (de facto standard) but also simple txt line oriented input/output. We may need to file a ticket to deal with the async crawling as part of this.","type":"feature","status":"open","priority":1,"labels":null,"assignee":"","created":"2026-02-14T16:45:04.739162003Z","updated":"2026-02-14T16:45:04.739162003Z"} {"id":"NK-ric1zs","title":"Migrate frontend to /api/ endpoints","description":"The backend now provides a clean REST API at /api/. Update the frontend UI to use these new endpoints instead of the legacy backward-compatibility routes (/stream/, /feed/, etc.). This will allow for cleaner separation and better utilization of proper REST patterns.","type":"cleanup","status":"closed","priority":3,"labels":null,"assignee":"","created":"2026-02-13T04:26:55.864725765Z","updated":"2026-02-13T04:26:55.864725765Z"} +{"id":"NK-rn4nzp","title":"font themes","description":"in the v2 ui, let's offer a few different font stacks the user can switch through. primarily this should just change font-face, maybe size, but don't worry about colors or anything right now.\n\nthe current default (helvetica neue, palatino)\na fancy all serif stack\na no-nonsense modern san-serif stack\na terminal inspired fixed width stack","type":"feature","status":"open","priority":2,"labels":null,"assignee":"","created":"2026-02-14T17:10:02.185477382Z","updated":"2026-02-14T17:10:02.185477382Z"} {"id":"NK-rohuiq","title":"titles changing on read state and hover","description":"Titles are changing on read state from blue to grey. They should just stay blue all the time.\n\nTitles are getting underlined on hover. They should have no underline regardless of hover state.","type":"bug","status":"closed","priority":0,"labels":null,"assignee":"","created":"2026-02-14T03:36:26.36373162Z","updated":"2026-02-14T03:37:50.73870586Z"} {"id":"NK-shpyxh","title":"add search to new ui","description":"","type":"epic","status":"closed","priority":1,"labels":null,"assignee":"","created":"2026-02-13T19:29:44.251257089Z","updated":"2026-02-14T01:02:58.547025683Z"} {"id":"NK-sne5ox","title":"Implement Export/Import UI","description":"Add UI in settings to download OPML export and upload OPML import. Use /export/ and /import/ (need to check if import exists).","type":"epic","status":"icebox","priority":3,"labels":null,"assignee":"","created":"2026-02-13T15:05:23.266731399Z","updated":"2026-02-13T15:05:23.266731399Z"} diff --git a/internal/crawler/crawler.go b/internal/crawler/crawler.go index 10253d8..fce2769 100644 --- a/internal/crawler/crawler.go +++ b/internal/crawler/crawler.go @@ -6,6 +6,7 @@ import ( "net/http" "time" + "adammathes.com/neko/internal/safehttp" "adammathes.com/neko/internal/vlog" "adammathes.com/neko/models/feed" "adammathes.com/neko/models/item" @@ -58,10 +59,7 @@ func GetFeedContent(feedURL string) string { // n := time.Duration(rand.Int63n(3)) // time.Sleep(n * time.Second) - c := &http.Client{ - // give up after 5 seconds - Timeout: 5 * time.Second, - } + c := safehttp.NewSafeClient(5 * time.Second) request, err := http.NewRequest("GET", feedURL, nil) if err != nil { @@ -100,10 +98,7 @@ func GetFeedContent(feedURL string) string { TODO: sanitize input on crawl */ func CrawlFeed(f *feed.Feed, ch chan<- string) { - c := &http.Client{ - // give up after 5 seconds - Timeout: 5 * time.Second, - } + c := safehttp.NewSafeClient(5 * time.Second) fp := gofeed.NewParser() fp.Client = c diff --git a/internal/safehttp/safehttp.go b/internal/safehttp/safehttp.go new file mode 100644 index 0000000..cfc70f1 --- /dev/null +++ b/internal/safehttp/safehttp.go @@ -0,0 +1,110 @@ +package safehttp + +import ( + "context" + "fmt" + "net" + "net/http" + "time" +) + +var privateIPBlocks []*net.IPNet + +func init() { + for _, cidr := range []string{ + "127.0.0.0/8", // IPv4 loopback + "10.0.0.0/8", // RFC1918 + "172.16.0.0/12", // RFC1918 + "192.168.0.0/16", // RFC1918 + "169.254.0.0/16", // IPv4 link-local + "::1/128", // IPv6 loopback + "fe80::/10", // IPv6 link-local + "fc00::/7", // IPv6 unique local addr + } { + _, block, _ := net.ParseCIDR(cidr) + privateIPBlocks = append(privateIPBlocks, block) + } +} + +func isPrivateIP(ip net.IP) bool { + if ip.IsLoopback() || ip.IsLinkLocalUnicast() || ip.IsLinkLocalMulticast() { + return true + } + for _, block := range privateIPBlocks { + if block.Contains(ip) { + return true + } + } + return false +} + +func SafeDialer(dialer *net.Dialer) func(context.Context, string, string) (net.Conn, error) { + return func(ctx context.Context, network, address string) (net.Conn, error) { + host, _, err := net.SplitHostPort(address) + if err != nil { + host = address + } + + if ip := net.ParseIP(host); ip != nil { + if isPrivateIP(ip) { + return nil, fmt.Errorf("connection to private IP %s is not allowed", ip) + } + } else { + ips, err := net.DefaultResolver.LookupIP(ctx, "ip", host) + if err != nil { + return nil, err + } + + for _, ip := range ips { + if isPrivateIP(ip) { + return nil, fmt.Errorf("connection to private IP %s is not allowed", ip) + } + } + } + + return dialer.DialContext(ctx, network, address) + } +} + +func NewSafeClient(timeout time.Duration) *http.Client { + dialer := &net.Dialer{ + Timeout: 30 * time.Second, + KeepAlive: 30 * time.Second, + } + + transport := http.DefaultTransport.(*http.Transport).Clone() + transport.DialContext = SafeDialer(dialer) + + return &http.Client{ + Timeout: timeout, + Transport: transport, + CheckRedirect: func(req *http.Request, via []*http.Request) error { + if len(via) >= 10 { + return fmt.Errorf("too many redirects") + } + + host, _, err := net.SplitHostPort(req.URL.Host) + if err != nil { + host = req.URL.Host + } + + if ip := net.ParseIP(host); ip != nil { + if isPrivateIP(ip) { + return fmt.Errorf("redirect to private IP %s is not allowed", ip) + } + } else { + ips, err := net.DefaultResolver.LookupIP(req.Context(), "ip", host) + if err != nil { + return err + } + + for _, ip := range ips { + if isPrivateIP(ip) { + return fmt.Errorf("redirect to private IP %s is not allowed", ip) + } + } + } + return nil + }, + } +} diff --git a/internal/safehttp/safehttp_test.go b/internal/safehttp/safehttp_test.go new file mode 100644 index 0000000..b2636da --- /dev/null +++ b/internal/safehttp/safehttp_test.go @@ -0,0 +1,53 @@ +package safehttp + +import ( + "net" + "testing" + "time" +) + +func TestSafeClient(t *testing.T) { + client := NewSafeClient(2 * time.Second) + + // Localhost should fail + t.Log("Testing localhost...") + _, err := client.Get("http://127.0.0.1:8080") + if err == nil { + t.Error("Expected error for localhost request, got nil") + } else { + t.Logf("Got expected error: %v", err) + } + + // Private IP should fail + t.Log("Testing private IP...") + _, err = client.Get("http://10.0.0.1") + if err == nil { + t.Error("Expected error for private IP request, got nil") + } else { + t.Logf("Got expected error: %v", err) + } +} + +func TestIsPrivateIP(t *testing.T) { + tests := []struct { + ip string + expected bool + }{ + {"127.0.0.1", true}, + {"10.0.0.1", true}, + {"172.16.0.1", true}, + {"192.168.1.1", true}, + {"169.254.1.1", true}, + {"8.8.8.8", false}, + {"1.1.1.1", false}, + {"::1", true}, + {"fe80::1", true}, + {"fc00::1", true}, + } + + for _, tc := range tests { + if res := isPrivateIP(net.ParseIP(tc.ip)); res != tc.expected { + t.Errorf("isPrivateIP(%s) = %v, want %v", tc.ip, res, tc.expected) + } + } +} diff --git a/models/feed/feed.go b/models/feed/feed.go index 95e7104..800e47c 100644 --- a/models/feed/feed.go +++ b/models/feed/feed.go @@ -6,6 +6,7 @@ import ( "strings" "time" + "adammathes.com/neko/internal/safehttp" "adammathes.com/neko/models" "github.com/PuerkitoBio/goquery" ) @@ -120,12 +121,7 @@ func (f *Feed) Create() error { // Given a string `url`, return to the best guess of the feed func ResolveFeedURL(url string) string { - c := &http.Client{ - Timeout: 10 * http.DefaultClient.Timeout, - } - if c.Timeout == 0 { - c.Timeout = 10 * time.Second - } + c := safehttp.NewSafeClient(10 * time.Second) req, err := http.NewRequest("GET", url, nil) if err != nil { @@ -20,6 +20,7 @@ import ( "adammathes.com/neko/api" "adammathes.com/neko/config" + "adammathes.com/neko/internal/safehttp" "golang.org/x/crypto/bcrypt" ) @@ -74,9 +75,7 @@ func imageProxyHandler(w http.ResponseWriter, r *http.Request) { } // grab the img - c := &http.Client{ - Timeout: 5 * time.Second, - } + c := safehttp.NewSafeClient(5 * time.Second) request, err := http.NewRequest("GET", string(decodedURL), nil) if err != nil { |