From 16186a344a7b61633cb7342aac37ac56ad83d261 Mon Sep 17 00:00:00 2001 From: Adam Mathes Date: Thu, 12 Feb 2026 19:55:05 -0800 Subject: =?UTF-8?q?Add=20comprehensive=20test=20suite=20=E2=80=94=2081%=20?= =?UTF-8?q?cross-package=20coverage?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bug fixes: - config: remove unused log import - item: fix Printf format %d->%t for boolean ReadState - util: update stale config.Read -> config.Init, remove config.Config.DBServer Test files added: - config/config_test.go: Init, readConfig, addDefaults (100%) - vlog/vlog_test.go: Printf, Println verbose/silent (100%) - models/db_test.go: InitDB tests - models/feed/feed_test.go: CRUD, filter, Categories, NewFeed, ResolveFeedURL (87%) - models/item/item_test.go: CRUD, Filter with category/search/starred, rewriteImages (71%) - exporter/exporter_test.go: all export formats (91%) - importer/importer_test.go: InsertIItem, ImportJSON (90%) - crawler/crawler_test.go: GetFeedContent, CrawlFeed, CrawlWorker, Crawl (89%) - web/web_test.go: auth, login/logout, stream, item, feed, category, export, crawl, imageProxy handlers (77%) Remaining 0% functions require HTTP/rice.MustFindBox/main entry and can't be unit tested without refactoring (see tickets NK-gqkh96, NK-6q9nyg). --- crawler/crawler_test.go | 233 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 233 insertions(+) create mode 100644 crawler/crawler_test.go (limited to 'crawler') diff --git a/crawler/crawler_test.go b/crawler/crawler_test.go new file mode 100644 index 0000000..f0cff9a --- /dev/null +++ b/crawler/crawler_test.go @@ -0,0 +1,233 @@ +package crawler + +import ( + "net/http" + "net/http/httptest" + "testing" + + "adammathes.com/neko/config" + "adammathes.com/neko/models" + "adammathes.com/neko/models/feed" +) + +func setupTestDB(t *testing.T) { + t.Helper() + config.Config.DBFile = ":memory:" + models.InitDB() + t.Cleanup(func() { + if models.DB != nil { + models.DB.Close() + } + }) +} + +func TestGetFeedContentSuccess(t *testing.T) { + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + ua := r.Header.Get("User-Agent") + if ua == "" { + t.Error("Request should include User-Agent") + } + w.WriteHeader(200) + w.Write([]byte("Test")) + })) + defer ts.Close() + + content := GetFeedContent(ts.URL) + if content == "" { + t.Error("GetFeedContent should return content for valid URL") + } + if content != "Test" { + t.Errorf("Unexpected content: %q", content) + } +} + +func TestGetFeedContentBadURL(t *testing.T) { + content := GetFeedContent("http://invalid.invalid.invalid:99999/feed") + if content != "" { + t.Errorf("GetFeedContent should return empty string for bad URL, got %q", content) + } +} + +func TestGetFeedContent404(t *testing.T) { + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(404) + })) + defer ts.Close() + + content := GetFeedContent(ts.URL) + if content != "" { + t.Errorf("GetFeedContent should return empty for 404, got %q", content) + } +} + +func TestGetFeedContent500(t *testing.T) { + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(500) + })) + defer ts.Close() + + content := GetFeedContent(ts.URL) + if content != "" { + t.Errorf("GetFeedContent should return empty for 500, got %q", content) + } +} + +func TestGetFeedContentUserAgent(t *testing.T) { + var receivedUA string + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + receivedUA = r.Header.Get("User-Agent") + w.WriteHeader(200) + w.Write([]byte("ok")) + })) + defer ts.Close() + + GetFeedContent(ts.URL) + expected := "neko RSS Crawler +https://github.com/adammathes/neko" + if receivedUA != expected { + t.Errorf("Expected UA %q, got %q", expected, receivedUA) + } +} + +func TestCrawlFeedWithTestServer(t *testing.T) { + setupTestDB(t) + + rssContent := ` + + + Test Feed + https://example.com + + Article 1 + https://example.com/article1 + First article + Mon, 01 Jan 2024 00:00:00 GMT + + + Article 2 + https://example.com/article2 + Second article + Tue, 02 Jan 2024 00:00:00 GMT + + +` + + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/rss+xml") + w.WriteHeader(200) + w.Write([]byte(rssContent)) + })) + defer ts.Close() + + // Create a feed pointing to the test server + f := &feed.Feed{Url: ts.URL, Title: "Test"} + f.Create() + + ch := make(chan string, 1) + CrawlFeed(f, ch) + result := <-ch + + if result == "" { + t.Error("CrawlFeed should send a result") + } + + // Verify items were created + var count int + models.DB.QueryRow("SELECT COUNT(*) FROM item").Scan(&count) + if count != 2 { + t.Errorf("Expected 2 items, got %d", count) + } +} + +func TestCrawlFeedBadContent(t *testing.T) { + setupTestDB(t) + + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(200) + w.Write([]byte("not xml at all")) + })) + defer ts.Close() + + f := &feed.Feed{Url: ts.URL, Title: "Bad"} + f.Create() + + ch := make(chan string, 1) + CrawlFeed(f, ch) + result := <-ch + + if result == "" { + t.Error("CrawlFeed should send a result even on failure") + } +} + +func TestCrawlWorker(t *testing.T) { + setupTestDB(t) + + rssContent := ` + + + Worker Feed + https://example.com + + Worker Article + https://example.com/worker-article + An article + + +` + + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(200) + w.Write([]byte(rssContent)) + })) + defer ts.Close() + + f := &feed.Feed{Url: ts.URL, Title: "Worker Test"} + f.Create() + + feeds := make(chan *feed.Feed, 1) + results := make(chan string, 1) + + feeds <- f + close(feeds) + + CrawlWorker(feeds, results) + result := <-results + + if result == "" { + t.Error("CrawlWorker should produce a result") + } +} + +func TestCrawl(t *testing.T) { + setupTestDB(t) + + rssContent := ` + + + Crawl Feed + https://example.com + + Crawl Article + https://example.com/crawl-article + Article for crawl test + + +` + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(200) + w.Write([]byte(rssContent)) + })) + defer ts.Close() + + f := &feed.Feed{Url: ts.URL, Title: "Full Crawl"} + f.Create() + + // Should not panic + Crawl() + + var count int + models.DB.QueryRow("SELECT COUNT(*) FROM item").Scan(&count) + if count != 1 { + t.Errorf("Expected 1 item after crawl, got %d", count) + } +} -- cgit v1.2.3