package crawler import ( "net/http" "net/http/httptest" "testing" "adammathes.com/neko/config" "adammathes.com/neko/internal/safehttp" "adammathes.com/neko/models" "adammathes.com/neko/models/feed" "github.com/mmcdole/gofeed" ) const testRSSFeed = ` Bench Test Feed https://example.com A feed for benchmarking Article One https://example.com/1 <p>First article with <b>bold</b> and <a href="https://example.com">link</a></p> Mon, 01 Jan 2024 00:00:00 +0000 Article Two https://example.com/2 <p>Second article with some content</p> Tue, 02 Jan 2024 00:00:00 +0000 Article Three https://example.com/3 <p>Third article</p> Wed, 03 Jan 2024 00:00:00 +0000 Article Four https://example.com/4 <p>Fourth article with <img src="https://example.com/img.jpg"></p> Thu, 04 Jan 2024 00:00:00 +0000 Article Five https://example.com/5 <p>Fifth article</p> Fri, 05 Jan 2024 00:00:00 +0000 Article Six https://example.com/6 <p>Sixth article</p> Sat, 06 Jan 2024 00:00:00 +0000 Article Seven https://example.com/7 <p>Seventh article</p> Sun, 07 Jan 2024 00:00:00 +0000 Article Eight https://example.com/8 <p>Eighth article</p> Mon, 08 Jan 2024 00:00:00 +0000 Article Nine https://example.com/9 <p>Ninth article with longer content to simulate a real feed item that has more text in it</p> Tue, 09 Jan 2024 00:00:00 +0000 Article Ten https://example.com/10 <p>Tenth article</p> Wed, 10 Jan 2024 00:00:00 +0000 ` func BenchmarkParseFeed(b *testing.B) { fp := gofeed.NewParser() b.ResetTimer() for i := 0; i < b.N; i++ { _, err := fp.ParseString(testRSSFeed) if err != nil { b.Fatal(err) } } } func BenchmarkCrawlFeedMocked(b *testing.B) { safehttp.AllowLocal = true ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/rss+xml") w.WriteHeader(200) w.Write([]byte(testRSSFeed)) })) defer ts.Close() b.ResetTimer() for i := 0; i < b.N; i++ { // Each iteration needs a fresh DB since CrawlFeed inserts items config.Config.DBFile = ":memory:" models.InitDB() f := &feed.Feed{Url: ts.URL, Title: "Bench Feed"} f.Create() ch := make(chan string, 1) CrawlFeed(f, ch) <-ch models.DB.Close() } } func BenchmarkGetFeedContent(b *testing.B) { safehttp.AllowLocal = true ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/rss+xml") w.WriteHeader(200) w.Write([]byte(testRSSFeed)) })) defer ts.Close() b.ResetTimer() for i := 0; i < b.N; i++ { content := GetFeedContent(ts.URL) if content == "" { b.Fatal("empty content") } } }