package dataset import ( "context" "errors" "net/url" "os" "testing" "github.com/stretchr/testify/require" "senlinai-agent/backend/internal/models" ) func TestParseFeedDetectsJSONFeedFromContent(t *testing.T) { feed, err := ParseFeed([]byte(`{ "version": "https://jsonfeed.org/version/1.1", "items": [{ "id": "json-1", "url": "https://example.com/json-1", "title": "JSON item", "summary": "JSON summary", "content_html": "

JSON content

", "date_published": "2026-07-23T08:00:00Z" }] }`), "text/plain") require.NoError(t, err) require.Equal(t, "json_feed", feed.Format) require.Len(t, feed.Items, 1) require.Equal(t, "JSON item", feed.Items[0].Title) require.Equal(t, "JSON content", feed.Items[0].Content) require.Equal(t, "https://example.com/json-1", feed.Items[0].URL) require.NotEmpty(t, feed.Items[0].ExternalID) require.NotNil(t, feed.Items[0].PublishedAt) } func TestParseFeedDetectsRSSXML(t *testing.T) { feed, err := ParseFeed([]byte(` RSS item https://example.com/rss-1 rss-1 RSS summary

]]>
RSS content

]]>
Thu, 23 Jul 2026 08:00:00 +0000
`), "application/xml") require.NoError(t, err) require.Equal(t, "rss", feed.Format) require.Len(t, feed.Items, 1) require.Equal(t, "RSS summary", feed.Items[0].Summary) require.Equal(t, "RSS content", feed.Items[0].Content) require.NotNil(t, feed.Items[0].PublishedAt) } func TestParseFeedDetectsAtomXML(t *testing.T) { feed, err := ParseFeed([]byte(` Atom item atom-1 Atom summary Atom content 2026-07-23T08:00:00Z `), "application/atom+xml") require.NoError(t, err) require.Equal(t, "atom", feed.Format) require.Len(t, feed.Items, 1) require.Equal(t, "https://example.com/atom-1", feed.Items[0].URL) require.Equal(t, "Atom content", feed.Items[0].Content) } func TestParseFeedRejectsUnknownContent(t *testing.T) { _, err := ParseFeed([]byte("not a feed"), "text/plain") require.ErrorIs(t, err, ErrFeedFormat) } func TestValidateFeedURLRejectsLocalAddressesAndCredentials(t *testing.T) { for _, value := range []string{ "http://127.0.0.1/feed", "http://[::1]/feed", "http://169.254.169.254/latest/meta-data", "https://user:password@example.com/feed", } { parsed, err := url.Parse(value) require.NoError(t, err) require.Error(t, validateFeedURL(parsed), value) } parsed, err := url.Parse("https://rsshub.app/cls/hot") require.NoError(t, err) require.NoError(t, validateFeedURL(parsed)) } func TestLiveFeedCollectionFormats(t *testing.T) { if os.Getenv("TEST_LIVE_FEEDS") != "1" { t.Skip("set TEST_LIVE_FEEDS=1 to verify external feeds") } fetcher := NewHTTPFeedFetcher() for _, test := range []struct { name string url string wantFormat string }{ {name: "JSON Feed", url: "https://bbg.buzzing.cc/feed.json", wantFormat: "json_feed"}, {name: "RSS XML", url: "https://rsshub.ktachibana.party/cls/hot", wantFormat: "rss"}, {name: "second RSS XML", url: "https://rsshub.ktachibana.party/jin10", wantFormat: "rss"}, } { t.Run(test.name, func(t *testing.T) { feed, err := fetcher.Fetch(context.Background(), test.url) require.NoError(t, err) require.Equal(t, test.wantFormat, feed.Format) require.NotEmpty(t, feed.Items) }) } } func TestSyncSourcesRecordsFeedFailure(t *testing.T) { database := newDatasetTestDatabase(t) user := createDatasetTestUser(t, database, "feed-owner@example.com") source := models.SaDatasetSource{ CreatedBy: user.ID, Name: "Broken feed", Kind: "rss", URL: "https://example.com/feed", Enabled: true, } require.NoError(t, database.Create(&source).Error) crons, err := newServiceWithFetcher(database, failingFeedFetcher{}).SyncSources(context.Background(), user.ID) require.NoError(t, err) require.Len(t, crons, 1) require.Equal(t, "failed", crons[0].Status) require.False(t, crons[0].Enabled) require.Contains(t, crons[0].LastResult, "feed unavailable") } func TestSyncAllSourcesCollectsEnabledRSSSourcesForEveryUser(t *testing.T) { database := newDatasetTestDatabase(t) firstUser := createDatasetTestUser(t, database, "first-feed-owner@example.com") secondUser := createDatasetTestUser(t, database, "second-feed-owner@example.com") for _, source := range []models.SaDatasetSource{ {CreatedBy: firstUser.ID, Name: "First RSS", Kind: "rss", URL: "https://example.com/first", Enabled: true}, {CreatedBy: secondUser.ID, Name: "Second RSS", Kind: "rss", URL: "https://example.com/second", Enabled: true}, {CreatedBy: firstUser.ID, Name: "Manual", Kind: "manual", Enabled: true}, } { require.NoError(t, database.Create(&source).Error) } crons, err := newServiceWithFetcher(database, staticFeedFetcher{}).SyncAllSources(context.Background()) require.NoError(t, err) require.Len(t, crons, 2) var itemCount int64 require.NoError(t, database.Model(&models.SaDatasetItem{}).Count(&itemCount).Error) require.Equal(t, int64(2), itemCount) } type failingFeedFetcher struct{} func (failingFeedFetcher) Fetch(context.Context, string) (ParsedFeed, error) { return ParsedFeed{}, errors.New("feed unavailable") }