agent/backend/internal/logic/dataset/feed_test.go

package dataset

import (
	"context"
	"errors"
	"fmt"
	"net/url"
	"os"
	"testing"
	"time"

	"github.com/stretchr/testify/require"
	"senlinai-agent/backend/internal/models"
)

func TestParseFeedDetectsJSONFeedFromContent(t *testing.T) {
	feed, err := ParseFeed([]byte(`{
		"version": "https://jsonfeed.org/version/1.1",
		"items": [{
			"id": "json-1",
			"url": "https://example.com/json-1",
			"title": "JSON item",
			"summary": "JSON summary",
			"content_html": "<p>JSON <strong>content</strong></p>",
			"date_published": "2026-07-23T08:00:00Z"
		}]
	}`), "text/plain")

	require.NoError(t, err)
	require.Equal(t, "json_feed", feed.Format)
	require.Len(t, feed.Items, 1)
	require.Equal(t, "JSON item", feed.Items[0].Title)
	require.Equal(t, "JSON content", feed.Items[0].Content)
	require.Equal(t, "https://example.com/json-1", feed.Items[0].URL)
	require.NotEmpty(t, feed.Items[0].ExternalID)
	require.NotNil(t, feed.Items[0].PublishedAt)
}

func TestParseFeedDetectsRSSXML(t *testing.T) {
	feed, err := ParseFeed([]byte(`<?xml version="1.0"?>
	<rss version="2.0">
		<channel>
			<item>
				<title>RSS item</title>
				<link>https://example.com/rss-1</link>
				<guid>rss-1</guid>
				<description><![CDATA[<p>RSS summary</p>]]></description>
				<content:encoded xmlns:content="http://purl.org/rss/1.0/modules/content/"><![CDATA[<p>RSS content</p>]]></content:encoded>
				<pubDate>Thu, 23 Jul 2026 08:00:00 +0000</pubDate>
			</item>
		</channel>
	</rss>`), "application/xml")

	require.NoError(t, err)
	require.Equal(t, "rss", feed.Format)
	require.Len(t, feed.Items, 1)
	require.Equal(t, "RSS summary", feed.Items[0].Summary)
	require.Equal(t, "RSS content", feed.Items[0].Content)
	require.NotNil(t, feed.Items[0].PublishedAt)
}

func TestParseFeedDetectsAtomXML(t *testing.T) {
	feed, err := ParseFeed([]byte(`<?xml version="1.0"?>
	<feed xmlns="http://www.w3.org/2005/Atom">
		<entry>
			<title>Atom item</title>
			<id>atom-1</id>
			<link rel="alternate" href="https://example.com/atom-1"/>
			<summary>Atom summary</summary>
			<content>Atom content</content>
			<updated>2026-07-23T08:00:00Z</updated>
		</entry>
	</feed>`), "application/atom+xml")

	require.NoError(t, err)
	require.Equal(t, "atom", feed.Format)
	require.Len(t, feed.Items, 1)
	require.Equal(t, "https://example.com/atom-1", feed.Items[0].URL)
	require.Equal(t, "Atom content", feed.Items[0].Content)
}

func TestParseFeedRejectsUnknownContent(t *testing.T) {
	_, err := ParseFeed([]byte("not a feed"), "text/plain")
	require.ErrorIs(t, err, ErrFeedFormat)
}

func TestValidateFeedURLRejectsLocalAddressesAndCredentials(t *testing.T) {
	for _, value := range []string{
		"http://127.0.0.1/feed",
		"http://[::1]/feed",
		"http://169.254.169.254/latest/meta-data",
		"https://user:password@example.com/feed",
	} {
		parsed, err := url.Parse(value)
		require.NoError(t, err)
		require.Error(t, validateFeedURL(parsed), value)
	}

	parsed, err := url.Parse("https://rsshub.app/cls/hot")
	require.NoError(t, err)
	require.NoError(t, validateFeedURL(parsed))
}

func TestLiveFeedCollectionFormats(t *testing.T) {
	if os.Getenv("TEST_LIVE_FEEDS") != "1" {
		t.Skip("set TEST_LIVE_FEEDS=1 to verify external feeds")
	}
	fetcher := NewHTTPFeedFetcher()
	for _, test := range []struct {
		name       string
		url        string
		wantFormat string
	}{
		{name: "JSON Feed", url: "https://bbg.buzzing.cc/feed.json", wantFormat: "json_feed"},
		{name: "RSS XML", url: "https://rsshub.ktachibana.party/cls/hot", wantFormat: "rss"},
		{name: "second RSS XML", url: "https://rsshub.ktachibana.party/jin10", wantFormat: "rss"},
	} {
		t.Run(test.name, func(t *testing.T) {
			feed, err := fetcher.Fetch(context.Background(), test.url)
			require.NoError(t, err)
			require.Equal(t, test.wantFormat, feed.Format)
			require.NotEmpty(t, feed.Items)
		})
	}
}

func TestSyncSourcesRecordsFeedFailure(t *testing.T) {
	database := newDatasetTestDatabase(t)
	user := createDatasetTestUser(t, database, "feed-owner@example.com")
	source := models.SaDatasetSource{
		CreatedBy: user.ID,
		Name:      "Broken feed",
		Kind:      "rss",
		URL:       "https://example.com/feed",
		Enabled:   true,
	}
	require.NoError(t, database.Create(&source).Error)

	crons, err := newServiceWithFetcher(database, failingFeedFetcher{}).SyncSources(context.Background(), user.ID)

	require.NoError(t, err)
	require.Len(t, crons, 1)
	require.Equal(t, "failed", crons[0].Status)
	require.False(t, crons[0].Enabled)
	require.Contains(t, crons[0].LastResult, "feed unavailable")
}

func TestSyncAllSourcesCollectsEnabledRSSSourcesForEveryUser(t *testing.T) {
	database := newDatasetTestDatabase(t)
	firstUser := createDatasetTestUser(t, database, "first-feed-owner@example.com")
	secondUser := createDatasetTestUser(t, database, "second-feed-owner@example.com")
	for _, source := range []models.SaDatasetSource{
		{CreatedBy: firstUser.ID, Name: "First RSS", Kind: "rss", URL: "https://example.com/first", Enabled: true},
		{CreatedBy: secondUser.ID, Name: "Second RSS", Kind: "rss", URL: "https://example.com/second", Enabled: true},
		{CreatedBy: firstUser.ID, Name: "Manual", Kind: "manual", Enabled: true},
	} {
		require.NoError(t, database.Create(&source).Error)
	}

	crons, err := newServiceWithFetcher(database, staticFeedFetcher{}).SyncAllSources(context.Background())

	require.NoError(t, err)
	require.Len(t, crons, 2)
	var itemCount int64
	require.NoError(t, database.Model(&models.SaDatasetItem{}).Count(&itemCount).Error)
	require.Equal(t, int64(2), itemCount)
}

func TestSyncSourcesCollectsMultipleSourcesForOneOwner(t *testing.T) {
	database := newDatasetTestDatabase(t)
	user := createDatasetTestUser(t, database, "multi-feed-owner@example.com")
	for index := 0; index < 5; index++ {
		require.NoError(t, database.Create(&models.SaDatasetSource{
			CreatedBy: user.ID, Name: fmt.Sprintf("Feed %d", index), Kind: "rss",
			URL: fmt.Sprintf("https://example.com/feed-%d", index), Enabled: true,
		}).Error)
	}

	runs, err := newServiceWithFetcher(database, staticFeedFetcher{}).SyncSources(context.Background(), user.ID)

	require.NoError(t, err)
	require.Len(t, runs, 5)
	for _, run := range runs {
		require.Equal(t, "completed", run.Status)
	}
}

func TestQueueSourcesRunsInBackgroundAndRejectsOverlap(t *testing.T) {
	database := newDatasetTestDatabase(t)
	user := createDatasetTestUser(t, database, "queued-feed-owner@example.com")
	source := models.SaDatasetSource{
		CreatedBy: user.ID, Name: "Queued RSS", Kind: "rss",
		URL: "https://example.com/feed", Enabled: true,
	}
	require.NoError(t, database.Create(&source).Error)
	fetcher := &blockingFeedFetcher{
		started: make(chan struct{}),
		release: make(chan struct{}),
	}
	service := newServiceWithFetcher(database, fetcher)

	queued, err := service.QueueSources(user.ID)

	require.NoError(t, err)
	require.Len(t, queued, 1)
	require.Equal(t, "pending", queued[0].Status)
	<-fetcher.started
	_, err = service.QueueSources(user.ID)
	require.ErrorIs(t, err, ErrSyncInProgress)
	require.ErrorIs(t, service.DeleteSource(user.ID, source.Identity), ErrSyncInProgress)
	close(fetcher.release)
	require.Eventually(t, func() bool {
		runs, listErr := service.ListCrons(user.ID)
		return listErr == nil && len(runs) == 1 && runs[0].Status == "completed"
	}, time.Second, 10*time.Millisecond)
}

type failingFeedFetcher struct{}

func (failingFeedFetcher) Fetch(context.Context, string) (ParsedFeed, error) {
	return ParsedFeed{}, errors.New("feed unavailable")
}

type blockingFeedFetcher struct {
	started chan struct{}
	release chan struct{}
}

func (f *blockingFeedFetcher) Fetch(context.Context, string) (ParsedFeed, error) {
	close(f.started)
	<-f.release
	return staticFeedFetcher{}.Fetch(context.Background(), "")
}