feat: collect JSON and XML feeds

2026-07-23 20:47:54 +08:00
parent 4b38a42607
commit 5da343cc71
8 changed files with 627 additions and 42 deletions
--- a/backend/internal/logic/dataset/feed_test.go
+++ b/backend/internal/logic/dataset/feed_test.go
@@ -0,0 +1,126 @@
+package dataset
+
+import (
+	"context"
+	"errors"
+	"net/url"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+	"senlinai-agent/backend/internal/models"
+)
+
+func TestParseFeedDetectsJSONFeedFromContent(t *testing.T) {
+	feed, err := ParseFeed([]byte(`{
+		"version": "https://jsonfeed.org/version/1.1",
+		"items": [{
+			"id": "json-1",
+			"url": "https://example.com/json-1",
+			"title": "JSON item",
+			"summary": "JSON summary",
+			"content_html": "<p>JSON <strong>content</strong></p>",
+			"date_published": "2026-07-23T08:00:00Z"
+		}]
+	}`), "text/plain")
+
+	require.NoError(t, err)
+	require.Equal(t, "json_feed", feed.Format)
+	require.Len(t, feed.Items, 1)
+	require.Equal(t, "JSON item", feed.Items[0].Title)
+	require.Equal(t, "JSON content", feed.Items[0].Content)
+	require.Equal(t, "https://example.com/json-1", feed.Items[0].URL)
+	require.NotEmpty(t, feed.Items[0].ExternalID)
+	require.NotNil(t, feed.Items[0].PublishedAt)
+}
+
+func TestParseFeedDetectsRSSXML(t *testing.T) {
+	feed, err := ParseFeed([]byte(`<?xml version="1.0"?>
+	<rss version="2.0">
+		<channel>
+			<item>
+				<title>RSS item</title>
+				<link>https://example.com/rss-1</link>
+				<guid>rss-1</guid>
+				<description><![CDATA[<p>RSS summary</p>]]></description>
+				<content:encoded xmlns:content="http://purl.org/rss/1.0/modules/content/"><![CDATA[<p>RSS content</p>]]></content:encoded>
+				<pubDate>Thu, 23 Jul 2026 08:00:00 +0000</pubDate>
+			</item>
+		</channel>
+	</rss>`), "application/xml")
+
+	require.NoError(t, err)
+	require.Equal(t, "rss", feed.Format)
+	require.Len(t, feed.Items, 1)
+	require.Equal(t, "RSS summary", feed.Items[0].Summary)
+	require.Equal(t, "RSS content", feed.Items[0].Content)
+	require.NotNil(t, feed.Items[0].PublishedAt)
+}
+
+func TestParseFeedDetectsAtomXML(t *testing.T) {
+	feed, err := ParseFeed([]byte(`<?xml version="1.0"?>
+	<feed xmlns="http://www.w3.org/2005/Atom">
+		<entry>
+			<title>Atom item</title>
+			<id>atom-1</id>
+			<link rel="alternate" href="https://example.com/atom-1"/>
+			<summary>Atom summary</summary>
+			<content>Atom content</content>
+			<updated>2026-07-23T08:00:00Z</updated>
+		</entry>
+	</feed>`), "application/atom+xml")
+
+	require.NoError(t, err)
+	require.Equal(t, "atom", feed.Format)
+	require.Len(t, feed.Items, 1)
+	require.Equal(t, "https://example.com/atom-1", feed.Items[0].URL)
+	require.Equal(t, "Atom content", feed.Items[0].Content)
+}
+
+func TestParseFeedRejectsUnknownContent(t *testing.T) {
+	_, err := ParseFeed([]byte("not a feed"), "text/plain")
+	require.ErrorIs(t, err, ErrFeedFormat)
+}
+
+func TestValidateFeedURLRejectsLocalAddressesAndCredentials(t *testing.T) {
+	for _, value := range []string{
+		"http://127.0.0.1/feed",
+		"http://[::1]/feed",
+		"http://169.254.169.254/latest/meta-data",
+		"https://user:password@example.com/feed",
+	} {
+		parsed, err := url.Parse(value)
+		require.NoError(t, err)
+		require.Error(t, validateFeedURL(parsed), value)
+	}
+
+	parsed, err := url.Parse("https://rsshub.app/cls/hot")
+	require.NoError(t, err)
+	require.NoError(t, validateFeedURL(parsed))
+}
+
+func TestSyncSourcesRecordsFeedFailure(t *testing.T) {
+	database := newDatasetTestDatabase(t)
+	user := createDatasetTestUser(t, database, "feed-owner@example.com")
+	source := models.SaDatasetSource{
+		CreatedBy: user.ID,
+		Name:      "Broken feed",
+		Kind:      "rss",
+		URL:       "https://example.com/feed",
+		Enabled:   true,
+	}
+	require.NoError(t, database.Create(&source).Error)
+
+	crons, err := newServiceWithFetcher(database, failingFeedFetcher{}).SyncSources(context.Background(), user.ID)
+
+	require.NoError(t, err)
+	require.Len(t, crons, 1)
+	require.Equal(t, "failed", crons[0].Status)
+	require.False(t, crons[0].Enabled)
+	require.Contains(t, crons[0].LastResult, "feed unavailable")
+}
+
+type failingFeedFetcher struct{}
+
+func (failingFeedFetcher) Fetch(context.Context, string) (ParsedFeed, error) {
+	return ParsedFeed{}, errors.New("feed unavailable")
+}