fix: harden explore dataset collection

backend/internal/logic/dataset/feed_test.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"errors"
 	"net/url"
+	"os"
 	"testing"
 
 	"github.com/stretchr/testify/require"
@@ -98,6 +99,29 @@ func TestValidateFeedURLRejectsLocalAddressesAndCredentials(t *testing.T) {
 	require.NoError(t, validateFeedURL(parsed))
 }
 
+func TestLiveFeedCollectionFormats(t *testing.T) {
+	if os.Getenv("TEST_LIVE_FEEDS") != "1" {
+		t.Skip("set TEST_LIVE_FEEDS=1 to verify external feeds")
+	}
+	fetcher := NewHTTPFeedFetcher()
+	for _, test := range []struct {
+		name       string
+		url        string
+		wantFormat string
+	}{
+		{name: "JSON Feed", url: "https://bbg.buzzing.cc/feed.json", wantFormat: "json_feed"},
+		{name: "RSS XML", url: "https://rsshub.ktachibana.party/cls/hot", wantFormat: "rss"},
+		{name: "second RSS XML", url: "https://rsshub.ktachibana.party/jin10", wantFormat: "rss"},
+	} {
+		t.Run(test.name, func(t *testing.T) {
+			feed, err := fetcher.Fetch(context.Background(), test.url)
+			require.NoError(t, err)
+			require.Equal(t, test.wantFormat, feed.Format)
+			require.NotEmpty(t, feed.Items)
+		})
+	}
+}
+
 func TestSyncSourcesRecordsFeedFailure(t *testing.T) {
 	database := newDatasetTestDatabase(t)
 	user := createDatasetTestUser(t, database, "feed-owner@example.com")

backend/internal/logic/dataset/handlers_test.go

@@ -38,6 +38,10 @@ func TestDatasetAPIConnectsSourcesItemsAndSyncTasks(t *testing.T) {
 	require.NotEmpty(t, source.ID)
 	require.Equal(t, "Industry feed", source.Name)
 	require.Empty(t, source.IconURL)
+	var storedSource models.SaDatasetSource
+	require.NoError(t, database.Where("identity = ?", source.ID).First(&storedSource).Error)
+	require.Equal(t, owner.ID, storedSource.CreatedBy)
+	require.Equal(t, owner.Identity, storedSource.CreatedByIdentity)
 
 	createdItem := performDatasetRequest(t, ownerRouter, http.MethodPost, "/api/v1/dataset-items", map[string]any{
 		"sourceId": source.ID, "title": "A collected article", "summary": "Summary",
@@ -108,6 +112,10 @@ func newDatasetTestDatabase(t *testing.T) *gorm.DB {
 		&models.SaDatasetItem{},
 		&models.SaDatasetCron{},
 	))
+	require.NoError(t, database.Exec(`
+		CREATE UNIQUE INDEX idx_sa_dataset_item_source_external
+		ON sa_dataset_items (source_id, external_id)
+	`).Error)
 	return database
 }
 

backend/internal/logic/dataset/service.go

@@ -10,6 +10,7 @@ import (
 	"time"
 
 	"gorm.io/gorm"
+	"gorm.io/gorm/clause"
 	"senlinai-agent/backend/internal/models"
 )
 
@@ -326,24 +327,20 @@ func (s *Service) storeFeedItems(userID uint, source models.SaDatasetSource, ite
 	inserted := 0
 	err := s.db.Transaction(func(tx *gorm.DB) error {
 		for _, input := range items {
-			var count int64
-			if err := tx.Model(&models.SaDatasetItem{}).
-				Where("source_id = ? AND external_id = ?", source.ID, input.ExternalID).
-				Count(&count).Error; err != nil {
-				return err
-			}
-			if count > 0 {
-				continue
-			}
+			externalID := input.ExternalID
 			item := models.SaDatasetItem{
-				SourceID: source.ID, CreatedBy: userID, ExternalID: input.ExternalID,
+				SourceID: source.ID, CreatedBy: userID, ExternalID: &externalID,
 				Title: input.Title, Summary: input.Summary, Content: input.Content,
 				URL: input.URL, Status: "unread", PublishedAt: input.PublishedAt,
 			}
-			if err := tx.Create(&item).Error; err != nil {
-				return err
+			result := tx.Clauses(clause.OnConflict{
+				Columns:   []clause.Column{{Name: "source_id"}, {Name: "external_id"}},
+				DoNothing: true,
+			}).Create(&item)
+			if result.Error != nil {
+				return result.Error
 			}
-			inserted++
+			inserted += int(result.RowsAffected)
 		}
 		return nil
 	})