fix: harden explore dataset collection

2026-07-23 21:34:13 +08:00
parent 3749cf4543
commit 50d8246915
9 changed files with 254 additions and 87 deletions
--- a/backend/internal/initdb/dataset.go
+++ b/backend/internal/initdb/dataset.go
@@ -8,53 +8,109 @@ import (
 	"senlinai-agent/backend/internal/models"
 )

-var defaultDatasetSources = []struct {
-	Name    string
-	URL     string
-	IconURL string
-}{
-	{Name: "彭博社最新报道", URL: "https://bbg.buzzing.cc/feed.json", IconURL: "https://bbg.buzzing.cc/apple-touch-icon.png"},
-	{Name: "国外科技头条", URL: "https://tech.buzzing.cc/feed.json", IconURL: "https://tech.buzzing.cc/apple-touch-icon.png"},
-	{Name: "国外财经新闻", URL: "https://finance.buzzing.cc/feed.json", IconURL: "https://finance.buzzing.cc/apple-touch-icon.png"},
-	{Name: "华尔街日报热门", URL: "https://wsj.buzzing.cc/feed.json", IconURL: "https://wsj.buzzing.cc/apple-touch-icon.png"},
-	{Name: "Product Hunt", URL: "https://ph.buzzing.cc/feed.json", IconURL: "https://ph.buzzing.cc/apple-touch-icon.png"},
-	{Name: "Dev.to", URL: "https://dev.buzzing.cc/feed.json", IconURL: "https://dev.buzzing.cc/apple-touch-icon.png"},
-	{Name: "财联社-热门", URL: "https://rsshub.app/cls/hot"},
-	{Name: "金十数据-快讯", URL: "https://rsshub.app/jin10"},
+type defaultDatasetSource struct {
+	Key        string
+	Name       string
+	URL        string
+	IconURL    string
+	LegacyURLs []string
 }

-// InitDataset creates the default Buzzing RSS sources for the default account when the source table is empty.
-func InitDataset(database *gorm.DB) error {
-	var count int64
-	if err := database.Model(&models.SaDatasetSource{}).Count(&count).Error; err != nil {
-		return err
-	}
-	if count > 0 {
-		return nil
-	}
+var defaultDatasetSources = []defaultDatasetSource{
+	{Key: "buzzing-bloomberg", Name: "彭博社最新报道", URL: "https://bbg.buzzing.cc/feed.json", IconURL: "https://bbg.buzzing.cc/apple-touch-icon.png"},
+	{Key: "buzzing-tech", Name: "国外科技头条", URL: "https://tech.buzzing.cc/feed.json", IconURL: "https://tech.buzzing.cc/apple-touch-icon.png"},
+	{Key: "buzzing-finance", Name: "国外财经新闻", URL: "https://finance.buzzing.cc/feed.json", IconURL: "https://finance.buzzing.cc/apple-touch-icon.png"},
+	{Key: "buzzing-wsj", Name: "华尔街日报热门", URL: "https://wsj.buzzing.cc/feed.json", IconURL: "https://wsj.buzzing.cc/apple-touch-icon.png"},
+	{Key: "buzzing-product-hunt", Name: "Product Hunt", URL: "https://ph.buzzing.cc/feed.json", IconURL: "https://ph.buzzing.cc/apple-touch-icon.png"},
+	{Key: "buzzing-devto", Name: "Dev.to", URL: "https://dev.buzzing.cc/feed.json", IconURL: "https://dev.buzzing.cc/apple-touch-icon.png"},
+	{
+		Key: "rsshub-cls-hot", Name: "财联社-热门",
+		URL:        "https://rsshub.ktachibana.party/cls/hot",
+		LegacyURLs: []string{"https://rsshub.app/cls/hot"},
+	},
+	{
+		Key: "rsshub-jin10", Name: "金十数据-快讯",
+		URL:        "https://rsshub.ktachibana.party/jin10",
+		LegacyURLs: []string{"https://rsshub.app/jin10"},
+	},
+}

-	var owner models.SaUser
-	err := database.Where("email = ?", rootUsername).First(&owner).Error
-	if err != nil && !errors.Is(err, gorm.ErrRecordNotFound) {
+// InitDataset inserts or repairs root's built-in dataset sources without touching user-created sources.
+func InitDataset(database *gorm.DB) error {
+	var root models.SaUser
+	if err := database.Where("email = ? AND role = ?", rootUsername, rootRole).First(&root).Error; err != nil {
 		return fmt.Errorf("find root user for default dataset sources: %w", err)
 	}
-	if errors.Is(err, gorm.ErrRecordNotFound) {
-		if err := database.Order("id asc").First(&owner).Error; err != nil {
-			return fmt.Errorf("find owner for default dataset sources: %w", err)
+
+	return database.Transaction(func(tx *gorm.DB) error {
+		if err := prepareDatasetItems(tx); err != nil {
+			return err
 		}
+		for _, definition := range defaultDatasetSources {
+			if err := upsertDefaultDatasetSource(tx, root, definition); err != nil {
+				return err
+			}
+		}
+		return nil
+	})
+}
+
+func prepareDatasetItems(tx *gorm.DB) error {
+	if tx.Migrator().HasIndex(&models.SaDatasetItem{}, "idx_sa_dataset_item_source_external") {
+		return nil
+	}
+	if err := tx.Exec(
+		"UPDATE sa_dataset_items SET external_id = NULL WHERE external_id = ''",
+	).Error; err != nil {
+		return err
+	}
+	if err := tx.Exec(`
+		DELETE FROM sa_dataset_items
+		WHERE external_id IS NOT NULL
+		  AND EXISTS (
+			SELECT 1
+			FROM sa_dataset_items AS earlier
+			WHERE earlier.source_id = sa_dataset_items.source_id
+			  AND earlier.external_id = sa_dataset_items.external_id
+			  AND earlier.id < sa_dataset_items.id
+		  )
+	`).Error; err != nil {
+		return err
+	}
+	return tx.Exec(`
+		CREATE UNIQUE INDEX IF NOT EXISTS idx_sa_dataset_item_source_external
+		ON sa_dataset_items (source_id, external_id)
+	`).Error
+}
+
+func upsertDefaultDatasetSource(tx *gorm.DB, root models.SaUser, definition defaultDatasetSource) error {
+	var source models.SaDatasetSource
+	err := tx.Where("created_by = ? AND seed_key = ?", root.ID, definition.Key).First(&source).Error
+	if err == nil {
+		return tx.Model(&source).Updates(map[string]any{
+			"url": definition.URL, "icon_url": definition.IconURL, "kind": "rss",
+		}).Error
+	}
+	if !errors.Is(err, gorm.ErrRecordNotFound) {
+		return err
 	}

-	sources := make([]models.SaDatasetSource, 0, len(defaultDatasetSources))
-	for _, source := range defaultDatasetSources {
-		sources = append(sources, models.SaDatasetSource{
-			CreatedBy:   owner.ID,
-			Name:        source.Name,
-			Kind:        "rss",
-			URL:         source.URL,
-			IconURL:     source.IconURL,
-			Description: "Buzzing 中文聚合订阅",
-			Enabled:     true,
-		})
+	knownURLs := append([]string{definition.URL}, definition.LegacyURLs...)
+	err = tx.Where("created_by = ? AND url IN ?", root.ID, knownURLs).First(&source).Error
+	if err == nil {
+		return tx.Model(&source).Updates(map[string]any{
+			"seed_key": definition.Key, "url": definition.URL,
+			"icon_url": definition.IconURL, "kind": "rss",
+		}).Error
 	}
-	return database.Create(&sources).Error
+	if !errors.Is(err, gorm.ErrRecordNotFound) {
+		return err
+	}
+
+	seedKey := definition.Key
+	return tx.Create(&models.SaDatasetSource{
+		CreatedBy: root.ID, SeedKey: &seedKey, Name: definition.Name,
+		Kind: "rss", URL: definition.URL, IconURL: definition.IconURL,
+		Description: "默认资讯订阅", Enabled: true,
+	}).Error
 }
--- a/backend/internal/initdb/dataset_test.go
+++ b/backend/internal/initdb/dataset_test.go
@@ -3,6 +3,7 @@ package initdb
 import (
 	"fmt"
 	"testing"
+	"time"

 	"github.com/glebarez/sqlite"
 	"github.com/stretchr/testify/require"
@@ -26,6 +27,8 @@ func TestInitDatasetCreatesDefaultSourcesWhenTableIsEmpty(t *testing.T) {
 		require.Equal(t, expected.Name, sources[index].Name)
 		require.Equal(t, expected.URL, sources[index].URL)
 		require.Equal(t, expected.IconURL, sources[index].IconURL)
+		require.NotNil(t, sources[index].SeedKey)
+		require.Equal(t, expected.Key, *sources[index].SeedKey)
 		require.Equal(t, "rss", sources[index].Kind)
 		require.True(t, sources[index].Enabled)
 		require.Equal(t, root.ID, sources[index].CreatedBy)
@@ -33,7 +36,7 @@ func TestInitDatasetCreatesDefaultSourcesWhenTableIsEmpty(t *testing.T) {
 	}
 }

-func TestInitDatasetDoesNothingWhenSourceTableIsNotEmpty(t *testing.T) {
+func TestInitDatasetAddsMissingDefaultsWithoutChangingCustomSources(t *testing.T) {
 	database := newDatasetInitTestDatabase(t)
 	require.NoError(t, InitUser(database))
 	var root models.SaUser
@@ -50,11 +53,14 @@ func TestInitDatasetDoesNothingWhenSourceTableIsNotEmpty(t *testing.T) {

 	var sources []models.SaDatasetSource
 	require.NoError(t, database.Find(&sources).Error)
-	require.Len(t, sources, 1)
-	require.Equal(t, existing.Identity, sources[0].Identity)
+	require.Len(t, sources, len(defaultDatasetSources)+1)
+	var preserved models.SaDatasetSource
+	require.NoError(t, database.Where("identity = ?", existing.Identity).First(&preserved).Error)
+	require.Equal(t, existing.Name, preserved.Name)
+	require.Nil(t, preserved.SeedKey)
 }

-func TestInitDatasetUsesFirstExistingUserWhenRootIsAbsent(t *testing.T) {
+func TestInitDatasetRejectsMissingRootInsteadOfAssigningDefaultsToAnotherUser(t *testing.T) {
 	database := newDatasetInitTestDatabase(t)
 	user := models.SaUser{
 		Email:        "existing@example.com",
@@ -64,21 +70,93 @@ func TestInitDatasetUsesFirstExistingUserWhenRootIsAbsent(t *testing.T) {
 	}
 	require.NoError(t, database.Create(&user).Error)

-	require.NoError(t, InitDataset(database))
+	require.Error(t, InitDataset(database))

 	var sources []models.SaDatasetSource
 	require.NoError(t, database.Find(&sources).Error)
-	require.Len(t, sources, len(defaultDatasetSources))
-	for _, source := range sources {
-		require.Equal(t, user.ID, source.CreatedBy)
-		require.Equal(t, user.Identity, source.CreatedByIdentity)
+	require.Empty(t, sources)
+}
+
+func TestInitDatasetRepairsLegacyRSSHubURLsForRootOnly(t *testing.T) {
+	database := newDatasetInitTestDatabase(t)
+	require.NoError(t, InitUser(database))
+	var root models.SaUser
+	require.NoError(t, database.Where("email = ?", rootUsername).First(&root).Error)
+	other := models.SaUser{
+		Email: "other@example.com", DisplayName: "Other",
+		PasswordHash: "hash", Role: "user",
 	}
+	require.NoError(t, database.Create(&other).Error)
+
+	rootLegacy := models.SaDatasetSource{
+		CreatedBy: root.ID, Name: "财联社-热门", Kind: "rss",
+		URL: "https://rsshub.app/cls/hot", Enabled: true,
+	}
+	otherLegacy := models.SaDatasetSource{
+		CreatedBy: other.ID, Name: "财联社-热门", Kind: "rss",
+		URL: "https://rsshub.app/cls/hot", Enabled: true,
+	}
+	require.NoError(t, database.Create(&rootLegacy).Error)
+	require.NoError(t, database.Create(&otherLegacy).Error)
+
+	require.NoError(t, InitDataset(database))
+
+	require.NoError(t, database.First(&rootLegacy, rootLegacy.ID).Error)
+	require.Equal(t, "https://rsshub.ktachibana.party/cls/hot", rootLegacy.URL)
+	require.NotNil(t, rootLegacy.SeedKey)
+	require.Equal(t, "rsshub-cls-hot", *rootLegacy.SeedKey)
+	require.NoError(t, database.First(&otherLegacy, otherLegacy.ID).Error)
+	require.Equal(t, "https://rsshub.app/cls/hot", otherLegacy.URL)
+	require.Nil(t, otherLegacy.SeedKey)
+}
+
+func TestInitDatasetPreparesCollectedItemUniqueness(t *testing.T) {
+	database := newDatasetInitTestDatabase(t)
+	require.NoError(t, InitUser(database))
+	var root models.SaUser
+	require.NoError(t, database.Where("email = ?", rootUsername).First(&root).Error)
+	source := models.SaDatasetSource{
+		CreatedBy: root.ID, Name: "Existing feed", Kind: "rss",
+		URL: "https://example.com/feed", Enabled: true,
+	}
+	require.NoError(t, database.Create(&source).Error)
+	now := time.Now()
+	for index, externalID := range []string{"duplicate", "duplicate", "", ""} {
+		require.NoError(t, database.Exec(`
+			INSERT INTO sa_dataset_items
+				(identity, source_id, created_by, external_id, title, status, created_at, updated_at)
+			VALUES (?, ?, ?, ?, ?, ?, ?, ?)
+		`, fmt.Sprintf("item-%d", index), source.ID, root.ID, externalID,
+			fmt.Sprintf("Item %d", index), "unread", now, now).Error)
+	}
+
+	require.NoError(t, InitDataset(database))
+
+	var collectedCount int64
+	require.NoError(t, database.Model(&models.SaDatasetItem{}).
+		Where("source_id = ? AND external_id = ?", source.ID, "duplicate").
+		Count(&collectedCount).Error)
+	require.Equal(t, int64(1), collectedCount)
+	var manualCount int64
+	require.NoError(t, database.Model(&models.SaDatasetItem{}).
+		Where("source_id = ? AND external_id IS NULL", source.ID).
+		Count(&manualCount).Error)
+	require.Equal(t, int64(2), manualCount)
+	require.Error(t, database.Exec(`
+		INSERT INTO sa_dataset_items
+			(identity, source_id, created_by, external_id, title, status, created_at, updated_at)
+		VALUES (?, ?, ?, ?, ?, ?, ?, ?)
+	`, "item-new", source.ID, root.ID, "duplicate", "Duplicate", "unread", now, now).Error)
 }

 func newDatasetInitTestDatabase(t *testing.T) *gorm.DB {
 	t.Helper()
 	database, err := gorm.Open(sqlite.Open(fmt.Sprintf("file:%s?mode=memory&cache=shared", t.Name())), &gorm.Config{})
 	require.NoError(t, err)
-	require.NoError(t, database.AutoMigrate(&models.SaUser{}, &models.SaDatasetSource{}))
+	require.NoError(t, database.AutoMigrate(
+		&models.SaUser{},
+		&models.SaDatasetSource{},
+		&models.SaDatasetItem{},
+	))
 	return database
 }
--- a/backend/internal/initdb/expert_test.go
+++ b/backend/internal/initdb/expert_test.go
@@ -60,6 +60,7 @@ func newExpertTestDatabase(t *testing.T) *gorm.DB {
 	require.NoError(t, database.AutoMigrate(
 		&models.SaUser{},
 		&models.SaDatasetSource{},
+		&models.SaDatasetItem{},
 		&models.SaAIExpertCategory{},
 		&models.SaAIExpertItem{},
 	))