fix: harden explore dataset collection

2026-07-23 21:34:13 +08:00
parent 3749cf4543
commit 50d8246915
9 changed files with 254 additions and 87 deletions
--- a/backend/internal/initdb/dataset.go
+++ b/backend/internal/initdb/dataset.go
@@ -8,53 +8,109 @@ import (
 	"senlinai-agent/backend/internal/models"
 )

-var defaultDatasetSources = []struct {
-	Name    string
-	URL     string
-	IconURL string
-}{
-	{Name: "彭博社最新报道", URL: "https://bbg.buzzing.cc/feed.json", IconURL: "https://bbg.buzzing.cc/apple-touch-icon.png"},
-	{Name: "国外科技头条", URL: "https://tech.buzzing.cc/feed.json", IconURL: "https://tech.buzzing.cc/apple-touch-icon.png"},
-	{Name: "国外财经新闻", URL: "https://finance.buzzing.cc/feed.json", IconURL: "https://finance.buzzing.cc/apple-touch-icon.png"},
-	{Name: "华尔街日报热门", URL: "https://wsj.buzzing.cc/feed.json", IconURL: "https://wsj.buzzing.cc/apple-touch-icon.png"},
-	{Name: "Product Hunt", URL: "https://ph.buzzing.cc/feed.json", IconURL: "https://ph.buzzing.cc/apple-touch-icon.png"},
-	{Name: "Dev.to", URL: "https://dev.buzzing.cc/feed.json", IconURL: "https://dev.buzzing.cc/apple-touch-icon.png"},
-	{Name: "财联社-热门", URL: "https://rsshub.app/cls/hot"},
-	{Name: "金十数据-快讯", URL: "https://rsshub.app/jin10"},
+type defaultDatasetSource struct {
+	Key        string
+	Name       string
+	URL        string
+	IconURL    string
+	LegacyURLs []string
 }

-// InitDataset creates the default Buzzing RSS sources for the default account when the source table is empty.
-func InitDataset(database *gorm.DB) error {
-	var count int64
-	if err := database.Model(&models.SaDatasetSource{}).Count(&count).Error; err != nil {
-		return err
-	}
-	if count > 0 {
-		return nil
-	}
+var defaultDatasetSources = []defaultDatasetSource{
+	{Key: "buzzing-bloomberg", Name: "彭博社最新报道", URL: "https://bbg.buzzing.cc/feed.json", IconURL: "https://bbg.buzzing.cc/apple-touch-icon.png"},
+	{Key: "buzzing-tech", Name: "国外科技头条", URL: "https://tech.buzzing.cc/feed.json", IconURL: "https://tech.buzzing.cc/apple-touch-icon.png"},
+	{Key: "buzzing-finance", Name: "国外财经新闻", URL: "https://finance.buzzing.cc/feed.json", IconURL: "https://finance.buzzing.cc/apple-touch-icon.png"},
+	{Key: "buzzing-wsj", Name: "华尔街日报热门", URL: "https://wsj.buzzing.cc/feed.json", IconURL: "https://wsj.buzzing.cc/apple-touch-icon.png"},
+	{Key: "buzzing-product-hunt", Name: "Product Hunt", URL: "https://ph.buzzing.cc/feed.json", IconURL: "https://ph.buzzing.cc/apple-touch-icon.png"},
+	{Key: "buzzing-devto", Name: "Dev.to", URL: "https://dev.buzzing.cc/feed.json", IconURL: "https://dev.buzzing.cc/apple-touch-icon.png"},
+	{
+		Key: "rsshub-cls-hot", Name: "财联社-热门",
+		URL:        "https://rsshub.ktachibana.party/cls/hot",
+		LegacyURLs: []string{"https://rsshub.app/cls/hot"},
+	},
+	{
+		Key: "rsshub-jin10", Name: "金十数据-快讯",
+		URL:        "https://rsshub.ktachibana.party/jin10",
+		LegacyURLs: []string{"https://rsshub.app/jin10"},
+	},
+}

-	var owner models.SaUser
-	err := database.Where("email = ?", rootUsername).First(&owner).Error
-	if err != nil && !errors.Is(err, gorm.ErrRecordNotFound) {
+// InitDataset inserts or repairs root's built-in dataset sources without touching user-created sources.
+func InitDataset(database *gorm.DB) error {
+	var root models.SaUser
+	if err := database.Where("email = ? AND role = ?", rootUsername, rootRole).First(&root).Error; err != nil {
 		return fmt.Errorf("find root user for default dataset sources: %w", err)
 	}
-	if errors.Is(err, gorm.ErrRecordNotFound) {
-		if err := database.Order("id asc").First(&owner).Error; err != nil {
-			return fmt.Errorf("find owner for default dataset sources: %w", err)
+
+	return database.Transaction(func(tx *gorm.DB) error {
+		if err := prepareDatasetItems(tx); err != nil {
+			return err
 		}
+		for _, definition := range defaultDatasetSources {
+			if err := upsertDefaultDatasetSource(tx, root, definition); err != nil {
+				return err
+			}
+		}
+		return nil
+	})
+}
+
+func prepareDatasetItems(tx *gorm.DB) error {
+	if tx.Migrator().HasIndex(&models.SaDatasetItem{}, "idx_sa_dataset_item_source_external") {
+		return nil
+	}
+	if err := tx.Exec(
+		"UPDATE sa_dataset_items SET external_id = NULL WHERE external_id = ''",
+	).Error; err != nil {
+		return err
+	}
+	if err := tx.Exec(`
+		DELETE FROM sa_dataset_items
+		WHERE external_id IS NOT NULL
+		  AND EXISTS (
+			SELECT 1
+			FROM sa_dataset_items AS earlier
+			WHERE earlier.source_id = sa_dataset_items.source_id
+			  AND earlier.external_id = sa_dataset_items.external_id
+			  AND earlier.id < sa_dataset_items.id
+		  )
+	`).Error; err != nil {
+		return err
+	}
+	return tx.Exec(`
+		CREATE UNIQUE INDEX IF NOT EXISTS idx_sa_dataset_item_source_external
+		ON sa_dataset_items (source_id, external_id)
+	`).Error
+}
+
+func upsertDefaultDatasetSource(tx *gorm.DB, root models.SaUser, definition defaultDatasetSource) error {
+	var source models.SaDatasetSource
+	err := tx.Where("created_by = ? AND seed_key = ?", root.ID, definition.Key).First(&source).Error
+	if err == nil {
+		return tx.Model(&source).Updates(map[string]any{
+			"url": definition.URL, "icon_url": definition.IconURL, "kind": "rss",
+		}).Error
+	}
+	if !errors.Is(err, gorm.ErrRecordNotFound) {
+		return err
 	}

-	sources := make([]models.SaDatasetSource, 0, len(defaultDatasetSources))
-	for _, source := range defaultDatasetSources {
-		sources = append(sources, models.SaDatasetSource{
-			CreatedBy:   owner.ID,
-			Name:        source.Name,
-			Kind:        "rss",
-			URL:         source.URL,
-			IconURL:     source.IconURL,
-			Description: "Buzzing 中文聚合订阅",
-			Enabled:     true,
-		})
+	knownURLs := append([]string{definition.URL}, definition.LegacyURLs...)
+	err = tx.Where("created_by = ? AND url IN ?", root.ID, knownURLs).First(&source).Error
+	if err == nil {
+		return tx.Model(&source).Updates(map[string]any{
+			"seed_key": definition.Key, "url": definition.URL,
+			"icon_url": definition.IconURL, "kind": "rss",
+		}).Error
 	}
-	return database.Create(&sources).Error
+	if !errors.Is(err, gorm.ErrRecordNotFound) {
+		return err
+	}
+
+	seedKey := definition.Key
+	return tx.Create(&models.SaDatasetSource{
+		CreatedBy: root.ID, SeedKey: &seedKey, Name: definition.Name,
+		Kind: "rss", URL: definition.URL, IconURL: definition.IconURL,
+		Description: "默认资讯订阅", Enabled: true,
+	}).Error
 }