From 50d824691520373fafb631b85720623d28b3ba3b Mon Sep 17 00:00:00 2001 From: yanweidong Date: Thu, 23 Jul 2026 21:34:13 +0800 Subject: [PATCH] fix: harden explore dataset collection --- apps/web_v1/scripts/visual-check.mjs | 4 +- backend/internal/initdb/dataset.go | 136 ++++++++++++------ backend/internal/initdb/dataset_test.go | 98 +++++++++++-- backend/internal/initdb/expert_test.go | 1 + backend/internal/logic/dataset/feed_test.go | 24 ++++ .../internal/logic/dataset/handlers_test.go | 8 ++ backend/internal/logic/dataset/service.go | 23 ++- backend/internal/models/dataset_item.go | 26 ++-- backend/internal/models/dataset_source.go | 21 +-- 9 files changed, 254 insertions(+), 87 deletions(-) diff --git a/apps/web_v1/scripts/visual-check.mjs b/apps/web_v1/scripts/visual-check.mjs index 8098751..b9cb3da 100644 --- a/apps/web_v1/scripts/visual-check.mjs +++ b/apps/web_v1/scripts/visual-check.mjs @@ -415,7 +415,9 @@ const editSourceModalCheck = await page.evaluate(() => ({ })) await page.locator('.explore-source-modal input').first().fill('手动整理') await page.locator('.explore-source-modal .arco-modal-footer .arco-btn-primary').click() -const editedSourceVisible = await page.locator('.explore-source-card', { hasText: '手动整理' }).count() === 1 +const editedSourceCard = page.locator('.explore-source-card', { hasText: '手动整理' }) +await editedSourceCard.waitFor({ state: 'visible' }) +const editedSourceVisible = await editedSourceCard.count() === 1 await page.locator('.project-button').first().click() await page.waitForTimeout(500) diff --git a/backend/internal/initdb/dataset.go b/backend/internal/initdb/dataset.go index 30d1b16..c3b11c2 100644 --- a/backend/internal/initdb/dataset.go +++ b/backend/internal/initdb/dataset.go @@ -8,53 +8,109 @@ import ( "senlinai-agent/backend/internal/models" ) -var defaultDatasetSources = []struct { - Name string - URL string - IconURL string -}{ - {Name: "彭博社最新报道", URL: "https://bbg.buzzing.cc/feed.json", IconURL: "https://bbg.buzzing.cc/apple-touch-icon.png"}, - {Name: "国外科技头条", URL: "https://tech.buzzing.cc/feed.json", IconURL: "https://tech.buzzing.cc/apple-touch-icon.png"}, - {Name: "国外财经新闻", URL: "https://finance.buzzing.cc/feed.json", IconURL: "https://finance.buzzing.cc/apple-touch-icon.png"}, - {Name: "华尔街日报热门", URL: "https://wsj.buzzing.cc/feed.json", IconURL: "https://wsj.buzzing.cc/apple-touch-icon.png"}, - {Name: "Product Hunt", URL: "https://ph.buzzing.cc/feed.json", IconURL: "https://ph.buzzing.cc/apple-touch-icon.png"}, - {Name: "Dev.to", URL: "https://dev.buzzing.cc/feed.json", IconURL: "https://dev.buzzing.cc/apple-touch-icon.png"}, - {Name: "财联社-热门", URL: "https://rsshub.app/cls/hot"}, - {Name: "金十数据-快讯", URL: "https://rsshub.app/jin10"}, +type defaultDatasetSource struct { + Key string + Name string + URL string + IconURL string + LegacyURLs []string } -// InitDataset creates the default Buzzing RSS sources for the default account when the source table is empty. -func InitDataset(database *gorm.DB) error { - var count int64 - if err := database.Model(&models.SaDatasetSource{}).Count(&count).Error; err != nil { - return err - } - if count > 0 { - return nil - } +var defaultDatasetSources = []defaultDatasetSource{ + {Key: "buzzing-bloomberg", Name: "彭博社最新报道", URL: "https://bbg.buzzing.cc/feed.json", IconURL: "https://bbg.buzzing.cc/apple-touch-icon.png"}, + {Key: "buzzing-tech", Name: "国外科技头条", URL: "https://tech.buzzing.cc/feed.json", IconURL: "https://tech.buzzing.cc/apple-touch-icon.png"}, + {Key: "buzzing-finance", Name: "国外财经新闻", URL: "https://finance.buzzing.cc/feed.json", IconURL: "https://finance.buzzing.cc/apple-touch-icon.png"}, + {Key: "buzzing-wsj", Name: "华尔街日报热门", URL: "https://wsj.buzzing.cc/feed.json", IconURL: "https://wsj.buzzing.cc/apple-touch-icon.png"}, + {Key: "buzzing-product-hunt", Name: "Product Hunt", URL: "https://ph.buzzing.cc/feed.json", IconURL: "https://ph.buzzing.cc/apple-touch-icon.png"}, + {Key: "buzzing-devto", Name: "Dev.to", URL: "https://dev.buzzing.cc/feed.json", IconURL: "https://dev.buzzing.cc/apple-touch-icon.png"}, + { + Key: "rsshub-cls-hot", Name: "财联社-热门", + URL: "https://rsshub.ktachibana.party/cls/hot", + LegacyURLs: []string{"https://rsshub.app/cls/hot"}, + }, + { + Key: "rsshub-jin10", Name: "金十数据-快讯", + URL: "https://rsshub.ktachibana.party/jin10", + LegacyURLs: []string{"https://rsshub.app/jin10"}, + }, +} - var owner models.SaUser - err := database.Where("email = ?", rootUsername).First(&owner).Error - if err != nil && !errors.Is(err, gorm.ErrRecordNotFound) { +// InitDataset inserts or repairs root's built-in dataset sources without touching user-created sources. +func InitDataset(database *gorm.DB) error { + var root models.SaUser + if err := database.Where("email = ? AND role = ?", rootUsername, rootRole).First(&root).Error; err != nil { return fmt.Errorf("find root user for default dataset sources: %w", err) } - if errors.Is(err, gorm.ErrRecordNotFound) { - if err := database.Order("id asc").First(&owner).Error; err != nil { - return fmt.Errorf("find owner for default dataset sources: %w", err) + + return database.Transaction(func(tx *gorm.DB) error { + if err := prepareDatasetItems(tx); err != nil { + return err } + for _, definition := range defaultDatasetSources { + if err := upsertDefaultDatasetSource(tx, root, definition); err != nil { + return err + } + } + return nil + }) +} + +func prepareDatasetItems(tx *gorm.DB) error { + if tx.Migrator().HasIndex(&models.SaDatasetItem{}, "idx_sa_dataset_item_source_external") { + return nil + } + if err := tx.Exec( + "UPDATE sa_dataset_items SET external_id = NULL WHERE external_id = ''", + ).Error; err != nil { + return err + } + if err := tx.Exec(` + DELETE FROM sa_dataset_items + WHERE external_id IS NOT NULL + AND EXISTS ( + SELECT 1 + FROM sa_dataset_items AS earlier + WHERE earlier.source_id = sa_dataset_items.source_id + AND earlier.external_id = sa_dataset_items.external_id + AND earlier.id < sa_dataset_items.id + ) + `).Error; err != nil { + return err + } + return tx.Exec(` + CREATE UNIQUE INDEX IF NOT EXISTS idx_sa_dataset_item_source_external + ON sa_dataset_items (source_id, external_id) + `).Error +} + +func upsertDefaultDatasetSource(tx *gorm.DB, root models.SaUser, definition defaultDatasetSource) error { + var source models.SaDatasetSource + err := tx.Where("created_by = ? AND seed_key = ?", root.ID, definition.Key).First(&source).Error + if err == nil { + return tx.Model(&source).Updates(map[string]any{ + "url": definition.URL, "icon_url": definition.IconURL, "kind": "rss", + }).Error + } + if !errors.Is(err, gorm.ErrRecordNotFound) { + return err } - sources := make([]models.SaDatasetSource, 0, len(defaultDatasetSources)) - for _, source := range defaultDatasetSources { - sources = append(sources, models.SaDatasetSource{ - CreatedBy: owner.ID, - Name: source.Name, - Kind: "rss", - URL: source.URL, - IconURL: source.IconURL, - Description: "Buzzing 中文聚合订阅", - Enabled: true, - }) + knownURLs := append([]string{definition.URL}, definition.LegacyURLs...) + err = tx.Where("created_by = ? AND url IN ?", root.ID, knownURLs).First(&source).Error + if err == nil { + return tx.Model(&source).Updates(map[string]any{ + "seed_key": definition.Key, "url": definition.URL, + "icon_url": definition.IconURL, "kind": "rss", + }).Error } - return database.Create(&sources).Error + if !errors.Is(err, gorm.ErrRecordNotFound) { + return err + } + + seedKey := definition.Key + return tx.Create(&models.SaDatasetSource{ + CreatedBy: root.ID, SeedKey: &seedKey, Name: definition.Name, + Kind: "rss", URL: definition.URL, IconURL: definition.IconURL, + Description: "默认资讯订阅", Enabled: true, + }).Error } diff --git a/backend/internal/initdb/dataset_test.go b/backend/internal/initdb/dataset_test.go index db3d1c4..428eda1 100644 --- a/backend/internal/initdb/dataset_test.go +++ b/backend/internal/initdb/dataset_test.go @@ -3,6 +3,7 @@ package initdb import ( "fmt" "testing" + "time" "github.com/glebarez/sqlite" "github.com/stretchr/testify/require" @@ -26,6 +27,8 @@ func TestInitDatasetCreatesDefaultSourcesWhenTableIsEmpty(t *testing.T) { require.Equal(t, expected.Name, sources[index].Name) require.Equal(t, expected.URL, sources[index].URL) require.Equal(t, expected.IconURL, sources[index].IconURL) + require.NotNil(t, sources[index].SeedKey) + require.Equal(t, expected.Key, *sources[index].SeedKey) require.Equal(t, "rss", sources[index].Kind) require.True(t, sources[index].Enabled) require.Equal(t, root.ID, sources[index].CreatedBy) @@ -33,7 +36,7 @@ func TestInitDatasetCreatesDefaultSourcesWhenTableIsEmpty(t *testing.T) { } } -func TestInitDatasetDoesNothingWhenSourceTableIsNotEmpty(t *testing.T) { +func TestInitDatasetAddsMissingDefaultsWithoutChangingCustomSources(t *testing.T) { database := newDatasetInitTestDatabase(t) require.NoError(t, InitUser(database)) var root models.SaUser @@ -50,11 +53,14 @@ func TestInitDatasetDoesNothingWhenSourceTableIsNotEmpty(t *testing.T) { var sources []models.SaDatasetSource require.NoError(t, database.Find(&sources).Error) - require.Len(t, sources, 1) - require.Equal(t, existing.Identity, sources[0].Identity) + require.Len(t, sources, len(defaultDatasetSources)+1) + var preserved models.SaDatasetSource + require.NoError(t, database.Where("identity = ?", existing.Identity).First(&preserved).Error) + require.Equal(t, existing.Name, preserved.Name) + require.Nil(t, preserved.SeedKey) } -func TestInitDatasetUsesFirstExistingUserWhenRootIsAbsent(t *testing.T) { +func TestInitDatasetRejectsMissingRootInsteadOfAssigningDefaultsToAnotherUser(t *testing.T) { database := newDatasetInitTestDatabase(t) user := models.SaUser{ Email: "existing@example.com", @@ -64,21 +70,93 @@ func TestInitDatasetUsesFirstExistingUserWhenRootIsAbsent(t *testing.T) { } require.NoError(t, database.Create(&user).Error) - require.NoError(t, InitDataset(database)) + require.Error(t, InitDataset(database)) var sources []models.SaDatasetSource require.NoError(t, database.Find(&sources).Error) - require.Len(t, sources, len(defaultDatasetSources)) - for _, source := range sources { - require.Equal(t, user.ID, source.CreatedBy) - require.Equal(t, user.Identity, source.CreatedByIdentity) + require.Empty(t, sources) +} + +func TestInitDatasetRepairsLegacyRSSHubURLsForRootOnly(t *testing.T) { + database := newDatasetInitTestDatabase(t) + require.NoError(t, InitUser(database)) + var root models.SaUser + require.NoError(t, database.Where("email = ?", rootUsername).First(&root).Error) + other := models.SaUser{ + Email: "other@example.com", DisplayName: "Other", + PasswordHash: "hash", Role: "user", } + require.NoError(t, database.Create(&other).Error) + + rootLegacy := models.SaDatasetSource{ + CreatedBy: root.ID, Name: "财联社-热门", Kind: "rss", + URL: "https://rsshub.app/cls/hot", Enabled: true, + } + otherLegacy := models.SaDatasetSource{ + CreatedBy: other.ID, Name: "财联社-热门", Kind: "rss", + URL: "https://rsshub.app/cls/hot", Enabled: true, + } + require.NoError(t, database.Create(&rootLegacy).Error) + require.NoError(t, database.Create(&otherLegacy).Error) + + require.NoError(t, InitDataset(database)) + + require.NoError(t, database.First(&rootLegacy, rootLegacy.ID).Error) + require.Equal(t, "https://rsshub.ktachibana.party/cls/hot", rootLegacy.URL) + require.NotNil(t, rootLegacy.SeedKey) + require.Equal(t, "rsshub-cls-hot", *rootLegacy.SeedKey) + require.NoError(t, database.First(&otherLegacy, otherLegacy.ID).Error) + require.Equal(t, "https://rsshub.app/cls/hot", otherLegacy.URL) + require.Nil(t, otherLegacy.SeedKey) +} + +func TestInitDatasetPreparesCollectedItemUniqueness(t *testing.T) { + database := newDatasetInitTestDatabase(t) + require.NoError(t, InitUser(database)) + var root models.SaUser + require.NoError(t, database.Where("email = ?", rootUsername).First(&root).Error) + source := models.SaDatasetSource{ + CreatedBy: root.ID, Name: "Existing feed", Kind: "rss", + URL: "https://example.com/feed", Enabled: true, + } + require.NoError(t, database.Create(&source).Error) + now := time.Now() + for index, externalID := range []string{"duplicate", "duplicate", "", ""} { + require.NoError(t, database.Exec(` + INSERT INTO sa_dataset_items + (identity, source_id, created_by, external_id, title, status, created_at, updated_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?) + `, fmt.Sprintf("item-%d", index), source.ID, root.ID, externalID, + fmt.Sprintf("Item %d", index), "unread", now, now).Error) + } + + require.NoError(t, InitDataset(database)) + + var collectedCount int64 + require.NoError(t, database.Model(&models.SaDatasetItem{}). + Where("source_id = ? AND external_id = ?", source.ID, "duplicate"). + Count(&collectedCount).Error) + require.Equal(t, int64(1), collectedCount) + var manualCount int64 + require.NoError(t, database.Model(&models.SaDatasetItem{}). + Where("source_id = ? AND external_id IS NULL", source.ID). + Count(&manualCount).Error) + require.Equal(t, int64(2), manualCount) + require.Error(t, database.Exec(` + INSERT INTO sa_dataset_items + (identity, source_id, created_by, external_id, title, status, created_at, updated_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?) + `, "item-new", source.ID, root.ID, "duplicate", "Duplicate", "unread", now, now).Error) } func newDatasetInitTestDatabase(t *testing.T) *gorm.DB { t.Helper() database, err := gorm.Open(sqlite.Open(fmt.Sprintf("file:%s?mode=memory&cache=shared", t.Name())), &gorm.Config{}) require.NoError(t, err) - require.NoError(t, database.AutoMigrate(&models.SaUser{}, &models.SaDatasetSource{})) + require.NoError(t, database.AutoMigrate( + &models.SaUser{}, + &models.SaDatasetSource{}, + &models.SaDatasetItem{}, + )) return database } diff --git a/backend/internal/initdb/expert_test.go b/backend/internal/initdb/expert_test.go index db07d04..766f8a8 100644 --- a/backend/internal/initdb/expert_test.go +++ b/backend/internal/initdb/expert_test.go @@ -60,6 +60,7 @@ func newExpertTestDatabase(t *testing.T) *gorm.DB { require.NoError(t, database.AutoMigrate( &models.SaUser{}, &models.SaDatasetSource{}, + &models.SaDatasetItem{}, &models.SaAIExpertCategory{}, &models.SaAIExpertItem{}, )) diff --git a/backend/internal/logic/dataset/feed_test.go b/backend/internal/logic/dataset/feed_test.go index ccbf134..3890dc1 100644 --- a/backend/internal/logic/dataset/feed_test.go +++ b/backend/internal/logic/dataset/feed_test.go @@ -4,6 +4,7 @@ import ( "context" "errors" "net/url" + "os" "testing" "github.com/stretchr/testify/require" @@ -98,6 +99,29 @@ func TestValidateFeedURLRejectsLocalAddressesAndCredentials(t *testing.T) { require.NoError(t, validateFeedURL(parsed)) } +func TestLiveFeedCollectionFormats(t *testing.T) { + if os.Getenv("TEST_LIVE_FEEDS") != "1" { + t.Skip("set TEST_LIVE_FEEDS=1 to verify external feeds") + } + fetcher := NewHTTPFeedFetcher() + for _, test := range []struct { + name string + url string + wantFormat string + }{ + {name: "JSON Feed", url: "https://bbg.buzzing.cc/feed.json", wantFormat: "json_feed"}, + {name: "RSS XML", url: "https://rsshub.ktachibana.party/cls/hot", wantFormat: "rss"}, + {name: "second RSS XML", url: "https://rsshub.ktachibana.party/jin10", wantFormat: "rss"}, + } { + t.Run(test.name, func(t *testing.T) { + feed, err := fetcher.Fetch(context.Background(), test.url) + require.NoError(t, err) + require.Equal(t, test.wantFormat, feed.Format) + require.NotEmpty(t, feed.Items) + }) + } +} + func TestSyncSourcesRecordsFeedFailure(t *testing.T) { database := newDatasetTestDatabase(t) user := createDatasetTestUser(t, database, "feed-owner@example.com") diff --git a/backend/internal/logic/dataset/handlers_test.go b/backend/internal/logic/dataset/handlers_test.go index dec7674..f1d7e3d 100644 --- a/backend/internal/logic/dataset/handlers_test.go +++ b/backend/internal/logic/dataset/handlers_test.go @@ -38,6 +38,10 @@ func TestDatasetAPIConnectsSourcesItemsAndSyncTasks(t *testing.T) { require.NotEmpty(t, source.ID) require.Equal(t, "Industry feed", source.Name) require.Empty(t, source.IconURL) + var storedSource models.SaDatasetSource + require.NoError(t, database.Where("identity = ?", source.ID).First(&storedSource).Error) + require.Equal(t, owner.ID, storedSource.CreatedBy) + require.Equal(t, owner.Identity, storedSource.CreatedByIdentity) createdItem := performDatasetRequest(t, ownerRouter, http.MethodPost, "/api/v1/dataset-items", map[string]any{ "sourceId": source.ID, "title": "A collected article", "summary": "Summary", @@ -108,6 +112,10 @@ func newDatasetTestDatabase(t *testing.T) *gorm.DB { &models.SaDatasetItem{}, &models.SaDatasetCron{}, )) + require.NoError(t, database.Exec(` + CREATE UNIQUE INDEX idx_sa_dataset_item_source_external + ON sa_dataset_items (source_id, external_id) + `).Error) return database } diff --git a/backend/internal/logic/dataset/service.go b/backend/internal/logic/dataset/service.go index 52ba3c8..a6169e5 100644 --- a/backend/internal/logic/dataset/service.go +++ b/backend/internal/logic/dataset/service.go @@ -10,6 +10,7 @@ import ( "time" "gorm.io/gorm" + "gorm.io/gorm/clause" "senlinai-agent/backend/internal/models" ) @@ -326,24 +327,20 @@ func (s *Service) storeFeedItems(userID uint, source models.SaDatasetSource, ite inserted := 0 err := s.db.Transaction(func(tx *gorm.DB) error { for _, input := range items { - var count int64 - if err := tx.Model(&models.SaDatasetItem{}). - Where("source_id = ? AND external_id = ?", source.ID, input.ExternalID). - Count(&count).Error; err != nil { - return err - } - if count > 0 { - continue - } + externalID := input.ExternalID item := models.SaDatasetItem{ - SourceID: source.ID, CreatedBy: userID, ExternalID: input.ExternalID, + SourceID: source.ID, CreatedBy: userID, ExternalID: &externalID, Title: input.Title, Summary: input.Summary, Content: input.Content, URL: input.URL, Status: "unread", PublishedAt: input.PublishedAt, } - if err := tx.Create(&item).Error; err != nil { - return err + result := tx.Clauses(clause.OnConflict{ + Columns: []clause.Column{{Name: "source_id"}, {Name: "external_id"}}, + DoNothing: true, + }).Create(&item) + if result.Error != nil { + return result.Error } - inserted++ + inserted += int(result.RowsAffected) } return nil }) diff --git a/backend/internal/models/dataset_item.go b/backend/internal/models/dataset_item.go index 8cf28b0..a35ddba 100644 --- a/backend/internal/models/dataset_item.go +++ b/backend/internal/models/dataset_item.go @@ -3,19 +3,19 @@ package models import "time" type SaDatasetItem struct { - ID uint `gorm:"primaryKey"` - Identity string `gorm:"type:char(36);uniqueIndex"` - SourceID uint `gorm:"index;not null"` - SourceIdentity string `gorm:"type:char(36);index"` - CreatedBy uint `gorm:"index;not null"` - CreatedByIdentity string `gorm:"type:char(36);index"` - ExternalID string `gorm:"size:64;index"` - Title string `gorm:"size:500;not null"` - Summary string `gorm:"type:text"` - Content string `gorm:"type:text"` - URL string `gorm:"size:2048"` - Status string `gorm:"size:32;not null;default:unread;index"` - Starred bool `gorm:"not null;default:false;index"` + ID uint `gorm:"primaryKey"` + Identity string `gorm:"type:char(36);uniqueIndex"` + SourceID uint `gorm:"index;not null"` + SourceIdentity string `gorm:"type:char(36);index"` + CreatedBy uint `gorm:"index;not null"` + CreatedByIdentity string `gorm:"type:char(36);index"` + ExternalID *string `gorm:"size:64;index"` + Title string `gorm:"size:500;not null"` + Summary string `gorm:"type:text"` + Content string `gorm:"type:text"` + URL string `gorm:"size:2048"` + Status string `gorm:"size:32;not null;default:unread;index"` + Starred bool `gorm:"not null;default:false;index"` PublishedAt *time.Time CreatedAt time.Time UpdatedAt time.Time diff --git a/backend/internal/models/dataset_source.go b/backend/internal/models/dataset_source.go index a594acd..a1370f0 100644 --- a/backend/internal/models/dataset_source.go +++ b/backend/internal/models/dataset_source.go @@ -3,16 +3,17 @@ package models import "time" type SaDatasetSource struct { - ID uint `gorm:"primaryKey"` - Identity string `gorm:"type:char(36);uniqueIndex"` - CreatedBy uint `gorm:"index;not null"` - CreatedByIdentity string `gorm:"type:char(36);index"` - Name string `gorm:"size:160;not null"` - Kind string `gorm:"size:32;not null"` - URL string `gorm:"size:2048"` - IconURL string `gorm:"size:2048"` - Description string `gorm:"type:text"` - Enabled bool `gorm:"not null;default:true;index"` + ID uint `gorm:"primaryKey"` + Identity string `gorm:"type:char(36);uniqueIndex"` + CreatedBy uint `gorm:"index;not null;uniqueIndex:idx_sa_dataset_source_seed,priority:1"` + CreatedByIdentity string `gorm:"type:char(36);index"` + SeedKey *string `gorm:"size:64;uniqueIndex:idx_sa_dataset_source_seed,priority:2"` + Name string `gorm:"size:160;not null"` + Kind string `gorm:"size:32;not null"` + URL string `gorm:"size:2048"` + IconURL string `gorm:"size:2048"` + Description string `gorm:"type:text"` + Enabled bool `gorm:"not null;default:true;index"` LastSyncedAt *time.Time CreatedAt time.Time UpdatedAt time.Time