senlinai/agent
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: harden explore dataset collection

Browse Source
This commit is contained in:
yanweidong
2026-07-23 21:34:13 +08:00
parent 3749cf4543
commit 50d8246915
9 changed files with 254 additions and 87 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
apps/web_v1/scripts/visual-check.mjs
View File

@@ -415,7 +415,9 @@ const editSourceModalCheck = await page.evaluate(() => ({
}))
await page.locator('.explore-source-modal input').first().fill('手动整理')
await page.locator('.explore-source-modal .arco-modal-footer .arco-btn-primary').click()
const editedSourceVisible = await page.locator('.explore-source-card', { hasText: '手动整理' }).count() === 1
const editedSourceCard = page.locator('.explore-source-card', { hasText: '手动整理' })
await editedSourceCard.waitFor({ state: 'visible' })
const editedSourceVisible = await editedSourceCard.count() === 1
await page.locator('.project-button').first().click()
await page.waitForTimeout(500)

136
backend/internal/initdb/dataset.go
View File

@@ -8,53 +8,109 @@ import (
"senlinai-agent/backend/internal/models"
)
var defaultDatasetSources = []struct {
Name string
URL string
IconURL string
}{
{Name: "彭博社最新报道", URL: "https://bbg.buzzing.cc/feed.json", IconURL: "https://bbg.buzzing.cc/apple-touch-icon.png"},
{Name: "国外科技头条", URL: "https://tech.buzzing.cc/feed.json", IconURL: "https://tech.buzzing.cc/apple-touch-icon.png"},
{Name: "国外财经新闻", URL: "https://finance.buzzing.cc/feed.json", IconURL: "https://finance.buzzing.cc/apple-touch-icon.png"},
{Name: "华尔街日报热门", URL: "https://wsj.buzzing.cc/feed.json", IconURL: "https://wsj.buzzing.cc/apple-touch-icon.png"},
{Name: "Product Hunt", URL: "https://ph.buzzing.cc/feed.json", IconURL: "https://ph.buzzing.cc/apple-touch-icon.png"},
{Name: "Dev.to", URL: "https://dev.buzzing.cc/feed.json", IconURL: "https://dev.buzzing.cc/apple-touch-icon.png"},
{Name: "财联社-热门", URL: "https://rsshub.app/cls/hot"},
{Name: "金十数据-快讯", URL: "https://rsshub.app/jin10"},
type defaultDatasetSource struct {
Key string
Name string
URL string
IconURL string
LegacyURLs []string
}
// InitDataset creates the default Buzzing RSS sources for the default account when the source table is empty.
func InitDataset(database *gorm.DB) error {
var count int64
if err := database.Model(&models.SaDatasetSource{}).Count(&count).Error; err != nil {
return err
}
if count > 0 {
return nil
}
var defaultDatasetSources = []defaultDatasetSource{
{Key: "buzzing-bloomberg", Name: "彭博社最新报道", URL: "https://bbg.buzzing.cc/feed.json", IconURL: "https://bbg.buzzing.cc/apple-touch-icon.png"},
{Key: "buzzing-tech", Name: "国外科技头条", URL: "https://tech.buzzing.cc/feed.json", IconURL: "https://tech.buzzing.cc/apple-touch-icon.png"},
{Key: "buzzing-finance", Name: "国外财经新闻", URL: "https://finance.buzzing.cc/feed.json", IconURL: "https://finance.buzzing.cc/apple-touch-icon.png"},
{Key: "buzzing-wsj", Name: "华尔街日报热门", URL: "https://wsj.buzzing.cc/feed.json", IconURL: "https://wsj.buzzing.cc/apple-touch-icon.png"},
{Key: "buzzing-product-hunt", Name: "Product Hunt", URL: "https://ph.buzzing.cc/feed.json", IconURL: "https://ph.buzzing.cc/apple-touch-icon.png"},
{Key: "buzzing-devto", Name: "Dev.to", URL: "https://dev.buzzing.cc/feed.json", IconURL: "https://dev.buzzing.cc/apple-touch-icon.png"},
{
Key: "rsshub-cls-hot", Name: "财联社-热门",
URL: "https://rsshub.ktachibana.party/cls/hot",
LegacyURLs: []string{"https://rsshub.app/cls/hot"},
},
{
Key: "rsshub-jin10", Name: "金十数据-快讯",
URL: "https://rsshub.ktachibana.party/jin10",
LegacyURLs: []string{"https://rsshub.app/jin10"},
},
}
var owner models.SaUser
err := database.Where("email = ?", rootUsername).First(&owner).Error
if err != nil && !errors.Is(err, gorm.ErrRecordNotFound) {
// InitDataset inserts or repairs root's built-in dataset sources without touching user-created sources.
func InitDataset(database *gorm.DB) error {
var root models.SaUser
if err := database.Where("email = ? AND role = ?", rootUsername, rootRole).First(&root).Error; err != nil {
return fmt.Errorf("find root user for default dataset sources: %w", err)
}
if errors.Is(err, gorm.ErrRecordNotFound) {
if err := database.Order("id asc").First(&owner).Error; err != nil {
return fmt.Errorf("find owner for default dataset sources: %w", err)
return database.Transaction(func(tx *gorm.DB) error {
if err := prepareDatasetItems(tx); err != nil {
return err
}
for _, definition := range defaultDatasetSources {
if err := upsertDefaultDatasetSource(tx, root, definition); err != nil {
return err
}
}
return nil
})
}
func prepareDatasetItems(tx *gorm.DB) error {
if tx.Migrator().HasIndex(&models.SaDatasetItem{}, "idx_sa_dataset_item_source_external") {
return nil
}
if err := tx.Exec(
"UPDATE sa_dataset_items SET external_id = NULL WHERE external_id = ''",
).Error; err != nil {
return err
}
if err := tx.Exec(`
DELETE FROM sa_dataset_items
WHERE external_id IS NOT NULL
AND EXISTS (
SELECT 1
FROM sa_dataset_items AS earlier
WHERE earlier.source_id = sa_dataset_items.source_id
AND earlier.external_id = sa_dataset_items.external_id
AND earlier.id < sa_dataset_items.id
)
`).Error; err != nil {
return err
}
return tx.Exec(`
CREATE UNIQUE INDEX IF NOT EXISTS idx_sa_dataset_item_source_external
ON sa_dataset_items (source_id, external_id)
`).Error
}
func upsertDefaultDatasetSource(tx *gorm.DB, root models.SaUser, definition defaultDatasetSource) error {
var source models.SaDatasetSource
err := tx.Where("created_by = ? AND seed_key = ?", root.ID, definition.Key).First(&source).Error
if err == nil {
return tx.Model(&source).Updates(map[string]any{
"url": definition.URL, "icon_url": definition.IconURL, "kind": "rss",
}).Error
}
if !errors.Is(err, gorm.ErrRecordNotFound) {
return err
}
sources := make([]models.SaDatasetSource, 0, len(defaultDatasetSources))
for _, source := range defaultDatasetSources {
sources = append(sources, models.SaDatasetSource{
CreatedBy: owner.ID,
Name: source.Name,
Kind: "rss",
URL: source.URL,
IconURL: source.IconURL,
Description: "Buzzing 中文聚合订阅",
Enabled: true,
})
knownURLs := append([]string{definition.URL}, definition.LegacyURLs...)
err = tx.Where("created_by = ? AND url IN ?", root.ID, knownURLs).First(&source).Error
if err == nil {
return tx.Model(&source).Updates(map[string]any{
"seed_key": definition.Key, "url": definition.URL,
"icon_url": definition.IconURL, "kind": "rss",
}).Error
}
return database.Create(&sources).Error
if !errors.Is(err, gorm.ErrRecordNotFound) {
return err
}
seedKey := definition.Key
return tx.Create(&models.SaDatasetSource{
CreatedBy: root.ID, SeedKey: &seedKey, Name: definition.Name,
Kind: "rss", URL: definition.URL, IconURL: definition.IconURL,
Description: "默认资讯订阅", Enabled: true,
}).Error
}

98
backend/internal/initdb/dataset_test.go
View File

@@ -3,6 +3,7 @@ package initdb
import (
"fmt"
"testing"
"time"
"github.com/glebarez/sqlite"
"github.com/stretchr/testify/require"
@@ -26,6 +27,8 @@ func TestInitDatasetCreatesDefaultSourcesWhenTableIsEmpty(t *testing.T) {
require.Equal(t, expected.Name, sources[index].Name)
require.Equal(t, expected.URL, sources[index].URL)
require.Equal(t, expected.IconURL, sources[index].IconURL)
require.NotNil(t, sources[index].SeedKey)
require.Equal(t, expected.Key, *sources[index].SeedKey)
require.Equal(t, "rss", sources[index].Kind)
require.True(t, sources[index].Enabled)
require.Equal(t, root.ID, sources[index].CreatedBy)
@@ -33,7 +36,7 @@ func TestInitDatasetCreatesDefaultSourcesWhenTableIsEmpty(t *testing.T) {
}
}
func TestInitDatasetDoesNothingWhenSourceTableIsNotEmpty(t *testing.T) {
func TestInitDatasetAddsMissingDefaultsWithoutChangingCustomSources(t *testing.T) {
database := newDatasetInitTestDatabase(t)
require.NoError(t, InitUser(database))
var root models.SaUser
@@ -50,11 +53,14 @@ func TestInitDatasetDoesNothingWhenSourceTableIsNotEmpty(t *testing.T) {
var sources []models.SaDatasetSource
require.NoError(t, database.Find(&sources).Error)
require.Len(t, sources, 1)
require.Equal(t, existing.Identity, sources[0].Identity)
require.Len(t, sources, len(defaultDatasetSources)+1)
var preserved models.SaDatasetSource
require.NoError(t, database.Where("identity = ?", existing.Identity).First(&preserved).Error)
require.Equal(t, existing.Name, preserved.Name)
require.Nil(t, preserved.SeedKey)
}
func TestInitDatasetUsesFirstExistingUserWhenRootIsAbsent(t *testing.T) {
func TestInitDatasetRejectsMissingRootInsteadOfAssigningDefaultsToAnotherUser(t *testing.T) {
database := newDatasetInitTestDatabase(t)
user := models.SaUser{
Email: "existing@example.com",
@@ -64,21 +70,93 @@ func TestInitDatasetUsesFirstExistingUserWhenRootIsAbsent(t *testing.T) {
}
require.NoError(t, database.Create(&user).Error)
require.NoError(t, InitDataset(database))
require.Error(t, InitDataset(database))
var sources []models.SaDatasetSource
require.NoError(t, database.Find(&sources).Error)
require.Len(t, sources, len(defaultDatasetSources))
for _, source := range sources {
require.Equal(t, user.ID, source.CreatedBy)
require.Equal(t, user.Identity, source.CreatedByIdentity)
require.Empty(t, sources)
}
func TestInitDatasetRepairsLegacyRSSHubURLsForRootOnly(t *testing.T) {
database := newDatasetInitTestDatabase(t)
require.NoError(t, InitUser(database))
var root models.SaUser
require.NoError(t, database.Where("email = ?", rootUsername).First(&root).Error)
other := models.SaUser{
Email: "other@example.com", DisplayName: "Other",
PasswordHash: "hash", Role: "user",
}
require.NoError(t, database.Create(&other).Error)
rootLegacy := models.SaDatasetSource{
CreatedBy: root.ID, Name: "财联社-热门", Kind: "rss",
URL: "https://rsshub.app/cls/hot", Enabled: true,
}
otherLegacy := models.SaDatasetSource{
CreatedBy: other.ID, Name: "财联社-热门", Kind: "rss",
URL: "https://rsshub.app/cls/hot", Enabled: true,
}
require.NoError(t, database.Create(&rootLegacy).Error)
require.NoError(t, database.Create(&otherLegacy).Error)
require.NoError(t, InitDataset(database))
require.NoError(t, database.First(&rootLegacy, rootLegacy.ID).Error)
require.Equal(t, "https://rsshub.ktachibana.party/cls/hot", rootLegacy.URL)
require.NotNil(t, rootLegacy.SeedKey)
require.Equal(t, "rsshub-cls-hot", *rootLegacy.SeedKey)
require.NoError(t, database.First(&otherLegacy, otherLegacy.ID).Error)
require.Equal(t, "https://rsshub.app/cls/hot", otherLegacy.URL)
require.Nil(t, otherLegacy.SeedKey)
}
func TestInitDatasetPreparesCollectedItemUniqueness(t *testing.T) {
database := newDatasetInitTestDatabase(t)
require.NoError(t, InitUser(database))
var root models.SaUser
require.NoError(t, database.Where("email = ?", rootUsername).First(&root).Error)
source := models.SaDatasetSource{
CreatedBy: root.ID, Name: "Existing feed", Kind: "rss",
URL: "https://example.com/feed", Enabled: true,
}
require.NoError(t, database.Create(&source).Error)
now := time.Now()
for index, externalID := range []string{"duplicate", "duplicate", "", ""} {
require.NoError(t, database.Exec(`
INSERT INTO sa_dataset_items
(identity, source_id, created_by, external_id, title, status, created_at, updated_at)
VALUES (?, ?, ?, ?, ?, ?, ?, ?)
`, fmt.Sprintf("item-%d", index), source.ID, root.ID, externalID,
fmt.Sprintf("Item %d", index), "unread", now, now).Error)
}
require.NoError(t, InitDataset(database))
var collectedCount int64
require.NoError(t, database.Model(&models.SaDatasetItem{}).
Where("source_id = ? AND external_id = ?", source.ID, "duplicate").
Count(&collectedCount).Error)
require.Equal(t, int64(1), collectedCount)
var manualCount int64
require.NoError(t, database.Model(&models.SaDatasetItem{}).
Where("source_id = ? AND external_id IS NULL", source.ID).
Count(&manualCount).Error)
require.Equal(t, int64(2), manualCount)
require.Error(t, database.Exec(`
INSERT INTO sa_dataset_items
(identity, source_id, created_by, external_id, title, status, created_at, updated_at)
VALUES (?, ?, ?, ?, ?, ?, ?, ?)
`, "item-new", source.ID, root.ID, "duplicate", "Duplicate", "unread", now, now).Error)
}
func newDatasetInitTestDatabase(t *testing.T) *gorm.DB {
t.Helper()
database, err := gorm.Open(sqlite.Open(fmt.Sprintf("file:%s?mode=memory&cache=shared", t.Name())), &gorm.Config{})
require.NoError(t, err)
require.NoError(t, database.AutoMigrate(&models.SaUser{}, &models.SaDatasetSource{}))
require.NoError(t, database.AutoMigrate(
&models.SaUser{},
&models.SaDatasetSource{},
&models.SaDatasetItem{},
))
return database
}

1
backend/internal/initdb/expert_test.go
View File

@@ -60,6 +60,7 @@ func newExpertTestDatabase(t *testing.T) *gorm.DB {
require.NoError(t, database.AutoMigrate(
&models.SaUser{},
&models.SaDatasetSource{},
&models.SaDatasetItem{},
&models.SaAIExpertCategory{},
&models.SaAIExpertItem{},
))

24
backend/internal/logic/dataset/feed_test.go
View File

@@ -4,6 +4,7 @@ import (
"context"
"errors"
"net/url"
"os"
"testing"
"github.com/stretchr/testify/require"
@@ -98,6 +99,29 @@ func TestValidateFeedURLRejectsLocalAddressesAndCredentials(t *testing.T) {
require.NoError(t, validateFeedURL(parsed))
}
func TestLiveFeedCollectionFormats(t *testing.T) {
if os.Getenv("TEST_LIVE_FEEDS") != "1" {
t.Skip("set TEST_LIVE_FEEDS=1 to verify external feeds")
}
fetcher := NewHTTPFeedFetcher()
for _, test := range []struct {
name string
url string
wantFormat string
}{
{name: "JSON Feed", url: "https://bbg.buzzing.cc/feed.json", wantFormat: "json_feed"},
{name: "RSS XML", url: "https://rsshub.ktachibana.party/cls/hot", wantFormat: "rss"},
{name: "second RSS XML", url: "https://rsshub.ktachibana.party/jin10", wantFormat: "rss"},
} {
t.Run(test.name, func(t *testing.T) {
feed, err := fetcher.Fetch(context.Background(), test.url)
require.NoError(t, err)
require.Equal(t, test.wantFormat, feed.Format)
require.NotEmpty(t, feed.Items)
})
}
}
func TestSyncSourcesRecordsFeedFailure(t *testing.T) {
database := newDatasetTestDatabase(t)
user := createDatasetTestUser(t, database, "feed-owner@example.com")

8
backend/internal/logic/dataset/handlers_test.go
View File

@@ -38,6 +38,10 @@ func TestDatasetAPIConnectsSourcesItemsAndSyncTasks(t *testing.T) {
require.NotEmpty(t, source.ID)
require.Equal(t, "Industry feed", source.Name)
require.Empty(t, source.IconURL)
var storedSource models.SaDatasetSource
require.NoError(t, database.Where("identity = ?", source.ID).First(&storedSource).Error)
require.Equal(t, owner.ID, storedSource.CreatedBy)
require.Equal(t, owner.Identity, storedSource.CreatedByIdentity)
createdItem := performDatasetRequest(t, ownerRouter, http.MethodPost, "/api/v1/dataset-items", map[string]any{
"sourceId": source.ID, "title": "A collected article", "summary": "Summary",
@@ -108,6 +112,10 @@ func newDatasetTestDatabase(t *testing.T) *gorm.DB {
&models.SaDatasetItem{},
&models.SaDatasetCron{},
))
require.NoError(t, database.Exec(`
CREATE UNIQUE INDEX idx_sa_dataset_item_source_external
ON sa_dataset_items (source_id, external_id)
`).Error)
return database
}

23
backend/internal/logic/dataset/service.go
View File

@@ -10,6 +10,7 @@ import (
"time"
"gorm.io/gorm"
"gorm.io/gorm/clause"
"senlinai-agent/backend/internal/models"
)
@@ -326,24 +327,20 @@ func (s *Service) storeFeedItems(userID uint, source models.SaDatasetSource, ite
inserted := 0
err := s.db.Transaction(func(tx *gorm.DB) error {
for _, input := range items {
var count int64
if err := tx.Model(&models.SaDatasetItem{}).
Where("source_id = ? AND external_id = ?", source.ID, input.ExternalID).
Count(&count).Error; err != nil {
return err
}
if count > 0 {
continue
}
externalID := input.ExternalID
item := models.SaDatasetItem{
SourceID: source.ID, CreatedBy: userID, ExternalID: input.ExternalID,
SourceID: source.ID, CreatedBy: userID, ExternalID: &externalID,
Title: input.Title, Summary: input.Summary, Content: input.Content,
URL: input.URL, Status: "unread", PublishedAt: input.PublishedAt,
}
if err := tx.Create(&item).Error; err != nil {
return err
result := tx.Clauses(clause.OnConflict{
Columns: []clause.Column{{Name: "source_id"}, {Name: "external_id"}},
DoNothing: true,
}).Create(&item)
if result.Error != nil {
return result.Error
}
inserted++
inserted += int(result.RowsAffected)
}
return nil
})

26
backend/internal/models/dataset_item.go
View File

@@ -3,19 +3,19 @@ package models
import "time"
type SaDatasetItem struct {
ID uint `gorm:"primaryKey"`
Identity string `gorm:"type:char(36);uniqueIndex"`
SourceID uint `gorm:"index;not null"`
SourceIdentity string `gorm:"type:char(36);index"`
CreatedBy uint `gorm:"index;not null"`
CreatedByIdentity string `gorm:"type:char(36);index"`
ExternalID string `gorm:"size:64;index"`
Title string `gorm:"size:500;not null"`
Summary string `gorm:"type:text"`
Content string `gorm:"type:text"`
URL string `gorm:"size:2048"`
Status string `gorm:"size:32;not null;default:unread;index"`
Starred bool `gorm:"not null;default:false;index"`
ID uint `gorm:"primaryKey"`
Identity string `gorm:"type:char(36);uniqueIndex"`
SourceID uint `gorm:"index;not null"`
SourceIdentity string `gorm:"type:char(36);index"`
CreatedBy uint `gorm:"index;not null"`
CreatedByIdentity string `gorm:"type:char(36);index"`
ExternalID *string `gorm:"size:64;index"`
Title string `gorm:"size:500;not null"`
Summary string `gorm:"type:text"`
Content string `gorm:"type:text"`
URL string `gorm:"size:2048"`
Status string `gorm:"size:32;not null;default:unread;index"`
Starred bool `gorm:"not null;default:false;index"`
PublishedAt *time.Time
CreatedAt time.Time
UpdatedAt time.Time

21
backend/internal/models/dataset_source.go
View File

@@ -3,16 +3,17 @@ package models
import "time"
type SaDatasetSource struct {
ID uint `gorm:"primaryKey"`
Identity string `gorm:"type:char(36);uniqueIndex"`
CreatedBy uint `gorm:"index;not null"`
CreatedByIdentity string `gorm:"type:char(36);index"`
Name string `gorm:"size:160;not null"`
Kind string `gorm:"size:32;not null"`
URL string `gorm:"size:2048"`
IconURL string `gorm:"size:2048"`
Description string `gorm:"type:text"`
Enabled bool `gorm:"not null;default:true;index"`
ID uint `gorm:"primaryKey"`
Identity string `gorm:"type:char(36);uniqueIndex"`
CreatedBy uint `gorm:"index;not null;uniqueIndex:idx_sa_dataset_source_seed,priority:1"`
CreatedByIdentity string `gorm:"type:char(36);index"`
SeedKey *string `gorm:"size:64;uniqueIndex:idx_sa_dataset_source_seed,priority:2"`
Name string `gorm:"size:160;not null"`
Kind string `gorm:"size:32;not null"`
URL string `gorm:"size:2048"`
IconURL string `gorm:"size:2048"`
Description string `gorm:"type:text"`
Enabled bool `gorm:"not null;default:true;index"`
LastSyncedAt *time.Time
CreatedAt time.Time
UpdatedAt time.Time