From b053c3874a9725347e6dd5d003fa3ddf49c79e79 Mon Sep 17 00:00:00 2001 From: yanweidong Date: Thu, 23 Jul 2026 15:14:31 +0800 Subject: [PATCH] feat: seed default dataset sources --- apps/web_v1/scripts/visual-check.mjs | 6 +- apps/web_v1/src/App.css | 7 ++ apps/web_v1/src/api/dataset.ts | 2 + apps/web_v1/src/pages/workspace-explore.tsx | 9 +- backend/internal/initdb/dataset.go | 58 +++++++++++++ backend/internal/initdb/dataset_test.go | 84 +++++++++++++++++++ backend/internal/initdb/expert_test.go | 1 + backend/internal/initdb/new.go | 3 + backend/internal/logic/dataset/handlers.go | 10 ++- .../internal/logic/dataset/handlers_test.go | 1 + backend/internal/logic/dataset/service.go | 10 ++- backend/internal/models/dataset_source.go | 1 + 12 files changed, 183 insertions(+), 9 deletions(-) create mode 100644 backend/internal/initdb/dataset.go create mode 100644 backend/internal/initdb/dataset_test.go diff --git a/apps/web_v1/scripts/visual-check.mjs b/apps/web_v1/scripts/visual-check.mjs index b1c760c..8f604f0 100644 --- a/apps/web_v1/scripts/visual-check.mjs +++ b/apps/web_v1/scripts/visual-check.mjs @@ -34,17 +34,17 @@ const visualExperts = [ ] let visualDatasetSources = [ { - id: '019b0000-0000-7000-8000-000000000030', name: '手动收集', kind: 'manual', url: '', + id: '019b0000-0000-7000-8000-000000000030', name: '手动收集', kind: 'manual', url: '', iconUrl: '', description: '手动收集的文章与线索', enabled: true, lastSyncedAt: null, itemCount: 1, createdAt: '2026-07-22T06:00:00Z', updatedAt: '2026-07-22T06:00:00Z', }, { - id: '019b0000-0000-7000-8000-000000000031', name: '需求文档', kind: 'link', url: 'https://example.com/requirements', + id: '019b0000-0000-7000-8000-000000000031', name: '需求文档', kind: 'link', url: 'https://example.com/requirements', iconUrl: '', description: '产品需求与业务文档', enabled: true, lastSyncedAt: null, itemCount: 1, createdAt: '2026-07-22T06:00:00Z', updatedAt: '2026-07-22T06:00:00Z', }, { - id: '019b0000-0000-7000-8000-000000000032', name: '架构讨论', kind: 'link', url: 'https://example.com/architecture', + id: '019b0000-0000-7000-8000-000000000032', name: '架构讨论', kind: 'link', url: 'https://example.com/architecture', iconUrl: '', description: '架构方案与系统设计讨论', enabled: true, lastSyncedAt: null, itemCount: 0, createdAt: '2026-07-22T06:00:00Z', updatedAt: '2026-07-22T06:00:00Z', }, diff --git a/apps/web_v1/src/App.css b/apps/web_v1/src/App.css index 91edaa3..cc7b1f1 100644 --- a/apps/web_v1/src/App.css +++ b/apps/web_v1/src/App.css @@ -753,6 +753,13 @@ font-size: 13px; } +.explore-source-image { + width: 100%; + height: 100%; + border-radius: inherit; + object-fit: cover; +} + .explore-source-icon.blue { background: rgb(var(--arcoblue-1)); color: rgb(var(--arcoblue-6)); diff --git a/apps/web_v1/src/api/dataset.ts b/apps/web_v1/src/api/dataset.ts index 084f5e5..3326dc6 100644 --- a/apps/web_v1/src/api/dataset.ts +++ b/apps/web_v1/src/api/dataset.ts @@ -7,6 +7,7 @@ export type DatasetSourceDTO = { name: string kind: DatasetSourceKind url: string + iconUrl: string description: string enabled: boolean lastSyncedAt: string | null @@ -46,6 +47,7 @@ export type DatasetSourceInput = { name: string kind: DatasetSourceKind url: string + iconUrl: string description: string enabled: boolean } diff --git a/apps/web_v1/src/pages/workspace-explore.tsx b/apps/web_v1/src/pages/workspace-explore.tsx index b4aded4..4e92f84 100644 --- a/apps/web_v1/src/pages/workspace-explore.tsx +++ b/apps/web_v1/src/pages/workspace-explore.tsx @@ -35,6 +35,7 @@ type SourceDraft = { name: string kind: DatasetSourceKind url: string + iconUrl: string description: string enabled: boolean } @@ -43,6 +44,7 @@ const EMPTY_SOURCE_DRAFT: SourceDraft = { name: '', kind: 'link', url: '', + iconUrl: '', description: '', enabled: true, } @@ -105,6 +107,7 @@ export function WorkspaceExplorePage({ name: '全部', kind: 'manual', url: '', + iconUrl: '', description: '全部探索内容', enabled: true, lastSyncedAt: null, @@ -118,7 +121,9 @@ export function WorkspaceExplorePage({ allSource, ...sourceRecords.map((source) => ({ ...source, - icon: sourceIcon(source.kind), + icon: source.iconUrl + ? + : sourceIcon(source.kind), color: sourceColor(source.kind), })), ] @@ -156,6 +161,7 @@ export function WorkspaceExplorePage({ name: source.name, kind: source.kind, url: source.url, + iconUrl: source.iconUrl, description: source.description, enabled: source.enabled, }) @@ -168,6 +174,7 @@ export function WorkspaceExplorePage({ name: sourceDraft.name.trim(), kind: sourceDraft.kind, url: sourceDraft.url.trim(), + iconUrl: sourceDraft.iconUrl, description: sourceDraft.description.trim(), enabled: sourceDraft.enabled, } diff --git a/backend/internal/initdb/dataset.go b/backend/internal/initdb/dataset.go new file mode 100644 index 0000000..8a7c06d --- /dev/null +++ b/backend/internal/initdb/dataset.go @@ -0,0 +1,58 @@ +package initdb + +import ( + "errors" + "fmt" + + "gorm.io/gorm" + "senlinai-agent/backend/internal/models" +) + +var defaultDatasetSources = []struct { + Name string + URL string + IconURL string +}{ + {Name: "彭博社最新报道", URL: "https://bbg.buzzing.cc/feed.json", IconURL: "https://bbg.buzzing.cc/apple-touch-icon.png"}, + {Name: "国外科技头条", URL: "https://tech.buzzing.cc/feed.json", IconURL: "https://tech.buzzing.cc/apple-touch-icon.png"}, + {Name: "国外财经新闻", URL: "https://finance.buzzing.cc/feed.json", IconURL: "https://finance.buzzing.cc/apple-touch-icon.png"}, + {Name: "华尔街日报热门", URL: "https://wsj.buzzing.cc/feed.json", IconURL: "https://wsj.buzzing.cc/apple-touch-icon.png"}, + {Name: "Product Hunt", URL: "https://ph.buzzing.cc/feed.json", IconURL: "https://ph.buzzing.cc/apple-touch-icon.png"}, + {Name: "Dev.to", URL: "https://dev.buzzing.cc/feed.json", IconURL: "https://dev.buzzing.cc/apple-touch-icon.png"}, +} + +// InitDataset creates the default Buzzing RSS sources for the default account when the source table is empty. +func InitDataset(database *gorm.DB) error { + var count int64 + if err := database.Model(&models.SaDatasetSource{}).Count(&count).Error; err != nil { + return err + } + if count > 0 { + return nil + } + + var owner models.SaUser + err := database.Where("email = ?", rootUsername).First(&owner).Error + if err != nil && !errors.Is(err, gorm.ErrRecordNotFound) { + return fmt.Errorf("find root user for default dataset sources: %w", err) + } + if errors.Is(err, gorm.ErrRecordNotFound) { + if err := database.Order("id asc").First(&owner).Error; err != nil { + return fmt.Errorf("find owner for default dataset sources: %w", err) + } + } + + sources := make([]models.SaDatasetSource, 0, len(defaultDatasetSources)) + for _, source := range defaultDatasetSources { + sources = append(sources, models.SaDatasetSource{ + CreatedBy: owner.ID, + Name: source.Name, + Kind: "rss", + URL: source.URL, + IconURL: source.IconURL, + Description: "Buzzing 中文聚合订阅", + Enabled: true, + }) + } + return database.Create(&sources).Error +} diff --git a/backend/internal/initdb/dataset_test.go b/backend/internal/initdb/dataset_test.go new file mode 100644 index 0000000..db3d1c4 --- /dev/null +++ b/backend/internal/initdb/dataset_test.go @@ -0,0 +1,84 @@ +package initdb + +import ( + "fmt" + "testing" + + "github.com/glebarez/sqlite" + "github.com/stretchr/testify/require" + "gorm.io/gorm" + "senlinai-agent/backend/internal/models" +) + +func TestInitDatasetCreatesDefaultSourcesWhenTableIsEmpty(t *testing.T) { + database := newDatasetInitTestDatabase(t) + require.NoError(t, InitUser(database)) + + require.NoError(t, InitDataset(database)) + require.NoError(t, InitDataset(database)) + + var root models.SaUser + require.NoError(t, database.Where("email = ?", rootUsername).First(&root).Error) + var sources []models.SaDatasetSource + require.NoError(t, database.Order("id asc").Find(&sources).Error) + require.Len(t, sources, len(defaultDatasetSources)) + for index, expected := range defaultDatasetSources { + require.Equal(t, expected.Name, sources[index].Name) + require.Equal(t, expected.URL, sources[index].URL) + require.Equal(t, expected.IconURL, sources[index].IconURL) + require.Equal(t, "rss", sources[index].Kind) + require.True(t, sources[index].Enabled) + require.Equal(t, root.ID, sources[index].CreatedBy) + require.Equal(t, root.Identity, sources[index].CreatedByIdentity) + } +} + +func TestInitDatasetDoesNothingWhenSourceTableIsNotEmpty(t *testing.T) { + database := newDatasetInitTestDatabase(t) + require.NoError(t, InitUser(database)) + var root models.SaUser + require.NoError(t, database.Where("email = ?", rootUsername).First(&root).Error) + existing := models.SaDatasetSource{ + CreatedBy: root.ID, + Name: "Existing source", + Kind: "manual", + Enabled: true, + } + require.NoError(t, database.Create(&existing).Error) + + require.NoError(t, InitDataset(database)) + + var sources []models.SaDatasetSource + require.NoError(t, database.Find(&sources).Error) + require.Len(t, sources, 1) + require.Equal(t, existing.Identity, sources[0].Identity) +} + +func TestInitDatasetUsesFirstExistingUserWhenRootIsAbsent(t *testing.T) { + database := newDatasetInitTestDatabase(t) + user := models.SaUser{ + Email: "existing@example.com", + DisplayName: "Existing", + PasswordHash: "hash", + Role: "user", + } + require.NoError(t, database.Create(&user).Error) + + require.NoError(t, InitDataset(database)) + + var sources []models.SaDatasetSource + require.NoError(t, database.Find(&sources).Error) + require.Len(t, sources, len(defaultDatasetSources)) + for _, source := range sources { + require.Equal(t, user.ID, source.CreatedBy) + require.Equal(t, user.Identity, source.CreatedByIdentity) + } +} + +func newDatasetInitTestDatabase(t *testing.T) *gorm.DB { + t.Helper() + database, err := gorm.Open(sqlite.Open(fmt.Sprintf("file:%s?mode=memory&cache=shared", t.Name())), &gorm.Config{}) + require.NoError(t, err) + require.NoError(t, database.AutoMigrate(&models.SaUser{}, &models.SaDatasetSource{})) + return database +} diff --git a/backend/internal/initdb/expert_test.go b/backend/internal/initdb/expert_test.go index b163cef..db07d04 100644 --- a/backend/internal/initdb/expert_test.go +++ b/backend/internal/initdb/expert_test.go @@ -59,6 +59,7 @@ func newExpertTestDatabase(t *testing.T) *gorm.DB { require.NoError(t, err) require.NoError(t, database.AutoMigrate( &models.SaUser{}, + &models.SaDatasetSource{}, &models.SaAIExpertCategory{}, &models.SaAIExpertItem{}, )) diff --git a/backend/internal/initdb/new.go b/backend/internal/initdb/new.go index 8e60356..ed9808e 100644 --- a/backend/internal/initdb/new.go +++ b/backend/internal/initdb/new.go @@ -8,6 +8,9 @@ func New(database *gorm.DB) error { if err := InitUser(tx); err != nil { return err } + if err := InitDataset(tx); err != nil { + return err + } return InitExpert(tx) }) } diff --git a/backend/internal/logic/dataset/handlers.go b/backend/internal/logic/dataset/handlers.go index cef81da..72130f5 100644 --- a/backend/internal/logic/dataset/handlers.go +++ b/backend/internal/logic/dataset/handlers.go @@ -22,6 +22,7 @@ type SourceDTO struct { Name string `json:"name"` Kind string `json:"kind"` URL string `json:"url"` + IconURL string `json:"iconUrl"` Description string `json:"description"` Enabled bool `json:"enabled"` LastSyncedAt *time.Time `json:"lastSyncedAt"` @@ -61,6 +62,7 @@ type sourceRequest struct { Name string `json:"name"` Kind string `json:"kind"` URL string `json:"url"` + IconURL string `json:"iconUrl"` Description string `json:"description"` Enabled *bool `json:"enabled"` } @@ -113,7 +115,8 @@ func (h *Handler) createSource(c *gin.Context) { enabled = *input.Enabled } source, err := h.service.CreateSource(userID, SourceInput{ - Name: input.Name, Kind: input.Kind, URL: input.URL, Description: input.Description, Enabled: enabled, + Name: input.Name, Kind: input.Kind, URL: input.URL, IconURL: input.IconURL, + Description: input.Description, Enabled: enabled, }) if err != nil { writeError(c, err) @@ -141,7 +144,8 @@ func (h *Handler) updateSource(c *gin.Context) { enabled = *input.Enabled } source, err := h.service.UpdateSource(userID, identity, SourceInput{ - Name: input.Name, Kind: input.Kind, URL: input.URL, Description: input.Description, Enabled: enabled, + Name: input.Name, Kind: input.Kind, URL: input.URL, IconURL: input.IconURL, + Description: input.Description, Enabled: enabled, }) if err != nil { writeError(c, err) @@ -308,7 +312,7 @@ func writeInvalidRequest(c *gin.Context) { func sourceDTO(source models.SaDatasetSource, itemCount int64) SourceDTO { return SourceDTO{ ID: source.Identity, Name: source.Name, Kind: source.Kind, URL: source.URL, - Description: source.Description, Enabled: source.Enabled, + IconURL: source.IconURL, Description: source.Description, Enabled: source.Enabled, LastSyncedAt: utcTime(source.LastSyncedAt), ItemCount: itemCount, CreatedAt: source.CreatedAt.UTC(), UpdatedAt: source.UpdatedAt.UTC(), } diff --git a/backend/internal/logic/dataset/handlers_test.go b/backend/internal/logic/dataset/handlers_test.go index d9655da..f384e29 100644 --- a/backend/internal/logic/dataset/handlers_test.go +++ b/backend/internal/logic/dataset/handlers_test.go @@ -36,6 +36,7 @@ func TestDatasetAPIConnectsSourcesItemsAndSyncTasks(t *testing.T) { require.NoError(t, json.Unmarshal(createdSource.Body.Bytes(), &source)) require.NotEmpty(t, source.ID) require.Equal(t, "Industry feed", source.Name) + require.Empty(t, source.IconURL) createdItem := performDatasetRequest(t, ownerRouter, http.MethodPost, "/api/v1/dataset-items", map[string]any{ "sourceId": source.ID, "title": "A collected article", "summary": "Summary", diff --git a/backend/internal/logic/dataset/service.go b/backend/internal/logic/dataset/service.go index 3ff28c1..fef6b54 100644 --- a/backend/internal/logic/dataset/service.go +++ b/backend/internal/logic/dataset/service.go @@ -26,6 +26,7 @@ type SourceInput struct { Name string Kind string URL string + IconURL string Description string Enabled bool } @@ -93,7 +94,8 @@ func (s *Service) CreateSource(userID uint, input SourceInput) (*models.SaDatase } source := &models.SaDatasetSource{ CreatedBy: userID, Name: normalized.Name, Kind: normalized.Kind, - URL: normalized.URL, Description: normalized.Description, Enabled: normalized.Enabled, + URL: normalized.URL, IconURL: normalized.IconURL, + Description: normalized.Description, Enabled: normalized.Enabled, } return source, s.db.Create(source).Error } @@ -109,7 +111,7 @@ func (s *Service) UpdateSource(userID uint, identity string, input SourceInput) } if err := s.db.Model(&source).Updates(map[string]any{ "name": normalized.Name, "kind": normalized.Kind, "url": normalized.URL, - "description": normalized.Description, "enabled": normalized.Enabled, + "icon_url": normalized.IconURL, "description": normalized.Description, "enabled": normalized.Enabled, }).Error; err != nil { return nil, err } @@ -248,6 +250,7 @@ func normalizeSourceInput(input SourceInput) (SourceInput, error) { input.Name = strings.TrimSpace(input.Name) input.Kind = strings.ToLower(strings.TrimSpace(input.Kind)) input.URL = strings.TrimSpace(input.URL) + input.IconURL = strings.TrimSpace(input.IconURL) input.Description = strings.TrimSpace(input.Description) if input.Name == "" { return input, ErrNameRequired @@ -258,6 +261,9 @@ func normalizeSourceInput(input SourceInput) (SourceInput, error) { if input.URL != "" && !validHTTPURL(input.URL) { return input, ErrURLInvalid } + if input.IconURL != "" && !validHTTPURL(input.IconURL) { + return input, ErrURLInvalid + } if input.Kind != "manual" && input.URL == "" { return input, ErrURLInvalid } diff --git a/backend/internal/models/dataset_source.go b/backend/internal/models/dataset_source.go index 442b70e..a594acd 100644 --- a/backend/internal/models/dataset_source.go +++ b/backend/internal/models/dataset_source.go @@ -10,6 +10,7 @@ type SaDatasetSource struct { Name string `gorm:"size:160;not null"` Kind string `gorm:"size:32;not null"` URL string `gorm:"size:2048"` + IconURL string `gorm:"size:2048"` Description string `gorm:"type:text"` Enabled bool `gorm:"not null;default:true;index"` LastSyncedAt *time.Time