senlinai/agent
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: seed default dataset sources

Browse Source
This commit is contained in:
yanweidong
2026-07-23 15:14:31 +08:00
parent 26dd3004a1
commit b053c3874a
12 changed files with 183 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
apps/web_v1/scripts/visual-check.mjs
View File

@@ -34,17 +34,17 @@ const visualExperts = [
]
let visualDatasetSources = [
{
id: '019b0000-0000-7000-8000-000000000030', name: '手动收集', kind: 'manual', url: '',
id: '019b0000-0000-7000-8000-000000000030', name: '手动收集', kind: 'manual', url: '', iconUrl: '',
description: '手动收集的文章与线索', enabled: true, lastSyncedAt: null, itemCount: 1,
createdAt: '2026-07-22T06:00:00Z', updatedAt: '2026-07-22T06:00:00Z',
},
{
id: '019b0000-0000-7000-8000-000000000031', name: '需求文档', kind: 'link', url: 'https://example.com/requirements',
id: '019b0000-0000-7000-8000-000000000031', name: '需求文档', kind: 'link', url: 'https://example.com/requirements', iconUrl: '',
description: '产品需求与业务文档', enabled: true, lastSyncedAt: null, itemCount: 1,
createdAt: '2026-07-22T06:00:00Z', updatedAt: '2026-07-22T06:00:00Z',
},
{
id: '019b0000-0000-7000-8000-000000000032', name: '架构讨论', kind: 'link', url: 'https://example.com/architecture',
id: '019b0000-0000-7000-8000-000000000032', name: '架构讨论', kind: 'link', url: 'https://example.com/architecture', iconUrl: '',
description: '架构方案与系统设计讨论', enabled: true, lastSyncedAt: null, itemCount: 0,
createdAt: '2026-07-22T06:00:00Z', updatedAt: '2026-07-22T06:00:00Z',
},

7
apps/web_v1/src/App.css
View File

@@ -753,6 +753,13 @@
font-size: 13px;
}
.explore-source-image {
width: 100%;
height: 100%;
border-radius: inherit;
object-fit: cover;
}
.explore-source-icon.blue {
background: rgb(var(--arcoblue-1));
color: rgb(var(--arcoblue-6));

2
apps/web_v1/src/api/dataset.ts
View File

@@ -7,6 +7,7 @@ export type DatasetSourceDTO = {
name: string
kind: DatasetSourceKind
url: string
iconUrl: string
description: string
enabled: boolean
lastSyncedAt: string | null
@@ -46,6 +47,7 @@ export type DatasetSourceInput = {
name: string
kind: DatasetSourceKind
url: string
iconUrl: string
description: string
enabled: boolean
}

9
apps/web_v1/src/pages/workspace-explore.tsx
View File

@@ -35,6 +35,7 @@ type SourceDraft = {
name: string
kind: DatasetSourceKind
url: string
iconUrl: string
description: string
enabled: boolean
}
@@ -43,6 +44,7 @@ const EMPTY_SOURCE_DRAFT: SourceDraft = {
name: '',
kind: 'link',
url: '',
iconUrl: '',
description: '',
enabled: true,
}
@@ -105,6 +107,7 @@ export function WorkspaceExplorePage({
name: '全部',
kind: 'manual',
url: '',
iconUrl: '',
description: '全部探索内容',
enabled: true,
lastSyncedAt: null,
@@ -118,7 +121,9 @@ export function WorkspaceExplorePage({
allSource,
...sourceRecords.map((source) => ({
...source,
icon: sourceIcon(source.kind),
icon: source.iconUrl
? <img className="explore-source-image" src={source.iconUrl} alt="" />
: sourceIcon(source.kind),
color: sourceColor(source.kind),
})),
]
@@ -156,6 +161,7 @@ export function WorkspaceExplorePage({
name: source.name,
kind: source.kind,
url: source.url,
iconUrl: source.iconUrl,
description: source.description,
enabled: source.enabled,
})
@@ -168,6 +174,7 @@ export function WorkspaceExplorePage({
name: sourceDraft.name.trim(),
kind: sourceDraft.kind,
url: sourceDraft.url.trim(),
iconUrl: sourceDraft.iconUrl,
description: sourceDraft.description.trim(),
enabled: sourceDraft.enabled,
}

58
backend/internal/initdb/dataset.go Normal file
View File

@@ -0,0 +1,58 @@
package initdb
import (
"errors"
"fmt"
"gorm.io/gorm"
"senlinai-agent/backend/internal/models"
)
var defaultDatasetSources = []struct {
Name string
URL string
IconURL string
}{
{Name: "彭博社最新报道", URL: "https://bbg.buzzing.cc/feed.json", IconURL: "https://bbg.buzzing.cc/apple-touch-icon.png"},
{Name: "国外科技头条", URL: "https://tech.buzzing.cc/feed.json", IconURL: "https://tech.buzzing.cc/apple-touch-icon.png"},
{Name: "国外财经新闻", URL: "https://finance.buzzing.cc/feed.json", IconURL: "https://finance.buzzing.cc/apple-touch-icon.png"},
{Name: "华尔街日报热门", URL: "https://wsj.buzzing.cc/feed.json", IconURL: "https://wsj.buzzing.cc/apple-touch-icon.png"},
{Name: "Product Hunt", URL: "https://ph.buzzing.cc/feed.json", IconURL: "https://ph.buzzing.cc/apple-touch-icon.png"},
{Name: "Dev.to", URL: "https://dev.buzzing.cc/feed.json", IconURL: "https://dev.buzzing.cc/apple-touch-icon.png"},
}
// InitDataset creates the default Buzzing RSS sources for the default account when the source table is empty.
func InitDataset(database *gorm.DB) error {
var count int64
if err := database.Model(&models.SaDatasetSource{}).Count(&count).Error; err != nil {
return err
}
if count > 0 {
return nil
}
var owner models.SaUser
err := database.Where("email = ?", rootUsername).First(&owner).Error
if err != nil && !errors.Is(err, gorm.ErrRecordNotFound) {
return fmt.Errorf("find root user for default dataset sources: %w", err)
}
if errors.Is(err, gorm.ErrRecordNotFound) {
if err := database.Order("id asc").First(&owner).Error; err != nil {
return fmt.Errorf("find owner for default dataset sources: %w", err)
}
}
sources := make([]models.SaDatasetSource, 0, len(defaultDatasetSources))
for _, source := range defaultDatasetSources {
sources = append(sources, models.SaDatasetSource{
CreatedBy: owner.ID,
Name: source.Name,
Kind: "rss",
URL: source.URL,
IconURL: source.IconURL,
Description: "Buzzing 中文聚合订阅",
Enabled: true,
})
}
return database.Create(&sources).Error
}

84
backend/internal/initdb/dataset_test.go Normal file
View File

@@ -0,0 +1,84 @@
package initdb
import (
"fmt"
"testing"
"github.com/glebarez/sqlite"
"github.com/stretchr/testify/require"
"gorm.io/gorm"
"senlinai-agent/backend/internal/models"
)
func TestInitDatasetCreatesDefaultSourcesWhenTableIsEmpty(t *testing.T) {
database := newDatasetInitTestDatabase(t)
require.NoError(t, InitUser(database))
require.NoError(t, InitDataset(database))
require.NoError(t, InitDataset(database))
var root models.SaUser
require.NoError(t, database.Where("email = ?", rootUsername).First(&root).Error)
var sources []models.SaDatasetSource
require.NoError(t, database.Order("id asc").Find(&sources).Error)
require.Len(t, sources, len(defaultDatasetSources))
for index, expected := range defaultDatasetSources {
require.Equal(t, expected.Name, sources[index].Name)
require.Equal(t, expected.URL, sources[index].URL)
require.Equal(t, expected.IconURL, sources[index].IconURL)
require.Equal(t, "rss", sources[index].Kind)
require.True(t, sources[index].Enabled)
require.Equal(t, root.ID, sources[index].CreatedBy)
require.Equal(t, root.Identity, sources[index].CreatedByIdentity)
}
}
func TestInitDatasetDoesNothingWhenSourceTableIsNotEmpty(t *testing.T) {
database := newDatasetInitTestDatabase(t)
require.NoError(t, InitUser(database))
var root models.SaUser
require.NoError(t, database.Where("email = ?", rootUsername).First(&root).Error)
existing := models.SaDatasetSource{
CreatedBy: root.ID,
Name: "Existing source",
Kind: "manual",
Enabled: true,
}
require.NoError(t, database.Create(&existing).Error)
require.NoError(t, InitDataset(database))
var sources []models.SaDatasetSource
require.NoError(t, database.Find(&sources).Error)
require.Len(t, sources, 1)
require.Equal(t, existing.Identity, sources[0].Identity)
}
func TestInitDatasetUsesFirstExistingUserWhenRootIsAbsent(t *testing.T) {
database := newDatasetInitTestDatabase(t)
user := models.SaUser{
Email: "existing@example.com",
DisplayName: "Existing",
PasswordHash: "hash",
Role: "user",
}
require.NoError(t, database.Create(&user).Error)
require.NoError(t, InitDataset(database))
var sources []models.SaDatasetSource
require.NoError(t, database.Find(&sources).Error)
require.Len(t, sources, len(defaultDatasetSources))
for _, source := range sources {
require.Equal(t, user.ID, source.CreatedBy)
require.Equal(t, user.Identity, source.CreatedByIdentity)
}
}
func newDatasetInitTestDatabase(t *testing.T) *gorm.DB {
t.Helper()
database, err := gorm.Open(sqlite.Open(fmt.Sprintf("file:%s?mode=memory&cache=shared", t.Name())), &gorm.Config{})
require.NoError(t, err)
require.NoError(t, database.AutoMigrate(&models.SaUser{}, &models.SaDatasetSource{}))
return database
}

1
backend/internal/initdb/expert_test.go
View File

@@ -59,6 +59,7 @@ func newExpertTestDatabase(t *testing.T) *gorm.DB {
require.NoError(t, err)
require.NoError(t, database.AutoMigrate(
&models.SaUser{},
&models.SaDatasetSource{},
&models.SaAIExpertCategory{},
&models.SaAIExpertItem{},
))

3
backend/internal/initdb/new.go
View File

@@ -8,6 +8,9 @@ func New(database *gorm.DB) error {
if err := InitUser(tx); err != nil {
return err
}
if err := InitDataset(tx); err != nil {
return err
}
return InitExpert(tx)
})
}

10
backend/internal/logic/dataset/handlers.go
View File

@@ -22,6 +22,7 @@ type SourceDTO struct {
Name string `json:"name"`
Kind string `json:"kind"`
URL string `json:"url"`
IconURL string `json:"iconUrl"`
Description string `json:"description"`
Enabled bool `json:"enabled"`
LastSyncedAt *time.Time `json:"lastSyncedAt"`
@@ -61,6 +62,7 @@ type sourceRequest struct {
Name string `json:"name"`
Kind string `json:"kind"`
URL string `json:"url"`
IconURL string `json:"iconUrl"`
Description string `json:"description"`
Enabled *bool `json:"enabled"`
}
@@ -113,7 +115,8 @@ func (h *Handler) createSource(c *gin.Context) {
enabled = *input.Enabled
}
source, err := h.service.CreateSource(userID, SourceInput{
Name: input.Name, Kind: input.Kind, URL: input.URL, Description: input.Description, Enabled: enabled,
Name: input.Name, Kind: input.Kind, URL: input.URL, IconURL: input.IconURL,
Description: input.Description, Enabled: enabled,
})
if err != nil {
writeError(c, err)
@@ -141,7 +144,8 @@ func (h *Handler) updateSource(c *gin.Context) {
enabled = *input.Enabled
}
source, err := h.service.UpdateSource(userID, identity, SourceInput{
Name: input.Name, Kind: input.Kind, URL: input.URL, Description: input.Description, Enabled: enabled,
Name: input.Name, Kind: input.Kind, URL: input.URL, IconURL: input.IconURL,
Description: input.Description, Enabled: enabled,
})
if err != nil {
writeError(c, err)
@@ -308,7 +312,7 @@ func writeInvalidRequest(c *gin.Context) {
func sourceDTO(source models.SaDatasetSource, itemCount int64) SourceDTO {
return SourceDTO{
ID: source.Identity, Name: source.Name, Kind: source.Kind, URL: source.URL,
Description: source.Description, Enabled: source.Enabled,
IconURL: source.IconURL, Description: source.Description, Enabled: source.Enabled,
LastSyncedAt: utcTime(source.LastSyncedAt), ItemCount: itemCount,
CreatedAt: source.CreatedAt.UTC(), UpdatedAt: source.UpdatedAt.UTC(),
}

1
backend/internal/logic/dataset/handlers_test.go
View File

@@ -36,6 +36,7 @@ func TestDatasetAPIConnectsSourcesItemsAndSyncTasks(t *testing.T) {
require.NoError(t, json.Unmarshal(createdSource.Body.Bytes(), &source))
require.NotEmpty(t, source.ID)
require.Equal(t, "Industry feed", source.Name)
require.Empty(t, source.IconURL)
createdItem := performDatasetRequest(t, ownerRouter, http.MethodPost, "/api/v1/dataset-items", map[string]any{
"sourceId": source.ID, "title": "A collected article", "summary": "Summary",

10
backend/internal/logic/dataset/service.go
View File

@@ -26,6 +26,7 @@ type SourceInput struct {
Name string
Kind string
URL string
IconURL string
Description string
Enabled bool
}
@@ -93,7 +94,8 @@ func (s *Service) CreateSource(userID uint, input SourceInput) (*models.SaDatase
}
source := &models.SaDatasetSource{
CreatedBy: userID, Name: normalized.Name, Kind: normalized.Kind,
URL: normalized.URL, Description: normalized.Description, Enabled: normalized.Enabled,
URL: normalized.URL, IconURL: normalized.IconURL,
Description: normalized.Description, Enabled: normalized.Enabled,
}
return source, s.db.Create(source).Error
}
@@ -109,7 +111,7 @@ func (s *Service) UpdateSource(userID uint, identity string, input SourceInput)
}
if err := s.db.Model(&source).Updates(map[string]any{
"name": normalized.Name, "kind": normalized.Kind, "url": normalized.URL,
"description": normalized.Description, "enabled": normalized.Enabled,
"icon_url": normalized.IconURL, "description": normalized.Description, "enabled": normalized.Enabled,
}).Error; err != nil {
return nil, err
}
@@ -248,6 +250,7 @@ func normalizeSourceInput(input SourceInput) (SourceInput, error) {
input.Name = strings.TrimSpace(input.Name)
input.Kind = strings.ToLower(strings.TrimSpace(input.Kind))
input.URL = strings.TrimSpace(input.URL)
input.IconURL = strings.TrimSpace(input.IconURL)
input.Description = strings.TrimSpace(input.Description)
if input.Name == "" {
return input, ErrNameRequired
@@ -258,6 +261,9 @@ func normalizeSourceInput(input SourceInput) (SourceInput, error) {
if input.URL != "" && !validHTTPURL(input.URL) {
return input, ErrURLInvalid
}
if input.IconURL != "" && !validHTTPURL(input.IconURL) {
return input, ErrURLInvalid
}
if input.Kind != "manual" && input.URL == "" {
return input, ErrURLInvalid
}

1
backend/internal/models/dataset_source.go
View File

@@ -10,6 +10,7 @@ type SaDatasetSource struct {
Name string `gorm:"size:160;not null"`
Kind string `gorm:"size:32;not null"`
URL string `gorm:"size:2048"`
IconURL string `gorm:"size:2048"`
Description string `gorm:"type:text"`
Enabled bool `gorm:"not null;default:true;index"`
LastSyncedAt *time.Time