senlinai/agent
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: connect explore datasets to backend

Browse Source
This commit is contained in:
yanweidong
2026-07-23 15:07:35 +08:00
parent 0ca2898ac0
commit 26dd3004a1
14 changed files with 1445 additions and 196 deletions
Download Patch File Download Diff File Expand all files Collapse all files

97
apps/web_v1/scripts/visual-check.mjs
View File

@@ -32,6 +32,37 @@ const visualExperts = [
name: '前端开发者', description: '负责现代 Web 应用实现与性能优化。', emoji: '💻', color: '#0FC6C2',
},
]
let visualDatasetSources = [
{
id: '019b0000-0000-7000-8000-000000000030', name: '手动收集', kind: 'manual', url: '',
description: '手动收集的文章与线索', enabled: true, lastSyncedAt: null, itemCount: 1,
createdAt: '2026-07-22T06:00:00Z', updatedAt: '2026-07-22T06:00:00Z',
},
{
id: '019b0000-0000-7000-8000-000000000031', name: '需求文档', kind: 'link', url: 'https://example.com/requirements',
description: '产品需求与业务文档', enabled: true, lastSyncedAt: null, itemCount: 1,
createdAt: '2026-07-22T06:00:00Z', updatedAt: '2026-07-22T06:00:00Z',
},
{
id: '019b0000-0000-7000-8000-000000000032', name: '架构讨论', kind: 'link', url: 'https://example.com/architecture',
description: '架构方案与系统设计讨论', enabled: true, lastSyncedAt: null, itemCount: 0,
createdAt: '2026-07-22T06:00:00Z', updatedAt: '2026-07-22T06:00:00Z',
},
]
let visualDatasetItems = [
{
id: '019b0000-0000-7000-8000-000000000040', sourceId: visualDatasetSources[0].id,
title: '森林项目体验优化需求', summary: '整理工作台信息层级，并沉淀为可跟进计划。', content: '探索数据条目的正文内容。',
url: 'https://example.com/article-1', status: 'unread', starred: false, publishedAt: '2026-07-22T08:00:00Z',
createdAt: '2026-07-22T08:00:00Z', updatedAt: '2026-07-22T08:00:00Z',
},
{
id: '019b0000-0000-7000-8000-000000000041', sourceId: visualDatasetSources[1].id,
title: '项目资料组织方案', summary: '围绕项目上下文统一组织任务、笔记、资料和 AI 会话。', content: '资料组织方案正文。',
url: 'https://example.com/article-2', status: 'read', starred: true, publishedAt: '2026-07-22T07:00:00Z',
createdAt: '2026-07-22T07:00:00Z', updatedAt: '2026-07-22T07:00:00Z',
},
]
page.on('console', (message) => {
if (message.type() === 'error') errors.push(message.text())
})
@@ -185,6 +216,71 @@ await page.route('http://localhost:9150/api/v1/**', async (route) => {
await route.fulfill({ json: visualExperts })
return
}
if (url.pathname === '/api/v1/dataset-sources' && route.request().method() === 'GET') {
await route.fulfill({ json: visualDatasetSources })
return
}
if (url.pathname === '/api/v1/dataset-sources' && route.request().method() === 'POST') {
const input = route.request().postDataJSON()
const created = {
id: `019b0000-0000-7000-8000-${String(50 + visualDatasetSources.length).padStart(12, '0')}`,
...input,
lastSyncedAt: null,
itemCount: 0,
createdAt: new Date().toISOString(),
updatedAt: new Date().toISOString(),
}
visualDatasetSources = [...visualDatasetSources, created]
await route.fulfill({ status: 201, json: created })
return
}
if (url.pathname.startsWith('/api/v1/dataset-sources/') && route.request().method() === 'PATCH') {
const id = url.pathname.split('/').at(-1)
const input = route.request().postDataJSON()
const current = visualDatasetSources.find((source) => source.id === id)
const updated = { ...current, ...input, updatedAt: new Date().toISOString() }
visualDatasetSources = visualDatasetSources.map((source) => source.id === id ? updated : source)
await route.fulfill({ json: updated })
return
}
if (url.pathname.startsWith('/api/v1/dataset-sources/') && route.request().method() === 'DELETE') {
const id = url.pathname.split('/').at(-1)
visualDatasetSources = visualDatasetSources.filter((source) => source.id !== id)
visualDatasetItems = visualDatasetItems.filter((item) => item.sourceId !== id)
await route.fulfill({ status: 204, body: '' })
return
}
if (url.pathname === '/api/v1/dataset-items' && route.request().method() === 'GET') {
await route.fulfill({ json: visualDatasetItems })
return
}
if (url.pathname.startsWith('/api/v1/dataset-items/') && route.request().method() === 'PATCH') {
const id = url.pathname.split('/').at(-1)
const input = route.request().postDataJSON()
const current = visualDatasetItems.find((item) => item.id === id)
const updated = { ...current, ...input, updatedAt: new Date().toISOString() }
visualDatasetItems = visualDatasetItems.map((item) => item.id === id ? updated : item)
await route.fulfill({ json: updated })
return
}
if (url.pathname === '/api/v1/dataset-crons/sync') {
await route.fulfill({
status: 202,
json: visualDatasetSources.map((source, index) => ({
id: `019b0000-0000-7000-8000-${String(70 + index).padStart(12, '0')}`,
sourceId: source.id,
schedule: '@once',
status: 'pending',
enabled: true,
nextRunAt: new Date().toISOString(),
lastRunAt: null,
lastResult: '',
createdAt: new Date().toISOString(),
updatedAt: new Date().toISOString(),
})),
})
return
}
await route.fulfill({ status: 404, json: { error: { code: 'not_found', message: '视觉检查未配置该接口' } } })
})
@@ -304,6 +400,7 @@ const addSourceModalCheck = await page.evaluate(() => ({
inputCount: document.querySelectorAll('.explore-source-modal input, .explore-source-modal textarea').length,
}))
await page.locator('.explore-source-modal input').first().fill('行业资讯')
await page.locator('.explore-source-modal input[placeholder="https://example.com/feed"]').fill('https://example.com/feed')
await page.locator('.explore-source-modal .arco-modal-footer .arco-btn-primary').click()
const addedSourceVisible = await page.locator('.explore-source-card', { hasText: '行业资讯' }).count() === 1

105
apps/web_v1/src/api/dataset.ts Normal file
View File

@@ -0,0 +1,105 @@
import { apiRequest, type ApiSession } from './client'
export type DatasetSourceKind = 'manual' | 'link' | 'rss'
export type DatasetSourceDTO = {
id: string
name: string
kind: DatasetSourceKind
url: string
description: string
enabled: boolean
lastSyncedAt: string | null
itemCount: number
createdAt: string
updatedAt: string
}
export type DatasetItemDTO = {
id: string
sourceId: string
title: string
summary: string
content: string
url: string
status: 'unread' | 'read' | 'archived'
starred: boolean
publishedAt: string | null
createdAt: string
updatedAt: string
}
export type DatasetCronDTO = {
id: string
sourceId: string
schedule: string
status: string
enabled: boolean
nextRunAt: string | null
lastRunAt: string | null
lastResult: string
createdAt: string
updatedAt: string
}
export type DatasetSourceInput = {
name: string
kind: DatasetSourceKind
url: string
description: string
enabled: boolean
}
export type DatasetItemUpdate = {
status: DatasetItemDTO['status']
starred: boolean
}
export function listDatasetSources(session: ApiSession, signal?: AbortSignal) {
return apiRequest<DatasetSourceDTO[]>('/api/v1/dataset-sources', { token: session.token, signal })
}
export function createDatasetSource(session: ApiSession, input: DatasetSourceInput) {
return apiRequest<DatasetSourceDTO>('/api/v1/dataset-sources', {
method: 'POST',
token: session.token,
body: input,
})
}
export function updateDatasetSource(session: ApiSession, sourceId: string, input: DatasetSourceInput) {
return apiRequest<DatasetSourceDTO>(`/api/v1/dataset-sources/${sourceId}`, {
method: 'PATCH',
token: session.token,
body: input,
})
}
export function deleteDatasetSource(session: ApiSession, sourceId: string) {
return apiRequest<void>(`/api/v1/dataset-sources/${sourceId}`, {
method: 'DELETE',
token: session.token,
responseType: 'void',
})
}
export function listDatasetItems(session: ApiSession, sourceId?: string, signal?: AbortSignal) {
const query = sourceId ? `?sourceId=${encodeURIComponent(sourceId)}` : ''
return apiRequest<DatasetItemDTO[]>(`/api/v1/dataset-items${query}`, { token: session.token, signal })
}
export function updateDatasetItem(session: ApiSession, itemId: string, input: DatasetItemUpdate) {
return apiRequest<DatasetItemDTO>(`/api/v1/dataset-items/${itemId}`, {
method: 'PATCH',
token: session.token,
body: input,
})
}
export function queueDatasetSync(session: ApiSession) {
return apiRequest<DatasetCronDTO[]>('/api/v1/dataset-crons/sync', {
method: 'POST',
token: session.token,
body: {},
})
}

53
apps/web_v1/src/app/App.tsx
View File

@@ -4,6 +4,17 @@ import '@arco-design/web-react/dist/css/arco.css'
import '../App.css'
import { ApiError, login, setApiBaseUrl, type ApiSession } from '../api/client'
import { createAISession, listAIExperts, listAISessions, type CreateAISessionInput } from '../api/ai'
import {
createDatasetSource,
deleteDatasetSource,
listDatasetItems,
listDatasetSources,
queueDatasetSync,
updateDatasetItem,
updateDatasetSource,
type DatasetItemUpdate,
type DatasetSourceInput,
} from '../api/dataset'
import { analyzeInboxItem, confirmInboxItem } from '../api/inbox'
import { mapWorkspace } from '../api/mappers'
import {
@@ -59,6 +70,41 @@ function App() {
return createAISession(session, projectId, input, signal)
}, [session])
const handleListDatasetSources = useCallback((signal?: AbortSignal) => {
if (!session) return Promise.reject(new Error('未登录'))
return listDatasetSources(session, signal)
}, [session])
const handleListDatasetItems = useCallback((signal?: AbortSignal) => {
if (!session) return Promise.reject(new Error('未登录'))
return listDatasetItems(session, undefined, signal)
}, [session])
const handleCreateDatasetSource = useCallback((input: DatasetSourceInput) => {
if (!session) return Promise.reject(new Error('未登录'))
return createDatasetSource(session, input)
}, [session])
const handleUpdateDatasetSource = useCallback((sourceId: string, input: DatasetSourceInput) => {
if (!session) return Promise.reject(new Error('未登录'))
return updateDatasetSource(session, sourceId, input)
}, [session])
const handleDeleteDatasetSource = useCallback((sourceId: string) => {
if (!session) return Promise.reject(new Error('未登录'))
return deleteDatasetSource(session, sourceId)
}, [session])
const handleUpdateDatasetItem = useCallback((itemId: string, input: DatasetItemUpdate) => {
if (!session) return Promise.reject(new Error('未登录'))
return updateDatasetItem(session, itemId, input)
}, [session])
const handleQueueDatasetSync = useCallback(() => {
if (!session) return Promise.reject(new Error('未登录'))
return queueDatasetSync(session)
}, [session])
const dark = theme === 'dark'
const activeWorkspace = workspaces.find((workspace) => workspace.project.id === activeProjectID) ?? workspaces[0]
const activeTagOptions = activeWorkspace?.tags.filter((tag) => tag !== 'all' && tag !== '全部') ?? []
@@ -365,6 +411,13 @@ function App() {
onListAISessions={handleListAISessions}
onListAIExperts={handleListAIExperts}
onCreateAISession={handleCreateAISession}
onListDatasetSources={handleListDatasetSources}
onListDatasetItems={handleListDatasetItems}
onCreateDatasetSource={handleCreateDatasetSource}
onUpdateDatasetSource={handleUpdateDatasetSource}
onDeleteDatasetSource={handleDeleteDatasetSource}
onUpdateDatasetItem={handleUpdateDatasetItem}
onQueueDatasetSync={handleQueueDatasetSync}
/>
) : (
<Spin loading />

480
apps/web_v1/src/pages/workspace-explore.tsx
View File

@@ -1,112 +1,147 @@
import { useEffect, useMemo, useState } from 'react'
import type { ReactNode } from 'react'
import { Alert, Button, Card, Empty, Grid, Input, Modal, Select, Space, Typography } from '@arco-design/web-react'
import { Alert, Button, Card, Empty, Grid, Input, Message, Modal, Select, Space, Spin, Switch, Typography } from '@arco-design/web-react'
import {
IconBook,
IconCheckCircle,
IconCompass,
IconDelete,
IconEdit,
IconFile,
IconLaunch,
IconLink,
IconRefresh,
IconStar,
IconStorage,
} from '@arco-design/web-react/icon'
import type { InboxItem, Project, ProjectWorkspace } from './projects/project-types'
import type {
DatasetCronDTO,
DatasetItemDTO,
DatasetItemUpdate,
DatasetSourceDTO,
DatasetSourceInput,
DatasetSourceKind,
} from '../api/dataset'
const { Row, Col } = Grid
const { Title, Text, Paragraph } = Typography
const { TextArea } = Input
type BuiltinDataSourceID = 'all' | 'manual' | 'requirements' | 'architecture'
type DataSourceID = BuiltinDataSourceID | `custom:${number}`
type DataSourceKind = 'manual' | 'link' | 'rss'
type ExploreArticle = InboxItem & {
project: Project
sourceID: BuiltinDataSourceID
}
type DataSourceConfig = {
id: DataSourceID
name: string
type DataSourceCard = DatasetSourceDTO & {
icon: ReactNode
color: string
kind: DataSourceKind
url: string
description: string
}
type DataSourceCard = DataSourceConfig & {
count: number
}
type SourceDraft = {
name: string
kind: DataSourceKind
kind: DatasetSourceKind
url: string
description: string
enabled: boolean
}
const INITIAL_SOURCES: DataSourceConfig[] = [
{ id: 'all', name: '全部', icon: <IconStorage />, color: 'blue', kind: 'manual', url: '', description: '全部探索内容' },
{ id: 'manual', name: '手动收集', icon: <IconCompass />, color: 'green', kind: 'manual', url: '', description: '手动收集的文章与线索' },
{ id: 'requirements', name: '需求文档', icon: <IconFile />, color: 'orange', kind: 'link', url: '', description: '产品需求与业务文档' },
{ id: 'architecture', name: '架构讨论', icon: <IconBook />, color: 'purple', kind: 'link', url: '', description: '架构方案与系统设计讨论' },
]
const EMPTY_SOURCE_DRAFT: SourceDraft = {
name: '',
kind: 'link',
url: '',
description: '',
enabled: true,
}
export function WorkspaceExplorePage({
workspaces,
onSelectItem,
onListSources,
onListItems,
onCreateSource,
onUpdateSource,
onDeleteSource,
onUpdateItem,
onQueueSync,
}: {
workspaces: ProjectWorkspace[]
onSelectItem: (title: string) => void
onListSources: (signal?: AbortSignal) => Promise<DatasetSourceDTO[]>
onListItems: (signal?: AbortSignal) => Promise<DatasetItemDTO[]>
onCreateSource: (input: DatasetSourceInput) => Promise<DatasetSourceDTO>
onUpdateSource: (sourceId: string, input: DatasetSourceInput) => Promise<DatasetSourceDTO>
onDeleteSource: (sourceId: string) => Promise<void>
onUpdateItem: (itemId: string, input: DatasetItemUpdate) => Promise<DatasetItemDTO>
onQueueSync: () => Promise<DatasetCronDTO[]>
}) {
const articles = useMemo(
() =>
workspaces.flatMap((workspace) =>
workspace.inbox.map((item) => ({
...item,
project: workspace.project,
sourceID: detectSource(item),
})),
),
[workspaces],
)
const [sourceConfigs, setSourceConfigs] = useState<DataSourceConfig[]>(INITIAL_SOURCES)
const [activeSourceID, setActiveSourceID] = useState<DataSourceID>('all')
const [sourceRecords, setSourceRecords] = useState<DatasetSourceDTO[]>([])
const [items, setItems] = useState<DatasetItemDTO[]>([])
const [activeSourceID, setActiveSourceID] = useState('all')
const [activeItemID, setActiveItemID] = useState<string | null>(null)
const [sourceModalMode, setSourceModalMode] = useState<'add' | 'edit' | null>(null)
const [editingSourceID, setEditingSourceID] = useState<DataSourceID | null>(null)
const [editingSourceID, setEditingSourceID] = useState<string | null>(null)
const [sourceDraft, setSourceDraft] = useState<SourceDraft>(EMPTY_SOURCE_DRAFT)
const [sourceError, setSourceError] = useState('')
const filteredArticles = useMemo(
() => (activeSourceID === 'all' ? articles : articles.filter((article) => article.sourceID === activeSourceID)),
[activeSourceID, articles],
)
const sources = useMemo(() => dataSources(articles, sourceConfigs), [articles, sourceConfigs])
const [activeArticleID, setActiveArticleID] = useState<string | null>(filteredArticles[0]?.id ?? null)
const [loadError, setLoadError] = useState('')
const [loading, setLoading] = useState(true)
const [saving, setSaving] = useState(false)
const [syncing, setSyncing] = useState(false)
useEffect(() => {
if (filteredArticles.length === 0) {
setActiveArticleID(null)
const controller = new AbortController()
setLoading(true)
setLoadError('')
Promise.all([onListSources(controller.signal), onListItems(controller.signal)])
.then(([nextSources, nextItems]) => {
setSourceRecords(nextSources)
setItems(nextItems)
})
.catch((error: unknown) => {
if (!controller.signal.aborted) {
setLoadError(error instanceof Error ? error.message : '探索数据加载失败')
}
})
.finally(() => {
if (!controller.signal.aborted) setLoading(false)
})
return () => controller.abort()
}, [onListItems, onListSources])
const sources = useMemo<DataSourceCard[]>(() => {
const allSource: DataSourceCard = {
id: 'all',
name: '全部',
kind: 'manual',
url: '',
description: '全部探索内容',
enabled: true,
lastSyncedAt: null,
itemCount: items.length,
createdAt: '',
updatedAt: '',
icon: <IconStorage />,
color: 'blue',
}
return [
allSource,
...sourceRecords.map((source) => ({
...source,
icon: sourceIcon(source.kind),
color: sourceColor(source.kind),
})),
]
}, [items.length, sourceRecords])
const filteredItems = useMemo(
() => activeSourceID === 'all' ? items : items.filter((item) => item.sourceId === activeSourceID),
[activeSourceID, items],
)
useEffect(() => {
if (filteredItems.length === 0) {
setActiveItemID(null)
return
}
if (!activeArticleID || !filteredArticles.some((article) => article.id === activeArticleID)) {
setActiveArticleID(filteredArticles[0].id)
if (!activeItemID || !filteredItems.some((item) => item.id === activeItemID)) {
setActiveItemID(filteredItems[0].id)
}
}, [activeArticleID, filteredArticles])
}, [activeItemID, filteredItems])
const selected = filteredArticles.find((article) => article.id === activeArticleID) ?? filteredArticles[0]
const selected = filteredItems.find((item) => item.id === activeItemID) ?? filteredItems[0]
const activeSource = sourceByID(sources, activeSourceID)
const selectedSource = selected ? sourceByID(sources, selected.sourceID) : activeSource
const selectedSource = selected ? sourceByID(sources, selected.sourceId) : activeSource
function openAddSource() {
setEditingSourceID(null)
@@ -122,46 +157,94 @@ export function WorkspaceExplorePage({
kind: source.kind,
url: source.url,
description: source.description,
enabled: source.enabled,
})
setSourceError('')
setSourceModalMode('edit')
}
function saveSource() {
const name = sourceDraft.name.trim()
if (!name) {
async function saveSource() {
const input: DatasetSourceInput = {
name: sourceDraft.name.trim(),
kind: sourceDraft.kind,
url: sourceDraft.url.trim(),
description: sourceDraft.description.trim(),
enabled: sourceDraft.enabled,
}
if (!input.name) {
setSourceError('请输入数据源名称')
return
}
const nextDraft = { ...sourceDraft, name, url: sourceDraft.url.trim(), description: sourceDraft.description.trim() }
if (sourceModalMode === 'edit' && editingSourceID) {
setSourceConfigs((current) => current.map((source) => source.id === editingSourceID ? { ...source, ...nextDraft } : source))
} else {
setSourceConfigs((current) => [
...current,
{
id: `custom:${Date.now()}`,
...nextDraft,
icon: sourceIcon(nextDraft.kind),
color: sourceColor(nextDraft.kind),
},
])
if (input.kind !== 'manual' && !input.url) {
setSourceError('链接和 RSS 数据源必须填写地址')
return
}
setSaving(true)
setSourceError('')
try {
if (sourceModalMode === 'edit' && editingSourceID) {
const updated = await onUpdateSource(editingSourceID, input)
setSourceRecords((current) => current.map((source) => source.id === updated.id ? updated : source))
Message.success('数据源已更新')
} else {
const created = await onCreateSource(input)
setSourceRecords((current) => [...current, created])
Message.success('数据源已添加')
}
setSourceModalMode(null)
} catch (error) {
setSourceError(error instanceof Error ? error.message : '数据源保存失败')
} finally {
setSaving(false)
}
setSourceModalMode(null)
}
function deleteSource(source: DataSourceCard) {
Modal.confirm({
title: `删除“${source.name}”数据源？`,
content: '删除后，该数据源将从探索页移除。',
content: '删除后，该数据源及其采集条目和待执行任务都会移除。',
okButtonProps: { status: 'danger' },
onOk: () => {
setSourceConfigs((current) => current.filter((item) => item.id !== source.id))
if (activeSourceID === source.id) setActiveSourceID('all')
onOk: async () => {
try {
await onDeleteSource(source.id)
setSourceRecords((current) => current.filter((item) => item.id !== source.id))
setItems((current) => current.filter((item) => item.sourceId !== source.id))
if (activeSourceID === source.id) setActiveSourceID('all')
Message.success('数据源已删除')
} catch (error) {
Message.error(error instanceof Error ? error.message : '数据源删除失败')
throw error
}
},
})
}
async function syncSources() {
setSyncing(true)
try {
const crons = await onQueueSync()
if (crons.length === 0) {
Message.info('暂无已启用的数据源')
} else {
Message.success(`已提交 ${crons.length} 个采集任务`)
}
} catch (error) {
Message.error(error instanceof Error ? error.message : '同步任务提交失败')
} finally {
setSyncing(false)
}
}
async function updateSelected(input: DatasetItemUpdate) {
if (!selected) return
try {
const updated = await onUpdateItem(selected.id, input)
setItems((current) => current.map((item) => item.id === updated.id ? updated : item))
} catch (error) {
Message.error(error instanceof Error ? error.message : '数据条目更新失败')
}
}
return (
<div className="workspace-explore-page overview-page">
<div className="overview-head">
@@ -170,109 +253,112 @@ export function WorkspaceExplorePage({
<Text type="secondary">多个外部数据源的集中阅读页，采集文章并沉淀到项目。</Text>
</div>
<Space>
<Button icon={<IconRefresh />}>同步数据源</Button>
<Button type="primary" icon={<IconLink />} onClick={openAddSource}>
添加数据源
</Button>
<Button icon={<IconRefresh />} loading={syncing} onClick={() => void syncSources()}>同步数据源</Button>
<Button type="primary" icon={<IconLink />} onClick={openAddSource}>添加数据源</Button>
</Space>
</div>
<Row gutter={10} className="explore-source-row">
{sources.map((source) => (
<Col span={6} key={source.id}>
<Card
className={activeSourceID === source.id ? 'compact-card explore-source-card active' : 'compact-card explore-source-card'}
bordered
onClick={() => setActiveSourceID(source.id)}
>
<span className={`explore-source-icon ${source.color}`}>{source.icon}</span>
<span className="explore-source-copy">
<Text className="explore-source-name">{source.name}</Text>
<Text type="secondary">{source.count} 条</Text>
</span>
{source.id !== 'all' ? (
<span className="explore-source-actions" onClick={(event) => event.stopPropagation()}>
<Button aria-label={`编辑${source.name}`} type="text" size="mini" icon={<IconEdit />} onClick={() => openEditSource(source)} />
<Button aria-label={`删除${source.name}`} type="text" size="mini" status="danger" icon={<IconDelete />} onClick={() => deleteSource(source)} />
{loadError ? <Alert type="error" content={loadError} /> : null}
<Spin loading={loading} style={{ width: '100%' }}>
<Row gutter={10} className="explore-source-row">
{sources.map((source) => (
<Col span={6} key={source.id}>
<Card
className={activeSourceID === source.id ? 'compact-card explore-source-card active' : 'compact-card explore-source-card'}
bordered
onClick={() => setActiveSourceID(source.id)}
>
<span className={`explore-source-icon ${source.color}`}>{source.icon}</span>
<span className="explore-source-copy">
<Text className="explore-source-name">{source.name}</Text>
<Text type="secondary">{source.itemCount} 条</Text>
</span>
) : null}
{source.id !== 'all' ? (
<span className="explore-source-actions" onClick={(event) => event.stopPropagation()}>
<Button aria-label={`编辑${source.name}`} type="text" size="mini" icon={<IconEdit />} onClick={() => openEditSource(source)} />
<Button aria-label={`删除${source.name}`} type="text" size="mini" status="danger" icon={<IconDelete />} onClick={() => deleteSource(source)} />
</span>
) : null}
</Card>
</Col>
))}
</Row>
{selected ? (
<section className="explore-reader-layout">
<Card className="explore-article-list queue-section" bordered>
<div className="explore-list-header">
<Title heading={5}>
<Space size={6}>
<span className={`explore-source-icon mini ${activeSource.color}`}>{activeSource.icon}</span>
<span>{activeSource.name}</span>
</Space>
</Title>
<Button type="text" icon={<IconRefresh />} loading={syncing} onClick={() => void syncSources()} />
</div>
<div className="explore-list-body">
{filteredItems.map((item) => {
const itemSource = sourceByID(sources, item.sourceId)
return (
<button
key={item.id}
className={item.id === selected.id ? 'explore-article-item active' : 'explore-article-item'}
onClick={() => {
setActiveItemID(item.id)
onSelectItem(item.title)
}}
>
<span className={`explore-source-logo ${itemSource.color}`}>{sourceInitial(itemSource.name)}</span>
<span className="explore-article-copy">
<Text type="secondary">{itemSource.name} · {itemTime(item)}</Text>
<Text className="explore-article-title">{item.title}</Text>
<Text className="explore-article-summary" type="secondary" ellipsis={{ showTooltip: true }}>
{item.summary || '暂无摘要'}
</Text>
</span>
</button>
)
})}
</div>
</Card>
</Col>
))}
</Row>
{selected ? (
<section className="explore-reader-layout">
<Card className="explore-article-list queue-section" bordered>
<div className="explore-list-header">
<Title heading={5}>
<Space size={6}>
<span className={`explore-source-icon mini ${activeSource.color}`}>{activeSource.icon}</span>
<span>{activeSource.name}</span>
<Card className="explore-article-detail queue-section" bordered>
<div className="explore-detail-toolbar">
<Title heading={5}>{selected.title}</Title>
<Space>
<Button
aria-label={selected.status === 'read' ? '标记未读' : '标记已读'}
type={selected.status === 'read' ? 'primary' : 'text'}
icon={<IconCheckCircle />}
onClick={() => void updateSelected({ status: selected.status === 'read' ? 'unread' : 'read', starred: selected.starred })}
/>
<Button
aria-label={selected.starred ? '取消收藏' : '收藏'}
type={selected.starred ? 'primary' : 'text'}
icon={<IconStar />}
onClick={() => void updateSelected({ status: selected.status, starred: !selected.starred })}
/>
<Button type="text" icon={<IconBook />} onClick={() => onSelectItem(selected.title)} />
{selected.url ? <Button type="text" icon={<IconLaunch />} href={selected.url} target="_blank" /> : null}
</Space>
</Title>
<Button type="text" icon={<IconRefresh />} />
</div>
<div className="explore-list-body">
{filteredArticles.map((article) => {
const articleSource = sourceByID(sources, article.sourceID)
return (
<button
key={`${article.project.id}-${article.id}`}
className={article.id === selected.id ? 'explore-article-item active' : 'explore-article-item'}
onClick={() => {
setActiveArticleID(article.id)
onSelectItem(article.title)
}}
>
<span className={`explore-source-logo ${articleSource.color}`}>
{sourceInitial(articleSource.name)}
</span>
<span className="explore-article-copy">
<Text type="secondary">
{articleSource.name} · {article.time}
</Text>
<Text className="explore-article-title">{article.title}</Text>
<Text className="explore-article-summary" type="secondary" ellipsis={{ showTooltip: true }}>
{article.summary || '暂无正文'}
</Text>
</span>
</button>
)
})}
</div>
</div>
<article className="explore-article-body">
<Space className="explore-article-meta" wrap>
<span className={`explore-source-logo small ${selectedSource.color}`}>{sourceInitial(selectedSource.name)}</span>
<Text type="secondary">{selectedSource.name}</Text>
<Text type="secondary">{itemTime(selected)}</Text>
</Space>
<Paragraph className="explore-article-content">{selected.content || selected.summary || '暂无正文'}</Paragraph>
</article>
</Card>
</section>
) : (
<Card className="queue-section" bordered>
<Empty description={sourceRecords.length === 0 ? '暂无数据源，请先添加数据源' : '暂无数据源文章'} />
</Card>
<Card className="explore-article-detail queue-section" bordered>
<div className="explore-detail-toolbar">
<Title heading={5}>{selected.title}</Title>
<Space>
<Button type="text" icon={<IconCheckCircle />} />
<Button type="text" icon={<IconStar />} />
<Button type="text" icon={<IconBook />} />
</Space>
</div>
<article className="explore-article-body">
<Space className="explore-article-meta" wrap>
<span className={`explore-source-logo small ${selectedSource.color}`}>
{sourceInitial(selectedSource.name)}
</span>
<Text type="secondary">{selectedSource.name}</Text>
<Text type="secondary">{selected.project.name}</Text>
<Text type="secondary">{selected.time}</Text>
</Space>
<Paragraph className="explore-article-content">{selected.summary || '暂无正文'}</Paragraph>
<blockquote>
推荐：将外部文章中的项目机会、风险提示和资料线索归档到对应项目，形成可追踪的计划和知识资产。
</blockquote>
</article>
</Card>
</section>
) : (
<Card className="queue-section" bordered>
<Empty description="暂无数据源文章" />
</Card>
)}
)}
</Spin>
<Modal
className="explore-source-modal"
@@ -280,18 +366,22 @@ export function WorkspaceExplorePage({
visible={sourceModalMode !== null}
okText="保存"
cancelText="取消"
confirmLoading={saving}
onCancel={() => setSourceModalMode(null)}
onOk={saveSource}
onOk={() => void saveSource()}
>
<Space direction="vertical" size={12} className="action-form">
{sourceError && <Alert type="error" content={sourceError} />}
{sourceError ? <Alert type="error" content={sourceError} /> : null}
<label>
<Text>数据源名称</Text>
<Input value={sourceDraft.name} placeholder="例如：行业资讯" onChange={(name) => setSourceDraft((current) => ({ ...current, name }))} />
</label>
<label>
<Text>数据源类型</Text>
<Select value={sourceDraft.kind} onChange={(kind) => setSourceDraft((current) => ({ ...current, kind }))}>
<Select
value={sourceDraft.kind}
onChange={(kind) => setSourceDraft((current) => ({ ...current, kind: kind as DatasetSourceKind }))}
>
<Select.Option value="link">网页链接</Select.Option>
<Select.Option value="rss">RSS 订阅</Select.Option>
<Select.Option value="manual">手动收集</Select.Option>
@@ -305,41 +395,41 @@ export function WorkspaceExplorePage({
<Text>说明</Text>
<TextArea rows={4} value={sourceDraft.description} placeholder="数据源用途或内容范围" onChange={(description) => setSourceDraft((current) => ({ ...current, description }))} />
</label>
<label>
<Space>
<Switch checked={sourceDraft.enabled} onChange={(enabled) => setSourceDraft((current) => ({ ...current, enabled }))} />
<Text>启用采集</Text>
</Space>
</label>
</Space>
</Modal>
</div>
)
}
function dataSources(articles: ExploreArticle[], configs: DataSourceConfig[]): DataSourceCard[] {
const counts = new Map<DataSourceID, number>(configs.map((source) => [source.id, source.id === 'all' ? articles.length : 0]))
articles.forEach((article) => counts.set(article.sourceID, (counts.get(article.sourceID) ?? 0) + 1))
return configs.map((source) => ({ ...source, count: counts.get(source.id) ?? 0 }))
}
function sourceByID(sources: DataSourceCard[], id: DataSourceID): DataSourceCard {
function sourceByID(sources: DataSourceCard[], id: string): DataSourceCard {
return sources.find((source) => source.id === id) ?? sources[0]
}
function detectSource(article: InboxItem): BuiltinDataSourceID {
const text = `${article.title} ${article.meta} ${article.tag} ${article.summary}`
if (/架构|技术方案|系统设计|architecture/i.test(text)) return 'architecture'
if (/需求|PRD|产品文档|requirement/i.test(text)) return 'requirements'
return 'manual'
}
function sourceIcon(kind: DataSourceKind) {
function sourceIcon(kind: DatasetSourceKind) {
if (kind === 'rss') return <IconStorage />
if (kind === 'manual') return <IconCompass />
return <IconLink />
}
function sourceColor(kind: DataSourceKind) {
function sourceColor(kind: DatasetSourceKind) {
if (kind === 'rss') return 'orange'
if (kind === 'manual') return 'green'
return 'blue'
}
function sourceInitial(name: string) {
return name.trim().slice(0, 1) || '源'
return Array.from(name.trim())[0] || '源'
}
function itemTime(item: DatasetItemDTO) {
const value = item.publishedAt ?? item.createdAt
const parsed = new Date(value)
if (Number.isNaN(parsed.getTime())) return ''
return new Intl.DateTimeFormat('zh-CN', { dateStyle: 'medium' }).format(parsed)
}

32
apps/web_v1/src/pages/workspace-home.tsx
View File

@@ -10,6 +10,13 @@ import { ProjectTopbar } from './projects/project-topbar'
import type { SearchResultDTO } from '../api/search'
import type { InboxSuggestionDTO } from '../api/inbox'
import type { AIExpertDTO, AISessionDTO, CreateAISessionInput } from '../api/ai'
import type {
DatasetCronDTO,
DatasetItemDTO,
DatasetItemUpdate,
DatasetSourceDTO,
DatasetSourceInput,
} from '../api/dataset'
import type { ChannelKey, InboxConfirmationOutcome, Project, ProjectWorkspace, Theme, WorkbenchView } from './projects/project-types'
const { Content } = Layout
@@ -48,6 +55,13 @@ export function ProjectPage({
onListAISessions,
onListAIExperts,
onCreateAISession,
onListDatasetSources,
onListDatasetItems,
onCreateDatasetSource,
onUpdateDatasetSource,
onDeleteDatasetSource,
onUpdateDatasetItem,
onQueueDatasetSync,
}: {
activeView: WorkbenchView
activeWorkspace: ProjectWorkspace
@@ -82,6 +96,13 @@ export function ProjectPage({
onListAISessions: (projectId: string, signal?: AbortSignal) => Promise<AISessionDTO[]>
onListAIExperts: (signal?: AbortSignal) => Promise<AIExpertDTO[]>
onCreateAISession: (projectId: string, input: CreateAISessionInput, signal?: AbortSignal) => Promise<AISessionDTO>
onListDatasetSources: (signal?: AbortSignal) => Promise<DatasetSourceDTO[]>
onListDatasetItems: (signal?: AbortSignal) => Promise<DatasetItemDTO[]>
onCreateDatasetSource: (input: DatasetSourceInput) => Promise<DatasetSourceDTO>
onUpdateDatasetSource: (sourceId: string, input: DatasetSourceInput) => Promise<DatasetSourceDTO>
onDeleteDatasetSource: (sourceId: string) => Promise<void>
onUpdateDatasetItem: (itemId: string, input: DatasetItemUpdate) => Promise<DatasetItemDTO>
onQueueDatasetSync: () => Promise<DatasetCronDTO[]>
}) {
const isProject = activeView === 'project'
const projects = workspaces.map((workspace) => workspace.project)
@@ -137,7 +158,16 @@ export function ProjectPage({
{activeView === 'workspace' ? (
<WorkspacePage workspaces={workspaces} onOpenTask={onOpenTask} onUpdateTask={onUpdateWorkspaceTask} />
) : activeView === 'workspace-explore' ? (
<WorkspaceExplorePage workspaces={workspaces} onSelectItem={onSelectItem} />
<WorkspaceExplorePage
onSelectItem={onSelectItem}
onListSources={onListDatasetSources}
onListItems={onListDatasetItems}
onCreateSource={onCreateDatasetSource}
onUpdateSource={onUpdateDatasetSource}
onDeleteSource={onDeleteDatasetSource}
onUpdateItem={onUpdateDatasetItem}
onQueueSync={onQueueDatasetSync}
/>
) : (
<ProjectChannelPage
activeChannel={activeChannel}

3
backend/cmd/api/main.go
View File

@@ -8,6 +8,7 @@ import (
"senlinai-agent/backend/internal/initdb"
"senlinai-agent/backend/internal/logic/ai"
"senlinai-agent/backend/internal/logic/auth"
"senlinai-agent/backend/internal/logic/dataset"
"senlinai-agent/backend/internal/logic/files"
"senlinai-agent/backend/internal/logic/inbox"
"senlinai-agent/backend/internal/logic/projects"
@@ -28,6 +29,7 @@ func main() {
}
authService := auth.NewService(cfg.AuthSecret)
datasetHandler := dataset.NewHandler(dataset.NewService(models.DBService))
projectService := projects.NewService()
fileService := files.NewService(cfg.StorageDir, models.DBService)
taskService := tasks.NewService(models.DBService)
@@ -45,6 +47,7 @@ func main() {
cfg,
authService.VerifySession,
authHandler,
datasetHandler,
projectHandler,
tagHandler,
cronHandler,

353
backend/internal/logic/dataset/handlers.go Normal file
View File

@@ -0,0 +1,353 @@
package dataset
import (
"errors"
"net/http"
"strings"
"time"
"github.com/gin-gonic/gin"
"gorm.io/gorm"
"senlinai-agent/backend/internal/httpx"
"senlinai-agent/backend/internal/logic/auth"
"senlinai-agent/backend/internal/models"
)
type Handler struct {
service *Service
}
type SourceDTO struct {
ID string `json:"id"`
Name string `json:"name"`
Kind string `json:"kind"`
URL string `json:"url"`
Description string `json:"description"`
Enabled bool `json:"enabled"`
LastSyncedAt *time.Time `json:"lastSyncedAt"`
ItemCount int64 `json:"itemCount"`
CreatedAt time.Time `json:"createdAt"`
UpdatedAt time.Time `json:"updatedAt"`
}
type ItemDTO struct {
ID string `json:"id"`
SourceID string `json:"sourceId"`
Title string `json:"title"`
Summary string `json:"summary"`
Content string `json:"content"`
URL string `json:"url"`
Status string `json:"status"`
Starred bool `json:"starred"`
PublishedAt *time.Time `json:"publishedAt"`
CreatedAt time.Time `json:"createdAt"`
UpdatedAt time.Time `json:"updatedAt"`
}
type CronDTO struct {
ID string `json:"id"`
SourceID string `json:"sourceId"`
Schedule string `json:"schedule"`
Status string `json:"status"`
Enabled bool `json:"enabled"`
NextRunAt *time.Time `json:"nextRunAt"`
LastRunAt *time.Time `json:"lastRunAt"`
LastResult string `json:"lastResult"`
CreatedAt time.Time `json:"createdAt"`
UpdatedAt time.Time `json:"updatedAt"`
}
type sourceRequest struct {
Name string `json:"name"`
Kind string `json:"kind"`
URL string `json:"url"`
Description string `json:"description"`
Enabled *bool `json:"enabled"`
}
func NewHandler(service *Service) *Handler {
return &Handler{service: service}
}
func (h *Handler) Register(router gin.IRouter) {
router.GET("/dataset-sources", h.listSources)
router.POST("/dataset-sources", h.createSource)
router.PATCH("/dataset-sources/:id", h.updateSource)
router.DELETE("/dataset-sources/:id", h.deleteSource)
router.GET("/dataset-items", h.listItems)
router.POST("/dataset-items", h.createItem)
router.PATCH("/dataset-items/:id", h.updateItem)
router.GET("/dataset-crons", h.listCrons)
router.POST("/dataset-crons/sync", h.queueSync)
}
func (h *Handler) listSources(c *gin.Context) {
userID, ok := currentUser(c)
if !ok {
return
}
records, err := h.service.ListSources(userID)
if err != nil {
writeError(c, err)
return
}
items := make([]SourceDTO, 0, len(records))
for _, record := range records {
items = append(items, sourceDTO(record.Source, record.ItemCount))
}
c.JSON(http.StatusOK, items)
}
func (h *Handler) createSource(c *gin.Context) {
userID, ok := currentUser(c)
if !ok {
return
}
var input sourceRequest
if err := c.ShouldBindJSON(&input); err != nil {
writeInvalidRequest(c)
return
}
enabled := true
if input.Enabled != nil {
enabled = *input.Enabled
}
source, err := h.service.CreateSource(userID, SourceInput{
Name: input.Name, Kind: input.Kind, URL: input.URL, Description: input.Description, Enabled: enabled,
})
if err != nil {
writeError(c, err)
return
}
c.JSON(http.StatusCreated, sourceDTO(*source, 0))
}
func (h *Handler) updateSource(c *gin.Context) {
userID, ok := currentUser(c)
if !ok {
return
}
identity, ok := httpx.IdentityParam(c, "id")
if !ok {
return
}
var input sourceRequest
if err := c.ShouldBindJSON(&input); err != nil {
writeInvalidRequest(c)
return
}
enabled := true
if input.Enabled != nil {
enabled = *input.Enabled
}
source, err := h.service.UpdateSource(userID, identity, SourceInput{
Name: input.Name, Kind: input.Kind, URL: input.URL, Description: input.Description, Enabled: enabled,
})
if err != nil {
writeError(c, err)
return
}
itemCount, err := h.service.CountItems(userID, source.ID)
if err != nil {
writeError(c, err)
return
}
c.JSON(http.StatusOK, sourceDTO(*source, itemCount))
}
func (h *Handler) deleteSource(c *gin.Context) {
userID, ok := currentUser(c)
if !ok {
return
}
identity, ok := httpx.IdentityParam(c, "id")
if !ok {
return
}
if err := h.service.DeleteSource(userID, identity); err != nil {
writeError(c, err)
return
}
c.Status(http.StatusNoContent)
}
func (h *Handler) listItems(c *gin.Context) {
userID, ok := currentUser(c)
if !ok {
return
}
items, err := h.service.ListItems(userID, c.Query("sourceId"))
if err != nil {
writeError(c, err)
return
}
result := make([]ItemDTO, 0, len(items))
for _, item := range items {
result = append(result, itemDTO(item))
}
c.JSON(http.StatusOK, result)
}
func (h *Handler) createItem(c *gin.Context) {
userID, ok := currentUser(c)
if !ok {
return
}
var input struct {
SourceID string `json:"sourceId"`
Title string `json:"title"`
Summary string `json:"summary"`
Content string `json:"content"`
URL string `json:"url"`
PublishedAt string `json:"publishedAt"`
}
if err := c.ShouldBindJSON(&input); err != nil {
writeInvalidRequest(c)
return
}
publishedAt, err := parseOptionalTime(input.PublishedAt)
if err != nil {
writeInvalidRequest(c)
return
}
item, err := h.service.CreateItem(userID, ItemInput{
SourceIdentity: input.SourceID, Title: input.Title, Summary: input.Summary,
Content: input.Content, URL: input.URL, PublishedAt: publishedAt,
})
if err != nil {
writeError(c, err)
return
}
c.JSON(http.StatusCreated, itemDTO(*item))
}
func (h *Handler) updateItem(c *gin.Context) {
userID, ok := currentUser(c)
if !ok {
return
}
identity, ok := httpx.IdentityParam(c, "id")
if !ok {
return
}
var input struct {
Status string `json:"status"`
Starred bool `json:"starred"`
}
if err := c.ShouldBindJSON(&input); err != nil {
writeInvalidRequest(c)
return
}
item, err := h.service.UpdateItem(userID, identity, ItemUpdate{Status: input.Status, Starred: input.Starred})
if err != nil {
writeError(c, err)
return
}
c.JSON(http.StatusOK, itemDTO(*item))
}
func (h *Handler) listCrons(c *gin.Context) {
userID, ok := currentUser(c)
if !ok {
return
}
crons, err := h.service.ListCrons(userID)
if err != nil {
writeError(c, err)
return
}
result := make([]CronDTO, 0, len(crons))
for _, cron := range crons {
result = append(result, cronDTO(cron))
}
c.JSON(http.StatusOK, result)
}
func (h *Handler) queueSync(c *gin.Context) {
userID, ok := currentUser(c)
if !ok {
return
}
crons, err := h.service.QueueSync(userID)
if err != nil {
writeError(c, err)
return
}
result := make([]CronDTO, 0, len(crons))
for _, cron := range crons {
result = append(result, cronDTO(cron))
}
c.JSON(http.StatusAccepted, result)
}
func currentUser(c *gin.Context) (uint, bool) {
userID, ok := auth.CurrentUserID(c)
if !ok {
httpx.Error(c, http.StatusUnauthorized, "unauthorized", "未登录或登录已失效")
return 0, false
}
return userID, true
}
func writeError(c *gin.Context, err error) {
switch {
case errors.Is(err, gorm.ErrRecordNotFound):
httpx.Error(c, http.StatusNotFound, "not_found", "数据源或数据条目不存在")
case errors.Is(err, ErrNameRequired), errors.Is(err, ErrKindInvalid),
errors.Is(err, ErrURLInvalid), errors.Is(err, ErrTitleRequired), errors.Is(err, ErrStatusInvalid):
writeInvalidRequest(c)
default:
httpx.Error(c, http.StatusInternalServerError, "internal_error", "探索数据操作失败")
}
}
func writeInvalidRequest(c *gin.Context) {
httpx.Error(c, http.StatusBadRequest, "invalid_request", "请求参数无效")
}
func sourceDTO(source models.SaDatasetSource, itemCount int64) SourceDTO {
return SourceDTO{
ID: source.Identity, Name: source.Name, Kind: source.Kind, URL: source.URL,
Description: source.Description, Enabled: source.Enabled,
LastSyncedAt: utcTime(source.LastSyncedAt), ItemCount: itemCount,
CreatedAt: source.CreatedAt.UTC(), UpdatedAt: source.UpdatedAt.UTC(),
}
}
func itemDTO(item models.SaDatasetItem) ItemDTO {
return ItemDTO{
ID: item.Identity, SourceID: item.SourceIdentity, Title: item.Title,
Summary: item.Summary, Content: item.Content, URL: item.URL, Status: item.Status,
Starred: item.Starred, PublishedAt: utcTime(item.PublishedAt),
CreatedAt: item.CreatedAt.UTC(), UpdatedAt: item.UpdatedAt.UTC(),
}
}
func cronDTO(cron models.SaDatasetCron) CronDTO {
return CronDTO{
ID: cron.Identity, SourceID: cron.SourceIdentity, Schedule: cron.Schedule,
Status: cron.Status, Enabled: cron.Enabled, NextRunAt: utcTime(cron.NextRunAt),
LastRunAt: utcTime(cron.LastRunAt), LastResult: cron.LastResult,
CreatedAt: cron.CreatedAt.UTC(), UpdatedAt: cron.UpdatedAt.UTC(),
}
}
func parseOptionalTime(value string) (*time.Time, error) {
if strings.TrimSpace(value) == "" {
return nil, nil
}
parsed, err := time.Parse(time.RFC3339, value)
if err != nil {
return nil, err
}
parsed = parsed.UTC()
return &parsed, nil
}
func utcTime(value *time.Time) *time.Time {
if value == nil {
return nil
}
result := value.UTC()
return &result
}

139
backend/internal/logic/dataset/handlers_test.go Normal file
View File

@@ -0,0 +1,139 @@
package dataset
import (
"bytes"
"encoding/json"
"fmt"
"net/http"
"net/http/httptest"
"testing"
"github.com/glebarez/sqlite"
"github.com/stretchr/testify/require"
"gorm.io/gorm"
"senlinai-agent/backend/internal/config"
"senlinai-agent/backend/internal/httpx"
"senlinai-agent/backend/internal/models"
)
func TestDatasetAPIConnectsSourcesItemsAndSyncTasks(t *testing.T) {
database := newDatasetTestDatabase(t)
owner := createDatasetTestUser(t, database, "owner@example.com")
other := createDatasetTestUser(t, database, "other@example.com")
ownerRouter := datasetTestRouter(database, owner.ID)
invalid := performDatasetRequest(t, ownerRouter, http.MethodPost, "/api/v1/dataset-sources", map[string]any{
"name": "Invalid RSS", "kind": "rss", "url": "",
})
require.Equal(t, http.StatusBadRequest, invalid.Code)
createdSource := performDatasetRequest(t, ownerRouter, http.MethodPost, "/api/v1/dataset-sources", map[string]any{
"name": "Industry feed", "kind": "rss", "url": "https://example.com/feed",
"description": "Industry updates", "enabled": true,
})
require.Equal(t, http.StatusCreated, createdSource.Code)
var source SourceDTO
require.NoError(t, json.Unmarshal(createdSource.Body.Bytes(), &source))
require.NotEmpty(t, source.ID)
require.Equal(t, "Industry feed", source.Name)
createdItem := performDatasetRequest(t, ownerRouter, http.MethodPost, "/api/v1/dataset-items", map[string]any{
"sourceId": source.ID, "title": "A collected article", "summary": "Summary",
"url": "https://example.com/article",
})
require.Equal(t, http.StatusCreated, createdItem.Code)
var item ItemDTO
require.NoError(t, json.Unmarshal(createdItem.Body.Bytes(), &item))
require.Equal(t, source.ID, item.SourceID)
listSources := performDatasetRequest(t, ownerRouter, http.MethodGet, "/api/v1/dataset-sources", nil)
require.Equal(t, http.StatusOK, listSources.Code)
var sources []SourceDTO
require.NoError(t, json.Unmarshal(listSources.Body.Bytes(), &sources))
require.Len(t, sources, 1)
require.Equal(t, int64(1), sources[0].ItemCount)
updatedItem := performDatasetRequest(t, ownerRouter, http.MethodPatch, "/api/v1/dataset-items/"+item.ID, map[string]any{
"status": "read", "starred": true,
})
require.Equal(t, http.StatusOK, updatedItem.Code)
require.NoError(t, json.Unmarshal(updatedItem.Body.Bytes(), &item))
require.Equal(t, "read", item.Status)
require.True(t, item.Starred)
firstSync := performDatasetRequest(t, ownerRouter, http.MethodPost, "/api/v1/dataset-crons/sync", map[string]any{})
require.Equal(t, http.StatusAccepted, firstSync.Code)
var firstCrons []CronDTO
require.NoError(t, json.Unmarshal(firstSync.Body.Bytes(), &firstCrons))
require.Len(t, firstCrons, 1)
require.Equal(t, "pending", firstCrons[0].Status)
secondSync := performDatasetRequest(t, ownerRouter, http.MethodPost, "/api/v1/dataset-crons/sync", map[string]any{})
require.Equal(t, http.StatusAccepted, secondSync.Code)
var secondCrons []CronDTO
require.NoError(t, json.Unmarshal(secondSync.Body.Bytes(), &secondCrons))
require.Equal(t, firstCrons[0].ID, secondCrons[0].ID)
otherRouter := datasetTestRouter(database, other.ID)
otherSources := performDatasetRequest(t, otherRouter, http.MethodGet, "/api/v1/dataset-sources", nil)
require.Equal(t, http.StatusOK, otherSources.Code)
require.JSONEq(t, `[]`, otherSources.Body.String())
otherUpdate := performDatasetRequest(t, otherRouter, http.MethodPatch, "/api/v1/dataset-sources/"+source.ID, map[string]any{
"name": "Stolen", "kind": "manual", "enabled": true,
})
require.Equal(t, http.StatusNotFound, otherUpdate.Code)
deleted := performDatasetRequest(t, ownerRouter, http.MethodDelete, "/api/v1/dataset-sources/"+source.ID, nil)
require.Equal(t, http.StatusNoContent, deleted.Code)
var itemCount, cronCount int64
require.NoError(t, database.Model(&models.SaDatasetItem{}).Count(&itemCount).Error)
require.NoError(t, database.Model(&models.SaDatasetCron{}).Count(&cronCount).Error)
require.Zero(t, itemCount)
require.Zero(t, cronCount)
}
func newDatasetTestDatabase(t *testing.T) *gorm.DB {
t.Helper()
database, err := gorm.Open(sqlite.Open(fmt.Sprintf("file:%s?mode=memory&cache=shared", t.Name())), &gorm.Config{})
require.NoError(t, err)
require.NoError(t, database.AutoMigrate(
&models.SaUser{},
&models.SaDatasetSource{},
&models.SaDatasetItem{},
&models.SaDatasetCron{},
))
return database
}
func createDatasetTestUser(t *testing.T, database *gorm.DB, email string) models.SaUser {
t.Helper()
user := models.SaUser{Email: email, DisplayName: email, PasswordHash: "hash", Role: "user"}
require.NoError(t, database.Create(&user).Error)
return user
}
func datasetTestRouter(database *gorm.DB, userID uint) http.Handler {
return httpx.NewProtectedRouter(
config.Config{Env: "test"},
func(string) (uint, error) { return userID, nil },
NewHandler(NewService(database)),
)
}
func performDatasetRequest(t *testing.T, router http.Handler, method, path string, body any) *httptest.ResponseRecorder {
t.Helper()
var encoded []byte
if body != nil {
var err error
encoded, err = json.Marshal(body)
require.NoError(t, err)
}
request := httptest.NewRequest(method, path, bytes.NewReader(encoded))
request.Header.Set("Authorization", "Bearer test-token")
if body != nil {
request.Header.Set("Content-Type", "application/json")
}
recorder := httptest.NewRecorder()
router.ServeHTTP(recorder, request)
return recorder
}

278
backend/internal/logic/dataset/service.go Normal file
View File

@@ -0,0 +1,278 @@
package dataset
import (
"errors"
"net/url"
"strings"
"time"
"gorm.io/gorm"
"senlinai-agent/backend/internal/models"
)
var (
ErrNameRequired = errors.New("dataset source name is required")
ErrKindInvalid = errors.New("dataset source kind is invalid")
ErrURLInvalid = errors.New("dataset source url is invalid")
ErrTitleRequired = errors.New("dataset item title is required")
ErrStatusInvalid = errors.New("dataset item status is invalid")
)
type Service struct {
db *gorm.DB
}
type SourceInput struct {
Name string
Kind string
URL string
Description string
Enabled bool
}
type ItemInput struct {
SourceIdentity string
Title string
Summary string
Content string
URL string
PublishedAt *time.Time
}
type ItemUpdate struct {
Status string
Starred bool
}
type SourceRecord struct {
Source models.SaDatasetSource
ItemCount int64
}
func NewService(database *gorm.DB) *Service {
return &Service{db: database}
}
func (s *Service) ListSources(userID uint) ([]SourceRecord, error) {
var sources []models.SaDatasetSource
if err := s.db.Where("created_by = ?", userID).Order("created_at asc, id asc").Find(&sources).Error; err != nil {
return nil, err
}
counts := make(map[uint]int64)
if len(sources) > 0 {
sourceIDs := make([]uint, 0, len(sources))
for _, source := range sources {
sourceIDs = append(sourceIDs, source.ID)
}
var rows []struct {
SourceID uint
Count int64
}
if err := s.db.Model(&models.SaDatasetItem{}).
Select("source_id, COUNT(*) AS count").
Where("created_by = ? AND source_id IN ?", userID, sourceIDs).
Group("source_id").
Scan(&rows).Error; err != nil {
return nil, err
}
for _, row := range rows {
counts[row.SourceID] = row.Count
}
}
result := make([]SourceRecord, 0, len(sources))
for _, source := range sources {
result = append(result, SourceRecord{Source: source, ItemCount: counts[source.ID]})
}
return result, nil
}
func (s *Service) CreateSource(userID uint, input SourceInput) (*models.SaDatasetSource, error) {
normalized, err := normalizeSourceInput(input)
if err != nil {
return nil, err
}
source := &models.SaDatasetSource{
CreatedBy: userID, Name: normalized.Name, Kind: normalized.Kind,
URL: normalized.URL, Description: normalized.Description, Enabled: normalized.Enabled,
}
return source, s.db.Create(source).Error
}
func (s *Service) UpdateSource(userID uint, identity string, input SourceInput) (*models.SaDatasetSource, error) {
normalized, err := normalizeSourceInput(input)
if err != nil {
return nil, err
}
var source models.SaDatasetSource
if err := s.db.Where("identity = ? AND created_by = ?", identity, userID).First(&source).Error; err != nil {
return nil, err
}
if err := s.db.Model(&source).Updates(map[string]any{
"name": normalized.Name, "kind": normalized.Kind, "url": normalized.URL,
"description": normalized.Description, "enabled": normalized.Enabled,
}).Error; err != nil {
return nil, err
}
return &source, nil
}
func (s *Service) DeleteSource(userID uint, identity string) error {
return s.db.Transaction(func(tx *gorm.DB) error {
var source models.SaDatasetSource
if err := tx.Where("identity = ? AND created_by = ?", identity, userID).First(&source).Error; err != nil {
return err
}
if err := tx.Where("source_id = ? AND created_by = ?", source.ID, userID).Delete(&models.SaDatasetCron{}).Error; err != nil {
return err
}
if err := tx.Where("source_id = ? AND created_by = ?", source.ID, userID).Delete(&models.SaDatasetItem{}).Error; err != nil {
return err
}
return tx.Delete(&source).Error
})
}
func (s *Service) ListItems(userID uint, sourceIdentity string) ([]models.SaDatasetItem, error) {
query := s.db.Where("created_by = ?", userID)
if sourceIdentity = strings.TrimSpace(sourceIdentity); sourceIdentity != "" {
source, err := s.findOwnedSource(userID, sourceIdentity)
if err != nil {
return nil, err
}
query = query.Where("source_id = ?", source.ID)
}
var items []models.SaDatasetItem
if err := query.Order("COALESCE(published_at, created_at) desc, id desc").Limit(200).Find(&items).Error; err != nil {
return nil, err
}
if items == nil {
items = []models.SaDatasetItem{}
}
return items, nil
}
func (s *Service) CountItems(userID uint, sourceID uint) (int64, error) {
var count int64
err := s.db.Model(&models.SaDatasetItem{}).
Where("created_by = ? AND source_id = ?", userID, sourceID).
Count(&count).Error
return count, err
}
func (s *Service) CreateItem(userID uint, input ItemInput) (*models.SaDatasetItem, error) {
title := strings.TrimSpace(input.Title)
if title == "" {
return nil, ErrTitleRequired
}
source, err := s.findOwnedSource(userID, input.SourceIdentity)
if err != nil {
return nil, err
}
itemURL := strings.TrimSpace(input.URL)
if itemURL != "" && !validHTTPURL(itemURL) {
return nil, ErrURLInvalid
}
item := &models.SaDatasetItem{
SourceID: source.ID, CreatedBy: userID, Title: title,
Summary: strings.TrimSpace(input.Summary), Content: strings.TrimSpace(input.Content),
URL: itemURL, Status: "unread", PublishedAt: utcOptionalTime(input.PublishedAt),
}
return item, s.db.Create(item).Error
}
func (s *Service) UpdateItem(userID uint, identity string, input ItemUpdate) (*models.SaDatasetItem, error) {
status := strings.TrimSpace(input.Status)
if status != "unread" && status != "read" && status != "archived" {
return nil, ErrStatusInvalid
}
var item models.SaDatasetItem
if err := s.db.Where("identity = ? AND created_by = ?", identity, userID).First(&item).Error; err != nil {
return nil, err
}
if err := s.db.Model(&item).Updates(map[string]any{"status": status, "starred": input.Starred}).Error; err != nil {
return nil, err
}
return &item, nil
}
func (s *Service) ListCrons(userID uint) ([]models.SaDatasetCron, error) {
var crons []models.SaDatasetCron
if err := s.db.Where("created_by = ?", userID).Order("created_at desc, id desc").Limit(100).Find(&crons).Error; err != nil {
return nil, err
}
if crons == nil {
crons = []models.SaDatasetCron{}
}
return crons, nil
}
func (s *Service) QueueSync(userID uint) ([]models.SaDatasetCron, error) {
result := make([]models.SaDatasetCron, 0)
err := s.db.Transaction(func(tx *gorm.DB) error {
var sources []models.SaDatasetSource
if err := tx.Where("created_by = ? AND enabled = ?", userID, true).Order("id asc").Find(&sources).Error; err != nil {
return err
}
now := time.Now().UTC()
for _, source := range sources {
var cron models.SaDatasetCron
err := tx.Where("source_id = ? AND created_by = ? AND status = ?", source.ID, userID, "pending").First(&cron).Error
if err == nil {
result = append(result, cron)
continue
}
if !errors.Is(err, gorm.ErrRecordNotFound) {
return err
}
cron = models.SaDatasetCron{
SourceID: source.ID, CreatedBy: userID, Schedule: "@once",
Status: "pending", Enabled: true, NextRunAt: &now,
}
if err := tx.Create(&cron).Error; err != nil {
return err
}
result = append(result, cron)
}
return nil
})
return result, err
}
func (s *Service) findOwnedSource(userID uint, identity string) (*models.SaDatasetSource, error) {
var source models.SaDatasetSource
err := s.db.Where("identity = ? AND created_by = ?", strings.TrimSpace(identity), userID).First(&source).Error
return &source, err
}
func normalizeSourceInput(input SourceInput) (SourceInput, error) {
input.Name = strings.TrimSpace(input.Name)
input.Kind = strings.ToLower(strings.TrimSpace(input.Kind))
input.URL = strings.TrimSpace(input.URL)
input.Description = strings.TrimSpace(input.Description)
if input.Name == "" {
return input, ErrNameRequired
}
if input.Kind != "manual" && input.Kind != "link" && input.Kind != "rss" {
return input, ErrKindInvalid
}
if input.URL != "" && !validHTTPURL(input.URL) {
return input, ErrURLInvalid
}
if input.Kind != "manual" && input.URL == "" {
return input, ErrURLInvalid
}
return input, nil
}
func validHTTPURL(value string) bool {
parsed, err := url.ParseRequestURI(value)
return err == nil && (parsed.Scheme == "http" || parsed.Scheme == "https") && parsed.Host != ""
}
func utcOptionalTime(value *time.Time) *time.Time {
if value == nil {
return nil
}
result := value.UTC()
return &result
}

24
backend/internal/models/dataset_cron.go Normal file
View File

@@ -0,0 +1,24 @@
package models
import "time"
type SaDatasetCron struct {
ID uint `gorm:"primaryKey"`
Identity string `gorm:"type:char(36);uniqueIndex"`
SourceID uint `gorm:"index;not null"`
SourceIdentity string `gorm:"type:char(36);index"`
CreatedBy uint `gorm:"index;not null"`
CreatedByIdentity string `gorm:"type:char(36);index"`
Schedule string `gorm:"size:100;not null"`
Status string `gorm:"size:32;not null;default:pending;index"`
Enabled bool `gorm:"not null;default:true;index"`
NextRunAt *time.Time
LastRunAt *time.Time
LastResult string `gorm:"type:text"`
CreatedAt time.Time
UpdatedAt time.Time
}
func (SaDatasetCron) TableName() string {
return "sa_dataset_crons"
}

25
backend/internal/models/dataset_item.go Normal file
View File

@@ -0,0 +1,25 @@
package models
import "time"
type SaDatasetItem struct {
ID uint `gorm:"primaryKey"`
Identity string `gorm:"type:char(36);uniqueIndex"`
SourceID uint `gorm:"index;not null"`
SourceIdentity string `gorm:"type:char(36);index"`
CreatedBy uint `gorm:"index;not null"`
CreatedByIdentity string `gorm:"type:char(36);index"`
Title string `gorm:"size:500;not null"`
Summary string `gorm:"type:text"`
Content string `gorm:"type:text"`
URL string `gorm:"size:2048"`
Status string `gorm:"size:32;not null;default:unread;index"`
Starred bool `gorm:"not null;default:false;index"`
PublishedAt *time.Time
CreatedAt time.Time
UpdatedAt time.Time
}
func (SaDatasetItem) TableName() string {
return "sa_dataset_items"
}

22
backend/internal/models/dataset_source.go Normal file
View File

@@ -0,0 +1,22 @@
package models
import "time"
type SaDatasetSource struct {
ID uint `gorm:"primaryKey"`
Identity string `gorm:"type:char(36);uniqueIndex"`
CreatedBy uint `gorm:"index;not null"`
CreatedByIdentity string `gorm:"type:char(36);index"`
Name string `gorm:"size:160;not null"`
Kind string `gorm:"size:32;not null"`
URL string `gorm:"size:2048"`
Description string `gorm:"type:text"`
Enabled bool `gorm:"not null;default:true;index"`
LastSyncedAt *time.Time
CreatedAt time.Time
UpdatedAt time.Time
}
func (SaDatasetSource) TableName() string {
return "sa_dataset_sources"
}

27
backend/internal/models/identity.go
View File

@@ -18,6 +18,33 @@ func (m *SaProject) BeforeCreate(tx *gorm.DB) error {
return resolveIdentity(tx, &SaUser{}, m.OwnerID, &m.OwnerIdentity)
}
func (m *SaDatasetSource) BeforeCreate(tx *gorm.DB) error {
if err := ensureIdentity(&m.Identity); err != nil {
return err
}
return resolveIdentity(tx, &SaUser{}, m.CreatedBy, &m.CreatedByIdentity)
}
func (m *SaDatasetItem) BeforeCreate(tx *gorm.DB) error {
if err := ensureIdentity(&m.Identity); err != nil {
return err
}
if err := resolveIdentity(tx, &SaDatasetSource{}, m.SourceID, &m.SourceIdentity); err != nil {
return err
}
return resolveIdentity(tx, &SaUser{}, m.CreatedBy, &m.CreatedByIdentity)
}
func (m *SaDatasetCron) BeforeCreate(tx *gorm.DB) error {
if err := ensureIdentity(&m.Identity); err != nil {
return err
}
if err := resolveIdentity(tx, &SaDatasetSource{}, m.SourceID, &m.SourceIdentity); err != nil {
return err
}
return resolveIdentity(tx, &SaUser{}, m.CreatedBy, &m.CreatedByIdentity)
}
func (m *SaInboxItem) BeforeCreate(tx *gorm.DB) error {
if err := ensureIdentity(&m.Identity); err != nil {
return err

3
backend/internal/models/new.go
View File

@@ -31,6 +31,9 @@ func AutoMigrate(database *gorm.DB) error {
&SaSchemaMigration{},
&SaUser{},
&SaProject{},
&SaDatasetSource{},
&SaDatasetItem{},
&SaDatasetCron{},
&SaInboxItem{},
&SaInboxSuggestion{},
&SaTask{},