fix: complete exploration data flow
This commit is contained in:
@@ -35,17 +35,17 @@ const visualExperts = [
|
|||||||
let visualDatasetSources = [
|
let visualDatasetSources = [
|
||||||
{
|
{
|
||||||
id: '019b0000-0000-7000-8000-000000000030', name: '手动收集', kind: 'manual', url: '', iconUrl: '',
|
id: '019b0000-0000-7000-8000-000000000030', name: '手动收集', kind: 'manual', url: '', iconUrl: '',
|
||||||
description: '手动收集的文章与线索', enabled: true, lastSyncedAt: null, itemCount: 1,
|
description: '手动收集的文章与线索', enabled: true, builtIn: false, lastSyncedAt: null, itemCount: 1,
|
||||||
createdAt: '2026-07-22T06:00:00Z', updatedAt: '2026-07-22T06:00:00Z',
|
createdAt: '2026-07-22T06:00:00Z', updatedAt: '2026-07-22T06:00:00Z',
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
id: '019b0000-0000-7000-8000-000000000031', name: '需求文档', kind: 'link', url: 'https://example.com/requirements', iconUrl: '',
|
id: '019b0000-0000-7000-8000-000000000031', name: '需求文档', kind: 'link', url: 'https://example.com/requirements', iconUrl: '',
|
||||||
description: '产品需求与业务文档', enabled: true, lastSyncedAt: null, itemCount: 1,
|
description: '产品需求与业务文档', enabled: true, builtIn: false, lastSyncedAt: null, itemCount: 1,
|
||||||
createdAt: '2026-07-22T06:00:00Z', updatedAt: '2026-07-22T06:00:00Z',
|
createdAt: '2026-07-22T06:00:00Z', updatedAt: '2026-07-22T06:00:00Z',
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
id: '019b0000-0000-7000-8000-000000000032', name: '架构讨论', kind: 'link', url: 'https://example.com/architecture', iconUrl: '',
|
id: '019b0000-0000-7000-8000-000000000032', name: '架构讨论', kind: 'link', url: 'https://example.com/architecture', iconUrl: '',
|
||||||
description: '架构方案与系统设计讨论', enabled: true, lastSyncedAt: null, itemCount: 0,
|
description: '架构方案与系统设计讨论', enabled: true, builtIn: false, lastSyncedAt: null, itemCount: 0,
|
||||||
createdAt: '2026-07-22T06:00:00Z', updatedAt: '2026-07-22T06:00:00Z',
|
createdAt: '2026-07-22T06:00:00Z', updatedAt: '2026-07-22T06:00:00Z',
|
||||||
},
|
},
|
||||||
]
|
]
|
||||||
@@ -225,6 +225,7 @@ await page.route('http://localhost:9150/api/v1/**', async (route) => {
|
|||||||
const created = {
|
const created = {
|
||||||
id: `019b0000-0000-7000-8000-${String(50 + visualDatasetSources.length).padStart(12, '0')}`,
|
id: `019b0000-0000-7000-8000-${String(50 + visualDatasetSources.length).padStart(12, '0')}`,
|
||||||
...input,
|
...input,
|
||||||
|
builtIn: false,
|
||||||
lastSyncedAt: null,
|
lastSyncedAt: null,
|
||||||
itemCount: 0,
|
itemCount: 0,
|
||||||
createdAt: new Date().toISOString(),
|
createdAt: new Date().toISOString(),
|
||||||
@@ -251,7 +252,24 @@ await page.route('http://localhost:9150/api/v1/**', async (route) => {
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
if (url.pathname === '/api/v1/dataset-items' && route.request().method() === 'GET') {
|
if (url.pathname === '/api/v1/dataset-items' && route.request().method() === 'GET') {
|
||||||
await route.fulfill({ json: visualDatasetItems })
|
const sourceId = url.searchParams.get('sourceId')
|
||||||
|
const offset = Number(url.searchParams.get('offset') ?? 0)
|
||||||
|
const limit = Number(url.searchParams.get('limit') ?? 50)
|
||||||
|
const matchingItems = sourceId
|
||||||
|
? visualDatasetItems.filter((item) => item.sourceId === sourceId)
|
||||||
|
: visualDatasetItems
|
||||||
|
await route.fulfill({
|
||||||
|
json: {
|
||||||
|
items: matchingItems.slice(offset, offset + limit),
|
||||||
|
total: matchingItems.length,
|
||||||
|
offset,
|
||||||
|
limit,
|
||||||
|
},
|
||||||
|
})
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if (url.pathname.endsWith('/deposit') && route.request().method() === 'POST') {
|
||||||
|
await route.fulfill({ status: 201, json: { noteId: '019b0000-0000-7000-8000-000000000099', projectId } })
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
if (url.pathname.startsWith('/api/v1/dataset-items/') && route.request().method() === 'PATCH') {
|
if (url.pathname.startsWith('/api/v1/dataset-items/') && route.request().method() === 'PATCH') {
|
||||||
@@ -281,6 +299,10 @@ await page.route('http://localhost:9150/api/v1/**', async (route) => {
|
|||||||
})
|
})
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
if (url.pathname === '/api/v1/dataset-crons' && route.request().method() === 'GET') {
|
||||||
|
await route.fulfill({ json: [] })
|
||||||
|
return
|
||||||
|
}
|
||||||
await route.fulfill({ status: 404, json: { error: { code: 'not_found', message: '视觉检查未配置该接口' } } })
|
await route.fulfill({ status: 404, json: { error: { code: 'not_found', message: '视觉检查未配置该接口' } } })
|
||||||
})
|
})
|
||||||
|
|
||||||
|
|||||||
@@ -10,6 +10,7 @@ export type DatasetSourceDTO = {
|
|||||||
iconUrl: string
|
iconUrl: string
|
||||||
description: string
|
description: string
|
||||||
enabled: boolean
|
enabled: boolean
|
||||||
|
builtIn: boolean
|
||||||
lastSyncedAt: string | null
|
lastSyncedAt: string | null
|
||||||
itemCount: number
|
itemCount: number
|
||||||
createdAt: string
|
createdAt: string
|
||||||
@@ -33,16 +34,25 @@ export type DatasetItemDTO = {
|
|||||||
export type DatasetCronDTO = {
|
export type DatasetCronDTO = {
|
||||||
id: string
|
id: string
|
||||||
sourceId: string
|
sourceId: string
|
||||||
schedule: string
|
|
||||||
status: string
|
status: string
|
||||||
enabled: boolean
|
|
||||||
nextRunAt: string | null
|
|
||||||
lastRunAt: string | null
|
lastRunAt: string | null
|
||||||
lastResult: string
|
lastResult: string
|
||||||
createdAt: string
|
createdAt: string
|
||||||
updatedAt: string
|
updatedAt: string
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export type DatasetItemPageDTO = {
|
||||||
|
items: DatasetItemDTO[]
|
||||||
|
total: number
|
||||||
|
limit: number
|
||||||
|
offset: number
|
||||||
|
}
|
||||||
|
|
||||||
|
export type DatasetDepositDTO = {
|
||||||
|
noteId: string
|
||||||
|
projectId: string
|
||||||
|
}
|
||||||
|
|
||||||
export type DatasetSourceInput = {
|
export type DatasetSourceInput = {
|
||||||
name: string
|
name: string
|
||||||
kind: DatasetSourceKind
|
kind: DatasetSourceKind
|
||||||
@@ -53,8 +63,8 @@ export type DatasetSourceInput = {
|
|||||||
}
|
}
|
||||||
|
|
||||||
export type DatasetItemUpdate = {
|
export type DatasetItemUpdate = {
|
||||||
status: DatasetItemDTO['status']
|
status?: DatasetItemDTO['status']
|
||||||
starred: boolean
|
starred?: boolean
|
||||||
}
|
}
|
||||||
|
|
||||||
export function listDatasetSources(session: ApiSession, signal?: AbortSignal) {
|
export function listDatasetSources(session: ApiSession, signal?: AbortSignal) {
|
||||||
@@ -85,9 +95,16 @@ export function deleteDatasetSource(session: ApiSession, sourceId: string) {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
export function listDatasetItems(session: ApiSession, sourceId?: string, signal?: AbortSignal) {
|
export function listDatasetItems(
|
||||||
const query = sourceId ? `?sourceId=${encodeURIComponent(sourceId)}` : ''
|
session: ApiSession,
|
||||||
return apiRequest<DatasetItemDTO[]>(`/api/v1/dataset-items${query}`, { token: session.token, signal })
|
sourceId?: string,
|
||||||
|
offset = 0,
|
||||||
|
limit = 50,
|
||||||
|
signal?: AbortSignal,
|
||||||
|
) {
|
||||||
|
const params = new URLSearchParams({ offset: String(offset), limit: String(limit) })
|
||||||
|
if (sourceId) params.set('sourceId', sourceId)
|
||||||
|
return apiRequest<DatasetItemPageDTO>(`/api/v1/dataset-items?${params}`, { token: session.token, signal })
|
||||||
}
|
}
|
||||||
|
|
||||||
export function updateDatasetItem(session: ApiSession, itemId: string, input: DatasetItemUpdate) {
|
export function updateDatasetItem(session: ApiSession, itemId: string, input: DatasetItemUpdate) {
|
||||||
@@ -105,3 +122,15 @@ export function queueDatasetSync(session: ApiSession) {
|
|||||||
body: {},
|
body: {},
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export function listDatasetCrons(session: ApiSession, signal?: AbortSignal) {
|
||||||
|
return apiRequest<DatasetCronDTO[]>('/api/v1/dataset-crons', { token: session.token, signal })
|
||||||
|
}
|
||||||
|
|
||||||
|
export function depositDatasetItem(session: ApiSession, itemId: string, projectId: string) {
|
||||||
|
return apiRequest<DatasetDepositDTO>(`/api/v1/dataset-items/${itemId}/deposit`, {
|
||||||
|
method: 'POST',
|
||||||
|
token: session.token,
|
||||||
|
body: { projectId },
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|||||||
@@ -7,6 +7,8 @@ import { createAISession, listAIExperts, listAISessions, type CreateAISessionInp
|
|||||||
import {
|
import {
|
||||||
createDatasetSource,
|
createDatasetSource,
|
||||||
deleteDatasetSource,
|
deleteDatasetSource,
|
||||||
|
depositDatasetItem,
|
||||||
|
listDatasetCrons,
|
||||||
listDatasetItems,
|
listDatasetItems,
|
||||||
listDatasetSources,
|
listDatasetSources,
|
||||||
queueDatasetSync,
|
queueDatasetSync,
|
||||||
@@ -75,9 +77,9 @@ function App() {
|
|||||||
return listDatasetSources(session, signal)
|
return listDatasetSources(session, signal)
|
||||||
}, [session])
|
}, [session])
|
||||||
|
|
||||||
const handleListDatasetItems = useCallback((signal?: AbortSignal) => {
|
const handleListDatasetItems = useCallback((sourceId?: string, offset = 0, limit = 50, signal?: AbortSignal) => {
|
||||||
if (!session) return Promise.reject(new Error('未登录'))
|
if (!session) return Promise.reject(new Error('未登录'))
|
||||||
return listDatasetItems(session, undefined, signal)
|
return listDatasetItems(session, sourceId, offset, limit, signal)
|
||||||
}, [session])
|
}, [session])
|
||||||
|
|
||||||
const handleCreateDatasetSource = useCallback((input: DatasetSourceInput) => {
|
const handleCreateDatasetSource = useCallback((input: DatasetSourceInput) => {
|
||||||
@@ -105,6 +107,27 @@ function App() {
|
|||||||
return queueDatasetSync(session)
|
return queueDatasetSync(session)
|
||||||
}, [session])
|
}, [session])
|
||||||
|
|
||||||
|
const handleListDatasetCrons = useCallback((signal?: AbortSignal) => {
|
||||||
|
if (!session) return Promise.reject(new Error('未登录'))
|
||||||
|
return listDatasetCrons(session, signal)
|
||||||
|
}, [session])
|
||||||
|
|
||||||
|
const handleDepositDatasetItem = useCallback(async (itemId: string, projectId: string) => {
|
||||||
|
if (!session) throw new Error('未登录')
|
||||||
|
const result = await depositDatasetItem(session, itemId, projectId)
|
||||||
|
const workspaceIndex = workspaces.findIndex((workspace) => workspace.project.id === projectId)
|
||||||
|
if (workspaceIndex >= 0) {
|
||||||
|
try {
|
||||||
|
const payload = await fetchProjectWorkspace(session, projectId)
|
||||||
|
const refreshed = mapWorkspace(payload, workspaceIndex)
|
||||||
|
setWorkspaces((current) => current.map((workspace) => workspace.project.id === projectId ? refreshed : workspace))
|
||||||
|
} catch {
|
||||||
|
return { ...result, refreshFailed: true }
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return result
|
||||||
|
}, [session, workspaces])
|
||||||
|
|
||||||
const dark = theme === 'dark'
|
const dark = theme === 'dark'
|
||||||
const activeWorkspace = workspaces.find((workspace) => workspace.project.id === activeProjectID) ?? workspaces[0]
|
const activeWorkspace = workspaces.find((workspace) => workspace.project.id === activeProjectID) ?? workspaces[0]
|
||||||
const activeTagOptions = activeWorkspace?.tags.filter((tag) => tag !== 'all' && tag !== '全部') ?? []
|
const activeTagOptions = activeWorkspace?.tags.filter((tag) => tag !== 'all' && tag !== '全部') ?? []
|
||||||
@@ -418,6 +441,8 @@ function App() {
|
|||||||
onDeleteDatasetSource={handleDeleteDatasetSource}
|
onDeleteDatasetSource={handleDeleteDatasetSource}
|
||||||
onUpdateDatasetItem={handleUpdateDatasetItem}
|
onUpdateDatasetItem={handleUpdateDatasetItem}
|
||||||
onQueueDatasetSync={handleQueueDatasetSync}
|
onQueueDatasetSync={handleQueueDatasetSync}
|
||||||
|
onListDatasetCrons={handleListDatasetCrons}
|
||||||
|
onDepositDatasetItem={handleDepositDatasetItem}
|
||||||
/>
|
/>
|
||||||
) : (
|
) : (
|
||||||
<Spin loading />
|
<Spin loading />
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
import { useEffect, useMemo, useState } from 'react'
|
import { useEffect, useMemo, useState } from 'react'
|
||||||
import type { ReactNode } from 'react'
|
import type { ReactNode } from 'react'
|
||||||
import { Alert, Button, Card, Empty, Grid, Input, Message, Modal, Select, Space, Spin, Switch, Typography } from '@arco-design/web-react'
|
import { Alert, Button, Card, Empty, Grid, Input, Message, Modal, Space, Spin, Switch, Typography } from '@arco-design/web-react'
|
||||||
import {
|
import {
|
||||||
IconBook,
|
IconBook,
|
||||||
IconCheckCircle,
|
IconCheckCircle,
|
||||||
@@ -15,7 +15,9 @@ import {
|
|||||||
} from '@arco-design/web-react/icon'
|
} from '@arco-design/web-react/icon'
|
||||||
import type {
|
import type {
|
||||||
DatasetCronDTO,
|
DatasetCronDTO,
|
||||||
|
DatasetDepositDTO,
|
||||||
DatasetItemDTO,
|
DatasetItemDTO,
|
||||||
|
DatasetItemPageDTO,
|
||||||
DatasetItemUpdate,
|
DatasetItemUpdate,
|
||||||
DatasetSourceDTO,
|
DatasetSourceDTO,
|
||||||
DatasetSourceInput,
|
DatasetSourceInput,
|
||||||
@@ -25,6 +27,7 @@ import type {
|
|||||||
const { Row, Col } = Grid
|
const { Row, Col } = Grid
|
||||||
const { Title, Text, Paragraph } = Typography
|
const { Title, Text, Paragraph } = Typography
|
||||||
const { TextArea } = Input
|
const { TextArea } = Input
|
||||||
|
const ITEM_PAGE_SIZE = 50
|
||||||
|
|
||||||
type DataSourceCard = DatasetSourceDTO & {
|
type DataSourceCard = DatasetSourceDTO & {
|
||||||
icon: ReactNode
|
icon: ReactNode
|
||||||
@@ -42,7 +45,7 @@ type SourceDraft = {
|
|||||||
|
|
||||||
const EMPTY_SOURCE_DRAFT: SourceDraft = {
|
const EMPTY_SOURCE_DRAFT: SourceDraft = {
|
||||||
name: '',
|
name: '',
|
||||||
kind: 'link',
|
kind: 'rss',
|
||||||
url: '',
|
url: '',
|
||||||
iconUrl: '',
|
iconUrl: '',
|
||||||
description: '',
|
description: '',
|
||||||
@@ -50,7 +53,8 @@ const EMPTY_SOURCE_DRAFT: SourceDraft = {
|
|||||||
}
|
}
|
||||||
|
|
||||||
export function WorkspaceExplorePage({
|
export function WorkspaceExplorePage({
|
||||||
onSelectItem,
|
activeProjectID,
|
||||||
|
activeProjectName,
|
||||||
onListSources,
|
onListSources,
|
||||||
onListItems,
|
onListItems,
|
||||||
onCreateSource,
|
onCreateSource,
|
||||||
@@ -58,18 +62,24 @@ export function WorkspaceExplorePage({
|
|||||||
onDeleteSource,
|
onDeleteSource,
|
||||||
onUpdateItem,
|
onUpdateItem,
|
||||||
onQueueSync,
|
onQueueSync,
|
||||||
|
onListCrons,
|
||||||
|
onDepositItem,
|
||||||
}: {
|
}: {
|
||||||
onSelectItem: (title: string) => void
|
activeProjectID: string
|
||||||
|
activeProjectName: string
|
||||||
onListSources: (signal?: AbortSignal) => Promise<DatasetSourceDTO[]>
|
onListSources: (signal?: AbortSignal) => Promise<DatasetSourceDTO[]>
|
||||||
onListItems: (signal?: AbortSignal) => Promise<DatasetItemDTO[]>
|
onListItems: (sourceId?: string, offset?: number, limit?: number, signal?: AbortSignal) => Promise<DatasetItemPageDTO>
|
||||||
onCreateSource: (input: DatasetSourceInput) => Promise<DatasetSourceDTO>
|
onCreateSource: (input: DatasetSourceInput) => Promise<DatasetSourceDTO>
|
||||||
onUpdateSource: (sourceId: string, input: DatasetSourceInput) => Promise<DatasetSourceDTO>
|
onUpdateSource: (sourceId: string, input: DatasetSourceInput) => Promise<DatasetSourceDTO>
|
||||||
onDeleteSource: (sourceId: string) => Promise<void>
|
onDeleteSource: (sourceId: string) => Promise<void>
|
||||||
onUpdateItem: (itemId: string, input: DatasetItemUpdate) => Promise<DatasetItemDTO>
|
onUpdateItem: (itemId: string, input: DatasetItemUpdate) => Promise<DatasetItemDTO>
|
||||||
onQueueSync: () => Promise<DatasetCronDTO[]>
|
onQueueSync: () => Promise<DatasetCronDTO[]>
|
||||||
|
onListCrons: (signal?: AbortSignal) => Promise<DatasetCronDTO[]>
|
||||||
|
onDepositItem: (itemId: string, projectId: string) => Promise<DatasetDepositDTO & { refreshFailed?: boolean }>
|
||||||
}) {
|
}) {
|
||||||
const [sourceRecords, setSourceRecords] = useState<DatasetSourceDTO[]>([])
|
const [sourceRecords, setSourceRecords] = useState<DatasetSourceDTO[]>([])
|
||||||
const [items, setItems] = useState<DatasetItemDTO[]>([])
|
const [items, setItems] = useState<DatasetItemDTO[]>([])
|
||||||
|
const [itemTotal, setItemTotal] = useState(0)
|
||||||
const [activeSourceID, setActiveSourceID] = useState('all')
|
const [activeSourceID, setActiveSourceID] = useState('all')
|
||||||
const [activeItemID, setActiveItemID] = useState<string | null>(null)
|
const [activeItemID, setActiveItemID] = useState<string | null>(null)
|
||||||
const [sourceModalMode, setSourceModalMode] = useState<'add' | 'edit' | null>(null)
|
const [sourceModalMode, setSourceModalMode] = useState<'add' | 'edit' | null>(null)
|
||||||
@@ -78,17 +88,18 @@ export function WorkspaceExplorePage({
|
|||||||
const [sourceError, setSourceError] = useState('')
|
const [sourceError, setSourceError] = useState('')
|
||||||
const [loadError, setLoadError] = useState('')
|
const [loadError, setLoadError] = useState('')
|
||||||
const [loading, setLoading] = useState(true)
|
const [loading, setLoading] = useState(true)
|
||||||
|
const [itemsLoading, setItemsLoading] = useState(true)
|
||||||
const [saving, setSaving] = useState(false)
|
const [saving, setSaving] = useState(false)
|
||||||
const [syncing, setSyncing] = useState(false)
|
const [syncing, setSyncing] = useState(false)
|
||||||
|
const [depositingItemID, setDepositingItemID] = useState<string | null>(null)
|
||||||
|
|
||||||
useEffect(() => {
|
useEffect(() => {
|
||||||
const controller = new AbortController()
|
const controller = new AbortController()
|
||||||
setLoading(true)
|
setLoading(true)
|
||||||
setLoadError('')
|
setLoadError('')
|
||||||
Promise.all([onListSources(controller.signal), onListItems(controller.signal)])
|
onListSources(controller.signal)
|
||||||
.then(([nextSources, nextItems]) => {
|
.then((nextSources) => {
|
||||||
setSourceRecords(nextSources)
|
setSourceRecords(nextSources)
|
||||||
setItems(nextItems)
|
|
||||||
})
|
})
|
||||||
.catch((error: unknown) => {
|
.catch((error: unknown) => {
|
||||||
if (!controller.signal.aborted) {
|
if (!controller.signal.aborted) {
|
||||||
@@ -99,7 +110,31 @@ export function WorkspaceExplorePage({
|
|||||||
if (!controller.signal.aborted) setLoading(false)
|
if (!controller.signal.aborted) setLoading(false)
|
||||||
})
|
})
|
||||||
return () => controller.abort()
|
return () => controller.abort()
|
||||||
}, [onListItems, onListSources])
|
}, [onListSources])
|
||||||
|
|
||||||
|
useEffect(() => {
|
||||||
|
const controller = new AbortController()
|
||||||
|
setItemsLoading(true)
|
||||||
|
setLoadError('')
|
||||||
|
setItems([])
|
||||||
|
setItemTotal(0)
|
||||||
|
setActiveItemID(null)
|
||||||
|
const sourceID = activeSourceID === 'all' ? undefined : activeSourceID
|
||||||
|
onListItems(sourceID, 0, ITEM_PAGE_SIZE, controller.signal)
|
||||||
|
.then((page) => {
|
||||||
|
setItems(page.items)
|
||||||
|
setItemTotal(page.total)
|
||||||
|
})
|
||||||
|
.catch((error: unknown) => {
|
||||||
|
if (!controller.signal.aborted) {
|
||||||
|
setLoadError(error instanceof Error ? error.message : '探索内容加载失败')
|
||||||
|
}
|
||||||
|
})
|
||||||
|
.finally(() => {
|
||||||
|
if (!controller.signal.aborted) setItemsLoading(false)
|
||||||
|
})
|
||||||
|
return () => controller.abort()
|
||||||
|
}, [activeSourceID, onListItems])
|
||||||
|
|
||||||
const sources = useMemo<DataSourceCard[]>(() => {
|
const sources = useMemo<DataSourceCard[]>(() => {
|
||||||
const allSource: DataSourceCard = {
|
const allSource: DataSourceCard = {
|
||||||
@@ -110,8 +145,9 @@ export function WorkspaceExplorePage({
|
|||||||
iconUrl: '',
|
iconUrl: '',
|
||||||
description: '全部探索内容',
|
description: '全部探索内容',
|
||||||
enabled: true,
|
enabled: true,
|
||||||
|
builtIn: true,
|
||||||
lastSyncedAt: null,
|
lastSyncedAt: null,
|
||||||
itemCount: items.length,
|
itemCount: sourceRecords.reduce((total, source) => total + source.itemCount, 0),
|
||||||
createdAt: '',
|
createdAt: '',
|
||||||
updatedAt: '',
|
updatedAt: '',
|
||||||
icon: <IconStorage />,
|
icon: <IconStorage />,
|
||||||
@@ -127,12 +163,9 @@ export function WorkspaceExplorePage({
|
|||||||
color: sourceColor(source.kind),
|
color: sourceColor(source.kind),
|
||||||
})),
|
})),
|
||||||
]
|
]
|
||||||
}, [items.length, sourceRecords])
|
}, [sourceRecords])
|
||||||
|
|
||||||
const filteredItems = useMemo(
|
const filteredItems = items
|
||||||
() => activeSourceID === 'all' ? items : items.filter((item) => item.sourceId === activeSourceID),
|
|
||||||
[activeSourceID, items],
|
|
||||||
)
|
|
||||||
|
|
||||||
useEffect(() => {
|
useEffect(() => {
|
||||||
if (filteredItems.length === 0) {
|
if (filteredItems.length === 0) {
|
||||||
@@ -147,6 +180,9 @@ export function WorkspaceExplorePage({
|
|||||||
const selected = filteredItems.find((item) => item.id === activeItemID) ?? filteredItems[0]
|
const selected = filteredItems.find((item) => item.id === activeItemID) ?? filteredItems[0]
|
||||||
const activeSource = sourceByID(sources, activeSourceID)
|
const activeSource = sourceByID(sources, activeSourceID)
|
||||||
const selectedSource = selected ? sourceByID(sources, selected.sourceId) : activeSource
|
const selectedSource = selected ? sourceByID(sources, selected.sourceId) : activeSource
|
||||||
|
const editingSource = editingSourceID
|
||||||
|
? sourceRecords.find((source) => source.id === editingSourceID)
|
||||||
|
: undefined
|
||||||
|
|
||||||
function openAddSource() {
|
function openAddSource() {
|
||||||
setEditingSourceID(null)
|
setEditingSourceID(null)
|
||||||
@@ -174,7 +210,7 @@ export function WorkspaceExplorePage({
|
|||||||
name: sourceDraft.name.trim(),
|
name: sourceDraft.name.trim(),
|
||||||
kind: sourceDraft.kind,
|
kind: sourceDraft.kind,
|
||||||
url: sourceDraft.url.trim(),
|
url: sourceDraft.url.trim(),
|
||||||
iconUrl: sourceDraft.iconUrl,
|
iconUrl: sourceDraft.iconUrl.trim(),
|
||||||
description: sourceDraft.description.trim(),
|
description: sourceDraft.description.trim(),
|
||||||
enabled: sourceDraft.enabled,
|
enabled: sourceDraft.enabled,
|
||||||
}
|
}
|
||||||
@@ -182,8 +218,8 @@ export function WorkspaceExplorePage({
|
|||||||
setSourceError('请输入数据源名称')
|
setSourceError('请输入数据源名称')
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
if (input.kind !== 'manual' && !input.url) {
|
if (input.kind === 'rss' && !input.url) {
|
||||||
setSourceError('链接和 RSS 数据源必须填写地址')
|
setSourceError('RSS 数据源必须填写地址')
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
setSaving(true)
|
setSaving(true)
|
||||||
@@ -229,14 +265,21 @@ export function WorkspaceExplorePage({
|
|||||||
async function syncSources() {
|
async function syncSources() {
|
||||||
setSyncing(true)
|
setSyncing(true)
|
||||||
try {
|
try {
|
||||||
const crons = await onQueueSync()
|
const queued = await onQueueSync()
|
||||||
if (crons.length === 0) {
|
if (queued.length === 0) {
|
||||||
Message.info('暂无已启用的 RSS 数据源')
|
Message.info('暂无已启用的 RSS 数据源')
|
||||||
} else {
|
} else {
|
||||||
const failed = crons.filter((cron) => cron.status === 'failed').length
|
Message.info(`已提交 ${queued.length} 个采集任务`)
|
||||||
const [nextSources, nextItems] = await Promise.all([onListSources(), onListItems()])
|
const crons = await waitForDatasetRuns(queued, onListCrons)
|
||||||
|
const sourceID = activeSourceID === 'all' ? undefined : activeSourceID
|
||||||
|
const [nextSources, nextPage] = await Promise.all([
|
||||||
|
onListSources(),
|
||||||
|
onListItems(sourceID, 0, ITEM_PAGE_SIZE),
|
||||||
|
])
|
||||||
setSourceRecords(nextSources)
|
setSourceRecords(nextSources)
|
||||||
setItems(nextItems)
|
setItems(nextPage.items)
|
||||||
|
setItemTotal(nextPage.total)
|
||||||
|
const failed = crons.filter((cron) => cron.status === 'failed').length
|
||||||
if (failed > 0) {
|
if (failed > 0) {
|
||||||
Message.warning(`${crons.length - failed} 个数据源采集完成,${failed} 个失败`)
|
Message.warning(`${crons.length - failed} 个数据源采集完成,${failed} 个失败`)
|
||||||
} else {
|
} else {
|
||||||
@@ -250,6 +293,21 @@ export function WorkspaceExplorePage({
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async function loadMoreItems() {
|
||||||
|
if (itemsLoading || items.length >= itemTotal) return
|
||||||
|
setItemsLoading(true)
|
||||||
|
try {
|
||||||
|
const sourceID = activeSourceID === 'all' ? undefined : activeSourceID
|
||||||
|
const page = await onListItems(sourceID, items.length, ITEM_PAGE_SIZE)
|
||||||
|
setItems((current) => [...current, ...page.items])
|
||||||
|
setItemTotal(page.total)
|
||||||
|
} catch (error) {
|
||||||
|
Message.error(error instanceof Error ? error.message : '更多探索内容加载失败')
|
||||||
|
} finally {
|
||||||
|
setItemsLoading(false)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
async function updateSelected(input: DatasetItemUpdate) {
|
async function updateSelected(input: DatasetItemUpdate) {
|
||||||
if (!selected) return
|
if (!selected) return
|
||||||
try {
|
try {
|
||||||
@@ -260,6 +318,23 @@ export function WorkspaceExplorePage({
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async function depositSelected() {
|
||||||
|
if (!selected || !activeProjectID) return
|
||||||
|
setDepositingItemID(selected.id)
|
||||||
|
try {
|
||||||
|
const result = await onDepositItem(selected.id, activeProjectID)
|
||||||
|
if (result.refreshFailed) {
|
||||||
|
Message.warning(`笔记已沉淀到项目“${activeProjectName}”,但项目数据刷新失败`)
|
||||||
|
} else {
|
||||||
|
Message.success(`已沉淀到项目“${activeProjectName}”的笔记资料`)
|
||||||
|
}
|
||||||
|
} catch (error) {
|
||||||
|
Message.error(error instanceof Error ? error.message : '沉淀到项目失败')
|
||||||
|
} finally {
|
||||||
|
setDepositingItemID(null)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
return (
|
return (
|
||||||
<div className="workspace-explore-page overview-page">
|
<div className="workspace-explore-page overview-page">
|
||||||
<div className="overview-head">
|
<div className="overview-head">
|
||||||
@@ -275,7 +350,7 @@ export function WorkspaceExplorePage({
|
|||||||
|
|
||||||
{loadError ? <Alert type="error" content={loadError} /> : null}
|
{loadError ? <Alert type="error" content={loadError} /> : null}
|
||||||
|
|
||||||
<Spin loading={loading} style={{ width: '100%' }}>
|
<Spin loading={loading || (itemsLoading && items.length === 0)} style={{ width: '100%' }}>
|
||||||
<Row gutter={10} className="explore-source-row">
|
<Row gutter={10} className="explore-source-row">
|
||||||
{sources.map((source) => (
|
{sources.map((source) => (
|
||||||
<Col span={6} key={source.id}>
|
<Col span={6} key={source.id}>
|
||||||
@@ -287,12 +362,14 @@ export function WorkspaceExplorePage({
|
|||||||
<span className={`explore-source-icon ${source.color}`}>{source.icon}</span>
|
<span className={`explore-source-icon ${source.color}`}>{source.icon}</span>
|
||||||
<span className="explore-source-copy">
|
<span className="explore-source-copy">
|
||||||
<Text className="explore-source-name">{source.name}</Text>
|
<Text className="explore-source-name">{source.name}</Text>
|
||||||
<Text type="secondary">{source.itemCount} 条</Text>
|
<Text type="secondary">{sourceSubtitle(source)}</Text>
|
||||||
</span>
|
</span>
|
||||||
{source.id !== 'all' ? (
|
{source.id !== 'all' ? (
|
||||||
<span className="explore-source-actions" onClick={(event) => event.stopPropagation()}>
|
<span className="explore-source-actions" onClick={(event) => event.stopPropagation()}>
|
||||||
<Button aria-label={`编辑${source.name}`} type="text" size="mini" icon={<IconEdit />} onClick={() => openEditSource(source)} />
|
<Button aria-label={`编辑${source.name}`} type="text" size="mini" icon={<IconEdit />} onClick={() => openEditSource(source)} />
|
||||||
<Button aria-label={`删除${source.name}`} type="text" size="mini" status="danger" icon={<IconDelete />} onClick={() => deleteSource(source)} />
|
{!source.builtIn ? (
|
||||||
|
<Button aria-label={`删除${source.name}`} type="text" size="mini" status="danger" icon={<IconDelete />} onClick={() => deleteSource(source)} />
|
||||||
|
) : null}
|
||||||
</span>
|
</span>
|
||||||
) : null}
|
) : null}
|
||||||
</Card>
|
</Card>
|
||||||
@@ -319,10 +396,7 @@ export function WorkspaceExplorePage({
|
|||||||
<button
|
<button
|
||||||
key={item.id}
|
key={item.id}
|
||||||
className={item.id === selected.id ? 'explore-article-item active' : 'explore-article-item'}
|
className={item.id === selected.id ? 'explore-article-item active' : 'explore-article-item'}
|
||||||
onClick={() => {
|
onClick={() => setActiveItemID(item.id)}
|
||||||
setActiveItemID(item.id)
|
|
||||||
onSelectItem(item.title)
|
|
||||||
}}
|
|
||||||
>
|
>
|
||||||
<span className={`explore-source-logo ${itemSource.color}`}>{sourceInitial(itemSource.name)}</span>
|
<span className={`explore-source-logo ${itemSource.color}`}>{sourceInitial(itemSource.name)}</span>
|
||||||
<span className="explore-article-copy">
|
<span className="explore-article-copy">
|
||||||
@@ -335,6 +409,11 @@ export function WorkspaceExplorePage({
|
|||||||
</button>
|
</button>
|
||||||
)
|
)
|
||||||
})}
|
})}
|
||||||
|
{items.length < itemTotal ? (
|
||||||
|
<Button long type="text" loading={itemsLoading} onClick={() => void loadMoreItems()}>
|
||||||
|
加载更多({items.length}/{itemTotal})
|
||||||
|
</Button>
|
||||||
|
) : null}
|
||||||
</div>
|
</div>
|
||||||
</Card>
|
</Card>
|
||||||
|
|
||||||
@@ -354,7 +433,13 @@ export function WorkspaceExplorePage({
|
|||||||
icon={<IconStar />}
|
icon={<IconStar />}
|
||||||
onClick={() => void updateSelected({ status: selected.status, starred: !selected.starred })}
|
onClick={() => void updateSelected({ status: selected.status, starred: !selected.starred })}
|
||||||
/>
|
/>
|
||||||
<Button type="text" icon={<IconBook />} onClick={() => onSelectItem(selected.title)} />
|
<Button
|
||||||
|
aria-label={`沉淀到项目${activeProjectName}`}
|
||||||
|
type="text"
|
||||||
|
icon={<IconBook />}
|
||||||
|
loading={depositingItemID === selected.id}
|
||||||
|
onClick={() => void depositSelected()}
|
||||||
|
/>
|
||||||
{selected.url ? <Button type="text" icon={<IconLaunch />} href={selected.url} target="_blank" /> : null}
|
{selected.url ? <Button type="text" icon={<IconLaunch />} href={selected.url} target="_blank" /> : null}
|
||||||
</Space>
|
</Space>
|
||||||
</div>
|
</div>
|
||||||
@@ -393,18 +478,25 @@ export function WorkspaceExplorePage({
|
|||||||
</label>
|
</label>
|
||||||
<label>
|
<label>
|
||||||
<Text>数据源类型</Text>
|
<Text>数据源类型</Text>
|
||||||
<Select
|
<Input value={sourceDraft.kind === 'rss' ? 'RSS 订阅' : '历史数据源(仅维护)'} disabled />
|
||||||
value={sourceDraft.kind}
|
|
||||||
onChange={(kind) => setSourceDraft((current) => ({ ...current, kind: kind as DatasetSourceKind }))}
|
|
||||||
>
|
|
||||||
<Select.Option value="link">网页链接</Select.Option>
|
|
||||||
<Select.Option value="rss">RSS 订阅</Select.Option>
|
|
||||||
<Select.Option value="manual">手动收集</Select.Option>
|
|
||||||
</Select>
|
|
||||||
</label>
|
</label>
|
||||||
<label>
|
<label>
|
||||||
<Text>链接地址</Text>
|
<Text>链接地址</Text>
|
||||||
<Input value={sourceDraft.url} placeholder="https://example.com/feed" onChange={(url) => setSourceDraft((current) => ({ ...current, url }))} />
|
<Input
|
||||||
|
value={sourceDraft.url}
|
||||||
|
disabled={Boolean(editingSource?.builtIn || (editingSource && editingSource.kind !== 'rss'))}
|
||||||
|
placeholder="https://example.com/feed"
|
||||||
|
onChange={(url) => setSourceDraft((current) => ({ ...current, url }))}
|
||||||
|
/>
|
||||||
|
</label>
|
||||||
|
<label>
|
||||||
|
<Text>图标地址</Text>
|
||||||
|
<Input
|
||||||
|
value={sourceDraft.iconUrl}
|
||||||
|
disabled={Boolean(editingSource?.builtIn || (editingSource && editingSource.kind !== 'rss'))}
|
||||||
|
placeholder="https://example.com/icon.png"
|
||||||
|
onChange={(iconUrl) => setSourceDraft((current) => ({ ...current, iconUrl }))}
|
||||||
|
/>
|
||||||
</label>
|
</label>
|
||||||
<label>
|
<label>
|
||||||
<Text>说明</Text>
|
<Text>说明</Text>
|
||||||
@@ -448,3 +540,26 @@ function itemTime(item: DatasetItemDTO) {
|
|||||||
if (Number.isNaN(parsed.getTime())) return ''
|
if (Number.isNaN(parsed.getTime())) return ''
|
||||||
return new Intl.DateTimeFormat('zh-CN', { dateStyle: 'medium' }).format(parsed)
|
return new Intl.DateTimeFormat('zh-CN', { dateStyle: 'medium' }).format(parsed)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function sourceSubtitle(source: DatasetSourceDTO) {
|
||||||
|
if (!source.lastSyncedAt) return `${source.itemCount} 条`
|
||||||
|
const parsed = new Date(source.lastSyncedAt)
|
||||||
|
if (Number.isNaN(parsed.getTime())) return `${source.itemCount} 条`
|
||||||
|
return `${source.itemCount} 条 · ${new Intl.DateTimeFormat('zh-CN', { dateStyle: 'short', timeStyle: 'short' }).format(parsed)}`
|
||||||
|
}
|
||||||
|
|
||||||
|
async function waitForDatasetRuns(
|
||||||
|
queued: DatasetCronDTO[],
|
||||||
|
listRuns: (signal?: AbortSignal) => Promise<DatasetCronDTO[]>,
|
||||||
|
) {
|
||||||
|
const queuedIDs = new Set(queued.map((run) => run.id))
|
||||||
|
const deadline = Date.now() + 10 * 60 * 1000
|
||||||
|
while (Date.now() < deadline) {
|
||||||
|
const runs = (await listRuns()).filter((run) => queuedIDs.has(run.id))
|
||||||
|
if (runs.length === queuedIDs.size && runs.every((run) => run.status === 'completed' || run.status === 'failed')) {
|
||||||
|
return runs
|
||||||
|
}
|
||||||
|
await new Promise((resolve) => setTimeout(resolve, 1500))
|
||||||
|
}
|
||||||
|
throw new Error('采集任务仍在后台运行,请稍后刷新')
|
||||||
|
}
|
||||||
|
|||||||
@@ -12,7 +12,9 @@ import type { InboxSuggestionDTO } from '../api/inbox'
|
|||||||
import type { AIExpertDTO, AISessionDTO, CreateAISessionInput } from '../api/ai'
|
import type { AIExpertDTO, AISessionDTO, CreateAISessionInput } from '../api/ai'
|
||||||
import type {
|
import type {
|
||||||
DatasetCronDTO,
|
DatasetCronDTO,
|
||||||
|
DatasetDepositDTO,
|
||||||
DatasetItemDTO,
|
DatasetItemDTO,
|
||||||
|
DatasetItemPageDTO,
|
||||||
DatasetItemUpdate,
|
DatasetItemUpdate,
|
||||||
DatasetSourceDTO,
|
DatasetSourceDTO,
|
||||||
DatasetSourceInput,
|
DatasetSourceInput,
|
||||||
@@ -62,6 +64,8 @@ export function ProjectPage({
|
|||||||
onDeleteDatasetSource,
|
onDeleteDatasetSource,
|
||||||
onUpdateDatasetItem,
|
onUpdateDatasetItem,
|
||||||
onQueueDatasetSync,
|
onQueueDatasetSync,
|
||||||
|
onListDatasetCrons,
|
||||||
|
onDepositDatasetItem,
|
||||||
}: {
|
}: {
|
||||||
activeView: WorkbenchView
|
activeView: WorkbenchView
|
||||||
activeWorkspace: ProjectWorkspace
|
activeWorkspace: ProjectWorkspace
|
||||||
@@ -97,12 +101,14 @@ export function ProjectPage({
|
|||||||
onListAIExperts: (signal?: AbortSignal) => Promise<AIExpertDTO[]>
|
onListAIExperts: (signal?: AbortSignal) => Promise<AIExpertDTO[]>
|
||||||
onCreateAISession: (projectId: string, input: CreateAISessionInput, signal?: AbortSignal) => Promise<AISessionDTO>
|
onCreateAISession: (projectId: string, input: CreateAISessionInput, signal?: AbortSignal) => Promise<AISessionDTO>
|
||||||
onListDatasetSources: (signal?: AbortSignal) => Promise<DatasetSourceDTO[]>
|
onListDatasetSources: (signal?: AbortSignal) => Promise<DatasetSourceDTO[]>
|
||||||
onListDatasetItems: (signal?: AbortSignal) => Promise<DatasetItemDTO[]>
|
onListDatasetItems: (sourceId?: string, offset?: number, limit?: number, signal?: AbortSignal) => Promise<DatasetItemPageDTO>
|
||||||
onCreateDatasetSource: (input: DatasetSourceInput) => Promise<DatasetSourceDTO>
|
onCreateDatasetSource: (input: DatasetSourceInput) => Promise<DatasetSourceDTO>
|
||||||
onUpdateDatasetSource: (sourceId: string, input: DatasetSourceInput) => Promise<DatasetSourceDTO>
|
onUpdateDatasetSource: (sourceId: string, input: DatasetSourceInput) => Promise<DatasetSourceDTO>
|
||||||
onDeleteDatasetSource: (sourceId: string) => Promise<void>
|
onDeleteDatasetSource: (sourceId: string) => Promise<void>
|
||||||
onUpdateDatasetItem: (itemId: string, input: DatasetItemUpdate) => Promise<DatasetItemDTO>
|
onUpdateDatasetItem: (itemId: string, input: DatasetItemUpdate) => Promise<DatasetItemDTO>
|
||||||
onQueueDatasetSync: () => Promise<DatasetCronDTO[]>
|
onQueueDatasetSync: () => Promise<DatasetCronDTO[]>
|
||||||
|
onListDatasetCrons: (signal?: AbortSignal) => Promise<DatasetCronDTO[]>
|
||||||
|
onDepositDatasetItem: (itemId: string, projectId: string) => Promise<DatasetDepositDTO & { refreshFailed?: boolean }>
|
||||||
}) {
|
}) {
|
||||||
const isProject = activeView === 'project'
|
const isProject = activeView === 'project'
|
||||||
const projects = workspaces.map((workspace) => workspace.project)
|
const projects = workspaces.map((workspace) => workspace.project)
|
||||||
@@ -159,7 +165,8 @@ export function ProjectPage({
|
|||||||
<WorkspacePage workspaces={workspaces} onOpenTask={onOpenTask} onUpdateTask={onUpdateWorkspaceTask} />
|
<WorkspacePage workspaces={workspaces} onOpenTask={onOpenTask} onUpdateTask={onUpdateWorkspaceTask} />
|
||||||
) : activeView === 'workspace-explore' ? (
|
) : activeView === 'workspace-explore' ? (
|
||||||
<WorkspaceExplorePage
|
<WorkspaceExplorePage
|
||||||
onSelectItem={onSelectItem}
|
activeProjectID={activeWorkspace.project.id}
|
||||||
|
activeProjectName={activeWorkspace.project.name}
|
||||||
onListSources={onListDatasetSources}
|
onListSources={onListDatasetSources}
|
||||||
onListItems={onListDatasetItems}
|
onListItems={onListDatasetItems}
|
||||||
onCreateSource={onCreateDatasetSource}
|
onCreateSource={onCreateDatasetSource}
|
||||||
@@ -167,6 +174,8 @@ export function ProjectPage({
|
|||||||
onDeleteSource={onDeleteDatasetSource}
|
onDeleteSource={onDeleteDatasetSource}
|
||||||
onUpdateItem={onUpdateDatasetItem}
|
onUpdateItem={onUpdateDatasetItem}
|
||||||
onQueueSync={onQueueDatasetSync}
|
onQueueSync={onQueueDatasetSync}
|
||||||
|
onListCrons={onListDatasetCrons}
|
||||||
|
onDepositItem={onDepositDatasetItem}
|
||||||
/>
|
/>
|
||||||
) : (
|
) : (
|
||||||
<ProjectChannelPage
|
<ProjectChannelPage
|
||||||
|
|||||||
@@ -8,6 +8,7 @@ import (
|
|||||||
"senlinai-agent/backend/internal/httpx"
|
"senlinai-agent/backend/internal/httpx"
|
||||||
"senlinai-agent/backend/internal/logic/ai"
|
"senlinai-agent/backend/internal/logic/ai"
|
||||||
"senlinai-agent/backend/internal/logic/auth"
|
"senlinai-agent/backend/internal/logic/auth"
|
||||||
|
"senlinai-agent/backend/internal/logic/dataset"
|
||||||
"senlinai-agent/backend/internal/logic/files"
|
"senlinai-agent/backend/internal/logic/files"
|
||||||
"senlinai-agent/backend/internal/logic/inbox"
|
"senlinai-agent/backend/internal/logic/inbox"
|
||||||
"senlinai-agent/backend/internal/logic/projects"
|
"senlinai-agent/backend/internal/logic/projects"
|
||||||
@@ -28,6 +29,7 @@ func TestMainRegistrarsRegisterTogether(t *testing.T) {
|
|||||||
cfg,
|
cfg,
|
||||||
authService.VerifySession,
|
authService.VerifySession,
|
||||||
auth.NewHandler(authService),
|
auth.NewHandler(authService),
|
||||||
|
dataset.NewHandler(dataset.NewService(nil)),
|
||||||
projects.NewHandler(projectService),
|
projects.NewHandler(projectService),
|
||||||
projects.NewTagHandler(projectService),
|
projects.NewTagHandler(projectService),
|
||||||
projects.NewCronHandler(projectService),
|
projects.NewCronHandler(projectService),
|
||||||
|
|||||||
@@ -3,9 +3,11 @@ package dataset
|
|||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"errors"
|
"errors"
|
||||||
|
"fmt"
|
||||||
"net/url"
|
"net/url"
|
||||||
"os"
|
"os"
|
||||||
"testing"
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
"github.com/stretchr/testify/require"
|
"github.com/stretchr/testify/require"
|
||||||
"senlinai-agent/backend/internal/models"
|
"senlinai-agent/backend/internal/models"
|
||||||
@@ -164,8 +166,68 @@ func TestSyncAllSourcesCollectsEnabledRSSSourcesForEveryUser(t *testing.T) {
|
|||||||
require.Equal(t, int64(2), itemCount)
|
require.Equal(t, int64(2), itemCount)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestSyncSourcesCollectsMultipleSourcesForOneOwner(t *testing.T) {
|
||||||
|
database := newDatasetTestDatabase(t)
|
||||||
|
user := createDatasetTestUser(t, database, "multi-feed-owner@example.com")
|
||||||
|
for index := 0; index < 5; index++ {
|
||||||
|
require.NoError(t, database.Create(&models.SaDatasetSource{
|
||||||
|
CreatedBy: user.ID, Name: fmt.Sprintf("Feed %d", index), Kind: "rss",
|
||||||
|
URL: fmt.Sprintf("https://example.com/feed-%d", index), Enabled: true,
|
||||||
|
}).Error)
|
||||||
|
}
|
||||||
|
|
||||||
|
runs, err := newServiceWithFetcher(database, staticFeedFetcher{}).SyncSources(context.Background(), user.ID)
|
||||||
|
|
||||||
|
require.NoError(t, err)
|
||||||
|
require.Len(t, runs, 5)
|
||||||
|
for _, run := range runs {
|
||||||
|
require.Equal(t, "completed", run.Status)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestQueueSourcesRunsInBackgroundAndRejectsOverlap(t *testing.T) {
|
||||||
|
database := newDatasetTestDatabase(t)
|
||||||
|
user := createDatasetTestUser(t, database, "queued-feed-owner@example.com")
|
||||||
|
source := models.SaDatasetSource{
|
||||||
|
CreatedBy: user.ID, Name: "Queued RSS", Kind: "rss",
|
||||||
|
URL: "https://example.com/feed", Enabled: true,
|
||||||
|
}
|
||||||
|
require.NoError(t, database.Create(&source).Error)
|
||||||
|
fetcher := &blockingFeedFetcher{
|
||||||
|
started: make(chan struct{}),
|
||||||
|
release: make(chan struct{}),
|
||||||
|
}
|
||||||
|
service := newServiceWithFetcher(database, fetcher)
|
||||||
|
|
||||||
|
queued, err := service.QueueSources(user.ID)
|
||||||
|
|
||||||
|
require.NoError(t, err)
|
||||||
|
require.Len(t, queued, 1)
|
||||||
|
require.Equal(t, "pending", queued[0].Status)
|
||||||
|
<-fetcher.started
|
||||||
|
_, err = service.QueueSources(user.ID)
|
||||||
|
require.ErrorIs(t, err, ErrSyncInProgress)
|
||||||
|
require.ErrorIs(t, service.DeleteSource(user.ID, source.Identity), ErrSyncInProgress)
|
||||||
|
close(fetcher.release)
|
||||||
|
require.Eventually(t, func() bool {
|
||||||
|
runs, listErr := service.ListCrons(user.ID)
|
||||||
|
return listErr == nil && len(runs) == 1 && runs[0].Status == "completed"
|
||||||
|
}, time.Second, 10*time.Millisecond)
|
||||||
|
}
|
||||||
|
|
||||||
type failingFeedFetcher struct{}
|
type failingFeedFetcher struct{}
|
||||||
|
|
||||||
func (failingFeedFetcher) Fetch(context.Context, string) (ParsedFeed, error) {
|
func (failingFeedFetcher) Fetch(context.Context, string) (ParsedFeed, error) {
|
||||||
return ParsedFeed{}, errors.New("feed unavailable")
|
return ParsedFeed{}, errors.New("feed unavailable")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
type blockingFeedFetcher struct {
|
||||||
|
started chan struct{}
|
||||||
|
release chan struct{}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f *blockingFeedFetcher) Fetch(context.Context, string) (ParsedFeed, error) {
|
||||||
|
close(f.started)
|
||||||
|
<-f.release
|
||||||
|
return staticFeedFetcher{}.Fetch(context.Background(), "")
|
||||||
|
}
|
||||||
|
|||||||
@@ -3,6 +3,7 @@ package dataset
|
|||||||
import (
|
import (
|
||||||
"errors"
|
"errors"
|
||||||
"net/http"
|
"net/http"
|
||||||
|
"strconv"
|
||||||
"strings"
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
@@ -25,6 +26,7 @@ type SourceDTO struct {
|
|||||||
IconURL string `json:"iconUrl"`
|
IconURL string `json:"iconUrl"`
|
||||||
Description string `json:"description"`
|
Description string `json:"description"`
|
||||||
Enabled bool `json:"enabled"`
|
Enabled bool `json:"enabled"`
|
||||||
|
BuiltIn bool `json:"builtIn"`
|
||||||
LastSyncedAt *time.Time `json:"lastSyncedAt"`
|
LastSyncedAt *time.Time `json:"lastSyncedAt"`
|
||||||
ItemCount int64 `json:"itemCount"`
|
ItemCount int64 `json:"itemCount"`
|
||||||
CreatedAt time.Time `json:"createdAt"`
|
CreatedAt time.Time `json:"createdAt"`
|
||||||
@@ -45,19 +47,28 @@ type ItemDTO struct {
|
|||||||
UpdatedAt time.Time `json:"updatedAt"`
|
UpdatedAt time.Time `json:"updatedAt"`
|
||||||
}
|
}
|
||||||
|
|
||||||
|
type ItemPageDTO struct {
|
||||||
|
Items []ItemDTO `json:"items"`
|
||||||
|
Total int64 `json:"total"`
|
||||||
|
Limit int `json:"limit"`
|
||||||
|
Offset int `json:"offset"`
|
||||||
|
}
|
||||||
|
|
||||||
type CronDTO struct {
|
type CronDTO struct {
|
||||||
ID string `json:"id"`
|
ID string `json:"id"`
|
||||||
SourceID string `json:"sourceId"`
|
SourceID string `json:"sourceId"`
|
||||||
Schedule string `json:"schedule"`
|
|
||||||
Status string `json:"status"`
|
Status string `json:"status"`
|
||||||
Enabled bool `json:"enabled"`
|
|
||||||
NextRunAt *time.Time `json:"nextRunAt"`
|
|
||||||
LastRunAt *time.Time `json:"lastRunAt"`
|
LastRunAt *time.Time `json:"lastRunAt"`
|
||||||
LastResult string `json:"lastResult"`
|
LastResult string `json:"lastResult"`
|
||||||
CreatedAt time.Time `json:"createdAt"`
|
CreatedAt time.Time `json:"createdAt"`
|
||||||
UpdatedAt time.Time `json:"updatedAt"`
|
UpdatedAt time.Time `json:"updatedAt"`
|
||||||
}
|
}
|
||||||
|
|
||||||
|
type DepositDTO struct {
|
||||||
|
NoteID string `json:"noteId"`
|
||||||
|
ProjectID string `json:"projectId"`
|
||||||
|
}
|
||||||
|
|
||||||
type sourceRequest struct {
|
type sourceRequest struct {
|
||||||
Name string `json:"name"`
|
Name string `json:"name"`
|
||||||
Kind string `json:"kind"`
|
Kind string `json:"kind"`
|
||||||
@@ -79,6 +90,7 @@ func (h *Handler) Register(router gin.IRouter) {
|
|||||||
router.GET("/dataset-items", h.listItems)
|
router.GET("/dataset-items", h.listItems)
|
||||||
router.POST("/dataset-items", h.createItem)
|
router.POST("/dataset-items", h.createItem)
|
||||||
router.PATCH("/dataset-items/:id", h.updateItem)
|
router.PATCH("/dataset-items/:id", h.updateItem)
|
||||||
|
router.POST("/dataset-items/:id/deposit", h.depositItem)
|
||||||
router.GET("/dataset-crons", h.listCrons)
|
router.GET("/dataset-crons", h.listCrons)
|
||||||
router.POST("/dataset-crons/sync", h.queueSync)
|
router.POST("/dataset-crons/sync", h.queueSync)
|
||||||
}
|
}
|
||||||
@@ -134,18 +146,21 @@ func (h *Handler) updateSource(c *gin.Context) {
|
|||||||
if !ok {
|
if !ok {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
var input sourceRequest
|
var input struct {
|
||||||
|
Name *string `json:"name"`
|
||||||
|
Kind *string `json:"kind"`
|
||||||
|
URL *string `json:"url"`
|
||||||
|
IconURL *string `json:"iconUrl"`
|
||||||
|
Description *string `json:"description"`
|
||||||
|
Enabled *bool `json:"enabled"`
|
||||||
|
}
|
||||||
if err := c.ShouldBindJSON(&input); err != nil {
|
if err := c.ShouldBindJSON(&input); err != nil {
|
||||||
writeInvalidRequest(c)
|
writeInvalidRequest(c)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
enabled := true
|
source, err := h.service.PatchSource(userID, identity, SourcePatch{
|
||||||
if input.Enabled != nil {
|
|
||||||
enabled = *input.Enabled
|
|
||||||
}
|
|
||||||
source, err := h.service.UpdateSource(userID, identity, SourceInput{
|
|
||||||
Name: input.Name, Kind: input.Kind, URL: input.URL, IconURL: input.IconURL,
|
Name: input.Name, Kind: input.Kind, URL: input.URL, IconURL: input.IconURL,
|
||||||
Description: input.Description, Enabled: enabled,
|
Description: input.Description, Enabled: input.Enabled,
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
writeError(c, err)
|
writeError(c, err)
|
||||||
@@ -180,16 +195,34 @@ func (h *Handler) listItems(c *gin.Context) {
|
|||||||
if !ok {
|
if !ok {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
items, err := h.service.ListItems(userID, c.Query("sourceId"))
|
limit, err := optionalNonNegativeInt(c.Query("limit"))
|
||||||
|
if err != nil {
|
||||||
|
writeInvalidRequest(c)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
offset, err := optionalNonNegativeInt(c.Query("offset"))
|
||||||
|
if err != nil {
|
||||||
|
writeInvalidRequest(c)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
page, err := h.service.ListItemsPage(userID, c.Query("sourceId"), limit, offset)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
writeError(c, err)
|
writeError(c, err)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
result := make([]ItemDTO, 0, len(items))
|
result := make([]ItemDTO, 0, len(page.Items))
|
||||||
for _, item := range items {
|
for _, item := range page.Items {
|
||||||
result = append(result, itemDTO(item))
|
result = append(result, itemDTO(item))
|
||||||
}
|
}
|
||||||
c.JSON(http.StatusOK, result)
|
_, hasLimit := c.GetQuery("limit")
|
||||||
|
_, hasOffset := c.GetQuery("offset")
|
||||||
|
if !hasLimit && !hasOffset {
|
||||||
|
c.JSON(http.StatusOK, result)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
c.JSON(http.StatusOK, ItemPageDTO{
|
||||||
|
Items: result, Total: page.Total, Limit: page.Limit, Offset: page.Offset,
|
||||||
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
func (h *Handler) createItem(c *gin.Context) {
|
func (h *Handler) createItem(c *gin.Context) {
|
||||||
@@ -235,14 +268,14 @@ func (h *Handler) updateItem(c *gin.Context) {
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
var input struct {
|
var input struct {
|
||||||
Status string `json:"status"`
|
Status *string `json:"status"`
|
||||||
Starred bool `json:"starred"`
|
Starred *bool `json:"starred"`
|
||||||
}
|
}
|
||||||
if err := c.ShouldBindJSON(&input); err != nil {
|
if err := c.ShouldBindJSON(&input); err != nil {
|
||||||
writeInvalidRequest(c)
|
writeInvalidRequest(c)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
item, err := h.service.UpdateItem(userID, identity, ItemUpdate{Status: input.Status, Starred: input.Starred})
|
item, err := h.service.PatchItem(userID, identity, ItemPatch{Status: input.Status, Starred: input.Starred})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
writeError(c, err)
|
writeError(c, err)
|
||||||
return
|
return
|
||||||
@@ -250,6 +283,32 @@ func (h *Handler) updateItem(c *gin.Context) {
|
|||||||
c.JSON(http.StatusOK, itemDTO(*item))
|
c.JSON(http.StatusOK, itemDTO(*item))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (h *Handler) depositItem(c *gin.Context) {
|
||||||
|
userID, ok := currentUser(c)
|
||||||
|
if !ok {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
identity, ok := httpx.IdentityParam(c, "id")
|
||||||
|
if !ok {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
var input struct {
|
||||||
|
ProjectID string `json:"projectId"`
|
||||||
|
}
|
||||||
|
if err := c.ShouldBindJSON(&input); err != nil || strings.TrimSpace(input.ProjectID) == "" {
|
||||||
|
writeInvalidRequest(c)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
result, err := h.service.DepositItem(userID, identity, input.ProjectID)
|
||||||
|
if err != nil {
|
||||||
|
writeError(c, err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
c.JSON(http.StatusCreated, DepositDTO{
|
||||||
|
NoteID: result.NoteIdentity, ProjectID: result.ProjectIdentity,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
func (h *Handler) listCrons(c *gin.Context) {
|
func (h *Handler) listCrons(c *gin.Context) {
|
||||||
userID, ok := currentUser(c)
|
userID, ok := currentUser(c)
|
||||||
if !ok {
|
if !ok {
|
||||||
@@ -272,7 +331,7 @@ func (h *Handler) queueSync(c *gin.Context) {
|
|||||||
if !ok {
|
if !ok {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
crons, err := h.service.SyncSources(c.Request.Context(), userID)
|
crons, err := h.service.QueueSources(userID)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
writeError(c, err)
|
writeError(c, err)
|
||||||
return
|
return
|
||||||
@@ -281,7 +340,7 @@ func (h *Handler) queueSync(c *gin.Context) {
|
|||||||
for _, cron := range crons {
|
for _, cron := range crons {
|
||||||
result = append(result, cronDTO(cron))
|
result = append(result, cronDTO(cron))
|
||||||
}
|
}
|
||||||
c.JSON(http.StatusOK, result)
|
c.JSON(http.StatusAccepted, result)
|
||||||
}
|
}
|
||||||
|
|
||||||
func currentUser(c *gin.Context) (uint, bool) {
|
func currentUser(c *gin.Context) (uint, bool) {
|
||||||
@@ -297,6 +356,10 @@ func writeError(c *gin.Context, err error) {
|
|||||||
switch {
|
switch {
|
||||||
case errors.Is(err, gorm.ErrRecordNotFound):
|
case errors.Is(err, gorm.ErrRecordNotFound):
|
||||||
httpx.Error(c, http.StatusNotFound, "not_found", "数据源或数据条目不存在")
|
httpx.Error(c, http.StatusNotFound, "not_found", "数据源或数据条目不存在")
|
||||||
|
case errors.Is(err, ErrBuiltInSource):
|
||||||
|
httpx.Error(c, http.StatusConflict, "built_in_source", "内置数据源不能删除,可以将其停用")
|
||||||
|
case errors.Is(err, ErrSyncInProgress):
|
||||||
|
httpx.Error(c, http.StatusConflict, "sync_in_progress", "已有数据源采集任务正在运行")
|
||||||
case errors.Is(err, ErrNameRequired), errors.Is(err, ErrKindInvalid),
|
case errors.Is(err, ErrNameRequired), errors.Is(err, ErrKindInvalid),
|
||||||
errors.Is(err, ErrURLInvalid), errors.Is(err, ErrTitleRequired), errors.Is(err, ErrStatusInvalid):
|
errors.Is(err, ErrURLInvalid), errors.Is(err, ErrTitleRequired), errors.Is(err, ErrStatusInvalid):
|
||||||
writeInvalidRequest(c)
|
writeInvalidRequest(c)
|
||||||
@@ -313,7 +376,7 @@ func sourceDTO(source models.SaDatasetSource, itemCount int64) SourceDTO {
|
|||||||
return SourceDTO{
|
return SourceDTO{
|
||||||
ID: source.Identity, Name: source.Name, Kind: source.Kind, URL: source.URL,
|
ID: source.Identity, Name: source.Name, Kind: source.Kind, URL: source.URL,
|
||||||
IconURL: source.IconURL, Description: source.Description, Enabled: source.Enabled,
|
IconURL: source.IconURL, Description: source.Description, Enabled: source.Enabled,
|
||||||
LastSyncedAt: utcTime(source.LastSyncedAt), ItemCount: itemCount,
|
BuiltIn: source.SeedKey != nil, LastSyncedAt: utcTime(source.LastSyncedAt), ItemCount: itemCount,
|
||||||
CreatedAt: source.CreatedAt.UTC(), UpdatedAt: source.UpdatedAt.UTC(),
|
CreatedAt: source.CreatedAt.UTC(), UpdatedAt: source.UpdatedAt.UTC(),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -329,13 +392,23 @@ func itemDTO(item models.SaDatasetItem) ItemDTO {
|
|||||||
|
|
||||||
func cronDTO(cron models.SaDatasetCron) CronDTO {
|
func cronDTO(cron models.SaDatasetCron) CronDTO {
|
||||||
return CronDTO{
|
return CronDTO{
|
||||||
ID: cron.Identity, SourceID: cron.SourceIdentity, Schedule: cron.Schedule,
|
ID: cron.Identity, SourceID: cron.SourceIdentity, Status: cron.Status,
|
||||||
Status: cron.Status, Enabled: cron.Enabled, NextRunAt: utcTime(cron.NextRunAt),
|
|
||||||
LastRunAt: utcTime(cron.LastRunAt), LastResult: cron.LastResult,
|
LastRunAt: utcTime(cron.LastRunAt), LastResult: cron.LastResult,
|
||||||
CreatedAt: cron.CreatedAt.UTC(), UpdatedAt: cron.UpdatedAt.UTC(),
|
CreatedAt: cron.CreatedAt.UTC(), UpdatedAt: cron.UpdatedAt.UTC(),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func optionalNonNegativeInt(value string) (int, error) {
|
||||||
|
if strings.TrimSpace(value) == "" {
|
||||||
|
return 0, nil
|
||||||
|
}
|
||||||
|
parsed, err := strconv.Atoi(value)
|
||||||
|
if err != nil || parsed < 0 {
|
||||||
|
return 0, errors.New("invalid non-negative integer")
|
||||||
|
}
|
||||||
|
return parsed, nil
|
||||||
|
}
|
||||||
|
|
||||||
func parseOptionalTime(value string) (*time.Time, error) {
|
func parseOptionalTime(value string) (*time.Time, error) {
|
||||||
if strings.TrimSpace(value) == "" {
|
if strings.TrimSpace(value) == "" {
|
||||||
return nil, nil
|
return nil, nil
|
||||||
|
|||||||
@@ -8,6 +8,7 @@ import (
|
|||||||
"net/http"
|
"net/http"
|
||||||
"net/http/httptest"
|
"net/http/httptest"
|
||||||
"testing"
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
"github.com/glebarez/sqlite"
|
"github.com/glebarez/sqlite"
|
||||||
"github.com/stretchr/testify/require"
|
"github.com/stretchr/testify/require"
|
||||||
@@ -21,12 +22,18 @@ func TestDatasetAPIConnectsSourcesItemsAndSyncTasks(t *testing.T) {
|
|||||||
database := newDatasetTestDatabase(t)
|
database := newDatasetTestDatabase(t)
|
||||||
owner := createDatasetTestUser(t, database, "owner@example.com")
|
owner := createDatasetTestUser(t, database, "owner@example.com")
|
||||||
other := createDatasetTestUser(t, database, "other@example.com")
|
other := createDatasetTestUser(t, database, "other@example.com")
|
||||||
|
project := models.SaProject{OwnerID: owner.ID, Name: "Research"}
|
||||||
|
require.NoError(t, database.Create(&project).Error)
|
||||||
ownerRouter := datasetTestRouter(database, owner.ID)
|
ownerRouter := datasetTestRouter(database, owner.ID)
|
||||||
|
|
||||||
invalid := performDatasetRequest(t, ownerRouter, http.MethodPost, "/api/v1/dataset-sources", map[string]any{
|
invalid := performDatasetRequest(t, ownerRouter, http.MethodPost, "/api/v1/dataset-sources", map[string]any{
|
||||||
"name": "Invalid RSS", "kind": "rss", "url": "",
|
"name": "Invalid RSS", "kind": "rss", "url": "",
|
||||||
})
|
})
|
||||||
require.Equal(t, http.StatusBadRequest, invalid.Code)
|
require.Equal(t, http.StatusBadRequest, invalid.Code)
|
||||||
|
unsupported := performDatasetRequest(t, ownerRouter, http.MethodPost, "/api/v1/dataset-sources", map[string]any{
|
||||||
|
"name": "Manual source", "kind": "manual", "enabled": true,
|
||||||
|
})
|
||||||
|
require.Equal(t, http.StatusBadRequest, unsupported.Code)
|
||||||
|
|
||||||
createdSource := performDatasetRequest(t, ownerRouter, http.MethodPost, "/api/v1/dataset-sources", map[string]any{
|
createdSource := performDatasetRequest(t, ownerRouter, http.MethodPost, "/api/v1/dataset-sources", map[string]any{
|
||||||
"name": "Industry feed", "kind": "rss", "url": "https://example.com/feed",
|
"name": "Industry feed", "kind": "rss", "url": "https://example.com/feed",
|
||||||
@@ -53,6 +60,11 @@ func TestDatasetAPIConnectsSourcesItemsAndSyncTasks(t *testing.T) {
|
|||||||
var item ItemDTO
|
var item ItemDTO
|
||||||
require.NoError(t, json.Unmarshal(createdItem.Body.Bytes(), &item))
|
require.NoError(t, json.Unmarshal(createdItem.Body.Bytes(), &item))
|
||||||
require.Equal(t, source.ID, item.SourceID)
|
require.Equal(t, source.ID, item.SourceID)
|
||||||
|
legacyItems := performDatasetRequest(t, ownerRouter, http.MethodGet, "/api/v1/dataset-items", nil)
|
||||||
|
require.Equal(t, http.StatusOK, legacyItems.Code)
|
||||||
|
var legacyItemList []ItemDTO
|
||||||
|
require.NoError(t, json.Unmarshal(legacyItems.Body.Bytes(), &legacyItemList))
|
||||||
|
require.Len(t, legacyItemList, 1)
|
||||||
|
|
||||||
listSources := performDatasetRequest(t, ownerRouter, http.MethodGet, "/api/v1/dataset-sources", nil)
|
listSources := performDatasetRequest(t, ownerRouter, http.MethodGet, "/api/v1/dataset-sources", nil)
|
||||||
require.Equal(t, http.StatusOK, listSources.Code)
|
require.Equal(t, http.StatusOK, listSources.Code)
|
||||||
@@ -68,20 +80,42 @@ func TestDatasetAPIConnectsSourcesItemsAndSyncTasks(t *testing.T) {
|
|||||||
require.NoError(t, json.Unmarshal(updatedItem.Body.Bytes(), &item))
|
require.NoError(t, json.Unmarshal(updatedItem.Body.Bytes(), &item))
|
||||||
require.Equal(t, "read", item.Status)
|
require.Equal(t, "read", item.Status)
|
||||||
require.True(t, item.Starred)
|
require.True(t, item.Starred)
|
||||||
|
partialItemUpdate := performDatasetRequest(t, ownerRouter, http.MethodPatch, "/api/v1/dataset-items/"+item.ID, map[string]any{
|
||||||
|
"starred": false,
|
||||||
|
})
|
||||||
|
require.Equal(t, http.StatusOK, partialItemUpdate.Code)
|
||||||
|
require.NoError(t, json.Unmarshal(partialItemUpdate.Body.Bytes(), &item))
|
||||||
|
require.Equal(t, "read", item.Status)
|
||||||
|
require.False(t, item.Starred)
|
||||||
|
|
||||||
|
deposited := performDatasetRequest(t, ownerRouter, http.MethodPost, "/api/v1/dataset-items/"+item.ID+"/deposit", map[string]any{
|
||||||
|
"projectId": project.Identity,
|
||||||
|
})
|
||||||
|
require.Equal(t, http.StatusCreated, deposited.Code)
|
||||||
|
var depositedNote models.SaNote
|
||||||
|
require.NoError(t, database.Where("project_id = ?", project.ID).First(&depositedNote).Error)
|
||||||
|
require.Equal(t, item.Title, depositedNote.Title)
|
||||||
|
require.Contains(t, depositedNote.Markdown, "原文链接")
|
||||||
|
|
||||||
firstSync := performDatasetRequest(t, ownerRouter, http.MethodPost, "/api/v1/dataset-crons/sync", map[string]any{})
|
firstSync := performDatasetRequest(t, ownerRouter, http.MethodPost, "/api/v1/dataset-crons/sync", map[string]any{})
|
||||||
require.Equal(t, http.StatusOK, firstSync.Code)
|
require.Equal(t, http.StatusAccepted, firstSync.Code)
|
||||||
var firstCrons []CronDTO
|
var firstCrons []CronDTO
|
||||||
require.NoError(t, json.Unmarshal(firstSync.Body.Bytes(), &firstCrons))
|
require.NoError(t, json.Unmarshal(firstSync.Body.Bytes(), &firstCrons))
|
||||||
require.Len(t, firstCrons, 1)
|
require.Len(t, firstCrons, 1)
|
||||||
require.Equal(t, "completed", firstCrons[0].Status)
|
require.Equal(t, "pending", firstCrons[0].Status)
|
||||||
require.Contains(t, firstCrons[0].LastResult, "format=rss")
|
firstCompleted := waitForDatasetRun(t, ownerRouter, firstCrons[0].ID)
|
||||||
|
require.Equal(t, "completed", firstCompleted.Status)
|
||||||
|
require.Contains(t, firstCompleted.LastResult, "format=rss")
|
||||||
|
|
||||||
secondSync := performDatasetRequest(t, ownerRouter, http.MethodPost, "/api/v1/dataset-crons/sync", map[string]any{})
|
var secondSync *httptest.ResponseRecorder
|
||||||
require.Equal(t, http.StatusOK, secondSync.Code)
|
require.Eventually(t, func() bool {
|
||||||
|
secondSync = performDatasetRequest(t, ownerRouter, http.MethodPost, "/api/v1/dataset-crons/sync", map[string]any{})
|
||||||
|
return secondSync.Code == http.StatusAccepted
|
||||||
|
}, time.Second, 10*time.Millisecond)
|
||||||
var secondCrons []CronDTO
|
var secondCrons []CronDTO
|
||||||
require.NoError(t, json.Unmarshal(secondSync.Body.Bytes(), &secondCrons))
|
require.NoError(t, json.Unmarshal(secondSync.Body.Bytes(), &secondCrons))
|
||||||
require.NotEqual(t, firstCrons[0].ID, secondCrons[0].ID)
|
require.NotEqual(t, firstCrons[0].ID, secondCrons[0].ID)
|
||||||
|
waitForDatasetRun(t, ownerRouter, secondCrons[0].ID)
|
||||||
var collectedItems []models.SaDatasetItem
|
var collectedItems []models.SaDatasetItem
|
||||||
require.NoError(t, database.Where("source_id = ?", 1).Find(&collectedItems).Error)
|
require.NoError(t, database.Where("source_id = ?", 1).Find(&collectedItems).Error)
|
||||||
require.Len(t, collectedItems, 2)
|
require.Len(t, collectedItems, 2)
|
||||||
@@ -121,6 +155,84 @@ func TestDatasetSourceAuthorizationUsesOwner(t *testing.T) {
|
|||||||
creatorSources, err := NewService(database).ListSources(creator.ID)
|
creatorSources, err := NewService(database).ListSources(creator.ID)
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
require.Empty(t, creatorSources)
|
require.Empty(t, creatorSources)
|
||||||
|
|
||||||
|
item := models.SaDatasetItem{
|
||||||
|
SourceID: source.ID, CreatedBy: creator.ID, Title: "Creator-authored item", Status: "unread",
|
||||||
|
}
|
||||||
|
require.NoError(t, database.Create(&item).Error)
|
||||||
|
page, err := NewService(database).ListItemsPage(owner.ID, source.Identity, 50, 0)
|
||||||
|
require.NoError(t, err)
|
||||||
|
require.Len(t, page.Items, 1)
|
||||||
|
creatorPage, err := NewService(database).ListItemsPage(creator.ID, "", 50, 0)
|
||||||
|
require.NoError(t, err)
|
||||||
|
require.Empty(t, creatorPage.Items)
|
||||||
|
creatorProject := models.SaProject{OwnerID: creator.ID, Name: "Creator project"}
|
||||||
|
require.NoError(t, database.Create(&creatorProject).Error)
|
||||||
|
_, err = NewService(database).DepositItem(owner.ID, item.Identity, creatorProject.Identity)
|
||||||
|
require.ErrorIs(t, err, gorm.ErrRecordNotFound)
|
||||||
|
require.NoError(t, NewService(database).DeleteSource(owner.ID, source.Identity))
|
||||||
|
var itemCount int64
|
||||||
|
require.NoError(t, database.Model(&models.SaDatasetItem{}).Count(&itemCount).Error)
|
||||||
|
require.Zero(t, itemCount)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestDatasetItemsArePaginatedByOwnedSource(t *testing.T) {
|
||||||
|
database := newDatasetTestDatabase(t)
|
||||||
|
owner := createDatasetTestUser(t, database, "owner@example.com")
|
||||||
|
source := models.SaDatasetSource{
|
||||||
|
OwnerID: owner.ID, CreatedBy: owner.ID,
|
||||||
|
Name: "Feed", Kind: "rss", URL: "https://example.com/feed", Enabled: true,
|
||||||
|
}
|
||||||
|
require.NoError(t, database.Create(&source).Error)
|
||||||
|
for index := 0; index < 3; index++ {
|
||||||
|
require.NoError(t, database.Create(&models.SaDatasetItem{
|
||||||
|
SourceID: source.ID, CreatedBy: owner.ID,
|
||||||
|
Title: fmt.Sprintf("Item %d", index), Status: "unread",
|
||||||
|
}).Error)
|
||||||
|
}
|
||||||
|
router := datasetTestRouter(database, owner.ID)
|
||||||
|
|
||||||
|
response := performDatasetRequest(t, router, http.MethodGet,
|
||||||
|
"/api/v1/dataset-items?sourceId="+source.Identity+"&limit=2&offset=1", nil)
|
||||||
|
|
||||||
|
require.Equal(t, http.StatusOK, response.Code)
|
||||||
|
var page ItemPageDTO
|
||||||
|
require.NoError(t, json.Unmarshal(response.Body.Bytes(), &page))
|
||||||
|
require.Equal(t, int64(3), page.Total)
|
||||||
|
require.Equal(t, 2, page.Limit)
|
||||||
|
require.Equal(t, 1, page.Offset)
|
||||||
|
require.Len(t, page.Items, 2)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestBuiltInDatasetSourceKeepsManagedFieldsAndCannotBeDeleted(t *testing.T) {
|
||||||
|
database := newDatasetTestDatabase(t)
|
||||||
|
owner := createDatasetTestUser(t, database, "owner@example.com")
|
||||||
|
seedKey := "built-in"
|
||||||
|
source := models.SaDatasetSource{
|
||||||
|
OwnerID: owner.ID, CreatedBy: owner.ID, SeedKey: &seedKey,
|
||||||
|
Name: "Built-in", Kind: "rss", URL: "https://example.com/feed",
|
||||||
|
IconURL: "https://example.com/icon.png", Enabled: true,
|
||||||
|
}
|
||||||
|
require.NoError(t, database.Create(&source).Error)
|
||||||
|
service := NewService(database)
|
||||||
|
|
||||||
|
updated, err := service.UpdateSource(owner.ID, source.Identity, SourceInput{
|
||||||
|
Name: "Renamed", Kind: "rss", URL: "https://attacker.example/feed",
|
||||||
|
IconURL: "https://attacker.example/icon.png", Enabled: false,
|
||||||
|
})
|
||||||
|
|
||||||
|
require.NoError(t, err)
|
||||||
|
require.Equal(t, "Renamed", updated.Name)
|
||||||
|
require.Equal(t, "https://example.com/feed", updated.URL)
|
||||||
|
require.Equal(t, "https://example.com/icon.png", updated.IconURL)
|
||||||
|
require.False(t, updated.Enabled)
|
||||||
|
renamed := "Renamed again"
|
||||||
|
updated, err = service.PatchSource(owner.ID, source.Identity, SourcePatch{Name: &renamed})
|
||||||
|
require.NoError(t, err)
|
||||||
|
require.Equal(t, renamed, updated.Name)
|
||||||
|
require.False(t, updated.Enabled)
|
||||||
|
require.Equal(t, "https://example.com/feed", updated.URL)
|
||||||
|
require.ErrorIs(t, service.DeleteSource(owner.ID, source.Identity), ErrBuiltInSource)
|
||||||
}
|
}
|
||||||
|
|
||||||
func newDatasetTestDatabase(t *testing.T) *gorm.DB {
|
func newDatasetTestDatabase(t *testing.T) *gorm.DB {
|
||||||
@@ -132,14 +244,39 @@ func newDatasetTestDatabase(t *testing.T) *gorm.DB {
|
|||||||
&models.SaDatasetSource{},
|
&models.SaDatasetSource{},
|
||||||
&models.SaDatasetItem{},
|
&models.SaDatasetItem{},
|
||||||
&models.SaDatasetCron{},
|
&models.SaDatasetCron{},
|
||||||
|
&models.SaProject{},
|
||||||
|
&models.SaNote{},
|
||||||
))
|
))
|
||||||
require.NoError(t, database.Exec(`
|
require.NoError(t, database.Exec(`
|
||||||
CREATE UNIQUE INDEX idx_sa_dataset_item_source_external
|
CREATE UNIQUE INDEX IF NOT EXISTS idx_sa_dataset_item_source_external
|
||||||
ON sa_dataset_items (source_id, external_id)
|
ON sa_dataset_items (source_id, external_id)
|
||||||
`).Error)
|
`).Error)
|
||||||
return database
|
return database
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func waitForDatasetRun(t *testing.T, router http.Handler, identity string) CronDTO {
|
||||||
|
t.Helper()
|
||||||
|
var completed CronDTO
|
||||||
|
require.Eventually(t, func() bool {
|
||||||
|
response := performDatasetRequest(t, router, http.MethodGet, "/api/v1/dataset-crons", nil)
|
||||||
|
if response.Code != http.StatusOK {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
var runs []CronDTO
|
||||||
|
if json.Unmarshal(response.Body.Bytes(), &runs) != nil {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
for _, run := range runs {
|
||||||
|
if run.ID == identity && (run.Status == "completed" || run.Status == "failed") {
|
||||||
|
completed = run
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}, time.Second, 10*time.Millisecond)
|
||||||
|
return completed
|
||||||
|
}
|
||||||
|
|
||||||
func createDatasetTestUser(t *testing.T, database *gorm.DB, email string) models.SaUser {
|
func createDatasetTestUser(t *testing.T, database *gorm.DB, email string) models.SaUser {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
user := models.SaUser{Email: email, DisplayName: email, PasswordHash: "hash", Role: "user"}
|
user := models.SaUser{Email: email, DisplayName: email, PasswordHash: "hash", Role: "user"}
|
||||||
|
|||||||
@@ -15,11 +15,20 @@ import (
|
|||||||
)
|
)
|
||||||
|
|
||||||
var (
|
var (
|
||||||
ErrNameRequired = errors.New("dataset source name is required")
|
ErrNameRequired = errors.New("dataset source name is required")
|
||||||
ErrKindInvalid = errors.New("dataset source kind is invalid")
|
ErrKindInvalid = errors.New("dataset source kind is invalid")
|
||||||
ErrURLInvalid = errors.New("dataset source url is invalid")
|
ErrURLInvalid = errors.New("dataset source url is invalid")
|
||||||
ErrTitleRequired = errors.New("dataset item title is required")
|
ErrTitleRequired = errors.New("dataset item title is required")
|
||||||
ErrStatusInvalid = errors.New("dataset item status is invalid")
|
ErrStatusInvalid = errors.New("dataset item status is invalid")
|
||||||
|
ErrBuiltInSource = errors.New("built-in dataset source cannot be deleted")
|
||||||
|
ErrSyncInProgress = errors.New("dataset sync is already in progress")
|
||||||
|
)
|
||||||
|
|
||||||
|
const (
|
||||||
|
defaultItemPageSize = 50
|
||||||
|
maxItemPageSize = 200
|
||||||
|
datasetRunRetention = 30 * 24 * time.Hour
|
||||||
|
maxConcurrentFetches = 4
|
||||||
)
|
)
|
||||||
|
|
||||||
type Service struct {
|
type Service struct {
|
||||||
@@ -37,6 +46,15 @@ type SourceInput struct {
|
|||||||
Enabled bool
|
Enabled bool
|
||||||
}
|
}
|
||||||
|
|
||||||
|
type SourcePatch struct {
|
||||||
|
Name *string
|
||||||
|
Kind *string
|
||||||
|
URL *string
|
||||||
|
IconURL *string
|
||||||
|
Description *string
|
||||||
|
Enabled *bool
|
||||||
|
}
|
||||||
|
|
||||||
type ItemInput struct {
|
type ItemInput struct {
|
||||||
SourceIdentity string
|
SourceIdentity string
|
||||||
Title string
|
Title string
|
||||||
@@ -51,11 +69,33 @@ type ItemUpdate struct {
|
|||||||
Starred bool
|
Starred bool
|
||||||
}
|
}
|
||||||
|
|
||||||
|
type ItemPatch struct {
|
||||||
|
Status *string
|
||||||
|
Starred *bool
|
||||||
|
}
|
||||||
|
|
||||||
|
type DepositResult struct {
|
||||||
|
NoteIdentity string
|
||||||
|
ProjectIdentity string
|
||||||
|
}
|
||||||
|
|
||||||
type SourceRecord struct {
|
type SourceRecord struct {
|
||||||
Source models.SaDatasetSource
|
Source models.SaDatasetSource
|
||||||
ItemCount int64
|
ItemCount int64
|
||||||
}
|
}
|
||||||
|
|
||||||
|
type ItemPage struct {
|
||||||
|
Items []models.SaDatasetItem
|
||||||
|
Total int64
|
||||||
|
Limit int
|
||||||
|
Offset int
|
||||||
|
}
|
||||||
|
|
||||||
|
type queuedSource struct {
|
||||||
|
Source models.SaDatasetSource
|
||||||
|
Run models.SaDatasetCron
|
||||||
|
}
|
||||||
|
|
||||||
func NewService(database *gorm.DB) *Service {
|
func NewService(database *gorm.DB) *Service {
|
||||||
return &Service{db: database, fetcher: NewHTTPFeedFetcher()}
|
return &Service{db: database, fetcher: NewHTTPFeedFetcher()}
|
||||||
}
|
}
|
||||||
@@ -81,7 +121,7 @@ func (s *Service) ListSources(userID uint) ([]SourceRecord, error) {
|
|||||||
}
|
}
|
||||||
if err := s.db.Model(&models.SaDatasetItem{}).
|
if err := s.db.Model(&models.SaDatasetItem{}).
|
||||||
Select("source_id, COUNT(*) AS count").
|
Select("source_id, COUNT(*) AS count").
|
||||||
Where("created_by = ? AND source_id IN ?", userID, sourceIDs).
|
Where("source_id IN ?", sourceIDs).
|
||||||
Group("source_id").
|
Group("source_id").
|
||||||
Scan(&rows).Error; err != nil {
|
Scan(&rows).Error; err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
@@ -111,14 +151,60 @@ func (s *Service) CreateSource(userID uint, input SourceInput) (*models.SaDatase
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (s *Service) UpdateSource(userID uint, identity string, input SourceInput) (*models.SaDatasetSource, error) {
|
func (s *Service) UpdateSource(userID uint, identity string, input SourceInput) (*models.SaDatasetSource, error) {
|
||||||
normalized, err := normalizeSourceInput(input)
|
return s.PatchSource(userID, identity, SourcePatch{
|
||||||
if err != nil {
|
Name: &input.Name, Kind: &input.Kind, URL: &input.URL, IconURL: &input.IconURL,
|
||||||
return nil, err
|
Description: &input.Description, Enabled: &input.Enabled,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *Service) PatchSource(userID uint, identity string, patch SourcePatch) (*models.SaDatasetSource, error) {
|
||||||
|
if !s.syncMu.TryLock() {
|
||||||
|
return nil, ErrSyncInProgress
|
||||||
}
|
}
|
||||||
|
defer s.syncMu.Unlock()
|
||||||
|
|
||||||
var source models.SaDatasetSource
|
var source models.SaDatasetSource
|
||||||
if err := s.db.Where("identity = ? AND owner_id = ?", identity, userID).First(&source).Error; err != nil {
|
if err := s.db.Where("identity = ? AND owner_id = ?", identity, userID).First(&source).Error; err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
input := SourceInput{
|
||||||
|
Name: source.Name, Kind: source.Kind, URL: source.URL, IconURL: source.IconURL,
|
||||||
|
Description: source.Description, Enabled: source.Enabled,
|
||||||
|
}
|
||||||
|
if patch.Name != nil {
|
||||||
|
input.Name = *patch.Name
|
||||||
|
}
|
||||||
|
if patch.Kind != nil {
|
||||||
|
input.Kind = *patch.Kind
|
||||||
|
}
|
||||||
|
if patch.URL != nil {
|
||||||
|
input.URL = *patch.URL
|
||||||
|
}
|
||||||
|
if patch.IconURL != nil {
|
||||||
|
input.IconURL = *patch.IconURL
|
||||||
|
}
|
||||||
|
if patch.Description != nil {
|
||||||
|
input.Description = *patch.Description
|
||||||
|
}
|
||||||
|
if patch.Enabled != nil {
|
||||||
|
input.Enabled = *patch.Enabled
|
||||||
|
}
|
||||||
|
managedFields := source.SeedKey != nil || source.Kind != "rss"
|
||||||
|
if managedFields {
|
||||||
|
input.Kind = source.Kind
|
||||||
|
input.URL = source.URL
|
||||||
|
input.IconURL = source.IconURL
|
||||||
|
}
|
||||||
|
var normalized SourceInput
|
||||||
|
var err error
|
||||||
|
if source.Kind == "rss" {
|
||||||
|
normalized, err = normalizeSourceInput(input)
|
||||||
|
} else {
|
||||||
|
normalized, err = normalizeLegacySourceInput(input)
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
if err := s.db.Model(&source).Updates(map[string]any{
|
if err := s.db.Model(&source).Updates(map[string]any{
|
||||||
"name": normalized.Name, "kind": normalized.Kind, "url": normalized.URL,
|
"name": normalized.Name, "kind": normalized.Kind, "url": normalized.URL,
|
||||||
"icon_url": normalized.IconURL, "description": normalized.Description, "enabled": normalized.Enabled,
|
"icon_url": normalized.IconURL, "description": normalized.Description, "enabled": normalized.Enabled,
|
||||||
@@ -129,15 +215,23 @@ func (s *Service) UpdateSource(userID uint, identity string, input SourceInput)
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (s *Service) DeleteSource(userID uint, identity string) error {
|
func (s *Service) DeleteSource(userID uint, identity string) error {
|
||||||
|
if !s.syncMu.TryLock() {
|
||||||
|
return ErrSyncInProgress
|
||||||
|
}
|
||||||
|
defer s.syncMu.Unlock()
|
||||||
|
|
||||||
return s.db.Transaction(func(tx *gorm.DB) error {
|
return s.db.Transaction(func(tx *gorm.DB) error {
|
||||||
var source models.SaDatasetSource
|
var source models.SaDatasetSource
|
||||||
if err := tx.Where("identity = ? AND owner_id = ?", identity, userID).First(&source).Error; err != nil {
|
if err := tx.Where("identity = ? AND owner_id = ?", identity, userID).First(&source).Error; err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
if err := tx.Where("source_id = ? AND created_by = ?", source.ID, userID).Delete(&models.SaDatasetCron{}).Error; err != nil {
|
if source.SeedKey != nil {
|
||||||
|
return ErrBuiltInSource
|
||||||
|
}
|
||||||
|
if err := tx.Where("source_id = ?", source.ID).Delete(&models.SaDatasetCron{}).Error; err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
if err := tx.Where("source_id = ? AND created_by = ?", source.ID, userID).Delete(&models.SaDatasetItem{}).Error; err != nil {
|
if err := tx.Where("source_id = ?", source.ID).Delete(&models.SaDatasetItem{}).Error; err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
return tx.Delete(&source).Error
|
return tx.Delete(&source).Error
|
||||||
@@ -145,28 +239,46 @@ func (s *Service) DeleteSource(userID uint, identity string) error {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (s *Service) ListItems(userID uint, sourceIdentity string) ([]models.SaDatasetItem, error) {
|
func (s *Service) ListItems(userID uint, sourceIdentity string) ([]models.SaDatasetItem, error) {
|
||||||
query := s.db.Where("created_by = ?", userID)
|
page, err := s.ListItemsPage(userID, sourceIdentity, maxItemPageSize, 0)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return page.Items, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *Service) ListItemsPage(userID uint, sourceIdentity string, limit, offset int) (ItemPage, error) {
|
||||||
|
limit, offset = normalizePage(limit, offset)
|
||||||
|
query := s.db.Model(&models.SaDatasetItem{}).
|
||||||
|
Joins("JOIN sa_dataset_sources ON sa_dataset_sources.id = sa_dataset_items.source_id").
|
||||||
|
Where("sa_dataset_sources.owner_id = ?", userID)
|
||||||
if sourceIdentity = strings.TrimSpace(sourceIdentity); sourceIdentity != "" {
|
if sourceIdentity = strings.TrimSpace(sourceIdentity); sourceIdentity != "" {
|
||||||
source, err := s.findOwnedSource(userID, sourceIdentity)
|
source, err := s.findOwnedSource(userID, sourceIdentity)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return ItemPage{}, err
|
||||||
}
|
}
|
||||||
query = query.Where("source_id = ?", source.ID)
|
query = query.Where("sa_dataset_items.source_id = ?", source.ID)
|
||||||
|
}
|
||||||
|
var total int64
|
||||||
|
if err := query.Session(&gorm.Session{}).Count(&total).Error; err != nil {
|
||||||
|
return ItemPage{}, err
|
||||||
}
|
}
|
||||||
var items []models.SaDatasetItem
|
var items []models.SaDatasetItem
|
||||||
if err := query.Order("COALESCE(published_at, created_at) desc, id desc").Limit(200).Find(&items).Error; err != nil {
|
if err := query.Select("sa_dataset_items.*").
|
||||||
return nil, err
|
Order("COALESCE(sa_dataset_items.published_at, sa_dataset_items.created_at) desc, sa_dataset_items.id desc").
|
||||||
|
Limit(limit).Offset(offset).Find(&items).Error; err != nil {
|
||||||
|
return ItemPage{}, err
|
||||||
}
|
}
|
||||||
if items == nil {
|
if items == nil {
|
||||||
items = []models.SaDatasetItem{}
|
items = []models.SaDatasetItem{}
|
||||||
}
|
}
|
||||||
return items, nil
|
return ItemPage{Items: items, Total: total, Limit: limit, Offset: offset}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s *Service) CountItems(userID uint, sourceID uint) (int64, error) {
|
func (s *Service) CountItems(userID uint, sourceID uint) (int64, error) {
|
||||||
var count int64
|
var count int64
|
||||||
err := s.db.Model(&models.SaDatasetItem{}).
|
err := s.db.Model(&models.SaDatasetItem{}).
|
||||||
Where("created_by = ? AND source_id = ?", userID, sourceID).
|
Joins("JOIN sa_dataset_sources ON sa_dataset_sources.id = sa_dataset_items.source_id").
|
||||||
|
Where("sa_dataset_sources.owner_id = ? AND sa_dataset_items.source_id = ?", userID, sourceID).
|
||||||
Count(&count).Error
|
Count(&count).Error
|
||||||
return count, err
|
return count, err
|
||||||
}
|
}
|
||||||
@@ -193,23 +305,72 @@ func (s *Service) CreateItem(userID uint, input ItemInput) (*models.SaDatasetIte
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (s *Service) UpdateItem(userID uint, identity string, input ItemUpdate) (*models.SaDatasetItem, error) {
|
func (s *Service) UpdateItem(userID uint, identity string, input ItemUpdate) (*models.SaDatasetItem, error) {
|
||||||
status := strings.TrimSpace(input.Status)
|
return s.PatchItem(userID, identity, ItemPatch{Status: &input.Status, Starred: &input.Starred})
|
||||||
if status != "unread" && status != "read" && status != "archived" {
|
}
|
||||||
return nil, ErrStatusInvalid
|
|
||||||
}
|
func (s *Service) PatchItem(userID uint, identity string, patch ItemPatch) (*models.SaDatasetItem, error) {
|
||||||
var item models.SaDatasetItem
|
item, err := s.findOwnedItem(userID, identity)
|
||||||
if err := s.db.Where("identity = ? AND created_by = ?", identity, userID).First(&item).Error; err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
if err := s.db.Model(&item).Updates(map[string]any{"status": status, "starred": input.Starred}).Error; err != nil {
|
updates := make(map[string]any, 2)
|
||||||
|
if patch.Status != nil {
|
||||||
|
status := strings.TrimSpace(*patch.Status)
|
||||||
|
if status != "unread" && status != "read" && status != "archived" {
|
||||||
|
return nil, ErrStatusInvalid
|
||||||
|
}
|
||||||
|
updates["status"] = status
|
||||||
|
}
|
||||||
|
if patch.Starred != nil {
|
||||||
|
updates["starred"] = *patch.Starred
|
||||||
|
}
|
||||||
|
if len(updates) == 0 {
|
||||||
|
return item, nil
|
||||||
|
}
|
||||||
|
if err := s.db.Model(item).Updates(updates).Error; err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
return &item, nil
|
return item, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *Service) DepositItem(userID uint, itemIdentity, projectIdentity string) (DepositResult, error) {
|
||||||
|
item, err := s.findOwnedItem(userID, itemIdentity)
|
||||||
|
if err != nil {
|
||||||
|
return DepositResult{}, err
|
||||||
|
}
|
||||||
|
var project models.SaProject
|
||||||
|
if err := s.db.Where("identity = ? AND owner_id = ?", strings.TrimSpace(projectIdentity), userID).
|
||||||
|
First(&project).Error; err != nil {
|
||||||
|
return DepositResult{}, err
|
||||||
|
}
|
||||||
|
body := strings.TrimSpace(item.Content)
|
||||||
|
if body == "" {
|
||||||
|
body = strings.TrimSpace(item.Summary)
|
||||||
|
}
|
||||||
|
if item.URL != "" {
|
||||||
|
if body != "" {
|
||||||
|
body += "\n\n"
|
||||||
|
}
|
||||||
|
body += "[原文链接](" + item.URL + ")"
|
||||||
|
}
|
||||||
|
note := models.SaNote{
|
||||||
|
ProjectID: project.ID, CreatedBy: userID,
|
||||||
|
Title: item.Title, Markdown: body,
|
||||||
|
}
|
||||||
|
if err := s.db.Create(¬e).Error; err != nil {
|
||||||
|
return DepositResult{}, err
|
||||||
|
}
|
||||||
|
return DepositResult{NoteIdentity: note.Identity, ProjectIdentity: project.Identity}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s *Service) ListCrons(userID uint) ([]models.SaDatasetCron, error) {
|
func (s *Service) ListCrons(userID uint) ([]models.SaDatasetCron, error) {
|
||||||
var crons []models.SaDatasetCron
|
var crons []models.SaDatasetCron
|
||||||
if err := s.db.Where("created_by = ?", userID).Order("created_at desc, id desc").Limit(100).Find(&crons).Error; err != nil {
|
if err := s.db.Model(&models.SaDatasetCron{}).
|
||||||
|
Select("sa_dataset_crons.*").
|
||||||
|
Joins("JOIN sa_dataset_sources ON sa_dataset_sources.id = sa_dataset_crons.source_id").
|
||||||
|
Where("sa_dataset_sources.owner_id = ?", userID).
|
||||||
|
Order("sa_dataset_crons.created_at desc, sa_dataset_crons.id desc").
|
||||||
|
Limit(100).Find(&crons).Error; err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
if crons == nil {
|
if crons == nil {
|
||||||
@@ -222,7 +383,36 @@ func (s *Service) SyncSources(ctx context.Context, userID uint) ([]models.SaData
|
|||||||
s.syncMu.Lock()
|
s.syncMu.Lock()
|
||||||
defer s.syncMu.Unlock()
|
defer s.syncMu.Unlock()
|
||||||
|
|
||||||
return s.syncSources(ctx, userID)
|
return s.syncSources(ctx, userID, "scheduled")
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *Service) QueueSources(userID uint) ([]models.SaDatasetCron, error) {
|
||||||
|
if !s.syncMu.TryLock() {
|
||||||
|
return nil, ErrSyncInProgress
|
||||||
|
}
|
||||||
|
sources, err := s.enabledRSSSources(userID)
|
||||||
|
if err != nil {
|
||||||
|
s.syncMu.Unlock()
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
queued, err := s.createDatasetRuns(userID, sources, "manual")
|
||||||
|
if err != nil {
|
||||||
|
s.syncMu.Unlock()
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
runs := make([]models.SaDatasetCron, 0, len(queued))
|
||||||
|
for _, entry := range queued {
|
||||||
|
runs = append(runs, entry.Run)
|
||||||
|
}
|
||||||
|
if len(queued) == 0 {
|
||||||
|
s.syncMu.Unlock()
|
||||||
|
return runs, nil
|
||||||
|
}
|
||||||
|
go func() {
|
||||||
|
defer s.syncMu.Unlock()
|
||||||
|
_, _ = s.executeDatasetRuns(context.Background(), queued)
|
||||||
|
}()
|
||||||
|
return runs, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s *Service) SyncAllSources(ctx context.Context) ([]models.SaDatasetCron, error) {
|
func (s *Service) SyncAllSources(ctx context.Context) ([]models.SaDatasetCron, error) {
|
||||||
@@ -250,103 +440,166 @@ func (s *Service) SyncAllSources(ctx context.Context) ([]models.SaDatasetCron, e
|
|||||||
return result, errors.Join(syncErrors...)
|
return result, errors.Join(syncErrors...)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s *Service) syncSources(ctx context.Context, userID uint) ([]models.SaDatasetCron, error) {
|
func (s *Service) syncSources(ctx context.Context, userID uint, trigger string) ([]models.SaDatasetCron, error) {
|
||||||
result := make([]models.SaDatasetCron, 0)
|
sources, err := s.enabledRSSSources(userID)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
queued, err := s.createDatasetRuns(userID, sources, trigger)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return s.executeDatasetRuns(ctx, queued)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *Service) enabledRSSSources(userID uint) ([]models.SaDatasetSource, error) {
|
||||||
var sources []models.SaDatasetSource
|
var sources []models.SaDatasetSource
|
||||||
if err := s.db.Where("owner_id = ? AND enabled = ? AND kind = ?", userID, true, "rss").
|
if err := s.db.Where("owner_id = ? AND enabled = ? AND kind = ?", userID, true, "rss").
|
||||||
Order("id asc").Find(&sources).Error; err != nil {
|
Order("id asc").Find(&sources).Error; err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
for _, source := range sources {
|
return sources, nil
|
||||||
if err := ctx.Err(); err != nil {
|
}
|
||||||
return result, err
|
|
||||||
}
|
|
||||||
now := time.Now().UTC()
|
|
||||||
cron := models.SaDatasetCron{
|
|
||||||
SourceID: source.ID, CreatedBy: userID, Schedule: "@once",
|
|
||||||
Status: "running", Enabled: true, NextRunAt: &now, LastRunAt: &now,
|
|
||||||
}
|
|
||||||
if err := s.db.Create(&cron).Error; err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
|
|
||||||
feed, fetchErr := s.fetcher.Fetch(ctx, source.URL)
|
func (s *Service) createDatasetRuns(userID uint, sources []models.SaDatasetSource, trigger string) ([]queuedSource, error) {
|
||||||
if fetchErr != nil {
|
queued := make([]queuedSource, 0, len(sources))
|
||||||
cron.Status = "failed"
|
now := time.Now().UTC()
|
||||||
cron.Enabled = false
|
err := s.db.Transaction(func(tx *gorm.DB) error {
|
||||||
cron.NextRunAt = nil
|
if err := tx.Where("created_at < ?", now.Add(-datasetRunRetention)).
|
||||||
cron.LastResult = truncateResult(fetchErr.Error())
|
Delete(&models.SaDatasetCron{}).Error; err != nil {
|
||||||
if err := s.db.Model(&cron).Updates(map[string]any{
|
return err
|
||||||
"status": cron.Status, "last_result": cron.LastResult, "enabled": false, "next_run_at": nil,
|
|
||||||
}).Error; err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
result = append(result, cron)
|
|
||||||
if err := ctx.Err(); err != nil {
|
|
||||||
return result, err
|
|
||||||
}
|
|
||||||
continue
|
|
||||||
}
|
}
|
||||||
|
for _, source := range sources {
|
||||||
|
run := models.SaDatasetCron{
|
||||||
|
SourceID: source.ID, CreatedBy: userID, Schedule: trigger,
|
||||||
|
Status: "pending", Enabled: true, NextRunAt: &now,
|
||||||
|
}
|
||||||
|
if err := tx.Create(&run).Error; err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
queued = append(queued, queuedSource{Source: source, Run: run})
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
})
|
||||||
|
return queued, err
|
||||||
|
}
|
||||||
|
|
||||||
inserted, err := s.storeFeedItems(userID, source, feed.Items)
|
func (s *Service) executeDatasetRuns(ctx context.Context, queued []queuedSource) ([]models.SaDatasetCron, error) {
|
||||||
if err != nil {
|
result := make([]models.SaDatasetCron, len(queued))
|
||||||
cron.Status = "failed"
|
runErrors := make([]error, len(queued))
|
||||||
cron.Enabled = false
|
concurrency := maxConcurrentFetches
|
||||||
cron.NextRunAt = nil
|
if s.db.Dialector.Name() == "sqlite" {
|
||||||
cron.LastResult = truncateResult("store feed items: " + err.Error())
|
concurrency = 1
|
||||||
if updateErr := s.db.Model(&cron).Updates(map[string]any{
|
}
|
||||||
"status": cron.Status, "last_result": cron.LastResult, "enabled": false, "next_run_at": nil,
|
semaphore := make(chan struct{}, concurrency)
|
||||||
}).Error; updateErr != nil {
|
var workers sync.WaitGroup
|
||||||
return nil, updateErr
|
for index, entry := range queued {
|
||||||
}
|
workers.Add(1)
|
||||||
result = append(result, cron)
|
go func() {
|
||||||
continue
|
defer workers.Done()
|
||||||
|
semaphore <- struct{}{}
|
||||||
|
defer func() { <-semaphore }()
|
||||||
|
result[index], runErrors[index] = s.executeDatasetRun(ctx, entry)
|
||||||
|
}()
|
||||||
|
}
|
||||||
|
workers.Wait()
|
||||||
|
return result, errors.Join(runErrors...)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *Service) executeDatasetRun(ctx context.Context, entry queuedSource) (models.SaDatasetCron, error) {
|
||||||
|
source := entry.Source
|
||||||
|
cron := entry.Run
|
||||||
|
if err := ctx.Err(); err != nil {
|
||||||
|
return cron, err
|
||||||
|
}
|
||||||
|
now := time.Now().UTC()
|
||||||
|
cron.Status = "running"
|
||||||
|
cron.LastRunAt = &now
|
||||||
|
if err := s.db.Model(&cron).Updates(map[string]any{
|
||||||
|
"status": "running", "last_run_at": now,
|
||||||
|
}).Error; err != nil {
|
||||||
|
return cron, err
|
||||||
|
}
|
||||||
|
|
||||||
|
feed, fetchErr := s.fetcher.Fetch(ctx, source.URL)
|
||||||
|
if fetchErr != nil {
|
||||||
|
cron.Status = "failed"
|
||||||
|
cron.Enabled = false
|
||||||
|
cron.NextRunAt = nil
|
||||||
|
cron.LastResult = truncateResult(fetchErr.Error())
|
||||||
|
if err := s.db.Model(&cron).Updates(map[string]any{
|
||||||
|
"status": cron.Status, "last_result": cron.LastResult, "enabled": false, "next_run_at": nil,
|
||||||
|
}).Error; err != nil {
|
||||||
|
return cron, err
|
||||||
|
}
|
||||||
|
return cron, ctx.Err()
|
||||||
|
}
|
||||||
|
|
||||||
|
inserted := 0
|
||||||
|
completedAt := time.Now().UTC()
|
||||||
|
err := s.db.Transaction(func(tx *gorm.DB) error {
|
||||||
|
var storeErr error
|
||||||
|
inserted, storeErr = storeFeedItems(tx, cron.CreatedBy, source, feed.Items)
|
||||||
|
if storeErr != nil {
|
||||||
|
return storeErr
|
||||||
}
|
}
|
||||||
completedAt := time.Now().UTC()
|
|
||||||
cron.Status = "completed"
|
cron.Status = "completed"
|
||||||
cron.Enabled = false
|
cron.Enabled = false
|
||||||
cron.NextRunAt = nil
|
cron.NextRunAt = nil
|
||||||
cron.LastResult = fmt.Sprintf("format=%s fetched=%d inserted=%d", feed.Format, len(feed.Items), inserted)
|
cron.LastResult = fmt.Sprintf("format=%s fetched=%d inserted=%d", feed.Format, len(feed.Items), inserted)
|
||||||
if err := s.db.Transaction(func(tx *gorm.DB) error {
|
if err := tx.Model(&cron).Updates(map[string]any{
|
||||||
if err := tx.Model(&cron).Updates(map[string]any{
|
"status": cron.Status, "last_result": cron.LastResult, "enabled": false, "next_run_at": nil,
|
||||||
"status": cron.Status, "last_result": cron.LastResult, "enabled": false, "next_run_at": nil,
|
}).Error; err != nil {
|
||||||
}).Error; err != nil {
|
return err
|
||||||
return err
|
|
||||||
}
|
|
||||||
return tx.Model(&source).Update("last_synced_at", completedAt).Error
|
|
||||||
}); err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
}
|
||||||
source.LastSyncedAt = &completedAt
|
return tx.Model(&source).Update("last_synced_at", completedAt).Error
|
||||||
result = append(result, cron)
|
})
|
||||||
|
if err == nil {
|
||||||
|
return cron, nil
|
||||||
}
|
}
|
||||||
return result, nil
|
cron.Status = "failed"
|
||||||
|
cron.Enabled = false
|
||||||
|
cron.NextRunAt = nil
|
||||||
|
cron.LastResult = truncateResult("store feed items: " + err.Error())
|
||||||
|
if updateErr := s.db.Model(&cron).Updates(map[string]any{
|
||||||
|
"status": cron.Status, "last_result": cron.LastResult, "enabled": false, "next_run_at": nil,
|
||||||
|
}).Error; updateErr != nil {
|
||||||
|
return cron, updateErr
|
||||||
|
}
|
||||||
|
return cron, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s *Service) storeFeedItems(userID uint, source models.SaDatasetSource, items []FeedItem) (int, error) {
|
func (s *Service) storeFeedItems(userID uint, source models.SaDatasetSource, items []FeedItem) (int, error) {
|
||||||
inserted := 0
|
inserted := 0
|
||||||
err := s.db.Transaction(func(tx *gorm.DB) error {
|
err := s.db.Transaction(func(tx *gorm.DB) error {
|
||||||
for _, input := range items {
|
var err error
|
||||||
externalID := input.ExternalID
|
inserted, err = storeFeedItems(tx, userID, source, items)
|
||||||
item := models.SaDatasetItem{
|
return err
|
||||||
SourceID: source.ID, CreatedBy: userID, ExternalID: &externalID,
|
|
||||||
Title: input.Title, Summary: input.Summary, Content: input.Content,
|
|
||||||
URL: input.URL, Status: "unread", PublishedAt: input.PublishedAt,
|
|
||||||
}
|
|
||||||
result := tx.Clauses(clause.OnConflict{
|
|
||||||
Columns: []clause.Column{{Name: "source_id"}, {Name: "external_id"}},
|
|
||||||
DoNothing: true,
|
|
||||||
}).Create(&item)
|
|
||||||
if result.Error != nil {
|
|
||||||
return result.Error
|
|
||||||
}
|
|
||||||
inserted += int(result.RowsAffected)
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
})
|
})
|
||||||
return inserted, err
|
return inserted, err
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func storeFeedItems(tx *gorm.DB, userID uint, source models.SaDatasetSource, items []FeedItem) (int, error) {
|
||||||
|
inserted := 0
|
||||||
|
for _, input := range items {
|
||||||
|
externalID := input.ExternalID
|
||||||
|
item := models.SaDatasetItem{
|
||||||
|
SourceID: source.ID, CreatedBy: userID, ExternalID: &externalID,
|
||||||
|
Title: input.Title, Summary: input.Summary, Content: input.Content,
|
||||||
|
URL: input.URL, Status: "unread", PublishedAt: input.PublishedAt,
|
||||||
|
}
|
||||||
|
result := tx.Clauses(clause.OnConflict{
|
||||||
|
Columns: []clause.Column{{Name: "source_id"}, {Name: "external_id"}},
|
||||||
|
DoNothing: true,
|
||||||
|
}).Create(&item)
|
||||||
|
if result.Error != nil {
|
||||||
|
return inserted, result.Error
|
||||||
|
}
|
||||||
|
inserted += int(result.RowsAffected)
|
||||||
|
}
|
||||||
|
return inserted, nil
|
||||||
|
}
|
||||||
|
|
||||||
func truncateResult(value string) string {
|
func truncateResult(value string) string {
|
||||||
return truncateRunes(value, 2000)
|
return truncateRunes(value, 2000)
|
||||||
}
|
}
|
||||||
@@ -365,6 +618,16 @@ func (s *Service) findOwnedSource(userID uint, identity string) (*models.SaDatas
|
|||||||
return &source, err
|
return &source, err
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (s *Service) findOwnedItem(userID uint, identity string) (*models.SaDatasetItem, error) {
|
||||||
|
var item models.SaDatasetItem
|
||||||
|
err := s.db.Model(&models.SaDatasetItem{}).
|
||||||
|
Select("sa_dataset_items.*").
|
||||||
|
Joins("JOIN sa_dataset_sources ON sa_dataset_sources.id = sa_dataset_items.source_id").
|
||||||
|
Where("sa_dataset_items.identity = ? AND sa_dataset_sources.owner_id = ?", strings.TrimSpace(identity), userID).
|
||||||
|
First(&item).Error
|
||||||
|
return &item, err
|
||||||
|
}
|
||||||
|
|
||||||
func normalizeSourceInput(input SourceInput) (SourceInput, error) {
|
func normalizeSourceInput(input SourceInput) (SourceInput, error) {
|
||||||
input.Name = strings.TrimSpace(input.Name)
|
input.Name = strings.TrimSpace(input.Name)
|
||||||
input.Kind = strings.ToLower(strings.TrimSpace(input.Kind))
|
input.Kind = strings.ToLower(strings.TrimSpace(input.Kind))
|
||||||
@@ -374,7 +637,7 @@ func normalizeSourceInput(input SourceInput) (SourceInput, error) {
|
|||||||
if input.Name == "" {
|
if input.Name == "" {
|
||||||
return input, ErrNameRequired
|
return input, ErrNameRequired
|
||||||
}
|
}
|
||||||
if input.Kind != "manual" && input.Kind != "link" && input.Kind != "rss" {
|
if input.Kind != "rss" {
|
||||||
return input, ErrKindInvalid
|
return input, ErrKindInvalid
|
||||||
}
|
}
|
||||||
if input.URL != "" && !validHTTPURL(input.URL) {
|
if input.URL != "" && !validHTTPURL(input.URL) {
|
||||||
@@ -383,12 +646,34 @@ func normalizeSourceInput(input SourceInput) (SourceInput, error) {
|
|||||||
if input.IconURL != "" && !validHTTPURL(input.IconURL) {
|
if input.IconURL != "" && !validHTTPURL(input.IconURL) {
|
||||||
return input, ErrURLInvalid
|
return input, ErrURLInvalid
|
||||||
}
|
}
|
||||||
if input.Kind != "manual" && input.URL == "" {
|
if input.URL == "" {
|
||||||
return input, ErrURLInvalid
|
return input, ErrURLInvalid
|
||||||
}
|
}
|
||||||
return input, nil
|
return input, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func normalizeLegacySourceInput(input SourceInput) (SourceInput, error) {
|
||||||
|
input.Name = strings.TrimSpace(input.Name)
|
||||||
|
input.Description = strings.TrimSpace(input.Description)
|
||||||
|
if input.Name == "" {
|
||||||
|
return input, ErrNameRequired
|
||||||
|
}
|
||||||
|
return input, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func normalizePage(limit, offset int) (int, int) {
|
||||||
|
if limit <= 0 {
|
||||||
|
limit = defaultItemPageSize
|
||||||
|
}
|
||||||
|
if limit > maxItemPageSize {
|
||||||
|
limit = maxItemPageSize
|
||||||
|
}
|
||||||
|
if offset < 0 {
|
||||||
|
offset = 0
|
||||||
|
}
|
||||||
|
return limit, offset
|
||||||
|
}
|
||||||
|
|
||||||
func validHTTPURL(value string) bool {
|
func validHTTPURL(value string) bool {
|
||||||
parsed, err := url.ParseRequestURI(value)
|
parsed, err := url.ParseRequestURI(value)
|
||||||
return err == nil && (parsed.Scheme == "http" || parsed.Scheme == "https") &&
|
return err == nil && (parsed.Scheme == "http" || parsed.Scheme == "https") &&
|
||||||
|
|||||||
@@ -14,8 +14,8 @@ type SaDatasetCron struct {
|
|||||||
Enabled bool `gorm:"not null;default:true;index"`
|
Enabled bool `gorm:"not null;default:true;index"`
|
||||||
NextRunAt *time.Time
|
NextRunAt *time.Time
|
||||||
LastRunAt *time.Time
|
LastRunAt *time.Time
|
||||||
LastResult string `gorm:"type:text"`
|
LastResult string `gorm:"type:text"`
|
||||||
CreatedAt time.Time
|
CreatedAt time.Time `gorm:"index"`
|
||||||
UpdatedAt time.Time
|
UpdatedAt time.Time
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -2,6 +2,7 @@ package models
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"database/sql"
|
"database/sql"
|
||||||
|
"errors"
|
||||||
|
|
||||||
"github.com/google/uuid"
|
"github.com/google/uuid"
|
||||||
"gorm.io/gorm"
|
"gorm.io/gorm"
|
||||||
@@ -28,6 +29,9 @@ func (m *SaDatasetSource) BeforeCreate(tx *gorm.DB) error {
|
|||||||
if m.CreatedBy == 0 {
|
if m.CreatedBy == 0 {
|
||||||
m.CreatedBy = m.OwnerID
|
m.CreatedBy = m.OwnerID
|
||||||
}
|
}
|
||||||
|
if m.OwnerID == 0 {
|
||||||
|
return errors.New("dataset source owner is required")
|
||||||
|
}
|
||||||
if err := resolveIdentity(tx, &SaUser{}, m.OwnerID, &m.OwnerIdentity); err != nil {
|
if err := resolveIdentity(tx, &SaUser{}, m.OwnerID, &m.OwnerIdentity); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user