senlinai/agent
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: complete exploration data flow

Browse Source
This commit is contained in:
yanweidong
2026-07-23 22:25:12 +08:00
parent 55ff320cec
commit 5a1b04c4ed
12 changed files with 952 additions and 189 deletions
Download Patch File Download Diff File Expand all files Collapse all files

117
backend/internal/logic/dataset/handlers.go
View File

@@ -3,6 +3,7 @@ package dataset
import (
"errors"
"net/http"
"strconv"
"strings"
"time"
@@ -25,6 +26,7 @@ type SourceDTO struct {
IconURL string `json:"iconUrl"`
Description string `json:"description"`
Enabled bool `json:"enabled"`
BuiltIn bool `json:"builtIn"`
LastSyncedAt *time.Time `json:"lastSyncedAt"`
ItemCount int64 `json:"itemCount"`
CreatedAt time.Time `json:"createdAt"`
@@ -45,19 +47,28 @@ type ItemDTO struct {
UpdatedAt time.Time `json:"updatedAt"`
}
type ItemPageDTO struct {
Items []ItemDTO `json:"items"`
Total int64 `json:"total"`
Limit int `json:"limit"`
Offset int `json:"offset"`
}
type CronDTO struct {
ID string `json:"id"`
SourceID string `json:"sourceId"`
Schedule string `json:"schedule"`
Status string `json:"status"`
Enabled bool `json:"enabled"`
NextRunAt *time.Time `json:"nextRunAt"`
LastRunAt *time.Time `json:"lastRunAt"`
LastResult string `json:"lastResult"`
CreatedAt time.Time `json:"createdAt"`
UpdatedAt time.Time `json:"updatedAt"`
}
type DepositDTO struct {
NoteID string `json:"noteId"`
ProjectID string `json:"projectId"`
}
type sourceRequest struct {
Name string `json:"name"`
Kind string `json:"kind"`
@@ -79,6 +90,7 @@ func (h *Handler) Register(router gin.IRouter) {
router.GET("/dataset-items", h.listItems)
router.POST("/dataset-items", h.createItem)
router.PATCH("/dataset-items/:id", h.updateItem)
router.POST("/dataset-items/:id/deposit", h.depositItem)
router.GET("/dataset-crons", h.listCrons)
router.POST("/dataset-crons/sync", h.queueSync)
}
@@ -134,18 +146,21 @@ func (h *Handler) updateSource(c *gin.Context) {
if !ok {
return
}
var input sourceRequest
var input struct {
Name *string `json:"name"`
Kind *string `json:"kind"`
URL *string `json:"url"`
IconURL *string `json:"iconUrl"`
Description *string `json:"description"`
Enabled *bool `json:"enabled"`
}
if err := c.ShouldBindJSON(&input); err != nil {
writeInvalidRequest(c)
return
}
enabled := true
if input.Enabled != nil {
enabled = *input.Enabled
}
source, err := h.service.UpdateSource(userID, identity, SourceInput{
source, err := h.service.PatchSource(userID, identity, SourcePatch{
Name: input.Name, Kind: input.Kind, URL: input.URL, IconURL: input.IconURL,
Description: input.Description, Enabled: enabled,
Description: input.Description, Enabled: input.Enabled,
})
if err != nil {
writeError(c, err)
@@ -180,16 +195,34 @@ func (h *Handler) listItems(c *gin.Context) {
if !ok {
return
}
items, err := h.service.ListItems(userID, c.Query("sourceId"))
limit, err := optionalNonNegativeInt(c.Query("limit"))
if err != nil {
writeInvalidRequest(c)
return
}
offset, err := optionalNonNegativeInt(c.Query("offset"))
if err != nil {
writeInvalidRequest(c)
return
}
page, err := h.service.ListItemsPage(userID, c.Query("sourceId"), limit, offset)
if err != nil {
writeError(c, err)
return
}
result := make([]ItemDTO, 0, len(items))
for _, item := range items {
result := make([]ItemDTO, 0, len(page.Items))
for _, item := range page.Items {
result = append(result, itemDTO(item))
}
c.JSON(http.StatusOK, result)
_, hasLimit := c.GetQuery("limit")
_, hasOffset := c.GetQuery("offset")
if !hasLimit && !hasOffset {
c.JSON(http.StatusOK, result)
return
}
c.JSON(http.StatusOK, ItemPageDTO{
Items: result, Total: page.Total, Limit: page.Limit, Offset: page.Offset,
})
}
func (h *Handler) createItem(c *gin.Context) {
@@ -235,14 +268,14 @@ func (h *Handler) updateItem(c *gin.Context) {
return
}
var input struct {
Status string `json:"status"`
Starred bool `json:"starred"`
Status *string `json:"status"`
Starred *bool `json:"starred"`
}
if err := c.ShouldBindJSON(&input); err != nil {
writeInvalidRequest(c)
return
}
item, err := h.service.UpdateItem(userID, identity, ItemUpdate{Status: input.Status, Starred: input.Starred})
item, err := h.service.PatchItem(userID, identity, ItemPatch{Status: input.Status, Starred: input.Starred})
if err != nil {
writeError(c, err)
return
@@ -250,6 +283,32 @@ func (h *Handler) updateItem(c *gin.Context) {
c.JSON(http.StatusOK, itemDTO(*item))
}
func (h *Handler) depositItem(c *gin.Context) {
userID, ok := currentUser(c)
if !ok {
return
}
identity, ok := httpx.IdentityParam(c, "id")
if !ok {
return
}
var input struct {
ProjectID string `json:"projectId"`
}
if err := c.ShouldBindJSON(&input); err != nil || strings.TrimSpace(input.ProjectID) == "" {
writeInvalidRequest(c)
return
}
result, err := h.service.DepositItem(userID, identity, input.ProjectID)
if err != nil {
writeError(c, err)
return
}
c.JSON(http.StatusCreated, DepositDTO{
NoteID: result.NoteIdentity, ProjectID: result.ProjectIdentity,
})
}
func (h *Handler) listCrons(c *gin.Context) {
userID, ok := currentUser(c)
if !ok {
@@ -272,7 +331,7 @@ func (h *Handler) queueSync(c *gin.Context) {
if !ok {
return
}
crons, err := h.service.SyncSources(c.Request.Context(), userID)
crons, err := h.service.QueueSources(userID)
if err != nil {
writeError(c, err)
return
@@ -281,7 +340,7 @@ func (h *Handler) queueSync(c *gin.Context) {
for _, cron := range crons {
result = append(result, cronDTO(cron))
}
c.JSON(http.StatusOK, result)
c.JSON(http.StatusAccepted, result)
}
func currentUser(c *gin.Context) (uint, bool) {
@@ -297,6 +356,10 @@ func writeError(c *gin.Context, err error) {
switch {
case errors.Is(err, gorm.ErrRecordNotFound):
httpx.Error(c, http.StatusNotFound, "not_found", "数据源或数据条目不存在")
case errors.Is(err, ErrBuiltInSource):
httpx.Error(c, http.StatusConflict, "built_in_source", "内置数据源不能删除，可以将其停用")
case errors.Is(err, ErrSyncInProgress):
httpx.Error(c, http.StatusConflict, "sync_in_progress", "已有数据源采集任务正在运行")
case errors.Is(err, ErrNameRequired), errors.Is(err, ErrKindInvalid),
errors.Is(err, ErrURLInvalid), errors.Is(err, ErrTitleRequired), errors.Is(err, ErrStatusInvalid):
writeInvalidRequest(c)
@@ -313,7 +376,7 @@ func sourceDTO(source models.SaDatasetSource, itemCount int64) SourceDTO {
return SourceDTO{
ID: source.Identity, Name: source.Name, Kind: source.Kind, URL: source.URL,
IconURL: source.IconURL, Description: source.Description, Enabled: source.Enabled,
LastSyncedAt: utcTime(source.LastSyncedAt), ItemCount: itemCount,
BuiltIn: source.SeedKey != nil, LastSyncedAt: utcTime(source.LastSyncedAt), ItemCount: itemCount,
CreatedAt: source.CreatedAt.UTC(), UpdatedAt: source.UpdatedAt.UTC(),
}
}
@@ -329,13 +392,23 @@ func itemDTO(item models.SaDatasetItem) ItemDTO {
func cronDTO(cron models.SaDatasetCron) CronDTO {
return CronDTO{
ID: cron.Identity, SourceID: cron.SourceIdentity, Schedule: cron.Schedule,
Status: cron.Status, Enabled: cron.Enabled, NextRunAt: utcTime(cron.NextRunAt),
ID: cron.Identity, SourceID: cron.SourceIdentity, Status: cron.Status,
LastRunAt: utcTime(cron.LastRunAt), LastResult: cron.LastResult,
CreatedAt: cron.CreatedAt.UTC(), UpdatedAt: cron.UpdatedAt.UTC(),
}
}
func optionalNonNegativeInt(value string) (int, error) {
if strings.TrimSpace(value) == "" {
return 0, nil
}
parsed, err := strconv.Atoi(value)
if err != nil || parsed < 0 {
return 0, errors.New("invalid non-negative integer")
}
return parsed, nil
}
func parseOptionalTime(value string) (*time.Time, error) {
if strings.TrimSpace(value) == "" {
return nil, nil