From 56d96f23dd4ca068ebbf3bb3b6233b0b3208ccde Mon Sep 17 00:00:00 2001 From: yanweidong Date: Thu, 23 Jul 2026 23:15:43 +0800 Subject: [PATCH] feat: add dataset item images --- apps/web_v1/src/App.css | 9 +++ apps/web_v1/src/api/dataset.ts | 1 + apps/web_v1/src/pages/workspace-explore.tsx | 3 + backend/internal/logic/dataset/feed.go | 81 ++++++++++++++++--- backend/internal/logic/dataset/feed_test.go | 6 ++ backend/internal/logic/dataset/handlers.go | 8 +- .../internal/logic/dataset/handlers_test.go | 4 +- backend/internal/logic/dataset/service.go | 11 ++- backend/internal/models/dataset_item.go | 1 + 9 files changed, 109 insertions(+), 15 deletions(-) diff --git a/apps/web_v1/src/App.css b/apps/web_v1/src/App.css index cc7b1f1..4c8fb39 100644 --- a/apps/web_v1/src/App.css +++ b/apps/web_v1/src/App.css @@ -965,6 +965,15 @@ margin-bottom: 24px; } +.explore-article-image { + display: block; + width: 100%; + max-height: 420px; + margin: 0 0 24px; + border-radius: 12px; + object-fit: cover; +} + .explore-article-content { margin: 0; color: var(--senlin-text); diff --git a/apps/web_v1/src/api/dataset.ts b/apps/web_v1/src/api/dataset.ts index e264142..89e4e0a 100644 --- a/apps/web_v1/src/api/dataset.ts +++ b/apps/web_v1/src/api/dataset.ts @@ -24,6 +24,7 @@ export type DatasetItemDTO = { summary: string content: string url: string + imageUrl: string status: 'unread' | 'read' | 'archived' starred: boolean publishedAt: string | null diff --git a/apps/web_v1/src/pages/workspace-explore.tsx b/apps/web_v1/src/pages/workspace-explore.tsx index 8caafe4..413c27a 100644 --- a/apps/web_v1/src/pages/workspace-explore.tsx +++ b/apps/web_v1/src/pages/workspace-explore.tsx @@ -445,6 +445,9 @@ export function WorkspaceExplorePage({ {selectedSource.name} {itemTime(selected)} + {selected.imageUrl ? ( + + ) : null} {selected.content || selected.summary || '暂无正文'} diff --git a/backend/internal/logic/dataset/feed.go b/backend/internal/logic/dataset/feed.go index 9a0dd72..e1263d3 100644 --- a/backend/internal/logic/dataset/feed.go +++ b/backend/internal/logic/dataset/feed.go @@ -26,6 +26,7 @@ var ( ErrFeedTooLarge = errors.New("feed response is too large") ErrUnsafeFeedURL = errors.New("feed url resolves to a private or local address") htmlTagPattern = regexp.MustCompile(`<[^>]*>`) + htmlImagePattern = regexp.MustCompile(`(?i)]+src\s*=\s*["']([^"']+)["']`) ) type FeedItem struct { @@ -34,6 +35,7 @@ type FeedItem struct { Summary string Content string URL string + ImageURL string PublishedAt *time.Time } @@ -156,6 +158,8 @@ type jsonFeedItem struct { Summary string `json:"summary"` ContentText string `json:"content_text"` ContentHTML string `json:"content_html"` + Image string `json:"image"` + BannerImage string `json:"banner_image"` DatePublished string `json:"date_published"` } @@ -175,7 +179,8 @@ func parseJSONFeed(body []byte) (ParsedFeed, error) { } item := normalizeFeedItem(FeedItem{ ExternalID: input.ID, Title: plainText(input.Title), Summary: plainText(input.Summary), - Content: content, URL: input.URL, PublishedAt: parseFeedTime(input.DatePublished), + Content: content, URL: input.URL, ImageURL: firstNonEmpty(input.Image, input.BannerImage), + PublishedAt: parseFeedTime(input.DatePublished), }) if item.Title != "" { items = append(items, item) @@ -194,13 +199,16 @@ type xmlFeedDocument struct { } type xmlFeedItem struct { - Title string `xml:"title"` - Link string `xml:"link"` - GUID string `xml:"guid"` - Description string `xml:"description"` - Content string `xml:"encoded"` - PubDate string `xml:"pubDate"` - Date string `xml:"date"` + Title string `xml:"title"` + Link string `xml:"link"` + GUID string `xml:"guid"` + Description string `xml:"description"` + Content string `xml:"encoded"` + PubDate string `xml:"pubDate"` + Date string `xml:"date"` + Enclosure xmlMedia `xml:"enclosure"` + Thumbnail xmlMedia `xml:"thumbnail"` + Media []xmlMedia `xml:"content"` } type xmlAtomEntry struct { @@ -216,6 +224,13 @@ type xmlAtomEntry struct { type xmlAtomLink struct { Href string `xml:"href,attr"` Rel string `xml:"rel,attr"` + Type string `xml:"type,attr"` +} + +type xmlMedia struct { + URL string `xml:"url,attr"` + Type string `xml:"type,attr"` + Medium string `xml:"medium,attr"` } func parseXMLFeed(body []byte) (ParsedFeed, error) { @@ -238,7 +253,15 @@ func parseXMLFeed(body []byte) (ParsedFeed, error) { item := normalizeFeedItem(FeedItem{ ExternalID: input.GUID, Title: plainText(input.Title), Summary: plainText(input.Description), Content: content, - URL: strings.TrimSpace(input.Link), PublishedAt: parseFeedTime(firstNonEmpty(input.PubDate, input.Date)), + URL: strings.TrimSpace(input.Link), + ImageURL: firstNonEmpty( + imageMediaURL(input.Enclosure), + imageMediaURL(input.Thumbnail), + firstImageMediaURL(input.Media), + htmlImageURL(input.Content), + htmlImageURL(input.Description), + ), + PublishedAt: parseFeedTime(firstNonEmpty(input.PubDate, input.Date)), }) if item.Title != "" { items = append(items, item) @@ -251,7 +274,8 @@ func parseXMLFeed(body []byte) (ParsedFeed, error) { item := normalizeFeedItem(FeedItem{ ExternalID: input.ID, Title: plainText(input.Title), Summary: plainText(input.Summary), Content: plainText(input.Content), - URL: atomLink(input.Links), PublishedAt: parseFeedTime(firstNonEmpty(input.Published, input.Updated)), + URL: atomLink(input.Links), ImageURL: atomImageURL(input.Links), + PublishedAt: parseFeedTime(firstNonEmpty(input.Published, input.Updated)), }) if item.Title != "" { items = append(items, item) @@ -268,6 +292,7 @@ func normalizeFeedItem(item FeedItem) FeedItem { item.Summary = strings.TrimSpace(item.Summary) item.Content = strings.TrimSpace(item.Content) item.URL = truncateRunes(strings.TrimSpace(item.URL), 2048) + item.ImageURL = truncateRunes(strings.TrimSpace(item.ImageURL), 2048) key := firstNonEmpty(strings.TrimSpace(item.ExternalID), item.URL) if key == "" { published := "" @@ -305,6 +330,32 @@ func plainText(value string) string { return strings.Join(strings.Fields(html.UnescapeString(value)), " ") } +func htmlImageURL(value string) string { + match := htmlImagePattern.FindStringSubmatch(value) + if len(match) < 2 { + return "" + } + return strings.TrimSpace(html.UnescapeString(match[1])) +} + +func imageMediaURL(media xmlMedia) string { + mediaType := strings.ToLower(strings.TrimSpace(media.Type)) + medium := strings.ToLower(strings.TrimSpace(media.Medium)) + if media.URL == "" || (mediaType != "" && !strings.HasPrefix(mediaType, "image/") && medium != "image") { + return "" + } + return strings.TrimSpace(media.URL) +} + +func firstImageMediaURL(media []xmlMedia) string { + for _, item := range media { + if imageURL := imageMediaURL(item); imageURL != "" { + return imageURL + } + } + return "" +} + func atomLink(links []xmlAtomLink) string { for _, link := range links { if link.Rel == "" || link.Rel == "alternate" { @@ -317,6 +368,16 @@ func atomLink(links []xmlAtomLink) string { return "" } +func atomImageURL(links []xmlAtomLink) string { + for _, link := range links { + if strings.EqualFold(link.Rel, "enclosure") && + strings.HasPrefix(strings.ToLower(strings.TrimSpace(link.Type)), "image/") { + return strings.TrimSpace(link.Href) + } + } + return "" +} + func firstNonEmpty(values ...string) string { for _, value := range values { if strings.TrimSpace(value) != "" { diff --git a/backend/internal/logic/dataset/feed_test.go b/backend/internal/logic/dataset/feed_test.go index e7e090b..10b13e2 100644 --- a/backend/internal/logic/dataset/feed_test.go +++ b/backend/internal/logic/dataset/feed_test.go @@ -21,6 +21,7 @@ func TestParseFeedDetectsJSONFeedFromContent(t *testing.T) { "title": "JSON item", "summary": "JSON summary", "content_html": "

JSON content

", + "image": "https://example.com/json-image.jpg", "date_published": "2026-07-23T08:00:00Z" }] }`), "text/plain") @@ -31,6 +32,7 @@ func TestParseFeedDetectsJSONFeedFromContent(t *testing.T) { require.Equal(t, "JSON item", feed.Items[0].Title) require.Equal(t, "JSON content", feed.Items[0].Content) require.Equal(t, "https://example.com/json-1", feed.Items[0].URL) + require.Equal(t, "https://example.com/json-image.jpg", feed.Items[0].ImageURL) require.NotEmpty(t, feed.Items[0].ExternalID) require.NotNil(t, feed.Items[0].PublishedAt) } @@ -45,6 +47,7 @@ func TestParseFeedDetectsRSSXML(t *testing.T) { rss-1 RSS summary

]]> RSS content

]]> + Thu, 23 Jul 2026 08:00:00 +0000 @@ -55,6 +58,7 @@ func TestParseFeedDetectsRSSXML(t *testing.T) { require.Len(t, feed.Items, 1) require.Equal(t, "RSS summary", feed.Items[0].Summary) require.Equal(t, "RSS content", feed.Items[0].Content) + require.Equal(t, "https://example.com/rss-image.jpg", feed.Items[0].ImageURL) require.NotNil(t, feed.Items[0].PublishedAt) } @@ -65,6 +69,7 @@ func TestParseFeedDetectsAtomXML(t *testing.T) { Atom item atom-1 + Atom summary Atom content 2026-07-23T08:00:00Z @@ -75,6 +80,7 @@ func TestParseFeedDetectsAtomXML(t *testing.T) { require.Equal(t, "atom", feed.Format) require.Len(t, feed.Items, 1) require.Equal(t, "https://example.com/atom-1", feed.Items[0].URL) + require.Equal(t, "https://example.com/atom-image.jpg", feed.Items[0].ImageURL) require.Equal(t, "Atom content", feed.Items[0].Content) } diff --git a/backend/internal/logic/dataset/handlers.go b/backend/internal/logic/dataset/handlers.go index b8df89e..409ce24 100644 --- a/backend/internal/logic/dataset/handlers.go +++ b/backend/internal/logic/dataset/handlers.go @@ -40,6 +40,7 @@ type ItemDTO struct { Summary string `json:"summary"` Content string `json:"content"` URL string `json:"url"` + ImageURL string `json:"imageUrl"` Status string `json:"status"` Starred bool `json:"starred"` PublishedAt *time.Time `json:"publishedAt"` @@ -231,6 +232,7 @@ func (h *Handler) createItem(c *gin.Context) { Summary string `json:"summary"` Content string `json:"content"` URL string `json:"url"` + ImageURL string `json:"imageUrl"` PublishedAt string `json:"publishedAt"` } if err := c.ShouldBindJSON(&input); err != nil { @@ -244,7 +246,8 @@ func (h *Handler) createItem(c *gin.Context) { } item, err := h.service.CreateItem(userID, ItemInput{ SourceIdentity: input.SourceID, Title: input.Title, Summary: input.Summary, - Content: input.Content, URL: input.URL, PublishedAt: publishedAt, + Content: input.Content, URL: input.URL, ImageURL: input.ImageURL, + PublishedAt: publishedAt, }) if err != nil { writeError(c, err) @@ -366,7 +369,8 @@ func sourceDTO(source models.SaDatasetSource, itemCount int64) SourceDTO { func itemDTO(item models.SaDatasetItem) ItemDTO { return ItemDTO{ ID: item.Identity, SourceID: item.SourceIdentity, Title: item.Title, - Summary: item.Summary, Content: item.Content, URL: item.URL, Status: item.Status, + Summary: item.Summary, Content: item.Content, URL: item.URL, + ImageURL: item.ImageURL, Status: item.Status, Starred: item.Starred, PublishedAt: utcTime(item.PublishedAt), CreatedAt: item.CreatedAt.UTC(), UpdatedAt: item.UpdatedAt.UTC(), } diff --git a/backend/internal/logic/dataset/handlers_test.go b/backend/internal/logic/dataset/handlers_test.go index 745f6d3..a63c809 100644 --- a/backend/internal/logic/dataset/handlers_test.go +++ b/backend/internal/logic/dataset/handlers_test.go @@ -51,12 +51,13 @@ func TestDatasetAPIConnectsSourcesItemsAndSyncTasks(t *testing.T) { createdItem := performDatasetRequest(t, ownerRouter, http.MethodPost, "/api/v1/dataset-items", map[string]any{ "sourceId": source.ID, "title": "A collected article", "summary": "Summary", - "url": "https://example.com/article", + "url": "https://example.com/article", "imageUrl": "https://example.com/article.jpg", }) require.Equal(t, http.StatusCreated, createdItem.Code) var item ItemDTO require.NoError(t, json.Unmarshal(createdItem.Body.Bytes(), &item)) require.Equal(t, source.ID, item.SourceID) + require.Equal(t, "https://example.com/article.jpg", item.ImageURL) legacyItems := performDatasetRequest(t, ownerRouter, http.MethodGet, "/api/v1/dataset-items", nil) require.Equal(t, http.StatusOK, legacyItems.Code) var legacyItemList []ItemDTO @@ -269,6 +270,7 @@ func (staticFeedFetcher) Fetch(context.Context, string) (ParsedFeed, error) { Summary: "Collected summary", Content: "Collected content", URL: "https://example.com/collected", + ImageURL: "https://example.com/collected.jpg", }}, }, nil } diff --git a/backend/internal/logic/dataset/service.go b/backend/internal/logic/dataset/service.go index 9a82174..ec18c64 100644 --- a/backend/internal/logic/dataset/service.go +++ b/backend/internal/logic/dataset/service.go @@ -60,6 +60,7 @@ type ItemInput struct { Summary string Content string URL string + ImageURL string PublishedAt *time.Time } @@ -293,10 +294,15 @@ func (s *Service) CreateItem(userID uint, input ItemInput) (*models.SaDatasetIte if itemURL != "" && !validHTTPURL(itemURL) { return nil, ErrURLInvalid } + imageURL := strings.TrimSpace(input.ImageURL) + if imageURL != "" && !validHTTPURL(imageURL) { + return nil, ErrURLInvalid + } item := &models.SaDatasetItem{ SourceID: source.ID, Title: title, Summary: strings.TrimSpace(input.Summary), Content: strings.TrimSpace(input.Content), - URL: itemURL, Status: "unread", PublishedAt: utcOptionalTime(input.PublishedAt), + URL: itemURL, ImageURL: imageURL, Status: "unread", + PublishedAt: utcOptionalTime(input.PublishedAt), } return item, s.db.Create(item).Error } @@ -461,7 +467,8 @@ func storeFeedItems(tx *gorm.DB, source models.SaDatasetSource, items []FeedItem item := models.SaDatasetItem{ SourceID: source.ID, ExternalID: &externalID, Title: input.Title, Summary: input.Summary, Content: input.Content, - URL: input.URL, Status: "unread", PublishedAt: input.PublishedAt, + URL: input.URL, ImageURL: input.ImageURL, + Status: "unread", PublishedAt: input.PublishedAt, } result := tx.Clauses(clause.OnConflict{ Columns: []clause.Column{{Name: "source_id"}, {Name: "external_id"}}, diff --git a/backend/internal/models/dataset_item.go b/backend/internal/models/dataset_item.go index 8a045ba..51ec2e6 100644 --- a/backend/internal/models/dataset_item.go +++ b/backend/internal/models/dataset_item.go @@ -12,6 +12,7 @@ type SaDatasetItem struct { Summary string `gorm:"type:text"` Content string `gorm:"type:text"` URL string `gorm:"size:2048"` + ImageURL string `gorm:"size:2048"` Status string `gorm:"size:32;not null;default:unread;index"` Starred bool `gorm:"not null;default:false;index"` PublishedAt *time.Time