feat: add dataset item images

This commit is contained in:
2026-07-23 23:15:43 +08:00
parent fbd142803a
commit 56d96f23dd
9 changed files with 109 additions and 15 deletions

View File

@@ -965,6 +965,15 @@
margin-bottom: 24px; margin-bottom: 24px;
} }
.explore-article-image {
display: block;
width: 100%;
max-height: 420px;
margin: 0 0 24px;
border-radius: 12px;
object-fit: cover;
}
.explore-article-content { .explore-article-content {
margin: 0; margin: 0;
color: var(--senlin-text); color: var(--senlin-text);

View File

@@ -24,6 +24,7 @@ export type DatasetItemDTO = {
summary: string summary: string
content: string content: string
url: string url: string
imageUrl: string
status: 'unread' | 'read' | 'archived' status: 'unread' | 'read' | 'archived'
starred: boolean starred: boolean
publishedAt: string | null publishedAt: string | null

View File

@@ -445,6 +445,9 @@ export function WorkspaceExplorePage({
<Text type="secondary">{selectedSource.name}</Text> <Text type="secondary">{selectedSource.name}</Text>
<Text type="secondary">{itemTime(selected)}</Text> <Text type="secondary">{itemTime(selected)}</Text>
</Space> </Space>
{selected.imageUrl ? (
<img className="explore-article-image" src={selected.imageUrl} alt="" />
) : null}
<Paragraph className="explore-article-content">{selected.content || selected.summary || '暂无正文'}</Paragraph> <Paragraph className="explore-article-content">{selected.content || selected.summary || '暂无正文'}</Paragraph>
</article> </article>
</Card> </Card>

View File

@@ -26,6 +26,7 @@ var (
ErrFeedTooLarge = errors.New("feed response is too large") ErrFeedTooLarge = errors.New("feed response is too large")
ErrUnsafeFeedURL = errors.New("feed url resolves to a private or local address") ErrUnsafeFeedURL = errors.New("feed url resolves to a private or local address")
htmlTagPattern = regexp.MustCompile(`<[^>]*>`) htmlTagPattern = regexp.MustCompile(`<[^>]*>`)
htmlImagePattern = regexp.MustCompile(`(?i)<img[^>]+src\s*=\s*["']([^"']+)["']`)
) )
type FeedItem struct { type FeedItem struct {
@@ -34,6 +35,7 @@ type FeedItem struct {
Summary string Summary string
Content string Content string
URL string URL string
ImageURL string
PublishedAt *time.Time PublishedAt *time.Time
} }
@@ -156,6 +158,8 @@ type jsonFeedItem struct {
Summary string `json:"summary"` Summary string `json:"summary"`
ContentText string `json:"content_text"` ContentText string `json:"content_text"`
ContentHTML string `json:"content_html"` ContentHTML string `json:"content_html"`
Image string `json:"image"`
BannerImage string `json:"banner_image"`
DatePublished string `json:"date_published"` DatePublished string `json:"date_published"`
} }
@@ -175,7 +179,8 @@ func parseJSONFeed(body []byte) (ParsedFeed, error) {
} }
item := normalizeFeedItem(FeedItem{ item := normalizeFeedItem(FeedItem{
ExternalID: input.ID, Title: plainText(input.Title), Summary: plainText(input.Summary), ExternalID: input.ID, Title: plainText(input.Title), Summary: plainText(input.Summary),
Content: content, URL: input.URL, PublishedAt: parseFeedTime(input.DatePublished), Content: content, URL: input.URL, ImageURL: firstNonEmpty(input.Image, input.BannerImage),
PublishedAt: parseFeedTime(input.DatePublished),
}) })
if item.Title != "" { if item.Title != "" {
items = append(items, item) items = append(items, item)
@@ -201,6 +206,9 @@ type xmlFeedItem struct {
Content string `xml:"encoded"` Content string `xml:"encoded"`
PubDate string `xml:"pubDate"` PubDate string `xml:"pubDate"`
Date string `xml:"date"` Date string `xml:"date"`
Enclosure xmlMedia `xml:"enclosure"`
Thumbnail xmlMedia `xml:"thumbnail"`
Media []xmlMedia `xml:"content"`
} }
type xmlAtomEntry struct { type xmlAtomEntry struct {
@@ -216,6 +224,13 @@ type xmlAtomEntry struct {
type xmlAtomLink struct { type xmlAtomLink struct {
Href string `xml:"href,attr"` Href string `xml:"href,attr"`
Rel string `xml:"rel,attr"` Rel string `xml:"rel,attr"`
Type string `xml:"type,attr"`
}
type xmlMedia struct {
URL string `xml:"url,attr"`
Type string `xml:"type,attr"`
Medium string `xml:"medium,attr"`
} }
func parseXMLFeed(body []byte) (ParsedFeed, error) { func parseXMLFeed(body []byte) (ParsedFeed, error) {
@@ -238,7 +253,15 @@ func parseXMLFeed(body []byte) (ParsedFeed, error) {
item := normalizeFeedItem(FeedItem{ item := normalizeFeedItem(FeedItem{
ExternalID: input.GUID, Title: plainText(input.Title), ExternalID: input.GUID, Title: plainText(input.Title),
Summary: plainText(input.Description), Content: content, Summary: plainText(input.Description), Content: content,
URL: strings.TrimSpace(input.Link), PublishedAt: parseFeedTime(firstNonEmpty(input.PubDate, input.Date)), URL: strings.TrimSpace(input.Link),
ImageURL: firstNonEmpty(
imageMediaURL(input.Enclosure),
imageMediaURL(input.Thumbnail),
firstImageMediaURL(input.Media),
htmlImageURL(input.Content),
htmlImageURL(input.Description),
),
PublishedAt: parseFeedTime(firstNonEmpty(input.PubDate, input.Date)),
}) })
if item.Title != "" { if item.Title != "" {
items = append(items, item) items = append(items, item)
@@ -251,7 +274,8 @@ func parseXMLFeed(body []byte) (ParsedFeed, error) {
item := normalizeFeedItem(FeedItem{ item := normalizeFeedItem(FeedItem{
ExternalID: input.ID, Title: plainText(input.Title), ExternalID: input.ID, Title: plainText(input.Title),
Summary: plainText(input.Summary), Content: plainText(input.Content), Summary: plainText(input.Summary), Content: plainText(input.Content),
URL: atomLink(input.Links), PublishedAt: parseFeedTime(firstNonEmpty(input.Published, input.Updated)), URL: atomLink(input.Links), ImageURL: atomImageURL(input.Links),
PublishedAt: parseFeedTime(firstNonEmpty(input.Published, input.Updated)),
}) })
if item.Title != "" { if item.Title != "" {
items = append(items, item) items = append(items, item)
@@ -268,6 +292,7 @@ func normalizeFeedItem(item FeedItem) FeedItem {
item.Summary = strings.TrimSpace(item.Summary) item.Summary = strings.TrimSpace(item.Summary)
item.Content = strings.TrimSpace(item.Content) item.Content = strings.TrimSpace(item.Content)
item.URL = truncateRunes(strings.TrimSpace(item.URL), 2048) item.URL = truncateRunes(strings.TrimSpace(item.URL), 2048)
item.ImageURL = truncateRunes(strings.TrimSpace(item.ImageURL), 2048)
key := firstNonEmpty(strings.TrimSpace(item.ExternalID), item.URL) key := firstNonEmpty(strings.TrimSpace(item.ExternalID), item.URL)
if key == "" { if key == "" {
published := "" published := ""
@@ -305,6 +330,32 @@ func plainText(value string) string {
return strings.Join(strings.Fields(html.UnescapeString(value)), " ") return strings.Join(strings.Fields(html.UnescapeString(value)), " ")
} }
func htmlImageURL(value string) string {
match := htmlImagePattern.FindStringSubmatch(value)
if len(match) < 2 {
return ""
}
return strings.TrimSpace(html.UnescapeString(match[1]))
}
func imageMediaURL(media xmlMedia) string {
mediaType := strings.ToLower(strings.TrimSpace(media.Type))
medium := strings.ToLower(strings.TrimSpace(media.Medium))
if media.URL == "" || (mediaType != "" && !strings.HasPrefix(mediaType, "image/") && medium != "image") {
return ""
}
return strings.TrimSpace(media.URL)
}
func firstImageMediaURL(media []xmlMedia) string {
for _, item := range media {
if imageURL := imageMediaURL(item); imageURL != "" {
return imageURL
}
}
return ""
}
func atomLink(links []xmlAtomLink) string { func atomLink(links []xmlAtomLink) string {
for _, link := range links { for _, link := range links {
if link.Rel == "" || link.Rel == "alternate" { if link.Rel == "" || link.Rel == "alternate" {
@@ -317,6 +368,16 @@ func atomLink(links []xmlAtomLink) string {
return "" return ""
} }
func atomImageURL(links []xmlAtomLink) string {
for _, link := range links {
if strings.EqualFold(link.Rel, "enclosure") &&
strings.HasPrefix(strings.ToLower(strings.TrimSpace(link.Type)), "image/") {
return strings.TrimSpace(link.Href)
}
}
return ""
}
func firstNonEmpty(values ...string) string { func firstNonEmpty(values ...string) string {
for _, value := range values { for _, value := range values {
if strings.TrimSpace(value) != "" { if strings.TrimSpace(value) != "" {

View File

@@ -21,6 +21,7 @@ func TestParseFeedDetectsJSONFeedFromContent(t *testing.T) {
"title": "JSON item", "title": "JSON item",
"summary": "JSON summary", "summary": "JSON summary",
"content_html": "<p>JSON <strong>content</strong></p>", "content_html": "<p>JSON <strong>content</strong></p>",
"image": "https://example.com/json-image.jpg",
"date_published": "2026-07-23T08:00:00Z" "date_published": "2026-07-23T08:00:00Z"
}] }]
}`), "text/plain") }`), "text/plain")
@@ -31,6 +32,7 @@ func TestParseFeedDetectsJSONFeedFromContent(t *testing.T) {
require.Equal(t, "JSON item", feed.Items[0].Title) require.Equal(t, "JSON item", feed.Items[0].Title)
require.Equal(t, "JSON content", feed.Items[0].Content) require.Equal(t, "JSON content", feed.Items[0].Content)
require.Equal(t, "https://example.com/json-1", feed.Items[0].URL) require.Equal(t, "https://example.com/json-1", feed.Items[0].URL)
require.Equal(t, "https://example.com/json-image.jpg", feed.Items[0].ImageURL)
require.NotEmpty(t, feed.Items[0].ExternalID) require.NotEmpty(t, feed.Items[0].ExternalID)
require.NotNil(t, feed.Items[0].PublishedAt) require.NotNil(t, feed.Items[0].PublishedAt)
} }
@@ -45,6 +47,7 @@ func TestParseFeedDetectsRSSXML(t *testing.T) {
<guid>rss-1</guid> <guid>rss-1</guid>
<description><![CDATA[<p>RSS summary</p>]]></description> <description><![CDATA[<p>RSS summary</p>]]></description>
<content:encoded xmlns:content="http://purl.org/rss/1.0/modules/content/"><![CDATA[<p>RSS content</p>]]></content:encoded> <content:encoded xmlns:content="http://purl.org/rss/1.0/modules/content/"><![CDATA[<p>RSS content</p>]]></content:encoded>
<media:thumbnail xmlns:media="http://search.yahoo.com/mrss/" url="https://example.com/rss-image.jpg"/>
<pubDate>Thu, 23 Jul 2026 08:00:00 +0000</pubDate> <pubDate>Thu, 23 Jul 2026 08:00:00 +0000</pubDate>
</item> </item>
</channel> </channel>
@@ -55,6 +58,7 @@ func TestParseFeedDetectsRSSXML(t *testing.T) {
require.Len(t, feed.Items, 1) require.Len(t, feed.Items, 1)
require.Equal(t, "RSS summary", feed.Items[0].Summary) require.Equal(t, "RSS summary", feed.Items[0].Summary)
require.Equal(t, "RSS content", feed.Items[0].Content) require.Equal(t, "RSS content", feed.Items[0].Content)
require.Equal(t, "https://example.com/rss-image.jpg", feed.Items[0].ImageURL)
require.NotNil(t, feed.Items[0].PublishedAt) require.NotNil(t, feed.Items[0].PublishedAt)
} }
@@ -65,6 +69,7 @@ func TestParseFeedDetectsAtomXML(t *testing.T) {
<title>Atom item</title> <title>Atom item</title>
<id>atom-1</id> <id>atom-1</id>
<link rel="alternate" href="https://example.com/atom-1"/> <link rel="alternate" href="https://example.com/atom-1"/>
<link rel="enclosure" type="image/jpeg" href="https://example.com/atom-image.jpg"/>
<summary>Atom summary</summary> <summary>Atom summary</summary>
<content>Atom content</content> <content>Atom content</content>
<updated>2026-07-23T08:00:00Z</updated> <updated>2026-07-23T08:00:00Z</updated>
@@ -75,6 +80,7 @@ func TestParseFeedDetectsAtomXML(t *testing.T) {
require.Equal(t, "atom", feed.Format) require.Equal(t, "atom", feed.Format)
require.Len(t, feed.Items, 1) require.Len(t, feed.Items, 1)
require.Equal(t, "https://example.com/atom-1", feed.Items[0].URL) require.Equal(t, "https://example.com/atom-1", feed.Items[0].URL)
require.Equal(t, "https://example.com/atom-image.jpg", feed.Items[0].ImageURL)
require.Equal(t, "Atom content", feed.Items[0].Content) require.Equal(t, "Atom content", feed.Items[0].Content)
} }

View File

@@ -40,6 +40,7 @@ type ItemDTO struct {
Summary string `json:"summary"` Summary string `json:"summary"`
Content string `json:"content"` Content string `json:"content"`
URL string `json:"url"` URL string `json:"url"`
ImageURL string `json:"imageUrl"`
Status string `json:"status"` Status string `json:"status"`
Starred bool `json:"starred"` Starred bool `json:"starred"`
PublishedAt *time.Time `json:"publishedAt"` PublishedAt *time.Time `json:"publishedAt"`
@@ -231,6 +232,7 @@ func (h *Handler) createItem(c *gin.Context) {
Summary string `json:"summary"` Summary string `json:"summary"`
Content string `json:"content"` Content string `json:"content"`
URL string `json:"url"` URL string `json:"url"`
ImageURL string `json:"imageUrl"`
PublishedAt string `json:"publishedAt"` PublishedAt string `json:"publishedAt"`
} }
if err := c.ShouldBindJSON(&input); err != nil { if err := c.ShouldBindJSON(&input); err != nil {
@@ -244,7 +246,8 @@ func (h *Handler) createItem(c *gin.Context) {
} }
item, err := h.service.CreateItem(userID, ItemInput{ item, err := h.service.CreateItem(userID, ItemInput{
SourceIdentity: input.SourceID, Title: input.Title, Summary: input.Summary, SourceIdentity: input.SourceID, Title: input.Title, Summary: input.Summary,
Content: input.Content, URL: input.URL, PublishedAt: publishedAt, Content: input.Content, URL: input.URL, ImageURL: input.ImageURL,
PublishedAt: publishedAt,
}) })
if err != nil { if err != nil {
writeError(c, err) writeError(c, err)
@@ -366,7 +369,8 @@ func sourceDTO(source models.SaDatasetSource, itemCount int64) SourceDTO {
func itemDTO(item models.SaDatasetItem) ItemDTO { func itemDTO(item models.SaDatasetItem) ItemDTO {
return ItemDTO{ return ItemDTO{
ID: item.Identity, SourceID: item.SourceIdentity, Title: item.Title, ID: item.Identity, SourceID: item.SourceIdentity, Title: item.Title,
Summary: item.Summary, Content: item.Content, URL: item.URL, Status: item.Status, Summary: item.Summary, Content: item.Content, URL: item.URL,
ImageURL: item.ImageURL, Status: item.Status,
Starred: item.Starred, PublishedAt: utcTime(item.PublishedAt), Starred: item.Starred, PublishedAt: utcTime(item.PublishedAt),
CreatedAt: item.CreatedAt.UTC(), UpdatedAt: item.UpdatedAt.UTC(), CreatedAt: item.CreatedAt.UTC(), UpdatedAt: item.UpdatedAt.UTC(),
} }

View File

@@ -51,12 +51,13 @@ func TestDatasetAPIConnectsSourcesItemsAndSyncTasks(t *testing.T) {
createdItem := performDatasetRequest(t, ownerRouter, http.MethodPost, "/api/v1/dataset-items", map[string]any{ createdItem := performDatasetRequest(t, ownerRouter, http.MethodPost, "/api/v1/dataset-items", map[string]any{
"sourceId": source.ID, "title": "A collected article", "summary": "Summary", "sourceId": source.ID, "title": "A collected article", "summary": "Summary",
"url": "https://example.com/article", "url": "https://example.com/article", "imageUrl": "https://example.com/article.jpg",
}) })
require.Equal(t, http.StatusCreated, createdItem.Code) require.Equal(t, http.StatusCreated, createdItem.Code)
var item ItemDTO var item ItemDTO
require.NoError(t, json.Unmarshal(createdItem.Body.Bytes(), &item)) require.NoError(t, json.Unmarshal(createdItem.Body.Bytes(), &item))
require.Equal(t, source.ID, item.SourceID) require.Equal(t, source.ID, item.SourceID)
require.Equal(t, "https://example.com/article.jpg", item.ImageURL)
legacyItems := performDatasetRequest(t, ownerRouter, http.MethodGet, "/api/v1/dataset-items", nil) legacyItems := performDatasetRequest(t, ownerRouter, http.MethodGet, "/api/v1/dataset-items", nil)
require.Equal(t, http.StatusOK, legacyItems.Code) require.Equal(t, http.StatusOK, legacyItems.Code)
var legacyItemList []ItemDTO var legacyItemList []ItemDTO
@@ -269,6 +270,7 @@ func (staticFeedFetcher) Fetch(context.Context, string) (ParsedFeed, error) {
Summary: "Collected summary", Summary: "Collected summary",
Content: "Collected content", Content: "Collected content",
URL: "https://example.com/collected", URL: "https://example.com/collected",
ImageURL: "https://example.com/collected.jpg",
}}, }},
}, nil }, nil
} }

View File

@@ -60,6 +60,7 @@ type ItemInput struct {
Summary string Summary string
Content string Content string
URL string URL string
ImageURL string
PublishedAt *time.Time PublishedAt *time.Time
} }
@@ -293,10 +294,15 @@ func (s *Service) CreateItem(userID uint, input ItemInput) (*models.SaDatasetIte
if itemURL != "" && !validHTTPURL(itemURL) { if itemURL != "" && !validHTTPURL(itemURL) {
return nil, ErrURLInvalid return nil, ErrURLInvalid
} }
imageURL := strings.TrimSpace(input.ImageURL)
if imageURL != "" && !validHTTPURL(imageURL) {
return nil, ErrURLInvalid
}
item := &models.SaDatasetItem{ item := &models.SaDatasetItem{
SourceID: source.ID, Title: title, SourceID: source.ID, Title: title,
Summary: strings.TrimSpace(input.Summary), Content: strings.TrimSpace(input.Content), Summary: strings.TrimSpace(input.Summary), Content: strings.TrimSpace(input.Content),
URL: itemURL, Status: "unread", PublishedAt: utcOptionalTime(input.PublishedAt), URL: itemURL, ImageURL: imageURL, Status: "unread",
PublishedAt: utcOptionalTime(input.PublishedAt),
} }
return item, s.db.Create(item).Error return item, s.db.Create(item).Error
} }
@@ -461,7 +467,8 @@ func storeFeedItems(tx *gorm.DB, source models.SaDatasetSource, items []FeedItem
item := models.SaDatasetItem{ item := models.SaDatasetItem{
SourceID: source.ID, ExternalID: &externalID, SourceID: source.ID, ExternalID: &externalID,
Title: input.Title, Summary: input.Summary, Content: input.Content, Title: input.Title, Summary: input.Summary, Content: input.Content,
URL: input.URL, Status: "unread", PublishedAt: input.PublishedAt, URL: input.URL, ImageURL: input.ImageURL,
Status: "unread", PublishedAt: input.PublishedAt,
} }
result := tx.Clauses(clause.OnConflict{ result := tx.Clauses(clause.OnConflict{
Columns: []clause.Column{{Name: "source_id"}, {Name: "external_id"}}, Columns: []clause.Column{{Name: "source_id"}, {Name: "external_id"}},

View File

@@ -12,6 +12,7 @@ type SaDatasetItem struct {
Summary string `gorm:"type:text"` Summary string `gorm:"type:text"`
Content string `gorm:"type:text"` Content string `gorm:"type:text"`
URL string `gorm:"size:2048"` URL string `gorm:"size:2048"`
ImageURL string `gorm:"size:2048"`
Status string `gorm:"size:32;not null;default:unread;index"` Status string `gorm:"size:32;not null;default:unread;index"`
Starred bool `gorm:"not null;default:false;index"` Starred bool `gorm:"not null;default:false;index"`
PublishedAt *time.Time PublishedAt *time.Time