senlinai/agent
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: add dataset item images

Browse Source
This commit is contained in:
yanweidong
2026-07-23 23:15:43 +08:00
parent fbd142803a
commit 56d96f23dd
9 changed files with 109 additions and 15 deletions
Download Patch File Download Diff File Expand all files Collapse all files

81
backend/internal/logic/dataset/feed.go
View File

@@ -26,6 +26,7 @@ var (
ErrFeedTooLarge = errors.New("feed response is too large")
ErrUnsafeFeedURL = errors.New("feed url resolves to a private or local address")
htmlTagPattern = regexp.MustCompile(`<[^>]*>`)
htmlImagePattern = regexp.MustCompile(`(?i)<img[^>]+src\s*=\s*["']([^"']+)["']`)
)
type FeedItem struct {
@@ -34,6 +35,7 @@ type FeedItem struct {
Summary string
Content string
URL string
ImageURL string
PublishedAt *time.Time
}
@@ -156,6 +158,8 @@ type jsonFeedItem struct {
Summary string `json:"summary"`
ContentText string `json:"content_text"`
ContentHTML string `json:"content_html"`
Image string `json:"image"`
BannerImage string `json:"banner_image"`
DatePublished string `json:"date_published"`
}
@@ -175,7 +179,8 @@ func parseJSONFeed(body []byte) (ParsedFeed, error) {
}
item := normalizeFeedItem(FeedItem{
ExternalID: input.ID, Title: plainText(input.Title), Summary: plainText(input.Summary),
Content: content, URL: input.URL, PublishedAt: parseFeedTime(input.DatePublished),
Content: content, URL: input.URL, ImageURL: firstNonEmpty(input.Image, input.BannerImage),
PublishedAt: parseFeedTime(input.DatePublished),
})
if item.Title != "" {
items = append(items, item)
@@ -194,13 +199,16 @@ type xmlFeedDocument struct {
}
type xmlFeedItem struct {
Title string `xml:"title"`
Link string `xml:"link"`
GUID string `xml:"guid"`
Description string `xml:"description"`
Content string `xml:"encoded"`
PubDate string `xml:"pubDate"`
Date string `xml:"date"`
Title string `xml:"title"`
Link string `xml:"link"`
GUID string `xml:"guid"`
Description string `xml:"description"`
Content string `xml:"encoded"`
PubDate string `xml:"pubDate"`
Date string `xml:"date"`
Enclosure xmlMedia `xml:"enclosure"`
Thumbnail xmlMedia `xml:"thumbnail"`
Media []xmlMedia `xml:"content"`
}
type xmlAtomEntry struct {
@@ -216,6 +224,13 @@ type xmlAtomEntry struct {
type xmlAtomLink struct {
Href string `xml:"href,attr"`
Rel string `xml:"rel,attr"`
Type string `xml:"type,attr"`
}
type xmlMedia struct {
URL string `xml:"url,attr"`
Type string `xml:"type,attr"`
Medium string `xml:"medium,attr"`
}
func parseXMLFeed(body []byte) (ParsedFeed, error) {
@@ -238,7 +253,15 @@ func parseXMLFeed(body []byte) (ParsedFeed, error) {
item := normalizeFeedItem(FeedItem{
ExternalID: input.GUID, Title: plainText(input.Title),
Summary: plainText(input.Description), Content: content,
URL: strings.TrimSpace(input.Link), PublishedAt: parseFeedTime(firstNonEmpty(input.PubDate, input.Date)),
URL: strings.TrimSpace(input.Link),
ImageURL: firstNonEmpty(
imageMediaURL(input.Enclosure),
imageMediaURL(input.Thumbnail),
firstImageMediaURL(input.Media),
htmlImageURL(input.Content),
htmlImageURL(input.Description),
),
PublishedAt: parseFeedTime(firstNonEmpty(input.PubDate, input.Date)),
})
if item.Title != "" {
items = append(items, item)
@@ -251,7 +274,8 @@ func parseXMLFeed(body []byte) (ParsedFeed, error) {
item := normalizeFeedItem(FeedItem{
ExternalID: input.ID, Title: plainText(input.Title),
Summary: plainText(input.Summary), Content: plainText(input.Content),
URL: atomLink(input.Links), PublishedAt: parseFeedTime(firstNonEmpty(input.Published, input.Updated)),
URL: atomLink(input.Links), ImageURL: atomImageURL(input.Links),
PublishedAt: parseFeedTime(firstNonEmpty(input.Published, input.Updated)),
})
if item.Title != "" {
items = append(items, item)
@@ -268,6 +292,7 @@ func normalizeFeedItem(item FeedItem) FeedItem {
item.Summary = strings.TrimSpace(item.Summary)
item.Content = strings.TrimSpace(item.Content)
item.URL = truncateRunes(strings.TrimSpace(item.URL), 2048)
item.ImageURL = truncateRunes(strings.TrimSpace(item.ImageURL), 2048)
key := firstNonEmpty(strings.TrimSpace(item.ExternalID), item.URL)
if key == "" {
published := ""
@@ -305,6 +330,32 @@ func plainText(value string) string {
return strings.Join(strings.Fields(html.UnescapeString(value)), " ")
}
func htmlImageURL(value string) string {
match := htmlImagePattern.FindStringSubmatch(value)
if len(match) < 2 {
return ""
}
return strings.TrimSpace(html.UnescapeString(match[1]))
}
func imageMediaURL(media xmlMedia) string {
mediaType := strings.ToLower(strings.TrimSpace(media.Type))
medium := strings.ToLower(strings.TrimSpace(media.Medium))
if media.URL == "" || (mediaType != "" && !strings.HasPrefix(mediaType, "image/") && medium != "image") {
return ""
}
return strings.TrimSpace(media.URL)
}
func firstImageMediaURL(media []xmlMedia) string {
for _, item := range media {
if imageURL := imageMediaURL(item); imageURL != "" {
return imageURL
}
}
return ""
}
func atomLink(links []xmlAtomLink) string {
for _, link := range links {
if link.Rel == "" || link.Rel == "alternate" {
@@ -317,6 +368,16 @@ func atomLink(links []xmlAtomLink) string {
return ""
}
func atomImageURL(links []xmlAtomLink) string {
for _, link := range links {
if strings.EqualFold(link.Rel, "enclosure") &&
strings.HasPrefix(strings.ToLower(strings.TrimSpace(link.Type)), "image/") {
return strings.TrimSpace(link.Href)
}
}
return ""
}
func firstNonEmpty(values ...string) string {
for _, value := range values {
if strings.TrimSpace(value) != "" {

6
backend/internal/logic/dataset/feed_test.go
View File

@@ -21,6 +21,7 @@ func TestParseFeedDetectsJSONFeedFromContent(t *testing.T) {
"title": "JSON item",
"summary": "JSON summary",
"content_html": "<p>JSON <strong>content</strong></p>",
"image": "https://example.com/json-image.jpg",
"date_published": "2026-07-23T08:00:00Z"
}]
}`), "text/plain")
@@ -31,6 +32,7 @@ func TestParseFeedDetectsJSONFeedFromContent(t *testing.T) {
require.Equal(t, "JSON item", feed.Items[0].Title)
require.Equal(t, "JSON content", feed.Items[0].Content)
require.Equal(t, "https://example.com/json-1", feed.Items[0].URL)
require.Equal(t, "https://example.com/json-image.jpg", feed.Items[0].ImageURL)
require.NotEmpty(t, feed.Items[0].ExternalID)
require.NotNil(t, feed.Items[0].PublishedAt)
}
@@ -45,6 +47,7 @@ func TestParseFeedDetectsRSSXML(t *testing.T) {
<guid>rss-1</guid>
<description><![CDATA[<p>RSS summary</p>]]></description>
<content:encoded xmlns:content="http://purl.org/rss/1.0/modules/content/"><![CDATA[<p>RSS content</p>]]></content:encoded>
<media:thumbnail xmlns:media="http://search.yahoo.com/mrss/" url="https://example.com/rss-image.jpg"/>
<pubDate>Thu, 23 Jul 2026 08:00:00 +0000</pubDate>
</item>
</channel>
@@ -55,6 +58,7 @@ func TestParseFeedDetectsRSSXML(t *testing.T) {
require.Len(t, feed.Items, 1)
require.Equal(t, "RSS summary", feed.Items[0].Summary)
require.Equal(t, "RSS content", feed.Items[0].Content)
require.Equal(t, "https://example.com/rss-image.jpg", feed.Items[0].ImageURL)
require.NotNil(t, feed.Items[0].PublishedAt)
}
@@ -65,6 +69,7 @@ func TestParseFeedDetectsAtomXML(t *testing.T) {
<title>Atom item</title>
<id>atom-1</id>
<link rel="alternate" href="https://example.com/atom-1"/>
<link rel="enclosure" type="image/jpeg" href="https://example.com/atom-image.jpg"/>
<summary>Atom summary</summary>
<content>Atom content</content>
<updated>2026-07-23T08:00:00Z</updated>
@@ -75,6 +80,7 @@ func TestParseFeedDetectsAtomXML(t *testing.T) {
require.Equal(t, "atom", feed.Format)
require.Len(t, feed.Items, 1)
require.Equal(t, "https://example.com/atom-1", feed.Items[0].URL)
require.Equal(t, "https://example.com/atom-image.jpg", feed.Items[0].ImageURL)
require.Equal(t, "Atom content", feed.Items[0].Content)
}

8
backend/internal/logic/dataset/handlers.go
View File

@@ -40,6 +40,7 @@ type ItemDTO struct {
Summary string `json:"summary"`
Content string `json:"content"`
URL string `json:"url"`
ImageURL string `json:"imageUrl"`
Status string `json:"status"`
Starred bool `json:"starred"`
PublishedAt *time.Time `json:"publishedAt"`
@@ -231,6 +232,7 @@ func (h *Handler) createItem(c *gin.Context) {
Summary string `json:"summary"`
Content string `json:"content"`
URL string `json:"url"`
ImageURL string `json:"imageUrl"`
PublishedAt string `json:"publishedAt"`
}
if err := c.ShouldBindJSON(&input); err != nil {
@@ -244,7 +246,8 @@ func (h *Handler) createItem(c *gin.Context) {
}
item, err := h.service.CreateItem(userID, ItemInput{
SourceIdentity: input.SourceID, Title: input.Title, Summary: input.Summary,
Content: input.Content, URL: input.URL, PublishedAt: publishedAt,
Content: input.Content, URL: input.URL, ImageURL: input.ImageURL,
PublishedAt: publishedAt,
})
if err != nil {
writeError(c, err)
@@ -366,7 +369,8 @@ func sourceDTO(source models.SaDatasetSource, itemCount int64) SourceDTO {
func itemDTO(item models.SaDatasetItem) ItemDTO {
return ItemDTO{
ID: item.Identity, SourceID: item.SourceIdentity, Title: item.Title,
Summary: item.Summary, Content: item.Content, URL: item.URL, Status: item.Status,
Summary: item.Summary, Content: item.Content, URL: item.URL,
ImageURL: item.ImageURL, Status: item.Status,
Starred: item.Starred, PublishedAt: utcTime(item.PublishedAt),
CreatedAt: item.CreatedAt.UTC(), UpdatedAt: item.UpdatedAt.UTC(),
}

4
backend/internal/logic/dataset/handlers_test.go
View File

@@ -51,12 +51,13 @@ func TestDatasetAPIConnectsSourcesItemsAndSyncTasks(t *testing.T) {
createdItem := performDatasetRequest(t, ownerRouter, http.MethodPost, "/api/v1/dataset-items", map[string]any{
"sourceId": source.ID, "title": "A collected article", "summary": "Summary",
"url": "https://example.com/article",
"url": "https://example.com/article", "imageUrl": "https://example.com/article.jpg",
})
require.Equal(t, http.StatusCreated, createdItem.Code)
var item ItemDTO
require.NoError(t, json.Unmarshal(createdItem.Body.Bytes(), &item))
require.Equal(t, source.ID, item.SourceID)
require.Equal(t, "https://example.com/article.jpg", item.ImageURL)
legacyItems := performDatasetRequest(t, ownerRouter, http.MethodGet, "/api/v1/dataset-items", nil)
require.Equal(t, http.StatusOK, legacyItems.Code)
var legacyItemList []ItemDTO
@@ -269,6 +270,7 @@ func (staticFeedFetcher) Fetch(context.Context, string) (ParsedFeed, error) {
Summary: "Collected summary",
Content: "Collected content",
URL: "https://example.com/collected",
ImageURL: "https://example.com/collected.jpg",
}},
}, nil
}

11
backend/internal/logic/dataset/service.go
View File

@@ -60,6 +60,7 @@ type ItemInput struct {
Summary string
Content string
URL string
ImageURL string
PublishedAt *time.Time
}
@@ -293,10 +294,15 @@ func (s *Service) CreateItem(userID uint, input ItemInput) (*models.SaDatasetIte
if itemURL != "" && !validHTTPURL(itemURL) {
return nil, ErrURLInvalid
}
imageURL := strings.TrimSpace(input.ImageURL)
if imageURL != "" && !validHTTPURL(imageURL) {
return nil, ErrURLInvalid
}
item := &models.SaDatasetItem{
SourceID: source.ID, Title: title,
Summary: strings.TrimSpace(input.Summary), Content: strings.TrimSpace(input.Content),
URL: itemURL, Status: "unread", PublishedAt: utcOptionalTime(input.PublishedAt),
URL: itemURL, ImageURL: imageURL, Status: "unread",
PublishedAt: utcOptionalTime(input.PublishedAt),
}
return item, s.db.Create(item).Error
}
@@ -461,7 +467,8 @@ func storeFeedItems(tx *gorm.DB, source models.SaDatasetSource, items []FeedItem
item := models.SaDatasetItem{
SourceID: source.ID, ExternalID: &externalID,
Title: input.Title, Summary: input.Summary, Content: input.Content,
URL: input.URL, Status: "unread", PublishedAt: input.PublishedAt,
URL: input.URL, ImageURL: input.ImageURL,
Status: "unread", PublishedAt: input.PublishedAt,
}
result := tx.Clauses(clause.OnConflict{
Columns: []clause.Column{{Name: "source_id"}, {Name: "external_id"}},

1
backend/internal/models/dataset_item.go
View File

@@ -12,6 +12,7 @@ type SaDatasetItem struct {
Summary string `gorm:"type:text"`
Content string `gorm:"type:text"`
URL string `gorm:"size:2048"`
ImageURL string `gorm:"size:2048"`
Status string `gorm:"size:32;not null;default:unread;index"`
Starred bool `gorm:"not null;default:false;index"`
PublishedAt *time.Time