senlinai/agent
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: add dataset item images

Browse Source
This commit is contained in:
yanweidong
2026-07-23 23:15:43 +08:00
parent fbd142803a
commit 56d96f23dd
9 changed files with 109 additions and 15 deletions
Download Patch File Download Diff File Expand all files Collapse all files

81
backend/internal/logic/dataset/feed.go
View File

@@ -26,6 +26,7 @@ var (
ErrFeedTooLarge = errors.New("feed response is too large")
ErrUnsafeFeedURL = errors.New("feed url resolves to a private or local address")
htmlTagPattern = regexp.MustCompile(`<[^>]*>`)
htmlImagePattern = regexp.MustCompile(`(?i)<img[^>]+src\s*=\s*["']([^"']+)["']`)
)
type FeedItem struct {
@@ -34,6 +35,7 @@ type FeedItem struct {
Summary string
Content string
URL string
ImageURL string
PublishedAt *time.Time
}
@@ -156,6 +158,8 @@ type jsonFeedItem struct {
Summary string `json:"summary"`
ContentText string `json:"content_text"`
ContentHTML string `json:"content_html"`
Image string `json:"image"`
BannerImage string `json:"banner_image"`
DatePublished string `json:"date_published"`
}
@@ -175,7 +179,8 @@ func parseJSONFeed(body []byte) (ParsedFeed, error) {
}
item := normalizeFeedItem(FeedItem{
ExternalID: input.ID, Title: plainText(input.Title), Summary: plainText(input.Summary),
Content: content, URL: input.URL, PublishedAt: parseFeedTime(input.DatePublished),
Content: content, URL: input.URL, ImageURL: firstNonEmpty(input.Image, input.BannerImage),
PublishedAt: parseFeedTime(input.DatePublished),
})
if item.Title != "" {
items = append(items, item)
@@ -194,13 +199,16 @@ type xmlFeedDocument struct {
}
type xmlFeedItem struct {
Title string `xml:"title"`
Link string `xml:"link"`
GUID string `xml:"guid"`
Description string `xml:"description"`
Content string `xml:"encoded"`
PubDate string `xml:"pubDate"`
Date string `xml:"date"`
Title string `xml:"title"`
Link string `xml:"link"`
GUID string `xml:"guid"`
Description string `xml:"description"`
Content string `xml:"encoded"`
PubDate string `xml:"pubDate"`
Date string `xml:"date"`
Enclosure xmlMedia `xml:"enclosure"`
Thumbnail xmlMedia `xml:"thumbnail"`
Media []xmlMedia `xml:"content"`
}
type xmlAtomEntry struct {
@@ -216,6 +224,13 @@ type xmlAtomEntry struct {
type xmlAtomLink struct {
Href string `xml:"href,attr"`
Rel string `xml:"rel,attr"`
Type string `xml:"type,attr"`
}
type xmlMedia struct {
URL string `xml:"url,attr"`
Type string `xml:"type,attr"`
Medium string `xml:"medium,attr"`
}
func parseXMLFeed(body []byte) (ParsedFeed, error) {
@@ -238,7 +253,15 @@ func parseXMLFeed(body []byte) (ParsedFeed, error) {
item := normalizeFeedItem(FeedItem{
ExternalID: input.GUID, Title: plainText(input.Title),
Summary: plainText(input.Description), Content: content,
URL: strings.TrimSpace(input.Link), PublishedAt: parseFeedTime(firstNonEmpty(input.PubDate, input.Date)),
URL: strings.TrimSpace(input.Link),
ImageURL: firstNonEmpty(
imageMediaURL(input.Enclosure),
imageMediaURL(input.Thumbnail),
firstImageMediaURL(input.Media),
htmlImageURL(input.Content),
htmlImageURL(input.Description),
),
PublishedAt: parseFeedTime(firstNonEmpty(input.PubDate, input.Date)),
})
if item.Title != "" {
items = append(items, item)
@@ -251,7 +274,8 @@ func parseXMLFeed(body []byte) (ParsedFeed, error) {
item := normalizeFeedItem(FeedItem{
ExternalID: input.ID, Title: plainText(input.Title),
Summary: plainText(input.Summary), Content: plainText(input.Content),
URL: atomLink(input.Links), PublishedAt: parseFeedTime(firstNonEmpty(input.Published, input.Updated)),
URL: atomLink(input.Links), ImageURL: atomImageURL(input.Links),
PublishedAt: parseFeedTime(firstNonEmpty(input.Published, input.Updated)),
})
if item.Title != "" {
items = append(items, item)
@@ -268,6 +292,7 @@ func normalizeFeedItem(item FeedItem) FeedItem {
item.Summary = strings.TrimSpace(item.Summary)
item.Content = strings.TrimSpace(item.Content)
item.URL = truncateRunes(strings.TrimSpace(item.URL), 2048)
item.ImageURL = truncateRunes(strings.TrimSpace(item.ImageURL), 2048)
key := firstNonEmpty(strings.TrimSpace(item.ExternalID), item.URL)
if key == "" {
published := ""
@@ -305,6 +330,32 @@ func plainText(value string) string {
return strings.Join(strings.Fields(html.UnescapeString(value)), " ")
}
func htmlImageURL(value string) string {
match := htmlImagePattern.FindStringSubmatch(value)
if len(match) < 2 {
return ""
}
return strings.TrimSpace(html.UnescapeString(match[1]))
}
func imageMediaURL(media xmlMedia) string {
mediaType := strings.ToLower(strings.TrimSpace(media.Type))
medium := strings.ToLower(strings.TrimSpace(media.Medium))
if media.URL == "" || (mediaType != "" && !strings.HasPrefix(mediaType, "image/") && medium != "image") {
return ""
}
return strings.TrimSpace(media.URL)
}
func firstImageMediaURL(media []xmlMedia) string {
for _, item := range media {
if imageURL := imageMediaURL(item); imageURL != "" {
return imageURL
}
}
return ""
}
func atomLink(links []xmlAtomLink) string {
for _, link := range links {
if link.Rel == "" || link.Rel == "alternate" {
@@ -317,6 +368,16 @@ func atomLink(links []xmlAtomLink) string {
return ""
}
func atomImageURL(links []xmlAtomLink) string {
for _, link := range links {
if strings.EqualFold(link.Rel, "enclosure") &&
strings.HasPrefix(strings.ToLower(strings.TrimSpace(link.Type)), "image/") {
return strings.TrimSpace(link.Href)
}
}
return ""
}
func firstNonEmpty(values ...string) string {
for _, value := range values {
if strings.TrimSpace(value) != "" {