senlinai/agent
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: harden exploration data flow

Browse Source
This commit is contained in:
yanweidong
2026-07-23 23:29:14 +08:00
parent 400d768ecf
commit bfcfd8d32e
7 changed files with 163 additions and 24 deletions
Download Patch File Download Diff File Expand all files Collapse all files

41
backend/internal/logic/dataset/feed.go
View File

@@ -152,15 +152,21 @@ type jsonFeedDocument struct {
}
type jsonFeedItem struct {
ID string `json:"id"`
URL string `json:"url"`
Title string `json:"title"`
Summary string `json:"summary"`
ContentText string `json:"content_text"`
ContentHTML string `json:"content_html"`
Image string `json:"image"`
BannerImage string `json:"banner_image"`
DatePublished string `json:"date_published"`
ID string `json:"id"`
URL string `json:"url"`
Title string `json:"title"`
Summary string `json:"summary"`
ContentText string `json:"content_text"`
ContentHTML string `json:"content_html"`
Image string `json:"image"`
BannerImage string `json:"banner_image"`
Attachments []jsonFeedAttachment `json:"attachments"`
DatePublished string `json:"date_published"`
}
type jsonFeedAttachment struct {
URL string `json:"url"`
MIMEType string `json:"mime_type"`
}
func parseJSONFeed(body []byte) (ParsedFeed, error) {
@@ -179,7 +185,13 @@ func parseJSONFeed(body []byte) (ParsedFeed, error) {
}
item := normalizeFeedItem(FeedItem{
ExternalID: input.ID, Title: plainText(input.Title), Summary: plainText(input.Summary),
Content: content, URL: input.URL, ImageURL: firstNonEmpty(input.Image, input.BannerImage),
Content: content, URL: input.URL,
ImageURL: firstNonEmpty(
input.Image,
input.BannerImage,
firstJSONImageAttachment(input.Attachments),
htmlImageURL(input.ContentHTML),
),
PublishedAt: parseFeedTime(input.DatePublished),
})
if item.Title != "" {
@@ -338,6 +350,15 @@ func htmlImageURL(value string) string {
return strings.TrimSpace(html.UnescapeString(match[1]))
}
func firstJSONImageAttachment(attachments []jsonFeedAttachment) string {
for _, attachment := range attachments {
if strings.HasPrefix(strings.ToLower(strings.TrimSpace(attachment.MIMEType)), "image/") {
return strings.TrimSpace(attachment.URL)
}
}
return ""
}
func imageMediaURL(media xmlMedia) string {
mediaType := strings.ToLower(strings.TrimSpace(media.Type))
medium := strings.ToLower(strings.TrimSpace(media.Medium))