fix: harden exploration data flow
This commit is contained in:
@@ -350,6 +350,13 @@ func (s *Service) DepositItem(userID uint, itemIdentity, projectIdentity string)
|
||||
if body == "" {
|
||||
body = strings.TrimSpace(item.Summary)
|
||||
}
|
||||
if item.ImageURL != "" {
|
||||
if body != "" {
|
||||
body = "\n\n" + body
|
||||
} else {
|
||||
body = ""
|
||||
}
|
||||
}
|
||||
if item.URL != "" {
|
||||
if body != "" {
|
||||
body += "\n\n"
|
||||
@@ -464,10 +471,12 @@ func storeFeedItems(tx *gorm.DB, source models.SaDatasetSource, items []FeedItem
|
||||
inserted := 0
|
||||
for _, input := range items {
|
||||
externalID := input.ExternalID
|
||||
itemURL := resolveFeedReference(source.URL, input.URL)
|
||||
imageURL := resolveFeedReference(source.URL, input.ImageURL)
|
||||
item := models.SaDatasetItem{
|
||||
SourceID: source.ID, ExternalID: &externalID,
|
||||
Title: input.Title, Summary: input.Summary, Content: input.Content,
|
||||
URL: input.URL, ImageURL: input.ImageURL,
|
||||
URL: itemURL, ImageURL: imageURL,
|
||||
Status: "unread", PublishedAt: input.PublishedAt,
|
||||
}
|
||||
result := tx.Clauses(clause.OnConflict{
|
||||
@@ -478,10 +487,58 @@ func storeFeedItems(tx *gorm.DB, source models.SaDatasetSource, items []FeedItem
|
||||
return inserted, result.Error
|
||||
}
|
||||
inserted += int(result.RowsAffected)
|
||||
if result.RowsAffected == 0 {
|
||||
updates := map[string]any{
|
||||
"title": item.Title,
|
||||
}
|
||||
if item.Summary != "" {
|
||||
updates["summary"] = item.Summary
|
||||
}
|
||||
if item.Content != "" {
|
||||
updates["content"] = item.Content
|
||||
}
|
||||
if item.URL != "" {
|
||||
updates["url"] = item.URL
|
||||
}
|
||||
if item.ImageURL != "" {
|
||||
updates["image_url"] = item.ImageURL
|
||||
}
|
||||
if item.PublishedAt != nil {
|
||||
updates["published_at"] = item.PublishedAt
|
||||
}
|
||||
if err := tx.Model(&models.SaDatasetItem{}).
|
||||
Where("source_id = ? AND external_id = ?", source.ID, externalID).
|
||||
Updates(updates).Error; err != nil {
|
||||
return inserted, err
|
||||
}
|
||||
}
|
||||
}
|
||||
return inserted, nil
|
||||
}
|
||||
|
||||
func resolveFeedReference(baseURL, reference string) string {
|
||||
reference = strings.TrimSpace(reference)
|
||||
if reference == "" {
|
||||
return ""
|
||||
}
|
||||
parsedReference, err := url.Parse(reference)
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
if !parsedReference.IsAbs() {
|
||||
parsedBase, err := url.Parse(baseURL)
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
parsedReference = parsedBase.ResolveReference(parsedReference)
|
||||
}
|
||||
resolved := parsedReference.String()
|
||||
if !validHTTPURL(resolved) {
|
||||
return ""
|
||||
}
|
||||
return truncateRunes(resolved, 2048)
|
||||
}
|
||||
|
||||
func truncateResult(value string) string {
|
||||
return truncateRunes(value, 2000)
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user