feat: collect JSON and XML feeds

backend/internal/logic/dataset/service.go

@@ -1,7 +1,9 @@
 package dataset
 
 import (
+	"context"
 	"errors"
+	"fmt"
 	"net/url"
 	"strings"
 	"time"
@@ -19,7 +21,8 @@ var (
 )
 
 type Service struct {
-	db *gorm.DB
+	db      *gorm.DB
+	fetcher FeedFetcher
 }
 
 type SourceInput struct {
@@ -51,7 +54,11 @@ type SourceRecord struct {
 }
 
 func NewService(database *gorm.DB) *Service {
-	return &Service{db: database}
+	return &Service{db: database, fetcher: NewHTTPFeedFetcher()}
+}
+
+func newServiceWithFetcher(database *gorm.DB, fetcher FeedFetcher) *Service {
+	return &Service{db: database, fetcher: fetcher}
 }
 
 func (s *Service) ListSources(userID uint) ([]SourceRecord, error) {
@@ -208,36 +215,111 @@ func (s *Service) ListCrons(userID uint) ([]models.SaDatasetCron, error) {
 	return crons, nil
 }
 
-func (s *Service) QueueSync(userID uint) ([]models.SaDatasetCron, error) {
+func (s *Service) SyncSources(ctx context.Context, userID uint) ([]models.SaDatasetCron, error) {
 	result := make([]models.SaDatasetCron, 0)
-	err := s.db.Transaction(func(tx *gorm.DB) error {
-		var sources []models.SaDatasetSource
-		if err := tx.Where("created_by = ? AND enabled = ?", userID, true).Order("id asc").Find(&sources).Error; err != nil {
-			return err
-		}
+	var sources []models.SaDatasetSource
+	if err := s.db.Where("created_by = ? AND enabled = ? AND kind = ?", userID, true, "rss").
+		Order("id asc").Find(&sources).Error; err != nil {
+		return nil, err
+	}
+	for _, source := range sources {
 		now := time.Now().UTC()
-		for _, source := range sources {
-			var cron models.SaDatasetCron
-			err := tx.Where("source_id = ? AND created_by = ? AND status = ?", source.ID, userID, "pending").First(&cron).Error
-			if err == nil {
-				result = append(result, cron)
-				continue
-			}
-			if !errors.Is(err, gorm.ErrRecordNotFound) {
-				return err
-			}
-			cron = models.SaDatasetCron{
-				SourceID: source.ID, CreatedBy: userID, Schedule: "@once",
-				Status: "pending", Enabled: true, NextRunAt: &now,
-			}
-			if err := tx.Create(&cron).Error; err != nil {
-				return err
+		cron := models.SaDatasetCron{
+			SourceID: source.ID, CreatedBy: userID, Schedule: "@once",
+			Status: "running", Enabled: true, NextRunAt: &now, LastRunAt: &now,
+		}
+		if err := s.db.Create(&cron).Error; err != nil {
+			return nil, err
+		}
+
+		feed, fetchErr := s.fetcher.Fetch(ctx, source.URL)
+		if fetchErr != nil {
+			cron.Status = "failed"
+			cron.Enabled = false
+			cron.NextRunAt = nil
+			cron.LastResult = truncateResult(fetchErr.Error())
+			if err := s.db.Model(&cron).Updates(map[string]any{
+				"status": cron.Status, "last_result": cron.LastResult, "enabled": false, "next_run_at": nil,
+			}).Error; err != nil {
+				return nil, err
 			}
 			result = append(result, cron)
+			continue
+		}
+
+		inserted, err := s.storeFeedItems(userID, source, feed.Items)
+		if err != nil {
+			cron.Status = "failed"
+			cron.Enabled = false
+			cron.NextRunAt = nil
+			cron.LastResult = truncateResult("store feed items: " + err.Error())
+			if updateErr := s.db.Model(&cron).Updates(map[string]any{
+				"status": cron.Status, "last_result": cron.LastResult, "enabled": false, "next_run_at": nil,
+			}).Error; updateErr != nil {
+				return nil, updateErr
+			}
+			result = append(result, cron)
+			continue
+		}
+		completedAt := time.Now().UTC()
+		cron.Status = "completed"
+		cron.Enabled = false
+		cron.NextRunAt = nil
+		cron.LastResult = fmt.Sprintf("format=%s fetched=%d inserted=%d", feed.Format, len(feed.Items), inserted)
+		if err := s.db.Transaction(func(tx *gorm.DB) error {
+			if err := tx.Model(&cron).Updates(map[string]any{
+				"status": cron.Status, "last_result": cron.LastResult, "enabled": false, "next_run_at": nil,
+			}).Error; err != nil {
+				return err
+			}
+			return tx.Model(&source).Update("last_synced_at", completedAt).Error
+		}); err != nil {
+			return nil, err
+		}
+		source.LastSyncedAt = &completedAt
+		result = append(result, cron)
+	}
+	return result, nil
+}
+
+func (s *Service) storeFeedItems(userID uint, source models.SaDatasetSource, items []FeedItem) (int, error) {
+	inserted := 0
+	err := s.db.Transaction(func(tx *gorm.DB) error {
+		for _, input := range items {
+			var count int64
+			if err := tx.Model(&models.SaDatasetItem{}).
+				Where("source_id = ? AND external_id = ?", source.ID, input.ExternalID).
+				Count(&count).Error; err != nil {
+				return err
+			}
+			if count > 0 {
+				continue
+			}
+			item := models.SaDatasetItem{
+				SourceID: source.ID, CreatedBy: userID, ExternalID: input.ExternalID,
+				Title: input.Title, Summary: input.Summary, Content: input.Content,
+				URL: input.URL, Status: "unread", PublishedAt: input.PublishedAt,
+			}
+			if err := tx.Create(&item).Error; err != nil {
+				return err
+			}
+			inserted++
 		}
 		return nil
 	})
-	return result, err
+	return inserted, err
+}
+
+func truncateResult(value string) string {
+	return truncateRunes(value, 2000)
+}
+
+func truncateRunes(value string, limit int) string {
+	runes := []rune(value)
+	if len(runes) <= limit {
+		return value
+	}
+	return string(runes[:limit])
 }
 
 func (s *Service) findOwnedSource(userID uint, identity string) (*models.SaDatasetSource, error) {
@@ -272,7 +354,8 @@ func normalizeSourceInput(input SourceInput) (SourceInput, error) {
 
 func validHTTPURL(value string) bool {
 	parsed, err := url.ParseRequestURI(value)
-	return err == nil && (parsed.Scheme == "http" || parsed.Scheme == "https") && parsed.Host != ""
+	return err == nil && (parsed.Scheme == "http" || parsed.Scheme == "https") &&
+		parsed.Host != "" && parsed.User == nil
 }
 
 func utcOptionalTime(value *time.Time) *time.Time {