feat: schedule dataset collection

backend/internal/logic/dataset/feed_test.go

@@ -119,6 +119,27 @@ func TestSyncSourcesRecordsFeedFailure(t *testing.T) {
 	require.Contains(t, crons[0].LastResult, "feed unavailable")
 }
 
+func TestSyncAllSourcesCollectsEnabledRSSSourcesForEveryUser(t *testing.T) {
+	database := newDatasetTestDatabase(t)
+	firstUser := createDatasetTestUser(t, database, "first-feed-owner@example.com")
+	secondUser := createDatasetTestUser(t, database, "second-feed-owner@example.com")
+	for _, source := range []models.SaDatasetSource{
+		{CreatedBy: firstUser.ID, Name: "First RSS", Kind: "rss", URL: "https://example.com/first", Enabled: true},
+		{CreatedBy: secondUser.ID, Name: "Second RSS", Kind: "rss", URL: "https://example.com/second", Enabled: true},
+		{CreatedBy: firstUser.ID, Name: "Manual", Kind: "manual", Enabled: true},
+	} {
+		require.NoError(t, database.Create(&source).Error)
+	}
+
+	crons, err := newServiceWithFetcher(database, staticFeedFetcher{}).SyncAllSources(context.Background())
+
+	require.NoError(t, err)
+	require.Len(t, crons, 2)
+	var itemCount int64
+	require.NoError(t, database.Model(&models.SaDatasetItem{}).Count(&itemCount).Error)
+	require.Equal(t, int64(2), itemCount)
+}
+
 type failingFeedFetcher struct{}
 
 func (failingFeedFetcher) Fetch(context.Context, string) (ParsedFeed, error) {

backend/internal/logic/dataset/service.go

@@ -6,6 +6,7 @@ import (
 	"fmt"
 	"net/url"
 	"strings"
+	"sync"
 	"time"
 
 	"gorm.io/gorm"
@@ -23,6 +24,7 @@ var (
 type Service struct {
 	db      *gorm.DB
 	fetcher FeedFetcher
+	syncMu  sync.Mutex
 }
 
 type SourceInput struct {
@@ -216,6 +218,38 @@ func (s *Service) ListCrons(userID uint) ([]models.SaDatasetCron, error) {
 }
 
 func (s *Service) SyncSources(ctx context.Context, userID uint) ([]models.SaDatasetCron, error) {
+	s.syncMu.Lock()
+	defer s.syncMu.Unlock()
+
+	return s.syncSources(ctx, userID)
+}
+
+func (s *Service) SyncAllSources(ctx context.Context) ([]models.SaDatasetCron, error) {
+	var userIDs []uint
+	if err := s.db.Model(&models.SaDatasetSource{}).
+		Distinct("created_by").
+		Where("enabled = ? AND kind = ?", true, "rss").
+		Order("created_by asc").
+		Pluck("created_by", &userIDs).Error; err != nil {
+		return nil, err
+	}
+
+	result := make([]models.SaDatasetCron, 0)
+	var syncErrors []error
+	for _, userID := range userIDs {
+		crons, err := s.SyncSources(ctx, userID)
+		result = append(result, crons...)
+		if err != nil {
+			syncErrors = append(syncErrors, fmt.Errorf("sync dataset sources for user %d: %w", userID, err))
+		}
+		if ctx.Err() != nil {
+			break
+		}
+	}
+	return result, errors.Join(syncErrors...)
+}
+
+func (s *Service) syncSources(ctx context.Context, userID uint) ([]models.SaDatasetCron, error) {
 	result := make([]models.SaDatasetCron, 0)
 	var sources []models.SaDatasetSource
 	if err := s.db.Where("created_by = ? AND enabled = ? AND kind = ?", userID, true, "rss").
@@ -223,6 +257,9 @@ func (s *Service) SyncSources(ctx context.Context, userID uint) ([]models.SaData
 		return nil, err
 	}
 	for _, source := range sources {
+		if err := ctx.Err(); err != nil {
+			return result, err
+		}
 		now := time.Now().UTC()
 		cron := models.SaDatasetCron{
 			SourceID: source.ID, CreatedBy: userID, Schedule: "@once",
@@ -244,6 +281,9 @@ func (s *Service) SyncSources(ctx context.Context, userID uint) ([]models.SaData
 				return nil, err
 			}
 			result = append(result, cron)
+			if err := ctx.Err(); err != nil {
+				return result, err
+			}
 			continue
 		}