fix: make log alert delivery durable

internal/ingest/replay.go

@@ -9,6 +9,7 @@ import (
 
 	"git.apinb.com/ops/logs/internal/impl"
 	"git.apinb.com/ops/logs/internal/models"
+	"gorm.io/gorm"
 )
 
 func BuildReplayRawEventPayload(ev models.LogEvent) (RawEventIngestBody, error) {
@@ -16,9 +17,10 @@ func BuildReplayRawEventPayload(ev models.LogEvent) (RawEventIngestBody, error)
 	if sourceType == "" {
 		return RawEventIngestBody{}, fmt.Errorf("unsupported source kind %q", ev.SourceKind)
 	}
+	now := time.Now().UTC()
 	rawObj := map[string]interface{}{
 		"source":       sourceType,
-		"replayed_at":  time.Now().UTC().Format(time.RFC3339),
+		"replayed_at":  now.Format(time.RFC3339Nano),
 		"log_entry_id": ev.ID,
 		"source_ip":    ev.SourceIP,
 		"remote_addr":  ev.RemoteAddr,
@@ -46,22 +48,26 @@ func BuildReplayRawEventPayload(ev models.LogEvent) (RawEventIngestBody, error)
 		"device":                 ev.DeviceName,
 		"job":                    "logs-replay",
 	}
-	if uid := replayResourceUID(ev); uid != "" {
-		labels["resource_uid"] = uid
-	}
+	resourceUID := replayResourceUID(ev)
+	category, identity := splitResourceUID(resourceUID)
+	labels["resource_uid"] = resourceUID
+	labels["resource_category"] = category
+	labels["service_identity"] = identity
 	return RawEventIngestBody{
-		SourceType:  sourceType,
-		ResourceUID: replayResourceUID(ev),
-		EventTime:   time.Now().UTC(),
-		Severity:    firstNonEmpty(ev.SeverityCode, "warning"),
-		Title:       replayTitle(ev),
-		Message:     firstNonEmpty(ev.NormalizedSummary, ev.RawPayload),
-		Labels:      labels,
+		SourceType:     sourceType,
+		SourceEventKey: fmt.Sprintf("logs:replay:%d:%d", ev.ID, now.UnixNano()),
+		State:          "firing",
+		ResourceUID:    resourceUID,
+		EventTime:      now,
+		Severity:       firstNonEmpty(ev.SeverityCode, "warning"),
+		Title:          replayTitle(ev),
+		Message:        firstNonEmpty(ev.NormalizedSummary, ev.RawPayload),
+		Labels:         labels,
 		Annotations: map[string]string{
 			"replay":          "true",
 			"dispatch_status": ev.DispatchStatus,
 		},
-		ParseStatus: "replayed",
+		ParseStatus: "parsed",
 		RawPayload:  rawBytes,
 	}, nil
 }
@@ -80,16 +86,42 @@ func EnqueueReplayLogEvent(ev models.LogEvent) (uint, error) {
 		PayloadJSON: string(payload),
 		Status:      outboxStatusPending,
 		RetryCount:  0,
-		NextRetryAt: time.Now(),
 	}
-	if err := enqueueOutboxRow(&row); err != nil {
+	if impl.DBService == nil {
+		return 0, fmt.Errorf("database is not initialized")
+	}
+	if err := impl.DBService.Transaction(func(tx *gorm.DB) error {
+		now, err := databaseNow(tx)
+		if err != nil {
+			return err
+		}
+		row.NextRetryAt = now
+		if err := enqueueOutboxRowWithDB(tx, &row); err != nil {
+			return err
+		}
+		result := tx.Model(&models.LogEvent{}).Where("id = ?", ev.ID).Updates(map[string]interface{}{
+			"dispatch_status":    "pending",
+			"dispatch_outbox_id": row.ID,
+			"alert_sent":         false,
+		})
+		if result.Error != nil {
+			return result.Error
+		}
+		if result.RowsAffected == 0 {
+			return fmt.Errorf("log event %d does not exist", ev.ID)
+		}
+		return nil
+	}); err != nil {
 		return 0, err
 	}
 	return row.ID, nil
 }
 
-func enqueueOutboxRow(row *models.AlertOutbox) error {
-	return impl.DBService.Create(row).Error
+func enqueueOutboxRowWithDB(db *gorm.DB, row *models.AlertOutbox) error {
+	if db == nil {
+		return fmt.Errorf("database is not initialized")
+	}
+	return db.Create(row).Error
 }
 
 func replaySourceType(kind string) string {
@@ -111,13 +143,32 @@ func replaySubtype(kind string) string {
 }
 
 func replayResourceUID(ev models.LogEvent) string {
-	if strings.Contains(ev.ResourceID, ":") {
-		return ev.ResourceID
+	if uid := strings.TrimSpace(ev.ResourceUID); uid != "" {
+		if category, identity, ok := splitKnownResourceUID(uid); ok {
+			return category + ":" + identity
+		}
+		return canonicalLogResourceCategory(ev.ResourceType, replaySubtype(ev.SourceKind)) + ":" + uid
+	}
+	if category, identity, ok := splitKnownResourceUID(ev.ResourceID); ok {
+		return category + ":" + identity
 	}
 	if ev.ResourceType != "" && ev.ResourceID != "" {
-		return ev.ResourceType + ":" + ev.ResourceID
+		return canonicalLogResourceCategory(ev.ResourceType, replaySubtype(ev.SourceKind)) + ":" + ev.ResourceID
 	}
-	return ""
+	category := canonicalLogResourceCategory("", replaySubtype(ev.SourceKind))
+	identity := firstNonEmpty(strings.TrimSpace(ev.SourceIP), strings.TrimSpace(ev.DeviceName))
+	if identity == "" {
+		identity = fmt.Sprintf("log-event-%d", ev.ID)
+	}
+	return category + ":" + identity
+}
+
+func splitResourceUID(uid string) (string, string) {
+	parts := strings.SplitN(strings.TrimSpace(uid), ":", 2)
+	if len(parts) != 2 {
+		return "log_source", strings.TrimSpace(uid)
+	}
+	return parts[0], parts[1]
 }
 
 func replayTitle(ev models.LogEvent) string {