fix: make log alert delivery durable

This commit is contained in:
zxr
2026-07-21 02:32:18 +08:00
parent ce559d5218
commit 6e66bad884
8 changed files with 709 additions and 305 deletions

View File

@@ -2,7 +2,9 @@ package ingest
import ( import (
"bytes" "bytes"
"encoding/hex"
"encoding/json" "encoding/json"
"errors"
"fmt" "fmt"
"io" "io"
"net/http" "net/http"
@@ -14,6 +16,13 @@ import (
const alertResponseBodyLimit = 1 << 20 const alertResponseBodyLimit = 1 << 20
var errAlertForwardDisabled = errors.New("Alert 转发未启用或 base_url 为空")
type alertForwardResponse struct {
Code *int32 `json:"code"`
Details json.RawMessage `json:"details"`
}
// AlertReceiveBody 与 alert ReceiveRequest 对齐(含必填 raw_data // AlertReceiveBody 与 alert ReceiveRequest 对齐(含必填 raw_data
type AlertReceiveBody struct { type AlertReceiveBody struct {
AlertName string `json:"alert_name"` AlertName string `json:"alert_name"`
@@ -25,11 +34,16 @@ type AlertReceiveBody struct {
Labels map[string]string `json:"labels"` Labels map[string]string `json:"labels"`
Agent string `json:"agent"` Agent string `json:"agent"`
PolicyID uint `json:"policy_id"` PolicyID uint `json:"policy_id"`
State string `json:"state,omitempty"`
SourceEventKey string `json:"source_event_key"`
OccurredAt time.Time `json:"occurred_at"`
RawData json.RawMessage `json:"raw_data"` RawData json.RawMessage `json:"raw_data"`
} }
type RawEventIngestBody struct { type RawEventIngestBody struct {
SourceType string `json:"source_type"` SourceType string `json:"source_type"`
SourceEventKey string `json:"source_event_key"`
State string `json:"state,omitempty"`
ResourceUID string `json:"resource_uid,omitempty"` ResourceUID string `json:"resource_uid,omitempty"`
EventTime time.Time `json:"event_time"` EventTime time.Time `json:"event_time"`
Severity string `json:"severity"` Severity string `json:"severity"`
@@ -45,7 +59,7 @@ type RawEventIngestBody struct {
func forwardAlert(body AlertReceiveBody) error { func forwardAlert(body AlertReceiveBody) error {
cfg := config.Spec.AlertForward cfg := config.Spec.AlertForward
if cfg == nil || !cfg.Enabled || cfg.BaseURL == "" { if cfg == nil || !cfg.Enabled || cfg.BaseURL == "" {
return nil return errAlertForwardDisabled
} }
if len(body.RawData) == 0 { if len(body.RawData) == 0 {
return fmt.Errorf("raw_data 不能为空") return fmt.Errorf("raw_data 不能为空")
@@ -60,13 +74,37 @@ func forwardAlert(body AlertReceiveBody) error {
if err != nil { if err != nil {
return err return err
} }
return postAlertPayload(cfg, "/Alert/v1/alerts/receive", raw) result, err := postAlertPayload(cfg, "/Alert/v1/alerts/receive", raw)
if err != nil {
return err
}
var details struct {
ID uint `json:"id"`
RawEventID uint `json:"raw_event_id"`
AlertRecordID uint `json:"alert_record_id"`
IncidentID uint `json:"incident_id"`
Status string `json:"status"`
Fingerprint string `json:"fingerprint"`
}
if err := json.Unmarshal(result.Details, &details); err != nil {
return fmt.Errorf("Alert 响应 details 无效:%v请稍后重试", err)
}
alertRecordID := details.AlertRecordID
if alertRecordID == 0 {
alertRecordID = details.ID
}
if details.RawEventID == 0 || alertRecordID == 0 || details.IncidentID == 0 ||
(details.ID != 0 && details.AlertRecordID != 0 && details.ID != details.AlertRecordID) ||
(details.Status != "firing" && details.Status != "resolved") || !validFingerprint(details.Fingerprint) {
return fmt.Errorf("Alert 响应缺少完整写入结果,无法确认转发成功;请稍后重试")
}
return nil
} }
func postAlertPayload(cfg *config.AlertForwardConf, path string, payload []byte) error { func postAlertPayload(cfg *config.AlertForwardConf, path string, payload []byte) (*alertForwardResponse, error) {
req, err := http.NewRequest(http.MethodPost, cfg.BaseURL+path, bytes.NewReader(payload)) req, err := http.NewRequest(http.MethodPost, cfg.BaseURL+path, bytes.NewReader(payload))
if err != nil { if err != nil {
return fmt.Errorf("创建 Alert 转发请求失败:%w", err) return nil, fmt.Errorf("创建 Alert 转发请求失败:%w", err)
} }
req.Header.Set("Content-Type", "application/json") req.Header.Set("Content-Type", "application/json")
if cfg.InternalKey != "" { if cfg.InternalKey != "" {
@@ -80,34 +118,35 @@ func postAlertPayload(cfg *config.AlertForwardConf, path string, payload []byte)
} }
resp, err := client.Do(req) resp, err := client.Do(req)
if err != nil { if err != nil {
return fmt.Errorf("发送 Alert 转发请求失败:%w", err) return nil, fmt.Errorf("发送 Alert 转发请求失败:%w", err)
} }
defer resp.Body.Close() defer resp.Body.Close()
responseBody, err := io.ReadAll(io.LimitReader(resp.Body, alertResponseBodyLimit+1)) responseBody, err := io.ReadAll(io.LimitReader(resp.Body, alertResponseBodyLimit+1))
if err != nil { if err != nil {
return fmt.Errorf("读取 Alert 响应失败:%w", err) return nil, fmt.Errorf("读取 Alert 响应失败:%w", err)
} }
if len(responseBody) > alertResponseBodyLimit { if len(responseBody) > alertResponseBodyLimit {
return fmt.Errorf("Alert 响应体超过 %d 字节限制,请稍后重试", alertResponseBodyLimit) return nil, fmt.Errorf("Alert 响应体超过 %d 字节限制,请稍后重试", alertResponseBodyLimit)
} }
if resp.StatusCode != http.StatusOK { if resp.StatusCode != http.StatusOK {
return fmt.Errorf("Alert 返回 HTTP %d请稍后重试", resp.StatusCode) return nil, fmt.Errorf("Alert 返回 HTTP %d请稍后重试", resp.StatusCode)
} }
var result struct { var result alertForwardResponse
Code *int32 `json:"code"`
}
if err := json.Unmarshal(responseBody, &result); err != nil { if err := json.Unmarshal(responseBody, &result); err != nil {
return fmt.Errorf("Alert 响应不是有效 JSON%v请稍后重试", err) return nil, fmt.Errorf("Alert 响应不是有效 JSON%v请稍后重试", err)
} }
if result.Code == nil { if result.Code == nil {
return fmt.Errorf("Alert 响应缺少 code无法确认转发成功请稍后重试") return nil, fmt.Errorf("Alert 响应缺少 code无法确认转发成功请稍后重试")
} }
if *result.Code != 0 { if *result.Code != 0 {
return fmt.Errorf("Alert 拒绝转发,业务 code=%d请稍后重试", *result.Code) return nil, fmt.Errorf("Alert 拒绝转发,业务 code=%d请稍后重试", *result.Code)
} }
return nil if len(result.Details) == 0 || string(result.Details) == "null" || string(result.Details) == `""` {
return nil, fmt.Errorf("Alert 响应缺少 details无法确认转发成功请稍后重试")
}
return &result, nil
} }
func buildRawEventIngestBody(body AlertReceiveBody, parseStatus string) RawEventIngestBody { func buildRawEventIngestBody(body AlertReceiveBody, parseStatus string) RawEventIngestBody {
@@ -123,8 +162,10 @@ func buildRawEventIngestBody(body AlertReceiveBody, parseStatus string) RawEvent
} }
return RawEventIngestBody{ return RawEventIngestBody{
SourceType: sourceType, SourceType: sourceType,
SourceEventKey: body.SourceEventKey,
State: body.State,
ResourceUID: rawEventResourceUID(body.Labels), ResourceUID: rawEventResourceUID(body.Labels),
EventTime: time.Now().UTC(), EventTime: body.OccurredAt,
Severity: body.SeverityCode, Severity: body.SeverityCode,
Title: firstNonEmpty(body.AlertName, "日志事件"), Title: firstNonEmpty(body.AlertName, "日志事件"),
Message: firstNonEmpty(body.Summary, body.Description), Message: firstNonEmpty(body.Summary, body.Description),
@@ -135,6 +176,14 @@ func buildRawEventIngestBody(body AlertReceiveBody, parseStatus string) RawEvent
} }
} }
func validFingerprint(value string) bool {
if len(value) != 64 {
return false
}
decoded, err := hex.DecodeString(value)
return err == nil && len(decoded) == 32
}
func rawEventSourceType(body AlertReceiveBody) string { func rawEventSourceType(body AlertReceiveBody) string {
if body.Labels != nil { if body.Labels != nil {
switch strings.TrimSpace(body.Labels["source_subtype"]) { switch strings.TrimSpace(body.Labels["source_subtype"]) {

View File

@@ -2,98 +2,179 @@ package ingest
import ( import (
"encoding/json" "encoding/json"
"errors"
"fmt" "fmt"
"log"
"os"
"strings" "strings"
"time" "time"
"git.apinb.com/ops/logs/internal/config" "git.apinb.com/ops/logs/internal/config"
"git.apinb.com/ops/logs/internal/impl" "git.apinb.com/ops/logs/internal/impl"
"git.apinb.com/ops/logs/internal/models" "git.apinb.com/ops/logs/internal/models"
"gorm.io/gorm"
"gorm.io/gorm/clause"
) )
const ( const (
outboxStatusPending = "pending" outboxStatusPending = "pending"
outboxStatusProcessing = "processing"
outboxStatusRetrying = "retrying" outboxStatusRetrying = "retrying"
outboxStatusSent = "sent" outboxStatusSent = "sent"
outboxStatusDead = "dead" outboxStatusDead = "dead"
) )
func enqueueAlert(logEventID uint, body AlertReceiveBody) error { func enqueueAlertWithDB(db *gorm.DB, logEventID uint, body AlertReceiveBody) (uint, error) {
payload, err := json.Marshal(body) payload, err := json.Marshal(body)
if err != nil { if err != nil {
return err return 0, err
} }
return enqueuePayload(logEventID, string(payload)) return enqueuePayloadWithDB(db, logEventID, string(payload))
} }
func enqueueRawEvent(logEventID uint, body AlertReceiveBody, parseStatus string) error { func enqueueRawEventWithDB(db *gorm.DB, logEventID uint, body AlertReceiveBody, parseStatus string) (uint, error) {
payload, err := json.Marshal(buildRawEventIngestBody(body, parseStatus)) payload, err := json.Marshal(buildRawEventIngestBody(body, parseStatus))
if err != nil { if err != nil {
return err return 0, err
} }
return enqueuePayload(logEventID, string(payload)) return enqueuePayloadWithDB(db, logEventID, string(payload))
} }
func enqueuePayload(logEventID uint, payloadJSON string) error { func enqueuePayloadWithDB(db *gorm.DB, logEventID uint, payloadJSON string) (uint, error) {
if db == nil {
return 0, fmt.Errorf("database is not initialized")
}
now, err := databaseNow(db)
if err != nil {
return 0, err
}
row := models.AlertOutbox{ row := models.AlertOutbox{
LogEventID: logEventID, LogEventID: logEventID,
PayloadJSON: payloadJSON, PayloadJSON: payloadJSON,
Status: outboxStatusPending, Status: outboxStatusPending,
RetryCount: 0, RetryCount: 0,
NextRetryAt: time.Now(), NextRetryAt: now,
LastError: "", LastError: "",
} }
return impl.DBService.Create(&row).Error if err := db.Create(&row).Error; err != nil {
return 0, err
}
return row.ID, nil
} }
func StartAlertDispatcher() { func StartAlertDispatcher() {
owner := dispatcherOwner()
go func() { go func() {
ticker := time.NewTicker(2 * time.Second) ticker := time.NewTicker(2 * time.Second)
defer ticker.Stop() defer ticker.Stop()
for range ticker.C { for range ticker.C {
processAlertOutboxBatch(20) if _, err := ProcessAlertOutboxBatch(impl.DBService, 20, owner); err != nil {
log.Printf("logs: alert outbox dispatch: %v", err)
}
} }
}() }()
} }
func processAlertOutboxBatch(limit int) { func dispatcherOwner() string {
host, _ := os.Hostname()
return fmt.Sprintf("%s:%d:%d", host, os.Getpid(), time.Now().UnixNano())
}
// ProcessAlertOutboxBatch 原子领取并处理一批任务,可安全用于多实例 worker。
func ProcessAlertOutboxBatch(db *gorm.DB, limit int, owner string) (int, error) {
if db == nil {
return 0, fmt.Errorf("database is not initialized")
}
if limit <= 0 { if limit <= 0 {
limit = 20 limit = 20
} }
var rows []models.AlertOutbox owner = strings.TrimSpace(owner)
now := time.Now() if owner == "" {
err := impl.DBService. return 0, fmt.Errorf("outbox lease owner is required")
Where("status IN ? AND next_retry_at <= ?", []string{outboxStatusPending, outboxStatusRetrying}, now).
Order("id asc").
Limit(limit).
Find(&rows).Error
if err != nil || len(rows) == 0 {
return
} }
for _, row := range rows { processed := 0
processOneOutbox(row) for processed < limit {
rows, err := claimAlertOutboxBatch(db, 1, owner)
if err != nil {
return processed, err
} }
if len(rows) == 0 {
break
}
if err := processOneOutbox(db, rows[0]); err != nil {
return processed, err
}
processed++
}
return processed, nil
} }
func processOneOutbox(row models.AlertOutbox) { func claimAlertOutboxBatch(db *gorm.DB, limit int, owner string) ([]models.AlertOutbox, error) {
var rows []models.AlertOutbox
now, err := databaseNow(db)
if err != nil {
return nil, err
}
leaseUntil := now.Add(30 * time.Second)
err = db.Transaction(func(tx *gorm.DB) error {
if err := tx.Clauses(clause.Locking{Strength: "UPDATE", Options: "SKIP LOCKED"}).
Where("(status IN ? AND next_retry_at <= ?) OR (status = ? AND lease_until IS NOT NULL AND lease_until <= ?)", []string{outboxStatusPending, outboxStatusRetrying}, now, outboxStatusProcessing, now).
Order("id asc").Limit(limit).Find(&rows).Error; err != nil {
return err
}
if len(rows) == 0 {
return nil
}
ids := make([]uint, 0, len(rows))
for i := range rows {
ids = append(ids, rows[i].ID)
}
if err := tx.Model(&models.AlertOutbox{}).Where("id IN ?", ids).Updates(map[string]interface{}{
"status": outboxStatusProcessing,
"lease_until": leaseUntil,
"lease_owner": owner,
}).Error; err != nil {
return err
}
for i := range rows {
rows[i].Status = outboxStatusProcessing
rows[i].LeaseUntil = &leaseUntil
rows[i].LeaseOwner = owner
}
return nil
})
return rows, err
}
func processOneOutbox(db *gorm.DB, row models.AlertOutbox) error {
var body AlertReceiveBody var body AlertReceiveBody
if err := json.Unmarshal([]byte(row.PayloadJSON), &body); err != nil { if err := json.Unmarshal([]byte(row.PayloadJSON), &body); err != nil {
markOutboxDead(row.ID, row.RetryCount, "invalid_payload: "+err.Error()) now, nowErr := databaseNow(db)
return if nowErr != nil {
return nowErr
} }
if err := forwardOutboxPayload(row.PayloadJSON, body); err != nil { return markOutboxDead(db, row, row.RetryCount, "invalid_payload: "+err.Error(), now)
markOutboxRetry(row, err.Error())
return
} }
_ = impl.DBService.Model(&models.AlertOutbox{}).Where("id = ?", row.ID).Updates(map[string]interface{}{ forwardErr := forwardOutboxPayload(row.PayloadJSON, body)
"status": outboxStatusSent, now, err := databaseNow(db)
"last_error": "", if err != nil {
"next_retry_at": time.Now(), return err
}).Error }
_ = impl.DBService.Model(&models.LogEvent{}).Where("id = ?", row.LogEventID).Updates(map[string]interface{}{ if forwardErr != nil {
"alert_sent": true, if errors.Is(forwardErr, errAlertForwardDisabled) {
"dispatch_status": "sent", return markOutboxDead(db, row, row.RetryCount, forwardErr.Error(), now)
}).Error }
return markOutboxRetry(db, row, forwardErr.Error(), now)
}
return markOutboxSent(db, row, now)
}
func databaseNow(db *gorm.DB) (time.Time, error) {
var now time.Time
if err := db.Raw("SELECT CURRENT_TIMESTAMP").Scan(&now).Error; err != nil {
return time.Time{}, err
}
return now.UTC(), nil
} }
func forwardOutboxPayload(payloadJSON string, legacyBody AlertReceiveBody) error { func forwardOutboxPayload(payloadJSON string, legacyBody AlertReceiveBody) error {
@@ -107,7 +188,7 @@ func forwardOutboxPayload(payloadJSON string, legacyBody AlertReceiveBody) error
func forwardRawEvent(body RawEventIngestBody) error { func forwardRawEvent(body RawEventIngestBody) error {
cfg := config.Spec.AlertForward cfg := config.Spec.AlertForward
if cfg == nil || !cfg.Enabled || cfg.BaseURL == "" { if cfg == nil || !cfg.Enabled || cfg.BaseURL == "" {
return nil return errAlertForwardDisabled
} }
if len(body.RawPayload) == 0 { if len(body.RawPayload) == 0 {
return fmt.Errorf("raw_payload 不能为空") return fmt.Errorf("raw_payload 不能为空")
@@ -116,46 +197,103 @@ func forwardRawEvent(body RawEventIngestBody) error {
if err != nil { if err != nil {
return err return err
} }
return postAlertPayload(cfg, "/Alert/v1/raw-events/ingest", raw) result, err := postAlertPayload(cfg, "/Alert/v1/raw-events/ingest", raw)
if err != nil {
return err
}
var details struct {
ID uint `json:"id"`
RawEventID uint `json:"raw_event_id"`
AlertRecordID uint `json:"alert_record_id"`
IncidentID uint `json:"incident_id"`
ParseStatus string `json:"parse_status"`
Status string `json:"status"`
Fingerprint string `json:"fingerprint"`
}
if err := json.Unmarshal(result.Details, &details); err != nil {
return fmt.Errorf("Alert 原始事件响应 details 无效:%v请稍后重试", err)
}
if body.ParseStatus == "unparsed" {
if details.ID == 0 || (details.ParseStatus != "unparsed" && details.ParseStatus != "replayed") {
return fmt.Errorf("Alert 原始事件响应缺少持久化结果,无法确认转发成功;请稍后重试")
}
return nil
}
rawEventID := details.RawEventID
if rawEventID == 0 {
rawEventID = details.ID
}
if rawEventID == 0 || details.AlertRecordID == 0 || details.IncidentID == 0 ||
(details.ID != 0 && details.RawEventID != 0 && details.ID != details.RawEventID) ||
(details.Status != "firing" && details.Status != "resolved") || !validFingerprint(details.Fingerprint) {
return fmt.Errorf("Alert 原始事件响应缺少完整处理结果,无法确认转发成功;请稍后重试")
}
return nil
} }
func markOutboxRetry(row models.AlertOutbox, msg string) { func markOutboxSent(db *gorm.DB, row models.AlertOutbox, now time.Time) error {
return db.Transaction(func(tx *gorm.DB) error {
result := tx.Model(&models.AlertOutbox{}).
Where("id = ? AND status = ? AND lease_owner = ?", row.ID, outboxStatusProcessing, row.LeaseOwner).
Updates(map[string]interface{}{"status": outboxStatusSent, "last_error": "", "next_retry_at": now, "lease_until": nil, "lease_owner": ""})
if result.Error != nil {
return result.Error
}
if result.RowsAffected == 0 {
return nil
}
return tx.Model(&models.LogEvent{}).Where("id = ? AND dispatch_outbox_id = ?", row.LogEventID, row.ID).Updates(map[string]interface{}{
"alert_sent": true, "dispatch_status": "sent",
}).Error
})
}
func markOutboxRetry(db *gorm.DB, row models.AlertOutbox, msg string, now time.Time) error {
retry := row.RetryCount + 1 retry := row.RetryCount + 1
const maxRetry = 5 const maxRetry = 5
if retry > maxRetry { if retry > maxRetry {
markOutboxDead(row.ID, retry, msg) return markOutboxDead(db, row, retry, msg, now)
return
} }
backoff := time.Duration(retry*retry) * time.Second backoff := time.Duration(retry*retry) * time.Second
if backoff > 60*time.Second { if backoff > 60*time.Second {
backoff = 60 * time.Second backoff = 60 * time.Second
} }
_ = impl.DBService.Model(&models.AlertOutbox{}).Where("id = ?", row.ID).Updates(map[string]interface{}{ return updateClaimedOutbox(db, row, map[string]interface{}{
"status": outboxStatusRetrying, "status": outboxStatusRetrying, "retry_count": retry, "next_retry_at": now.Add(backoff),
"retry_count": retry, "last_error": truncateError(msg, 1024), "lease_until": nil, "lease_owner": "",
"next_retry_at": time.Now().Add(backoff), }, "retrying")
"last_error": truncateError(msg, 1024),
}).Error
_ = impl.DBService.Model(&models.LogEvent{}).Where("id = ?", row.LogEventID).Update("dispatch_status", "retrying").Error
} }
func markOutboxDead(id uint, retry int, msg string) { func markOutboxDead(db *gorm.DB, row models.AlertOutbox, retry int, msg string, now time.Time) error {
_ = impl.DBService.Model(&models.AlertOutbox{}).Where("id = ?", id).Updates(map[string]interface{}{ return updateClaimedOutbox(db, row, map[string]interface{}{
"status": outboxStatusDead, "status": outboxStatusDead, "retry_count": retry, "next_retry_at": now,
"retry_count": retry, "last_error": truncateError(msg, 1024), "lease_until": nil, "lease_owner": "",
"next_retry_at": time.Now(), }, "dead")
"last_error": truncateError(msg, 1024), }
}).Error
var row models.AlertOutbox func updateClaimedOutbox(db *gorm.DB, row models.AlertOutbox, updates map[string]interface{}, eventStatus string) error {
if err := impl.DBService.Select("log_event_id").First(&row, id).Error; err == nil && row.LogEventID > 0 { return db.Transaction(func(tx *gorm.DB) error {
_ = impl.DBService.Model(&models.LogEvent{}).Where("id = ?", row.LogEventID).Update("dispatch_status", "dead").Error result := tx.Model(&models.AlertOutbox{}).
Where("id = ? AND status = ? AND lease_owner = ?", row.ID, outboxStatusProcessing, row.LeaseOwner).
Updates(updates)
if result.Error != nil {
return result.Error
} }
if result.RowsAffected == 0 {
return nil
}
return tx.Model(&models.LogEvent{}).Where("id = ? AND dispatch_outbox_id = ?", row.LogEventID, row.ID).Update("dispatch_status", eventStatus).Error
})
} }
func truncateError(s string, n int) string { func truncateError(s string, n int) string {
s = strings.TrimSpace(s) s = strings.TrimSpace(s)
if len(s) <= n { if n <= 0 {
return ""
}
runes := []rune(s)
if len(runes) <= n {
return s return s
} }
return s[:n] return string(runes[:n])
} }

View File

@@ -3,6 +3,7 @@ package ingest
import ( import (
"encoding/json" "encoding/json"
"fmt" "fmt"
"log"
"net" "net"
"regexp" "regexp"
"sort" "sort"
@@ -15,6 +16,7 @@ import (
"git.apinb.com/ops/logs/internal/impl" "git.apinb.com/ops/logs/internal/impl"
"git.apinb.com/ops/logs/internal/models" "git.apinb.com/ops/logs/internal/models"
"github.com/gosnmp/gosnmp" "github.com/gosnmp/gosnmp"
"gorm.io/gorm"
) )
type Engine struct { type Engine struct {
@@ -30,6 +32,9 @@ type Engine struct {
type resourceRef struct { type resourceRef struct {
ResourceType string ResourceType string
ResourceUID string
ResourceCategory string
ServiceIdentity string
ResourceID string ResourceID string
ResourceName string ResourceName string
} }
@@ -88,6 +93,12 @@ func (e *Engine) Refresh() error {
ResourceID: m.ResourceID, ResourceID: m.ResourceID,
ResourceName: m.ResourceName, ResourceName: m.ResourceName,
} }
var mappingLabels map[string]string
if err := json.Unmarshal([]byte(m.LabelsJSON), &mappingLabels); err == nil {
ref.ResourceUID = strings.TrimSpace(mappingLabels["resource_uid"])
ref.ResourceCategory = strings.TrimSpace(mappingLabels["resource_category"])
ref.ServiceIdentity = strings.TrimSpace(mappingLabels["service_identity"])
}
var ips []string var ips []string
if err := json.Unmarshal([]byte(m.IPsJSON), &ips); err == nil { if err := json.Unmarshal([]byte(m.IPsJSON), &ips); err == nil {
for _, ip := range ips { for _, ip := range ips {
@@ -146,6 +157,7 @@ func normOID(s string) string {
} }
func (e *Engine) HandleSyslog(addr *net.UDPAddr, payload []byte) { func (e *Engine) HandleSyslog(addr *net.UDPAddr, payload []byte) {
occurredAt := time.Now().UTC()
parsed := parseSyslogPayload(payload) parsed := parseSyslogPayload(payload)
device := parsed.Hostname device := parsed.Hostname
if device == "" { if device == "" {
@@ -193,22 +205,12 @@ func (e *Engine) HandleSyslog(addr *net.UDPAddr, payload []byte) {
} }
} }
if err := impl.DBService.Create(&ev).Error; err != nil {
return
}
if matched == nil { if matched == nil {
rawBytes, mErr := json.Marshal(string(payload)) rawBytes, mErr := json.Marshal(string(payload))
if mErr != nil { if mErr != nil {
return return
} }
body := AlertReceiveBody{ labels := map[string]string{
AlertName: "未解析 Syslog",
Summary: summary,
Description: parsed.RawLine,
SeverityCode: sev,
Value: parsed.Message,
Labels: map[string]string{
"source_type": "log", "source_type": "log",
"source_subtype": "syslog", "source_subtype": "syslog",
"device": device, "device": device,
@@ -216,12 +218,25 @@ func (e *Engine) HandleSyslog(addr *net.UDPAddr, payload []byte) {
"ip": addr.IP.String(), "ip": addr.IP.String(),
"instance": firstNonEmpty(device, addr.String()), "instance": firstNonEmpty(device, addr.String()),
"job": "logs-syslog", "job": "logs-syslog",
}, }
applyResourceIdentity(&ev, labels, "", ref, "syslog", addr.IP.String())
body := AlertReceiveBody{
AlertName: "未解析 Syslog",
Summary: summary,
Description: parsed.RawLine,
SeverityCode: sev,
Value: parsed.Message,
Labels: labels,
Agent: "logs-syslog", Agent: "logs-syslog",
RawData: rawBytes, RawData: rawBytes,
} }
if err := enqueueRawEvent(ev.ID, body, "unparsed"); err == nil { if err := persistLogEventAndOutbox(&ev, func(tx *gorm.DB, stored *models.LogEvent) (uint, error) {
_ = impl.DBService.Model(&ev).Update("dispatch_status", "pending").Error body.SourceEventKey = logSourceEventKey(stored, "raw")
body.OccurredAt = occurredAt
body.State = "firing"
return enqueueRawEventWithDB(tx, stored.ID, body, "unparsed")
}); err != nil {
log.Printf("logs: persist syslog raw event: %v", err)
} }
return return
} }
@@ -241,33 +256,26 @@ func (e *Engine) HandleSyslog(addr *net.UDPAddr, payload []byte) {
if matchDetails.ResourceUID != "" { if matchDetails.ResourceUID != "" {
labels["resource_uid"] = matchDetails.ResourceUID labels["resource_uid"] = matchDetails.ResourceUID
} }
applyResourceIdentity(&ev, labels, matchDetails.ResourceUID, ref, "syslog", addr.IP.String())
if err := persistLogEventAndOutbox(&ev, func(tx *gorm.DB, stored *models.LogEvent) (uint, error) {
rawObj := map[string]interface{}{ rawObj := map[string]interface{}{
"source": "syslog", "source": "syslog", "received_at": occurredAt.Format(time.RFC3339), "source_ip": addr.IP.String(),
"received_at": time.Now().UTC().Format(time.RFC3339), "rule_id": matched.ID, "log_entry_id": stored.ID, "raw_packet": string(payload),
"source_ip": addr.IP.String(), "parsed": detailObj, "match": matchDetails.Captures,
"rule_id": matched.ID,
"log_entry_id": ev.ID,
"raw_packet": string(payload),
"parsed": detailObj,
"match": matchDetails.Captures,
} }
rawBytes, mErr := json.Marshal(rawObj) rawBytes, err := json.Marshal(rawObj)
if mErr != nil { if err != nil {
return return 0, err
} }
body := AlertReceiveBody{ body := AlertReceiveBody{
AlertName: matched.AlertName, AlertName: matched.AlertName, Summary: summary, Description: summary,
Summary: summary,
Description: summary,
SeverityCode: firstNonEmpty(matchDetails.SeverityCode, firstNonEmpty(matched.SeverityCode, sev)), SeverityCode: firstNonEmpty(matchDetails.SeverityCode, firstNonEmpty(matched.SeverityCode, sev)),
Value: parsed.Message, Value: parsed.Message, Labels: labels, Agent: "logs-syslog", PolicyID: matched.PolicyID,
Labels: labels, State: "firing", SourceEventKey: logSourceEventKey(stored, "alert"), OccurredAt: occurredAt, RawData: rawBytes,
Agent: "logs-syslog",
PolicyID: matched.PolicyID,
RawData: rawBytes,
} }
if err := enqueueAlert(ev.ID, body); err == nil { return enqueueAlertWithDB(tx, stored.ID, body)
_ = impl.DBService.Model(&ev).Update("dispatch_status", "pending").Error }); err != nil {
log.Printf("logs: persist matched syslog event: %v", err)
} }
} }
@@ -372,9 +380,12 @@ func extractWithNamedRegex(pattern, groupName, message, rawLine string) string {
func normalizeExtractedResourceUID(uid string) string { func normalizeExtractedResourceUID(uid string) string {
uid = strings.TrimSpace(uid) uid = strings.TrimSpace(uid)
if uid == "" || strings.Contains(uid, ":") { if uid == "" {
return uid return uid
} }
if category, identity, ok := splitKnownResourceUID(uid); ok {
return category + ":" + identity
}
return "network:" + uid return "network:" + uid
} }
@@ -445,6 +456,7 @@ func lookupTrapDict(e *Engine, trapOID string) *models.TrapDictionaryEntry {
} }
func (e *Engine) HandleTrap(addr *net.UDPAddr, pkt *gosnmp.SnmpPacket) { func (e *Engine) HandleTrap(addr *net.UDPAddr, pkt *gosnmp.SnmpPacket) {
occurredAt := time.Now().UTC()
trapOID := extractTrapOID(pkt) trapOID := extractTrapOID(pkt)
if trapShielded(e, addr, trapOID, pkt) { if trapShielded(e, addr, trapOID, pkt) {
return return
@@ -490,10 +502,6 @@ func (e *Engine) HandleTrap(addr *net.UDPAddr, pkt *gosnmp.SnmpPacket) {
SeverityCode: sev, SeverityCode: sev,
TrapOID: trapOID, TrapOID: trapOID,
} }
if err := impl.DBService.Create(&ev).Error; err != nil {
return
}
e.mu.RLock() e.mu.RLock()
rules := e.trapRules rules := e.trapRules
e.mu.RUnlock() e.mu.RUnlock()
@@ -504,26 +512,28 @@ func (e *Engine) HandleTrap(addr *net.UDPAddr, pkt *gosnmp.SnmpPacket) {
if mErr != nil { if mErr != nil {
return return
} }
labels := map[string]string{
"source_type": "log", "source_subtype": "snmp_trap", "trap_oid": trapOID,
"remote_addr": addr.String(), "ip": addr.IP.String(), "instance": addr.IP.String(), "job": "logs-trap",
}
applyResourceIdentity(&ev, labels, "", ref, "snmp_trap", addr.IP.String())
body := AlertReceiveBody{ body := AlertReceiveBody{
AlertName: "未解析 SNMP Trap", AlertName: "未解析 SNMP Trap",
Summary: readable, Summary: readable,
Description: fp, Description: fp,
SeverityCode: sev, SeverityCode: sev,
Value: string(vbJSON), Value: string(vbJSON),
Labels: map[string]string{ Labels: labels,
"source_type": "log",
"source_subtype": "snmp_trap",
"trap_oid": trapOID,
"remote_addr": addr.String(),
"ip": addr.IP.String(),
"instance": addr.IP.String(),
"job": "logs-trap",
},
Agent: "logs-trap", Agent: "logs-trap",
RawData: rawBytes, RawData: rawBytes,
} }
if err := enqueueRawEvent(ev.ID, body, "unparsed"); err == nil { if err := persistLogEventAndOutbox(&ev, func(tx *gorm.DB, stored *models.LogEvent) (uint, error) {
_ = impl.DBService.Model(&ev).Update("dispatch_status", "pending").Error body.State = "firing"
body.SourceEventKey = logSourceEventKey(stored, "raw")
body.OccurredAt = occurredAt
return enqueueRawEventWithDB(tx, stored.ID, body, "unparsed")
}); err != nil {
log.Printf("logs: persist trap raw event: %v", err)
} }
return return
} }
@@ -551,6 +561,7 @@ func (e *Engine) HandleTrap(addr *net.UDPAddr, pkt *gosnmp.SnmpPacket) {
labels["resource_id"] = trapOID labels["resource_id"] = trapOID
} }
} }
applyResourceIdentity(&ev, labels, "", ref, "snmp_trap", addr.IP.String())
resolved := map[string]interface{}{} resolved := map[string]interface{}{}
if dict != nil { if dict != nil {
resolved["vendor"] = dict.Vendor resolved["vendor"] = dict.Vendor
@@ -561,36 +572,27 @@ func (e *Engine) HandleTrap(addr *net.UDPAddr, pkt *gosnmp.SnmpPacket) {
resolved["severity_mapping"] = dict.SeverityMappingJSON resolved["severity_mapping"] = dict.SeverityMappingJSON
resolved["parse_expression"] = dict.ParseExpression resolved["parse_expression"] = dict.ParseExpression
} }
if err := persistLogEventAndOutbox(&ev, func(tx *gorm.DB, stored *models.LogEvent) (uint, error) {
rawObj := map[string]interface{}{ rawObj := map[string]interface{}{
"source": "snmp_trap", "source": "snmp_trap", "received_at": occurredAt.Format(time.RFC3339), "source_ip": addr.IP.String(),
"received_at": time.Now().UTC().Format(time.RFC3339), "log_entry_id": stored.ID, "trap_oid": trapOID, "varbinds": trapVarbinds(pkt), "resolved": resolved, "pdu_summary": fp,
"source_ip": addr.IP.String(),
"log_entry_id": ev.ID,
"trap_oid": trapOID,
"varbinds": trapVarbinds(pkt),
"resolved": resolved,
"pdu_summary": fp,
} }
if matched.ID != 0 { if matched.ID != 0 {
rawObj["rule_id"] = matched.ID rawObj["rule_id"] = matched.ID
} }
rawBytes, mErr := json.Marshal(rawObj) rawBytes, err := json.Marshal(rawObj)
if mErr != nil { if err != nil {
return return 0, err
} }
body := AlertReceiveBody{ body := AlertReceiveBody{
AlertName: firstNonEmpty(matched.AlertName, "SNMP Trap"), AlertName: firstNonEmpty(matched.AlertName, "SNMP Trap"), Summary: readable, Description: desc,
Summary: readable, SeverityCode: firstNonEmpty(matched.SeverityCode, sev), Value: string(vbJSON), Labels: labels,
Description: desc, Agent: "logs-trap", PolicyID: matched.PolicyID, State: "firing",
SeverityCode: firstNonEmpty(matched.SeverityCode, sev), SourceEventKey: logSourceEventKey(stored, "alert"), OccurredAt: occurredAt, RawData: rawBytes,
Value: string(vbJSON),
Labels: labels,
Agent: "logs-trap",
PolicyID: matched.PolicyID,
RawData: rawBytes,
} }
if err := enqueueAlert(ev.ID, body); err == nil { return enqueueAlertWithDB(tx, stored.ID, body)
_ = impl.DBService.Model(&ev).Update("dispatch_status", "pending").Error }); err != nil {
log.Printf("logs: persist matched trap event: %v", err)
} }
} }
@@ -693,3 +695,116 @@ func (e *Engine) resolveResource(sourceIP, hostname string) (resourceRef, string
} }
return resourceRef{}, "none" return resourceRef{}, "none"
} }
func persistLogEventAndOutbox(ev *models.LogEvent, enqueue func(*gorm.DB, *models.LogEvent) (uint, error)) error {
if impl.DBService == nil {
return fmt.Errorf("database is not initialized")
}
if ev == nil || enqueue == nil {
return fmt.Errorf("log event and outbox builder are required")
}
return impl.DBService.Transaction(func(tx *gorm.DB) error {
if err := tx.Create(ev).Error; err != nil {
return err
}
outboxID, err := enqueue(tx, ev)
if err != nil {
return err
}
ev.DispatchStatus = "pending"
ev.DispatchOutboxID = outboxID
return tx.Model(ev).Updates(map[string]interface{}{
"dispatch_status": ev.DispatchStatus,
"dispatch_outbox_id": ev.DispatchOutboxID,
}).Error
})
}
func logSourceEventKey(ev *models.LogEvent, purpose string) string {
return fmt.Sprintf("logs:%s:%d:%s", strings.TrimSpace(ev.SourceKind), ev.ID, strings.TrimSpace(purpose))
}
func applyResourceIdentity(ev *models.LogEvent, labels map[string]string, explicitUID string, ref resourceRef, subtype, sourceIP string) {
uid := strings.TrimSpace(explicitUID)
category := ""
identity := ""
if uid == "" {
uid = strings.TrimSpace(ref.ResourceUID)
}
if uid != "" {
if knownCategory, knownIdentity, ok := splitKnownResourceUID(uid); ok {
category, identity = knownCategory, knownIdentity
uid = category + ":" + identity
} else {
category = strings.ToLower(strings.TrimSpace(ref.ResourceCategory))
if category == "" {
category = canonicalLogResourceCategory(ref.ResourceType, subtype)
}
identity = uid
uid = category + ":" + identity
}
}
if uid == "" && strings.TrimSpace(ref.ResourceCategory) != "" && strings.TrimSpace(ref.ServiceIdentity) != "" {
category = strings.ToLower(strings.TrimSpace(ref.ResourceCategory))
identity = strings.TrimSpace(ref.ServiceIdentity)
uid = category + ":" + identity
}
if uid == "" && strings.TrimSpace(ref.ResourceID) != "" {
identity = strings.TrimSpace(ref.ResourceID)
category = canonicalLogResourceCategory(ref.ResourceType, subtype)
if knownCategory, knownIdentity, ok := splitKnownResourceUID(identity); ok {
category, identity = knownCategory, knownIdentity
uid = category + ":" + identity
} else {
uid = category + ":" + identity
}
}
if uid == "" {
category = canonicalLogResourceCategory("", subtype)
identity = strings.TrimSpace(sourceIP)
uid = category + ":" + identity
}
labels["resource_uid"] = uid
labels["resource_category"] = category
labels["service_identity"] = identity
ev.ResourceUID = uid
if ev.ResourceType == "" || ev.ResourceID == "" {
ev.ResourceType = category
ev.ResourceID = identity
}
}
func canonicalLogResourceCategory(resourceType, subtype string) string {
switch strings.ToLower(strings.TrimSpace(resourceType)) {
case "server", "host":
return "host"
case "device", "network", "network_device":
return "network"
case "collector":
return "collector"
case "database", "middleware", "security", "storage", "room_device":
return strings.ToLower(strings.TrimSpace(resourceType))
}
if strings.TrimSpace(subtype) == "snmp_trap" || strings.TrimSpace(subtype) == "trap" {
return "network"
}
return "log_source"
}
func splitKnownResourceUID(value string) (string, string, bool) {
prefix, identity, found := strings.Cut(strings.TrimSpace(value), ":")
if !found {
return "", "", false
}
category := strings.ToLower(strings.TrimSpace(prefix))
identity = strings.TrimSpace(identity)
if identity == "" {
return "", "", false
}
switch category {
case "host", "network", "collector", "database", "middleware", "security", "storage", "room_device", "log_source":
return category, identity, true
default:
return "", "", false
}
}

View File

@@ -9,6 +9,7 @@ import (
"git.apinb.com/ops/logs/internal/impl" "git.apinb.com/ops/logs/internal/impl"
"git.apinb.com/ops/logs/internal/models" "git.apinb.com/ops/logs/internal/models"
"gorm.io/gorm"
) )
func BuildReplayRawEventPayload(ev models.LogEvent) (RawEventIngestBody, error) { func BuildReplayRawEventPayload(ev models.LogEvent) (RawEventIngestBody, error) {
@@ -16,9 +17,10 @@ func BuildReplayRawEventPayload(ev models.LogEvent) (RawEventIngestBody, error)
if sourceType == "" { if sourceType == "" {
return RawEventIngestBody{}, fmt.Errorf("unsupported source kind %q", ev.SourceKind) return RawEventIngestBody{}, fmt.Errorf("unsupported source kind %q", ev.SourceKind)
} }
now := time.Now().UTC()
rawObj := map[string]interface{}{ rawObj := map[string]interface{}{
"source": sourceType, "source": sourceType,
"replayed_at": time.Now().UTC().Format(time.RFC3339), "replayed_at": now.Format(time.RFC3339Nano),
"log_entry_id": ev.ID, "log_entry_id": ev.ID,
"source_ip": ev.SourceIP, "source_ip": ev.SourceIP,
"remote_addr": ev.RemoteAddr, "remote_addr": ev.RemoteAddr,
@@ -46,13 +48,17 @@ func BuildReplayRawEventPayload(ev models.LogEvent) (RawEventIngestBody, error)
"device": ev.DeviceName, "device": ev.DeviceName,
"job": "logs-replay", "job": "logs-replay",
} }
if uid := replayResourceUID(ev); uid != "" { resourceUID := replayResourceUID(ev)
labels["resource_uid"] = uid category, identity := splitResourceUID(resourceUID)
} labels["resource_uid"] = resourceUID
labels["resource_category"] = category
labels["service_identity"] = identity
return RawEventIngestBody{ return RawEventIngestBody{
SourceType: sourceType, SourceType: sourceType,
ResourceUID: replayResourceUID(ev), SourceEventKey: fmt.Sprintf("logs:replay:%d:%d", ev.ID, now.UnixNano()),
EventTime: time.Now().UTC(), State: "firing",
ResourceUID: resourceUID,
EventTime: now,
Severity: firstNonEmpty(ev.SeverityCode, "warning"), Severity: firstNonEmpty(ev.SeverityCode, "warning"),
Title: replayTitle(ev), Title: replayTitle(ev),
Message: firstNonEmpty(ev.NormalizedSummary, ev.RawPayload), Message: firstNonEmpty(ev.NormalizedSummary, ev.RawPayload),
@@ -61,7 +67,7 @@ func BuildReplayRawEventPayload(ev models.LogEvent) (RawEventIngestBody, error)
"replay": "true", "replay": "true",
"dispatch_status": ev.DispatchStatus, "dispatch_status": ev.DispatchStatus,
}, },
ParseStatus: "replayed", ParseStatus: "parsed",
RawPayload: rawBytes, RawPayload: rawBytes,
}, nil }, nil
} }
@@ -80,16 +86,42 @@ func EnqueueReplayLogEvent(ev models.LogEvent) (uint, error) {
PayloadJSON: string(payload), PayloadJSON: string(payload),
Status: outboxStatusPending, Status: outboxStatusPending,
RetryCount: 0, RetryCount: 0,
NextRetryAt: time.Now(),
} }
if err := enqueueOutboxRow(&row); err != nil { if impl.DBService == nil {
return 0, fmt.Errorf("database is not initialized")
}
if err := impl.DBService.Transaction(func(tx *gorm.DB) error {
now, err := databaseNow(tx)
if err != nil {
return err
}
row.NextRetryAt = now
if err := enqueueOutboxRowWithDB(tx, &row); err != nil {
return err
}
result := tx.Model(&models.LogEvent{}).Where("id = ?", ev.ID).Updates(map[string]interface{}{
"dispatch_status": "pending",
"dispatch_outbox_id": row.ID,
"alert_sent": false,
})
if result.Error != nil {
return result.Error
}
if result.RowsAffected == 0 {
return fmt.Errorf("log event %d does not exist", ev.ID)
}
return nil
}); err != nil {
return 0, err return 0, err
} }
return row.ID, nil return row.ID, nil
} }
func enqueueOutboxRow(row *models.AlertOutbox) error { func enqueueOutboxRowWithDB(db *gorm.DB, row *models.AlertOutbox) error {
return impl.DBService.Create(row).Error if db == nil {
return fmt.Errorf("database is not initialized")
}
return db.Create(row).Error
} }
func replaySourceType(kind string) string { func replaySourceType(kind string) string {
@@ -111,13 +143,32 @@ func replaySubtype(kind string) string {
} }
func replayResourceUID(ev models.LogEvent) string { func replayResourceUID(ev models.LogEvent) string {
if strings.Contains(ev.ResourceID, ":") { if uid := strings.TrimSpace(ev.ResourceUID); uid != "" {
return ev.ResourceID if category, identity, ok := splitKnownResourceUID(uid); ok {
return category + ":" + identity
}
return canonicalLogResourceCategory(ev.ResourceType, replaySubtype(ev.SourceKind)) + ":" + uid
}
if category, identity, ok := splitKnownResourceUID(ev.ResourceID); ok {
return category + ":" + identity
} }
if ev.ResourceType != "" && ev.ResourceID != "" { if ev.ResourceType != "" && ev.ResourceID != "" {
return ev.ResourceType + ":" + ev.ResourceID return canonicalLogResourceCategory(ev.ResourceType, replaySubtype(ev.SourceKind)) + ":" + ev.ResourceID
} }
return "" category := canonicalLogResourceCategory("", replaySubtype(ev.SourceKind))
identity := firstNonEmpty(strings.TrimSpace(ev.SourceIP), strings.TrimSpace(ev.DeviceName))
if identity == "" {
identity = fmt.Sprintf("log-event-%d", ev.ID)
}
return category + ":" + identity
}
func splitResourceUID(uid string) (string, string) {
parts := strings.SplitN(strings.TrimSpace(uid), ":", 2)
if len(parts) != 2 {
return "log_source", strings.TrimSpace(uid)
}
return parts[0], parts[1]
} }
func replayTitle(ev models.LogEvent) string { func replayTitle(ev models.LogEvent) string {

View File

@@ -344,10 +344,6 @@ func ReplayLogEvent(ctx *gin.Context) {
infra.Response.Error(ctx, err) infra.Response.Error(ctx, err)
return return
} }
if err := impl.DBService.Model(&models.LogEvent{}).Where("id = ?", id).Update("dispatch_status", "pending").Error; err != nil {
infra.Response.Error(ctx, err)
return
}
infra.Response.Success(ctx, gin.H{ infra.Response.Success(ctx, gin.H{
"log_event_id": id, "log_event_id": id,
"outbox_id": outboxID, "outbox_id": outboxID,

View File

@@ -4,21 +4,24 @@ import "time"
// AlertOutbox 表示待发送或重试中的告警任务。 // AlertOutbox 表示待发送或重试中的告警任务。
type AlertOutbox struct { type AlertOutbox struct {
ID uint `gorm:"primaryKey" json:"id"` ID uint `gorm:"primaryKey;index:idx_logs_alert_outbox_event_latest,priority:2" json:"id"`
CreatedAt time.Time `json:"created_at"` CreatedAt time.Time `json:"created_at"`
UpdatedAt time.Time `json:"updated_at"` UpdatedAt time.Time `json:"updated_at"`
// LogEventID 关联日志事件 ID。 // LogEventID 关联日志事件 ID。
LogEventID uint `gorm:"index" json:"log_event_id"` LogEventID uint `gorm:"index;index:idx_logs_alert_outbox_event_latest,priority:1" json:"log_event_id"`
// PayloadJSON 保存 AlertReceiveBody 的 JSON 文本。 // PayloadJSON 保存 AlertReceiveBody 的 JSON 文本。
PayloadJSON string `gorm:"type:text" json:"payload_json"` PayloadJSON string `gorm:"type:text" json:"payload_json"`
// Status 任务状态pending/retrying/sent/dead。 // Status 任务状态pending/processing/retrying/sent/dead。
Status string `gorm:"size:32;index" json:"status"` Status string `gorm:"size:32;index" json:"status"`
// RetryCount 已重试次数。 // RetryCount 已重试次数。
RetryCount int `json:"retry_count"` RetryCount int `json:"retry_count"`
// NextRetryAt 下一次可重试时间。 // NextRetryAt 下一次可重试时间。
NextRetryAt time.Time `gorm:"index" json:"next_retry_at"` NextRetryAt time.Time `gorm:"index" json:"next_retry_at"`
// LeaseUntil/LeaseOwner 用于多实例原子领取,避免同一任务并发发送。
LeaseUntil *time.Time `gorm:"index" json:"lease_until,omitempty"`
LeaseOwner string `gorm:"size:128;index" json:"lease_owner"`
// LastError 最近一次错误信息。 // LastError 最近一次错误信息。
LastError string `gorm:"type:text" json:"last_error"` LastError string `gorm:"type:text" json:"last_error"`
} }
@@ -26,4 +29,3 @@ type AlertOutbox struct {
func (AlertOutbox) TableName() string { func (AlertOutbox) TableName() string {
return "logs_alert_outbox" return "logs_alert_outbox"
} }

View File

@@ -24,6 +24,8 @@ type LogEvent struct {
SourceIP string `gorm:"size:64;index" json:"source_ip"` SourceIP string `gorm:"size:64;index" json:"source_ip"`
// ResourceType 表示关联到的资源类型。 // ResourceType 表示关联到的资源类型。
ResourceType string `gorm:"size:32;index" json:"resource_type"` ResourceType string `gorm:"size:32;index" json:"resource_type"`
// ResourceUID 是跨服务使用的规范资源标识。
ResourceUID string `gorm:"size:255;index" json:"resource_uid"`
// ResourceID 表示关联到的资源 ID。 // ResourceID 表示关联到的资源 ID。
ResourceID string `gorm:"size:128;index" json:"resource_id"` ResourceID string `gorm:"size:128;index" json:"resource_id"`
// ResourceName 表示关联到的资源名称。 // ResourceName 表示关联到的资源名称。
@@ -32,6 +34,8 @@ type LogEvent struct {
MatchMethod string `gorm:"size:32" json:"match_method"` MatchMethod string `gorm:"size:32" json:"match_method"`
// DispatchStatus 表示告警分发状态not_applicable/pending/retrying/sent/dead // DispatchStatus 表示告警分发状态not_applicable/pending/retrying/sent/dead
DispatchStatus string `gorm:"size:32;index" json:"dispatch_status"` DispatchStatus string `gorm:"size:32;index" json:"dispatch_status"`
// DispatchOutboxID 标识当前一次分发,防止旧任务覆盖较新的重放状态。
DispatchOutboxID uint `gorm:"not null;default:0;index" json:"dispatch_outbox_id"`
// SeverityCode 表示告警/严重度编码。 // SeverityCode 表示告警/严重度编码。
SeverityCode string `gorm:"size:32" json:"severity_code"` SeverityCode string `gorm:"size:32" json:"severity_code"`
// TrapOID 表示关联的 Trap OID若来源为 trap // TrapOID 表示关联的 Trap OID若来源为 trap

View File

@@ -23,6 +23,9 @@ func InitData(db *gorm.DB) error {
if db == nil { if db == nil {
return nil return nil
} }
if err := backfillDispatchOutboxIDs(db); err != nil {
return err
}
if err := seedDefaultSyslogRules(db); err != nil { if err := seedDefaultSyslogRules(db); err != nil {
return err return err
} }
@@ -35,6 +38,52 @@ func InitData(db *gorm.DB) error {
return nil return nil
} }
func backfillDispatchOutboxIDs(db *gorm.DB) error {
const postgresUpdate = `
WITH latest AS (
SELECT target_event.id AS log_event_id, current_outbox.id, current_outbox.status
FROM logs_events AS target_event
JOIN LATERAL (
SELECT id, status
FROM logs_alert_outbox
WHERE log_event_id = target_event.id
ORDER BY id DESC
LIMIT 1
) AS current_outbox ON TRUE
WHERE target_event.dispatch_outbox_id = 0
)
UPDATE logs_events AS target_event
SET dispatch_outbox_id = latest.id,
dispatch_status = CASE latest.status WHEN 'processing' THEN 'retrying' ELSE latest.status END,
alert_sent = CASE WHEN latest.status = 'sent' THEN TRUE ELSE FALSE END
FROM latest
WHERE target_event.id = latest.log_event_id
AND target_event.dispatch_outbox_id = 0`
const mysqlUpdate = `
UPDATE logs_events AS target_event
JOIN (
SELECT candidate.id AS log_event_id, MAX(outbox.id) AS id
FROM logs_events AS candidate
JOIN logs_alert_outbox AS outbox ON outbox.log_event_id = candidate.id
WHERE candidate.dispatch_outbox_id = 0
GROUP BY candidate.id
) AS latest_id ON latest_id.log_event_id = target_event.id
JOIN logs_alert_outbox AS latest ON latest.id = latest_id.id
SET dispatch_outbox_id = latest.id,
dispatch_status = CASE latest.status WHEN 'processing' THEN 'retrying' ELSE latest.status END,
alert_sent = CASE WHEN latest.status = 'sent' THEN TRUE ELSE FALSE END
WHERE target_event.dispatch_outbox_id = 0`
switch db.Dialector.Name() {
case "postgres":
return db.Exec(postgresUpdate).Error
case "mysql":
return db.Exec(mysqlUpdate).Error
default:
return gorm.ErrUnsupportedDriver
}
}
func seedDefaultSyslogRules(db *gorm.DB) error { func seedDefaultSyslogRules(db *gorm.DB) error {
var cnt int64 var cnt int64
if err := db.Model(&SyslogRule{}).Count(&cnt).Error; err != nil { if err := db.Model(&SyslogRule{}).Count(&cnt).Error; err != nil {