fix: correlate log alert recovery lifecycle

internal/ingest/alert_forward.go

@@ -38,6 +38,7 @@ type AlertReceiveBody struct {
 	Labels         map[string]string `json:"labels"`
 	Agent          string            `json:"agent"`
 	PolicyID       uint              `json:"policy_id"`
+	Fingerprint    string            `json:"fingerprint,omitempty"`
 	State          string            `json:"state,omitempty"`
 	SourceEventKey string            `json:"source_event_key"`
 	TraceID        string            `json:"trace_id"`
@@ -104,6 +105,9 @@ func forwardAlert(body AlertReceiveBody) error {
 		(details.Status != "firing" && details.Status != "resolved") || !validFingerprint(details.Fingerprint) {
 		return fmt.Errorf("Alert 响应缺少完整写入结果，无法确认转发成功；请稍后重试")
 	}
+	if body.Fingerprint != "" && details.Fingerprint != body.Fingerprint {
+		return fmt.Errorf("Alert 返回的告警指纹与请求不一致；请稍后重试")
+	}
 	return nil
 }
 

internal/ingest/engine.go

@@ -1,6 +1,8 @@
 package ingest
 
 import (
+	"crypto/sha256"
+	"encoding/hex"
 	"encoding/json"
 	"fmt"
 	"log"
@@ -267,16 +269,13 @@ func (e *Engine) HandleSyslog(addr *net.UDPAddr, payload []byte) {
 		if err != nil {
 			return 0, err
 		}
-		state := "firing"
-		if isRecoverySignal(parsed.Message, parsed.RawLine) {
-			state = "resolved"
-		}
 		body := AlertReceiveBody{
 			AlertName: matched.AlertName, Summary: summary, Description: summary,
 			SeverityCode: firstNonEmpty(matchDetails.SeverityCode, firstNonEmpty(matched.SeverityCode, sev)),
 			Value:        parsed.Message, Labels: labels, Agent: "logs-syslog", PolicyID: matched.PolicyID,
-			State: state, SourceEventKey: logSourceEventKey(stored, "alert"), OccurredAt: occurredAt, RawData: rawBytes,
+			State: matchDetails.State, SourceEventKey: logSourceEventKey(stored, "alert"), OccurredAt: occurredAt, RawData: rawBytes,
 		}
+		body.Fingerprint = alertLifecycleFingerprint(body, matched.LifecycleKey, "")
 		return enqueueAlertWithDB(tx, stored.ID, body)
 	}); err != nil {
 		log.Printf("logs: persist matched syslog event: %v", err)
@@ -285,6 +284,7 @@ func (e *Engine) HandleSyslog(addr *net.UDPAddr, payload []byte) {
 
 type syslogRuleMatch struct {
 	Matched      bool
+	State        string
 	ResourceUID  string
 	SeverityCode string
 	Captures     map[string]string
@@ -295,11 +295,12 @@ func syslogRuleMatches(rule *models.SyslogRule, device, message, rawLine string)
 }
 
 func syslogRuleMatchDetails(rule *models.SyslogRule, device, message, rawLine string) syslogRuleMatch {
-	result := syslogRuleMatch{Captures: map[string]string{}}
+	result := syslogRuleMatch{State: "firing", Captures: map[string]string{}}
 	deviceContains := strings.TrimSpace(rule.DeviceNameContains)
 	sourceMatch := strings.TrimSpace(rule.SourceMatch)
 	keywordRegex := strings.TrimSpace(rule.KeywordRegex)
 	messageRegex := strings.TrimSpace(rule.MessageRegex)
+	recoveryRegex := strings.TrimSpace(rule.RecoveryMatchRegex)
 	if deviceContains == "" && sourceMatch == "" && keywordRegex == "" && messageRegex == "" {
 		return result
 	}
@@ -316,6 +317,19 @@ func syslogRuleMatchDetails(rule *models.SyslogRule, device, message, rawLine st
 			return result
 		}
 	}
+	if recoveryRegex != "" {
+		re, err := regexp.Compile(recoveryRegex)
+		if err != nil {
+			return result
+		}
+		matches := firstRegexMatch(re, message, rawLine)
+		if matches != nil {
+			mergeNamedCaptures(result.Captures, re, matches)
+			result.State = "resolved"
+			result.Matched = true
+		}
+	}
+	if !result.Matched {
 		for _, pattern := range []string{keywordRegex, messageRegex} {
 			if pattern == "" {
 				continue
@@ -324,16 +338,14 @@ func syslogRuleMatchDetails(rule *models.SyslogRule, device, message, rawLine st
 			if err != nil {
 				return result
 			}
-		matches := re.FindStringSubmatch(message)
+			matches := firstRegexMatch(re, message, rawLine)
-		if matches == nil {
-			matches = re.FindStringSubmatch(rawLine)
-		}
 			if matches == nil {
 				return result
 			}
 			mergeNamedCaptures(result.Captures, re, matches)
 		}
 		result.Matched = true
+	}
 	if uid := extractWithNamedRegex(rule.ResourceUIDExtractRegex, "resource_uid", message, rawLine); uid != "" {
 		result.ResourceUID = normalizeExtractedResourceUID(uid)
 	} else if uid := result.Captures["resource_uid"]; uid != "" {
@@ -343,6 +355,15 @@ func syslogRuleMatchDetails(rule *models.SyslogRule, device, message, rawLine st
 	return result
 }
 
+func firstRegexMatch(re *regexp.Regexp, values ...string) []string {
+	for _, value := range values {
+		if matches := re.FindStringSubmatch(value); matches != nil {
+			return matches
+		}
+	}
+	return nil
+}
+
 func mergeNamedCaptures(dst map[string]string, re *regexp.Regexp, matches []string) {
 	names := re.SubexpNames()
 	for i, name := range names {
@@ -370,7 +391,9 @@ func extractWithNamedRegex(pattern, groupName, message, rawLine string) string {
 		names := re.SubexpNames()
 		for i, name := range names {
 			if i > 0 && name == groupName && i < len(matches) {
-				return strings.TrimSpace(matches[i])
+				if value := strings.TrimSpace(matches[i]); value != "" {
+					return value
+				}
 			}
 		}
 		for i := 1; i < len(matches); i++ {
@@ -510,8 +533,8 @@ func (e *Engine) HandleTrap(addr *net.UDPAddr, pkt *gosnmp.SnmpPacket) {
 	rules := e.trapRules
 	e.mu.RUnlock()
 
-	matched := firstMatchingTrapRule(rules, trapOID, fp)
+	match := firstMatchingTrapRule(rules, trapOID, fp)
-	if matched == nil {
+	if match.Rule == nil {
 		rawBytes, mErr := json.Marshal(fp)
 		if mErr != nil {
 			return
@@ -541,6 +564,7 @@ func (e *Engine) HandleTrap(addr *net.UDPAddr, pkt *gosnmp.SnmpPacket) {
 		}
 		return
 	}
+	matched := match.Rule
 
 	desc := readable
 	if dict != nil && dict.RecoveryMessage != "" {
@@ -555,6 +579,10 @@ func (e *Engine) HandleTrap(addr *net.UDPAddr, pkt *gosnmp.SnmpPacket) {
 		"instance":       addr.IP.String(),
 		"job":            "logs-trap",
 	}
+	trapInstance := trapInstanceKey(pkt)
+	if trapInstance != "" {
+		labels["trap_instance"] = trapInstance
+	}
 	if matched.ID != 0 {
 		labels["resource_type"] = "trap_rule"
 		labels["resource_id"] = strconv.FormatUint(uint64(matched.ID), 10)
@@ -588,20 +616,13 @@ func (e *Engine) HandleTrap(addr *net.UDPAddr, pkt *gosnmp.SnmpPacket) {
 		if err != nil {
 			return 0, err
 		}
-		state := "firing"
-		dictText := ""
-		if dict != nil {
-			dictText = firstNonEmpty(dict.Name, dict.Title)
-		}
-		if isRecoverySignal(readable, dictText, matched.Name, matched.AlertName) {
-			state = "resolved"
-		}
 		body := AlertReceiveBody{
 			AlertName: firstNonEmpty(matched.AlertName, "SNMP Trap"), Summary: readable, Description: desc,
 			SeverityCode: firstNonEmpty(matched.SeverityCode, sev), Value: string(vbJSON), Labels: labels,
-			Agent: "logs-trap", PolicyID: matched.PolicyID, State: state,
+			Agent: "logs-trap", PolicyID: matched.PolicyID, State: match.State,
 			SourceEventKey: logSourceEventKey(stored, "alert"), OccurredAt: occurredAt, RawData: rawBytes,
 		}
+		body.Fingerprint = alertLifecycleFingerprint(body, matched.LifecycleKey, trapInstance)
 		return enqueueAlertWithDB(tx, stored.ID, body)
 	}); err != nil {
 		log.Printf("logs: persist matched trap event: %v", err)
@@ -646,6 +667,27 @@ func trapVarbinds(pkt *gosnmp.SnmpPacket) []map[string]string {
 	return out
 }
 
+func trapInstanceKey(pkt *gosnmp.SnmpPacket) string {
+	if pkt == nil {
+		return ""
+	}
+	for _, prefix := range []string{
+		"1.3.6.1.2.1.31.1.1.1.1.",
+		"1.3.6.1.2.1.2.2.1.2.",
+		"1.3.6.1.2.1.2.2.1.1.",
+	} {
+		for _, variable := range pkt.Variables {
+			name := normOID(variable.Name)
+			if strings.HasPrefix(name, prefix) {
+				if index := strings.TrimSpace(strings.TrimPrefix(name, prefix)); index != "" {
+					return index
+				}
+			}
+		}
+	}
+	return ""
+}
+
 func buildTrapReadable(trapOID string, dict *models.TrapDictionaryEntry, varbindSummary string) string {
 	if dict != nil && firstNonEmpty(dict.Name, dict.Title) != "" {
 		return firstNonEmpty(dict.Name, dict.Title) + " (" + trapOID + ")"
@@ -656,34 +698,49 @@ func buildTrapReadable(trapOID string, dict *models.TrapDictionaryEntry, varbind
 	return truncate(varbindSummary, 256)
 }
 
-func trapRuleMatches(rule *models.TrapRule, trapOID, varbindFP string) bool {
+type trapRuleMatch struct {
+	Rule  *models.TrapRule
+	State string
+}
+
+func trapRuleState(rule *models.TrapRule, trapOID, varbindFP string) (string, bool) {
 	hasOID := strings.TrimSpace(rule.OIDPrefix) != ""
 	hasRE := strings.TrimSpace(rule.VarbindMatchRegex) != ""
+	hasRecoveryRE := strings.TrimSpace(rule.RecoveryMatchRegex) != ""
 	if !hasOID && !hasRE {
-		return false
+		return "", false
 	}
 	if hasOID && !strings.HasPrefix(normOID(trapOID), normOID(rule.OIDPrefix)) {
-		return false
+		return "", false
+	}
+	if hasRecoveryRE {
+		re, err := regexp.Compile(rule.RecoveryMatchRegex)
+		if err != nil {
+			return "", false
+		}
+		if re.MatchString(trapOID) || re.MatchString(varbindFP) {
+			return "resolved", true
+		}
 	}
 	if hasRE {
 		re, err := regexp.Compile(rule.VarbindMatchRegex)
 		if err != nil {
-			return false
+			return "", false
 		}
 		if !re.MatchString(varbindFP) {
-			return false
+			return "", false
 		}
 	}
-	return true
+	return "firing", true
 }
 
-func firstMatchingTrapRule(rules []models.TrapRule, trapOID, varbindFP string) *models.TrapRule {
+func firstMatchingTrapRule(rules []models.TrapRule, trapOID, varbindFP string) trapRuleMatch {
 	for i := range rules {
-		if trapRuleMatches(&rules[i], trapOID, varbindFP) {
+		if state, matched := trapRuleState(&rules[i], trapOID, varbindFP); matched {
-			return &rules[i]
+			return trapRuleMatch{Rule: &rules[i], State: state}
 		}
 	}
-	return nil
+	return trapRuleMatch{}
 }
 
 func firstNonEmpty(a, b string) string {
@@ -693,19 +750,27 @@ func firstNonEmpty(a, b string) string {
 	return b
 }
 
-func isRecoverySignal(values ...string) bool {
+func alertLifecycleFingerprint(body AlertReceiveBody, lifecycleKey, instance string) string {
-	for _, value := range values {
+	lifecycleKey = strings.TrimSpace(lifecycleKey)
-		text := strings.ToLower(strings.TrimSpace(value))
+	if lifecycleKey == "" {
-		if text == "" {
+		return ""
-			continue
 	}
-		for _, marker := range []string{"恢复", "recovered", "recovery", "ifup", "link up", "port up", "interface up", "normal"} {
+	resourceUID := ""
-			if strings.Contains(text, marker) {
+	sourceIP := ""
-				return true
+	if body.Labels != nil {
+		resourceUID = strings.TrimSpace(body.Labels["resource_uid"])
+		sourceIP = strings.TrimSpace(body.Labels["ip"])
 	}
-		}
+	identity := strings.Join([]string{
-	}
+		strings.TrimSpace(body.Agent),
-	return false
+		resourceUID,
+		sourceIP,
+		strconv.FormatUint(uint64(body.PolicyID), 10),
+		lifecycleKey,
+		strings.TrimSpace(instance),
+	}, "\x00")
+	sum := sha256.Sum256([]byte(identity))
+	return hex.EncodeToString(sum[:])
 }
 
 func (e *Engine) resolveResource(sourceIP, hostname string) (resourceRef, string) {

internal/logic/controllers/rule_validation.go

@@ -16,6 +16,7 @@ func validateSyslogRule(rule *models.SyslogRule) error {
 	}{
 		{name: "keyword_regex", pattern: rule.KeywordRegex},
 		{name: "message_regex", pattern: rule.MessageRegex},
+		{name: "recovery_match_regex", pattern: rule.RecoveryMatchRegex},
 		{name: "resource_uid_extract_regex", pattern: rule.ResourceUIDExtractRegex},
 	}
 	for _, field := range regexFields {
@@ -48,18 +49,25 @@ func validateSyslogRule(rule *models.SyslogRule) error {
 		strings.TrimSpace(rule.MessageRegex) == "" {
 		return fmt.Errorf("Syslog 规则的匹配条件全部为空，运行时永远不会命中；请至少填写 device_name_contains、source_match、keyword_regex、message_regex 中的一项")
 	}
+	if strings.TrimSpace(rule.RecoveryMatchRegex) != "" && strings.TrimSpace(rule.LifecycleKey) == "" {
+		return fmt.Errorf("配置 recovery_match_regex 时 lifecycle_key 不能为空")
+	}
 	return nil
 }
 
 func validateTrapRule(rule *models.TrapRule) error {
-	if strings.TrimSpace(rule.VarbindMatchRegex) != "" {
+	if err := validateOptionalRegex("varbind_match_regex", rule.VarbindMatchRegex); err != nil {
-		if _, err := regexp.Compile(rule.VarbindMatchRegex); err != nil {
+		return err
-			return fmt.Errorf("varbind_match_regex 不是有效正则表达式：%v；请修正 varbind_match_regex 后重试", err)
 	}
+	if err := validateOptionalRegex("recovery_match_regex", rule.RecoveryMatchRegex); err != nil {
+		return err
 	}
 	if strings.TrimSpace(rule.OIDPrefix) == "" && strings.TrimSpace(rule.VarbindMatchRegex) == "" {
 		return fmt.Errorf("Trap 规则的匹配条件全部为空，运行时永远不会命中；请至少填写 oid_prefix、varbind_match_regex 中的一项")
 	}
+	if strings.TrimSpace(rule.RecoveryMatchRegex) != "" && strings.TrimSpace(rule.LifecycleKey) == "" {
+		return fmt.Errorf("配置 recovery_match_regex 时 lifecycle_key 不能为空")
+	}
 	return nil
 }
 

internal/models/query.go

@@ -108,6 +108,8 @@ func seedDefaultSyslogRules(db *gorm.DB) error {
 			KeywordRegex:            "(?i)(link down|interface .* down|port .* down)",
 			SourceMatch:             "",
 			MessageRegex:            "(?i)(link down|interface .* down|port .* down|LINK_DOWN)",
+			RecoveryMatchRegex:      `(?i)(link[ _-]?up|interface .* up|port .* up|ifup)`,
+			LifecycleKey:            "syslog-link-state",
 			AlertName:               "Syslog链路中断",
 			SeverityCode:            "major",
 			SeverityMappingJSON:     `{"(?i)(critical|fatal|emergency)":"critical","(?i)(error|LINK_DOWN|down)":"major","(?i)(warning|warn)":"warning"}`,
@@ -118,8 +120,10 @@ func seedDefaultSyslogRules(db *gorm.DB) error {
 			Name:                    "H3C-Syslog-接口中断",
 			Enabled:                 true,
 			Priority:                120,
-			SourceMatch:             "h3c",
+			DeviceNameContains:      "h3c",
 			MessageRegex:            `(?i)(LINK_DOWN|Interface .* down|port .* down)`,
+			RecoveryMatchRegex:      `(?i)(link[ _-]?up|interface .* up|port .* up|ifup)`,
+			LifecycleKey:            "h3c-syslog-interface-state",
 			AlertName:               "H3C Syslog接口中断",
 			SeverityCode:            "major",
 			SeverityMappingJSON:     `{"(?i)(LINK_DOWN|down)":"major","(?i)(LINK_UP|up)":"info"}`,
@@ -147,6 +151,8 @@ func seedDefaultSyslogRules(db *gorm.DB) error {
 			"source_match",
 			"keyword_regex",
 			"message_regex",
+			"recovery_match_regex",
+			"lifecycle_key",
 			"alert_name",
 			"severity_code",
 			"severity_mapping_json",
@@ -166,7 +172,9 @@ func seedDefaultTrapRules(db *gorm.DB) error {
 			Enabled:            true,
 			Priority:           100,
 			OIDPrefix:          "1.3.6.1.6.3.1.1.5",
-			VarbindMatchRegex: "(?i)(linkdown|ifdown|down)",
+			VarbindMatchRegex:  `(?i)(1\.3\.6\.1\.6\.3\.1\.1\.5\.3([^0-9]|$)|\b(linkdown|ifdown|down)\b)`,
+			RecoveryMatchRegex: `(?i)(1\.3\.6\.1\.6\.3\.1\.1\.5\.4([^0-9]|$)|\b(linkup|ifup)\b)`,
+			LifecycleKey:       "snmp-interface-link-state",
 			AlertName:          "SNMP Trap链路中断",
 			SeverityCode:       "major",
 			PolicyID:           0,
@@ -190,6 +198,8 @@ func seedDefaultTrapRules(db *gorm.DB) error {
 			"priority",
 			"o_id_prefix",
 			"varbind_match_regex",
+			"recovery_match_regex",
+			"lifecycle_key",
 			"alert_name",
 			"severity_code",
 			"policy_id",

internal/models/syslog_rule.go

@@ -24,6 +24,10 @@ type SyslogRule struct {
 	KeywordRegex string `gorm:"size:512" json:"keyword_regex"`
 	// MessageRegex 表示消息正文匹配的正则表达式。
 	MessageRegex string `gorm:"size:1024" json:"message_regex"`
+	// RecoveryMatchRegex 匹配同一生命周期的恢复消息。
+	RecoveryMatchRegex string `gorm:"size:1024" json:"recovery_match_regex"`
+	// LifecycleKey 将故障和恢复事件绑定到同一告警生命周期。
+	LifecycleKey string `gorm:"size:256" json:"lifecycle_key"`
 	// AlertName 表示告警名称。
 	AlertName string `gorm:"size:256" json:"alert_name"`
 	// SeverityCode 表示严重级别编码。

internal/models/trap_rule.go

@@ -20,6 +20,10 @@ type TrapRule struct {
 	OIDPrefix string `gorm:"size:512" json:"oid_prefix"`
 	// VarbindMatchRegex 表示对 varbind 内容的正则匹配条件。
 	VarbindMatchRegex string `gorm:"size:512" json:"varbind_match_regex"`
+	// RecoveryMatchRegex 匹配同一生命周期的恢复 Trap OID 或 varbind。
+	RecoveryMatchRegex string `gorm:"size:1024" json:"recovery_match_regex"`
+	// LifecycleKey 将故障和恢复事件绑定到同一告警生命周期。
+	LifecycleKey string `gorm:"size:256" json:"lifecycle_key"`
 	// AlertName 表示告警名称。
 	AlertName string `gorm:"size:256" json:"alert_name"`
 	// SeverityCode 表示严重级别编码。