ops/logs
5
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: make log alert delivery durable

Browse Source
This commit is contained in:
zxr
2026-07-21 02:32:18 +08:00
parent ce559d5218
commit 6e66bad884
8 changed files with 709 additions and 305 deletions
Download Patch File Download Diff File Expand all files Collapse all files

305
internal/ingest/engine.go
View File

@@ -3,6 +3,7 @@ package ingest
import (
"encoding/json"
"fmt"
"log"
"net"
"regexp"
"sort"
@@ -15,6 +16,7 @@ import (
"git.apinb.com/ops/logs/internal/impl"
"git.apinb.com/ops/logs/internal/models"
"github.com/gosnmp/gosnmp"
"gorm.io/gorm"
)
type Engine struct {
@@ -29,9 +31,12 @@ type Engine struct {
}
type resourceRef struct {
ResourceType string
ResourceID string
ResourceName string
ResourceType string
ResourceUID string
ResourceCategory string
ServiceIdentity string
ResourceID string
ResourceName string
}
func resourceTypePriority(resourceType string) int {
@@ -88,6 +93,12 @@ func (e *Engine) Refresh() error {
ResourceID: m.ResourceID,
ResourceName: m.ResourceName,
}
var mappingLabels map[string]string
if err := json.Unmarshal([]byte(m.LabelsJSON), &mappingLabels); err == nil {
ref.ResourceUID = strings.TrimSpace(mappingLabels["resource_uid"])
ref.ResourceCategory = strings.TrimSpace(mappingLabels["resource_category"])
ref.ServiceIdentity = strings.TrimSpace(mappingLabels["service_identity"])
}
var ips []string
if err := json.Unmarshal([]byte(m.IPsJSON), &ips); err == nil {
for _, ip := range ips {
@@ -146,6 +157,7 @@ func normOID(s string) string {
}
func (e *Engine) HandleSyslog(addr *net.UDPAddr, payload []byte) {
occurredAt := time.Now().UTC()
parsed := parseSyslogPayload(payload)
device := parsed.Hostname
if device == "" {
@@ -193,35 +205,38 @@ func (e *Engine) HandleSyslog(addr *net.UDPAddr, payload []byte) {
}
}
if err := impl.DBService.Create(&ev).Error; err != nil {
return
}
if matched == nil {
rawBytes, mErr := json.Marshal(string(payload))
if mErr != nil {
return
}
labels := map[string]string{
"source_type": "log",
"source_subtype": "syslog",
"device": device,
"remote_addr": addr.String(),
"ip": addr.IP.String(),
"instance": firstNonEmpty(device, addr.String()),
"job": "logs-syslog",
}
applyResourceIdentity(&ev, labels, "", ref, "syslog", addr.IP.String())
body := AlertReceiveBody{
AlertName: "未解析 Syslog",
Summary: summary,
Description: parsed.RawLine,
SeverityCode: sev,
Value: parsed.Message,
Labels: map[string]string{
"source_type": "log",
"source_subtype": "syslog",
"device": device,
"remote_addr": addr.String(),
"ip": addr.IP.String(),
"instance": firstNonEmpty(device, addr.String()),
"job": "logs-syslog",
},
Agent: "logs-syslog",
RawData: rawBytes,
Labels: labels,
Agent: "logs-syslog",
RawData: rawBytes,
}
if err := enqueueRawEvent(ev.ID, body, "unparsed"); err == nil {
_ = impl.DBService.Model(&ev).Update("dispatch_status", "pending").Error
if err := persistLogEventAndOutbox(&ev, func(tx *gorm.DB, stored *models.LogEvent) (uint, error) {
body.SourceEventKey = logSourceEventKey(stored, "raw")
body.OccurredAt = occurredAt
body.State = "firing"
return enqueueRawEventWithDB(tx, stored.ID, body, "unparsed")
}); err != nil {
log.Printf("logs: persist syslog raw event: %v", err)
}
return
}
@@ -241,33 +256,26 @@ func (e *Engine) HandleSyslog(addr *net.UDPAddr, payload []byte) {
if matchDetails.ResourceUID != "" {
labels["resource_uid"] = matchDetails.ResourceUID
}
rawObj := map[string]interface{}{
"source": "syslog",
"received_at": time.Now().UTC().Format(time.RFC3339),
"source_ip": addr.IP.String(),
"rule_id": matched.ID,
"log_entry_id": ev.ID,
"raw_packet": string(payload),
"parsed": detailObj,
"match": matchDetails.Captures,
}
rawBytes, mErr := json.Marshal(rawObj)
if mErr != nil {
return
}
body := AlertReceiveBody{
AlertName: matched.AlertName,
Summary: summary,
Description: summary,
SeverityCode: firstNonEmpty(matchDetails.SeverityCode, firstNonEmpty(matched.SeverityCode, sev)),
Value: parsed.Message,
Labels: labels,
Agent: "logs-syslog",
PolicyID: matched.PolicyID,
RawData: rawBytes,
}
if err := enqueueAlert(ev.ID, body); err == nil {
_ = impl.DBService.Model(&ev).Update("dispatch_status", "pending").Error
applyResourceIdentity(&ev, labels, matchDetails.ResourceUID, ref, "syslog", addr.IP.String())
if err := persistLogEventAndOutbox(&ev, func(tx *gorm.DB, stored *models.LogEvent) (uint, error) {
rawObj := map[string]interface{}{
"source": "syslog", "received_at": occurredAt.Format(time.RFC3339), "source_ip": addr.IP.String(),
"rule_id": matched.ID, "log_entry_id": stored.ID, "raw_packet": string(payload),
"parsed": detailObj, "match": matchDetails.Captures,
}
rawBytes, err := json.Marshal(rawObj)
if err != nil {
return 0, err
}
body := AlertReceiveBody{
AlertName: matched.AlertName, Summary: summary, Description: summary,
SeverityCode: firstNonEmpty(matchDetails.SeverityCode, firstNonEmpty(matched.SeverityCode, sev)),
Value: parsed.Message, Labels: labels, Agent: "logs-syslog", PolicyID: matched.PolicyID,
State: "firing", SourceEventKey: logSourceEventKey(stored, "alert"), OccurredAt: occurredAt, RawData: rawBytes,
}
return enqueueAlertWithDB(tx, stored.ID, body)
}); err != nil {
log.Printf("logs: persist matched syslog event: %v", err)
}
}
@@ -372,9 +380,12 @@ func extractWithNamedRegex(pattern, groupName, message, rawLine string) string {
func normalizeExtractedResourceUID(uid string) string {
uid = strings.TrimSpace(uid)
if uid == "" || strings.Contains(uid, ":") {
if uid == "" {
return uid
}
if category, identity, ok := splitKnownResourceUID(uid); ok {
return category + ":" + identity
}
return "network:" + uid
}
@@ -445,6 +456,7 @@ func lookupTrapDict(e *Engine, trapOID string) *models.TrapDictionaryEntry {
}
func (e *Engine) HandleTrap(addr *net.UDPAddr, pkt *gosnmp.SnmpPacket) {
occurredAt := time.Now().UTC()
trapOID := extractTrapOID(pkt)
if trapShielded(e, addr, trapOID, pkt) {
return
@@ -490,10 +502,6 @@ func (e *Engine) HandleTrap(addr *net.UDPAddr, pkt *gosnmp.SnmpPacket) {
SeverityCode: sev,
TrapOID: trapOID,
}
if err := impl.DBService.Create(&ev).Error; err != nil {
return
}
e.mu.RLock()
rules := e.trapRules
e.mu.RUnlock()
@@ -504,26 +512,28 @@ func (e *Engine) HandleTrap(addr *net.UDPAddr, pkt *gosnmp.SnmpPacket) {
if mErr != nil {
return
}
labels := map[string]string{
"source_type": "log", "source_subtype": "snmp_trap", "trap_oid": trapOID,
"remote_addr": addr.String(), "ip": addr.IP.String(), "instance": addr.IP.String(), "job": "logs-trap",
}
applyResourceIdentity(&ev, labels, "", ref, "snmp_trap", addr.IP.String())
body := AlertReceiveBody{
AlertName: "未解析 SNMP Trap",
Summary: readable,
Description: fp,
SeverityCode: sev,
Value: string(vbJSON),
Labels: map[string]string{
"source_type": "log",
"source_subtype": "snmp_trap",
"trap_oid": trapOID,
"remote_addr": addr.String(),
"ip": addr.IP.String(),
"instance": addr.IP.String(),
"job": "logs-trap",
},
Agent: "logs-trap",
RawData: rawBytes,
Labels: labels,
Agent: "logs-trap",
RawData: rawBytes,
}
if err := enqueueRawEvent(ev.ID, body, "unparsed"); err == nil {
_ = impl.DBService.Model(&ev).Update("dispatch_status", "pending").Error
if err := persistLogEventAndOutbox(&ev, func(tx *gorm.DB, stored *models.LogEvent) (uint, error) {
body.State = "firing"
body.SourceEventKey = logSourceEventKey(stored, "raw")
body.OccurredAt = occurredAt
return enqueueRawEventWithDB(tx, stored.ID, body, "unparsed")
}); err != nil {
log.Printf("logs: persist trap raw event: %v", err)
}
return
}
@@ -551,6 +561,7 @@ func (e *Engine) HandleTrap(addr *net.UDPAddr, pkt *gosnmp.SnmpPacket) {
labels["resource_id"] = trapOID
}
}
applyResourceIdentity(&ev, labels, "", ref, "snmp_trap", addr.IP.String())
resolved := map[string]interface{}{}
if dict != nil {
resolved["vendor"] = dict.Vendor
@@ -561,36 +572,27 @@ func (e *Engine) HandleTrap(addr *net.UDPAddr, pkt *gosnmp.SnmpPacket) {
resolved["severity_mapping"] = dict.SeverityMappingJSON
resolved["parse_expression"] = dict.ParseExpression
}
rawObj := map[string]interface{}{
"source": "snmp_trap",
"received_at": time.Now().UTC().Format(time.RFC3339),
"source_ip": addr.IP.String(),
"log_entry_id": ev.ID,
"trap_oid": trapOID,
"varbinds": trapVarbinds(pkt),
"resolved": resolved,
"pdu_summary": fp,
}
if matched.ID != 0 {
rawObj["rule_id"] = matched.ID
}
rawBytes, mErr := json.Marshal(rawObj)
if mErr != nil {
return
}
body := AlertReceiveBody{
AlertName: firstNonEmpty(matched.AlertName, "SNMP Trap"),
Summary: readable,
Description: desc,
SeverityCode: firstNonEmpty(matched.SeverityCode, sev),
Value: string(vbJSON),
Labels: labels,
Agent: "logs-trap",
PolicyID: matched.PolicyID,
RawData: rawBytes,
}
if err := enqueueAlert(ev.ID, body); err == nil {
_ = impl.DBService.Model(&ev).Update("dispatch_status", "pending").Error
if err := persistLogEventAndOutbox(&ev, func(tx *gorm.DB, stored *models.LogEvent) (uint, error) {
rawObj := map[string]interface{}{
"source": "snmp_trap", "received_at": occurredAt.Format(time.RFC3339), "source_ip": addr.IP.String(),
"log_entry_id": stored.ID, "trap_oid": trapOID, "varbinds": trapVarbinds(pkt), "resolved": resolved, "pdu_summary": fp,
}
if matched.ID != 0 {
rawObj["rule_id"] = matched.ID
}
rawBytes, err := json.Marshal(rawObj)
if err != nil {
return 0, err
}
body := AlertReceiveBody{
AlertName: firstNonEmpty(matched.AlertName, "SNMP Trap"), Summary: readable, Description: desc,
SeverityCode: firstNonEmpty(matched.SeverityCode, sev), Value: string(vbJSON), Labels: labels,
Agent: "logs-trap", PolicyID: matched.PolicyID, State: "firing",
SourceEventKey: logSourceEventKey(stored, "alert"), OccurredAt: occurredAt, RawData: rawBytes,
}
return enqueueAlertWithDB(tx, stored.ID, body)
}); err != nil {
log.Printf("logs: persist matched trap event: %v", err)
}
}
@@ -693,3 +695,116 @@ func (e *Engine) resolveResource(sourceIP, hostname string) (resourceRef, string
}
return resourceRef{}, "none"
}
func persistLogEventAndOutbox(ev *models.LogEvent, enqueue func(*gorm.DB, *models.LogEvent) (uint, error)) error {
if impl.DBService == nil {
return fmt.Errorf("database is not initialized")
}
if ev == nil || enqueue == nil {
return fmt.Errorf("log event and outbox builder are required")
}
return impl.DBService.Transaction(func(tx *gorm.DB) error {
if err := tx.Create(ev).Error; err != nil {
return err
}
outboxID, err := enqueue(tx, ev)
if err != nil {
return err
}
ev.DispatchStatus = "pending"
ev.DispatchOutboxID = outboxID
return tx.Model(ev).Updates(map[string]interface{}{
"dispatch_status": ev.DispatchStatus,
"dispatch_outbox_id": ev.DispatchOutboxID,
}).Error
})
}
func logSourceEventKey(ev *models.LogEvent, purpose string) string {
return fmt.Sprintf("logs:%s:%d:%s", strings.TrimSpace(ev.SourceKind), ev.ID, strings.TrimSpace(purpose))
}
func applyResourceIdentity(ev *models.LogEvent, labels map[string]string, explicitUID string, ref resourceRef, subtype, sourceIP string) {
uid := strings.TrimSpace(explicitUID)
category := ""
identity := ""
if uid == "" {
uid = strings.TrimSpace(ref.ResourceUID)
}
if uid != "" {
if knownCategory, knownIdentity, ok := splitKnownResourceUID(uid); ok {
category, identity = knownCategory, knownIdentity
uid = category + ":" + identity
} else {
category = strings.ToLower(strings.TrimSpace(ref.ResourceCategory))
if category == "" {
category = canonicalLogResourceCategory(ref.ResourceType, subtype)
}
identity = uid
uid = category + ":" + identity
}
}
if uid == "" && strings.TrimSpace(ref.ResourceCategory) != "" && strings.TrimSpace(ref.ServiceIdentity) != "" {
category = strings.ToLower(strings.TrimSpace(ref.ResourceCategory))
identity = strings.TrimSpace(ref.ServiceIdentity)
uid = category + ":" + identity
}
if uid == "" && strings.TrimSpace(ref.ResourceID) != "" {
identity = strings.TrimSpace(ref.ResourceID)
category = canonicalLogResourceCategory(ref.ResourceType, subtype)
if knownCategory, knownIdentity, ok := splitKnownResourceUID(identity); ok {
category, identity = knownCategory, knownIdentity
uid = category + ":" + identity
} else {
uid = category + ":" + identity
}
}
if uid == "" {
category = canonicalLogResourceCategory("", subtype)
identity = strings.TrimSpace(sourceIP)
uid = category + ":" + identity
}
labels["resource_uid"] = uid
labels["resource_category"] = category
labels["service_identity"] = identity
ev.ResourceUID = uid
if ev.ResourceType == "" || ev.ResourceID == "" {
ev.ResourceType = category
ev.ResourceID = identity
}
}
func canonicalLogResourceCategory(resourceType, subtype string) string {
switch strings.ToLower(strings.TrimSpace(resourceType)) {
case "server", "host":
return "host"
case "device", "network", "network_device":
return "network"
case "collector":
return "collector"
case "database", "middleware", "security", "storage", "room_device":
return strings.ToLower(strings.TrimSpace(resourceType))
}
if strings.TrimSpace(subtype) == "snmp_trap" || strings.TrimSpace(subtype) == "trap" {
return "network"
}
return "log_source"
}
func splitKnownResourceUID(value string) (string, string, bool) {
prefix, identity, found := strings.Cut(strings.TrimSpace(value), ":")
if !found {
return "", "", false
}
category := strings.ToLower(strings.TrimSpace(prefix))
identity = strings.TrimSpace(identity)
if identity == "" {
return "", "", false
}
switch category {
case "host", "network", "collector", "database", "middleware", "security", "storage", "room_device", "log_source":
return category, identity, true
default:
return "", "", false
}
}