fix
This commit is contained in:
426
internal/ingest/engine.go
Normal file
426
internal/ingest/engine.go
Normal file
@@ -0,0 +1,426 @@
|
||||
package ingest
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"git.apinb.com/ops/logs/internal/config"
|
||||
"git.apinb.com/ops/logs/internal/impl"
|
||||
"git.apinb.com/ops/logs/internal/models"
|
||||
"github.com/gosnmp/gosnmp"
|
||||
)
|
||||
|
||||
type Engine struct {
|
||||
mu sync.RWMutex
|
||||
|
||||
trapDict []models.TrapDictionaryEntry
|
||||
syslogRules []models.SyslogRule
|
||||
trapRules []models.TrapRule
|
||||
shields []models.TrapShield
|
||||
}
|
||||
|
||||
var Global = &Engine{}
|
||||
|
||||
func (e *Engine) Refresh() error {
|
||||
var dict []models.TrapDictionaryEntry
|
||||
var syslog []models.SyslogRule
|
||||
var trap []models.TrapRule
|
||||
var shield []models.TrapShield
|
||||
|
||||
if err := impl.DBService.Where("enabled = ?", true).Find(&dict).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
sort.Slice(dict, func(i, j int) bool {
|
||||
return len(dict[i].OIDPrefix) > len(dict[j].OIDPrefix)
|
||||
})
|
||||
|
||||
if err := impl.DBService.Where("enabled = ?", true).Find(&syslog).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
sort.Slice(syslog, func(i, j int) bool { return syslog[i].Priority > syslog[j].Priority })
|
||||
|
||||
if err := impl.DBService.Where("enabled = ?", true).Find(&trap).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
sort.Slice(trap, func(i, j int) bool { return trap[i].Priority > trap[j].Priority })
|
||||
|
||||
if err := impl.DBService.Where("enabled = ?", true).Find(&shield).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
e.mu.Lock()
|
||||
e.trapDict = dict
|
||||
e.syslogRules = syslog
|
||||
e.trapRules = trap
|
||||
e.shields = shield
|
||||
e.mu.Unlock()
|
||||
return nil
|
||||
}
|
||||
|
||||
func StartRefresher() {
|
||||
interval := config.Spec.Ingest.RuleRefreshSecs
|
||||
if interval <= 0 {
|
||||
interval = 30
|
||||
}
|
||||
_ = Global.Refresh()
|
||||
go func() {
|
||||
t := time.NewTicker(time.Duration(interval) * time.Second)
|
||||
defer t.Stop()
|
||||
for range t.C {
|
||||
_ = Global.Refresh()
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
func normOID(s string) string {
|
||||
s = strings.TrimSpace(s)
|
||||
return strings.TrimPrefix(s, ".")
|
||||
}
|
||||
|
||||
func (e *Engine) HandleSyslog(addr *net.UDPAddr, payload []byte) {
|
||||
parsed := parseSyslogPayload(payload)
|
||||
device := parsed.Hostname
|
||||
if device == "" {
|
||||
device = addr.IP.String()
|
||||
}
|
||||
detailObj := map[string]interface{}{
|
||||
"priority": parsed.Priority,
|
||||
"hostname": parsed.Hostname,
|
||||
"tag": parsed.Tag,
|
||||
"message": parsed.Message,
|
||||
}
|
||||
detailBytes, _ := json.Marshal(detailObj)
|
||||
summary := formatSyslogSummary(parsed)
|
||||
sev := syslogPriorityToSeverity(parsed.Priority)
|
||||
|
||||
ev := models.LogEvent{
|
||||
SourceKind: "syslog",
|
||||
RemoteAddr: addr.String(),
|
||||
RawPayload: string(payload),
|
||||
NormalizedSummary: summary,
|
||||
NormalizedDetail: string(detailBytes),
|
||||
DeviceName: device,
|
||||
SeverityCode: sev,
|
||||
}
|
||||
|
||||
e.mu.RLock()
|
||||
rules := e.syslogRules
|
||||
e.mu.RUnlock()
|
||||
|
||||
var matched *models.SyslogRule
|
||||
for i := range rules {
|
||||
if syslogRuleMatches(&rules[i], device, parsed.Message, parsed.RawLine) {
|
||||
matched = &rules[i]
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
if err := impl.DBService.Create(&ev).Error; err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
if matched == nil {
|
||||
return
|
||||
}
|
||||
labels := map[string]string{
|
||||
"source": "syslog",
|
||||
"device": device,
|
||||
"rule_id": strconv.FormatUint(uint64(matched.ID), 10),
|
||||
"rule_name": matched.Name,
|
||||
"remote_addr": addr.String(),
|
||||
}
|
||||
rawObj := map[string]interface{}{
|
||||
"source": "syslog",
|
||||
"received_at": time.Now().UTC().Format(time.RFC3339),
|
||||
"source_ip": addr.IP.String(),
|
||||
"rule_id": matched.ID,
|
||||
"log_entry_id": ev.ID,
|
||||
"raw_packet": string(payload),
|
||||
"parsed": detailObj,
|
||||
}
|
||||
rawBytes, mErr := json.Marshal(rawObj)
|
||||
if mErr != nil {
|
||||
return
|
||||
}
|
||||
body := AlertReceiveBody{
|
||||
AlertName: matched.AlertName,
|
||||
Summary: summary,
|
||||
Description: summary,
|
||||
SeverityCode: firstNonEmpty(matched.SeverityCode, sev),
|
||||
Value: parsed.Message,
|
||||
Labels: labels,
|
||||
Agent: "logs-syslog",
|
||||
PolicyID: matched.PolicyID,
|
||||
RawData: rawBytes,
|
||||
}
|
||||
if err := forwardAlert(body); err == nil {
|
||||
_ = impl.DBService.Model(&ev).Update("alert_sent", true).Error
|
||||
}
|
||||
}
|
||||
|
||||
func syslogRuleMatches(rule *models.SyslogRule, device, message, rawLine string) bool {
|
||||
if strings.TrimSpace(rule.DeviceNameContains) == "" && strings.TrimSpace(rule.KeywordRegex) == "" {
|
||||
return false
|
||||
}
|
||||
deviceName := strings.ToLower(device)
|
||||
contains := strings.ToLower(rule.DeviceNameContains)
|
||||
if contains != "" && !strings.Contains(deviceName, contains) {
|
||||
return false
|
||||
}
|
||||
if rule.KeywordRegex != "" {
|
||||
re, err := regexp.Compile(rule.KeywordRegex)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
if !re.MatchString(message) && !re.MatchString(rawLine) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func trapShielded(e *Engine, addr *net.UDPAddr, trapOID string, pkt *gosnmp.SnmpPacket) bool {
|
||||
ip := addr.IP
|
||||
fp := varbindFingerprint(pkt)
|
||||
now := time.Now()
|
||||
e.mu.RLock()
|
||||
shields := e.shields
|
||||
e.mu.RUnlock()
|
||||
for i := range shields {
|
||||
s := &shields[i]
|
||||
if !s.Enabled {
|
||||
continue
|
||||
}
|
||||
if strings.TrimSpace(s.SourceIPCIDR) == "" {
|
||||
continue
|
||||
}
|
||||
if !ipMatchesCIDR(ip, s.SourceIPCIDR) {
|
||||
continue
|
||||
}
|
||||
if p := strings.TrimSpace(s.OIDPrefix); p != "" && !strings.HasPrefix(normOID(trapOID), normOID(p)) {
|
||||
continue
|
||||
}
|
||||
if h := strings.TrimSpace(s.InterfaceHint); h != "" && !strings.Contains(fp, h) {
|
||||
continue
|
||||
}
|
||||
if !inTimeWindows(now, s.TimeWindowsJSON) {
|
||||
continue
|
||||
}
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func lookupTrapDict(e *Engine, trapOID string) *models.TrapDictionaryEntry {
|
||||
t := normOID(trapOID)
|
||||
e.mu.RLock()
|
||||
dict := e.trapDict
|
||||
e.mu.RUnlock()
|
||||
for i := range dict {
|
||||
if strings.HasPrefix(t, normOID(dict[i].OIDPrefix)) {
|
||||
return &dict[i]
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (e *Engine) HandleTrap(addr *net.UDPAddr, pkt *gosnmp.SnmpPacket) {
|
||||
trapOID := extractTrapOID(pkt)
|
||||
if trapShielded(e, addr, trapOID, pkt) {
|
||||
return
|
||||
}
|
||||
|
||||
dict := lookupTrapDict(e, trapOID)
|
||||
fp := varbindFingerprint(pkt)
|
||||
vbJSON, _ := json.Marshal(trapVarbinds(pkt))
|
||||
|
||||
readable := buildTrapReadable(trapOID, dict, fp)
|
||||
detailObj := map[string]interface{}{
|
||||
"trap_oid": trapOID,
|
||||
"varbinds": trapVarbinds(pkt),
|
||||
"dict_title": "",
|
||||
"dict_description": "",
|
||||
"recovery": "",
|
||||
}
|
||||
sev := "warning"
|
||||
if dict != nil {
|
||||
detailObj["dict_title"] = dict.Title
|
||||
detailObj["dict_description"] = dict.Description
|
||||
detailObj["recovery"] = dict.RecoveryMessage
|
||||
if dict.SeverityCode != "" {
|
||||
sev = dict.SeverityCode
|
||||
}
|
||||
}
|
||||
detailBytes, _ := json.Marshal(detailObj)
|
||||
|
||||
ev := models.LogEvent{
|
||||
SourceKind: "snmp_trap",
|
||||
RemoteAddr: addr.String(),
|
||||
RawPayload: fp,
|
||||
NormalizedSummary: readable,
|
||||
NormalizedDetail: string(detailBytes),
|
||||
DeviceName: addr.IP.String(),
|
||||
SeverityCode: sev,
|
||||
TrapOID: trapOID,
|
||||
}
|
||||
if err := impl.DBService.Create(&ev).Error; err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
e.mu.RLock()
|
||||
rules := e.trapRules
|
||||
e.mu.RUnlock()
|
||||
|
||||
var matched *models.TrapRule
|
||||
for i := range rules {
|
||||
if trapRuleMatches(&rules[i], trapOID, fp) {
|
||||
matched = &rules[i]
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
if matched == nil && dict != nil && strings.TrimSpace(dict.SeverityCode) != "" {
|
||||
matched = &models.TrapRule{
|
||||
AlertName: firstNonEmpty(dict.Title, "SNMP Trap"),
|
||||
SeverityCode: dict.SeverityCode,
|
||||
PolicyID: 0,
|
||||
}
|
||||
}
|
||||
if matched == nil {
|
||||
return
|
||||
}
|
||||
|
||||
desc := readable
|
||||
if dict != nil && dict.RecoveryMessage != "" {
|
||||
desc = readable + "\n恢复建议: " + dict.RecoveryMessage
|
||||
}
|
||||
labels := map[string]string{
|
||||
"source": "snmp_trap",
|
||||
"trap_oid": trapOID,
|
||||
"remote_addr": addr.String(),
|
||||
}
|
||||
if matched.ID != 0 {
|
||||
labels["rule_id"] = strconv.FormatUint(uint64(matched.ID), 10)
|
||||
labels["rule_name"] = matched.Name
|
||||
}
|
||||
resolved := map[string]interface{}{}
|
||||
if dict != nil {
|
||||
resolved["title"] = dict.Title
|
||||
resolved["description"] = dict.Description
|
||||
resolved["recovery"] = dict.RecoveryMessage
|
||||
}
|
||||
rawObj := map[string]interface{}{
|
||||
"source": "snmp_trap",
|
||||
"received_at": time.Now().UTC().Format(time.RFC3339),
|
||||
"source_ip": addr.IP.String(),
|
||||
"log_entry_id": ev.ID,
|
||||
"trap_oid": trapOID,
|
||||
"varbinds": trapVarbinds(pkt),
|
||||
"resolved": resolved,
|
||||
"pdu_summary": fp,
|
||||
}
|
||||
if matched.ID != 0 {
|
||||
rawObj["rule_id"] = matched.ID
|
||||
}
|
||||
rawBytes, mErr := json.Marshal(rawObj)
|
||||
if mErr != nil {
|
||||
return
|
||||
}
|
||||
body := AlertReceiveBody{
|
||||
AlertName: firstNonEmpty(matched.AlertName, "SNMP Trap"),
|
||||
Summary: readable,
|
||||
Description: desc,
|
||||
SeverityCode: firstNonEmpty(matched.SeverityCode, sev),
|
||||
Value: string(vbJSON),
|
||||
Labels: labels,
|
||||
Agent: "logs-trap",
|
||||
PolicyID: matched.PolicyID,
|
||||
RawData: rawBytes,
|
||||
}
|
||||
if err := forwardAlert(body); err == nil {
|
||||
_ = impl.DBService.Model(&ev).Update("alert_sent", true).Error
|
||||
}
|
||||
}
|
||||
|
||||
func extractTrapOID(pkt *gosnmp.SnmpPacket) string {
|
||||
const snmpTrapOID = "1.3.6.1.6.3.1.1.4.1.0"
|
||||
for _, v := range pkt.Variables {
|
||||
if v.Name == snmpTrapOID || strings.HasSuffix(v.Name, ".1.3.6.1.6.3.1.1.4.1.0") {
|
||||
return oidToString(v.Value)
|
||||
}
|
||||
}
|
||||
for _, v := range pkt.Variables {
|
||||
if strings.Contains(v.Name, "1.3.6.1.6.3.1.1.4.1") {
|
||||
return oidToString(v.Value)
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func oidToString(val interface{}) string {
|
||||
switch x := val.(type) {
|
||||
case string:
|
||||
return x
|
||||
case []byte:
|
||||
return string(x)
|
||||
default:
|
||||
return fmt.Sprintf("%v", x)
|
||||
}
|
||||
}
|
||||
|
||||
func trapVarbinds(pkt *gosnmp.SnmpPacket) []map[string]string {
|
||||
out := make([]map[string]string, 0, len(pkt.Variables))
|
||||
for _, v := range pkt.Variables {
|
||||
out = append(out, map[string]string{
|
||||
"oid": v.Name,
|
||||
"type": fmt.Sprintf("%v", v.Type),
|
||||
"value": fmtVarbindValue(v),
|
||||
})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func buildTrapReadable(trapOID string, dict *models.TrapDictionaryEntry, varbindSummary string) string {
|
||||
if dict != nil && dict.Title != "" {
|
||||
return dict.Title + " (" + trapOID + ")"
|
||||
}
|
||||
if trapOID != "" {
|
||||
return "Trap " + trapOID
|
||||
}
|
||||
return truncate(varbindSummary, 256)
|
||||
}
|
||||
|
||||
func trapRuleMatches(rule *models.TrapRule, trapOID, varbindFP string) bool {
|
||||
hasOID := strings.TrimSpace(rule.OIDPrefix) != ""
|
||||
hasRE := strings.TrimSpace(rule.VarbindMatchRegex) != ""
|
||||
if !hasOID && !hasRE {
|
||||
return false
|
||||
}
|
||||
if hasOID && !strings.HasPrefix(normOID(trapOID), normOID(rule.OIDPrefix)) {
|
||||
return false
|
||||
}
|
||||
if rule.VarbindMatchRegex != "" {
|
||||
re, err := regexp.Compile(rule.VarbindMatchRegex)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
if !re.MatchString(varbindFP) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func firstNonEmpty(a, b string) string {
|
||||
if strings.TrimSpace(a) != "" {
|
||||
return a
|
||||
}
|
||||
return b
|
||||
}
|
||||
Reference in New Issue
Block a user