logs/internal/ingest/engine.go

package ingest

import (
	"encoding/json"
	"fmt"
	"net"
	"regexp"
	"sort"
	"strconv"
	"strings"
	"sync"
	"time"

	"git.apinb.com/ops/logs/internal/config"
	"git.apinb.com/ops/logs/internal/impl"
	"git.apinb.com/ops/logs/internal/models"
	"github.com/gosnmp/gosnmp"
)

type Engine struct {
	mu sync.RWMutex

	trapDict     []models.TrapDictionaryEntry
	syslogRules  []models.SyslogRule
	trapRules    []models.TrapRule
	shields      []models.TrapShield
	resourceByIP map[string]resourceRef
	resourceByHN map[string]resourceRef
}

type resourceRef struct {
	ResourceType string
	ResourceID   string
	ResourceName string
}

func resourceTypePriority(resourceType string) int {
	switch strings.ToLower(strings.TrimSpace(resourceType)) {
	case "server":
		return 3
	case "collector":
		return 2
	case "device":
		return 1
	default:
		return 0
	}
}

var Global = &Engine{}

func (e *Engine) Refresh() error {
	var dict []models.TrapDictionaryEntry
	var syslog []models.SyslogRule
	var trap []models.TrapRule
	var shield []models.TrapShield
	var mappings []models.ResourceMapping

	if err := impl.DBService.Where("enabled = ?", true).Find(&dict).Error; err != nil {
		return err
	}
	sort.Slice(dict, func(i, j int) bool {
		return len(dict[i].OIDPrefix) > len(dict[j].OIDPrefix)
	})

	if err := impl.DBService.Where("enabled = ?", true).Find(&syslog).Error; err != nil {
		return err
	}
	sort.Slice(syslog, func(i, j int) bool { return syslog[i].Priority > syslog[j].Priority })

	if err := impl.DBService.Where("enabled = ?", true).Find(&trap).Error; err != nil {
		return err
	}
	sort.Slice(trap, func(i, j int) bool { return trap[i].Priority > trap[j].Priority })

	if err := impl.DBService.Where("enabled = ?", true).Find(&shield).Error; err != nil {
		return err
	}
	if err := impl.DBService.Where("is_deleted = ?", false).Order("updated_at desc, id desc").Find(&mappings).Error; err != nil {
		return err
	}

	ipMap := make(map[string]resourceRef)
	hnMap := make(map[string]resourceRef)
	for _, m := range mappings {
		ref := resourceRef{
			ResourceType: m.ResourceType,
			ResourceID:   m.ResourceID,
			ResourceName: m.ResourceName,
		}
		var ips []string
		if err := json.Unmarshal([]byte(m.IPsJSON), &ips); err == nil {
			for _, ip := range ips {
				key := strings.TrimSpace(ip)
				if key == "" {
					continue
				}
				if cur, exists := ipMap[key]; !exists || resourceTypePriority(ref.ResourceType) > resourceTypePriority(cur.ResourceType) {
					ipMap[key] = ref
				}
			}
		}
		var hostnames []string
		if err := json.Unmarshal([]byte(m.HostnamesJSON), &hostnames); err == nil {
			for _, hn := range hostnames {
				key := strings.ToLower(strings.TrimSpace(hn))
				if key == "" {
					continue
				}
				if cur, exists := hnMap[key]; !exists || resourceTypePriority(ref.ResourceType) > resourceTypePriority(cur.ResourceType) {
					hnMap[key] = ref
				}
			}
		}
	}

	e.mu.Lock()
	e.trapDict = dict
	e.syslogRules = syslog
	e.trapRules = trap
	e.shields = shield
	e.resourceByIP = ipMap
	e.resourceByHN = hnMap
	e.mu.Unlock()
	return nil
}

func StartRefresher() {
	interval := config.Spec.Ingest.RuleRefreshSecs
	if interval <= 0 {
		interval = 30
	}
	_ = Global.Refresh()
	go func() {
		t := time.NewTicker(time.Duration(interval) * time.Second)
		defer t.Stop()
		for range t.C {
			_ = Global.Refresh()
		}
	}()
}

func normOID(s string) string {
	s = strings.TrimSpace(s)
	return strings.TrimPrefix(s, ".")
}

func (e *Engine) HandleSyslog(addr *net.UDPAddr, payload []byte) {
	parsed := parseSyslogPayload(payload)
	device := parsed.Hostname
	if device == "" {
		device = addr.IP.String()
	}
	detailObj := map[string]interface{}{
		"priority": parsed.Priority,
		"hostname": parsed.Hostname,
		"tag":      parsed.Tag,
		"message":  parsed.Message,
	}
	detailBytes, _ := json.Marshal(detailObj)
	summary := formatSyslogSummary(parsed)
	sev := syslogPriorityToSeverity(parsed.Priority)
	ref, method := e.resolveResource(addr.IP.String(), device)

	ev := models.LogEvent{
		SourceKind:        "syslog",
		RemoteAddr:        addr.String(),
		SourceIP:          addr.IP.String(),
		RawPayload:        string(payload),
		NormalizedSummary: summary,
		NormalizedDetail:  string(detailBytes),
		DeviceName:        device,
		ResourceType:      ref.ResourceType,
		ResourceID:        ref.ResourceID,
		ResourceName:      ref.ResourceName,
		MatchMethod:       method,
		DispatchStatus:    "not_applicable",
		SeverityCode:      sev,
	}

	e.mu.RLock()
	rules := e.syslogRules
	e.mu.RUnlock()

	var matched *models.SyslogRule
	var matchDetails syslogRuleMatch
	for i := range rules {
		details := syslogRuleMatchDetails(&rules[i], device, parsed.Message, parsed.RawLine)
		if details.Matched {
			matched = &rules[i]
			matchDetails = details
			break
		}
	}

	if err := impl.DBService.Create(&ev).Error; err != nil {
		return
	}

	if matched == nil {
		rawBytes, mErr := json.Marshal(string(payload))
		if mErr != nil {
			return
		}
		body := AlertReceiveBody{
			AlertName:    "未解析 Syslog",
			Summary:      summary,
			Description:  parsed.RawLine,
			SeverityCode: sev,
			Value:        parsed.Message,
			Labels: map[string]string{
				"source_type":    "log",
				"source_subtype": "syslog",
				"device":         device,
				"remote_addr":    addr.String(),
				"ip":             addr.IP.String(),
				"instance":       firstNonEmpty(device, addr.String()),
				"job":            "logs-syslog",
			},
			Agent:   "logs-syslog",
			RawData: rawBytes,
		}
		if err := enqueueRawEvent(ev.ID, body, "unparsed"); err == nil {
			_ = impl.DBService.Model(&ev).Update("dispatch_status", "pending").Error
		}
		return
	}
	// 与 alert/doc/17-resource-correlation 约定一致（字段映射）
	labels := map[string]string{
		"source_type":    "log",
		"source_subtype": "syslog",
		"resource_type":  "log_rule",
		"resource_id":    strconv.FormatUint(uint64(matched.ID), 10),
		"rule_name":      matched.Name,
		"device":         device,
		"remote_addr":    addr.String(),
		"ip":             addr.IP.String(),
		"instance":       firstNonEmpty(device, addr.String()),
		"job":            "logs-syslog",
	}
	if matchDetails.ResourceUID != "" {
		labels["resource_uid"] = matchDetails.ResourceUID
	}
	rawObj := map[string]interface{}{
		"source":       "syslog",
		"received_at":  time.Now().UTC().Format(time.RFC3339),
		"source_ip":    addr.IP.String(),
		"rule_id":      matched.ID,
		"log_entry_id": ev.ID,
		"raw_packet":   string(payload),
		"parsed":       detailObj,
		"match":        matchDetails.Captures,
	}
	rawBytes, mErr := json.Marshal(rawObj)
	if mErr != nil {
		return
	}
	body := AlertReceiveBody{
		AlertName:    matched.AlertName,
		Summary:      summary,
		Description:  summary,
		SeverityCode: firstNonEmpty(matchDetails.SeverityCode, firstNonEmpty(matched.SeverityCode, sev)),
		Value:        parsed.Message,
		Labels:       labels,
		Agent:        "logs-syslog",
		PolicyID:     matched.PolicyID,
		RawData:      rawBytes,
	}
	if err := enqueueAlert(ev.ID, body); err == nil {
		_ = impl.DBService.Model(&ev).Update("dispatch_status", "pending").Error
	}
}

type syslogRuleMatch struct {
	Matched      bool
	ResourceUID  string
	SeverityCode string
	Captures     map[string]string
}

func syslogRuleMatches(rule *models.SyslogRule, device, message, rawLine string) bool {
	return syslogRuleMatchDetails(rule, device, message, rawLine).Matched
}

func syslogRuleMatchDetails(rule *models.SyslogRule, device, message, rawLine string) syslogRuleMatch {
	result := syslogRuleMatch{Captures: map[string]string{}}
	deviceContains := strings.TrimSpace(rule.DeviceNameContains)
	sourceMatch := strings.TrimSpace(rule.SourceMatch)
	keywordRegex := strings.TrimSpace(rule.KeywordRegex)
	messageRegex := strings.TrimSpace(rule.MessageRegex)
	if deviceContains == "" && sourceMatch == "" && keywordRegex == "" && messageRegex == "" {
		return result
	}
	deviceName := strings.ToLower(device)
	contains := strings.ToLower(deviceContains)
	if contains != "" && !strings.Contains(deviceName, contains) {
		return result
	}
	if sourceMatch != "" {
		source := strings.ToLower(sourceMatch)
		rawLower := strings.ToLower(rawLine)
		msgLower := strings.ToLower(message)
		if !strings.Contains(deviceName, source) && !strings.Contains(rawLower, source) && !strings.Contains(msgLower, source) {
			return result
		}
	}
	for _, pattern := range []string{keywordRegex, messageRegex} {
		if pattern == "" {
			continue
		}
		re, err := regexp.Compile(pattern)
		if err != nil {
			return result
		}
		matches := re.FindStringSubmatch(message)
		if matches == nil {
			matches = re.FindStringSubmatch(rawLine)
		}
		if matches == nil {
			return result
		}
		mergeNamedCaptures(result.Captures, re, matches)
	}
	result.Matched = true
	if uid := extractWithNamedRegex(rule.ResourceUIDExtractRegex, "resource_uid", message, rawLine); uid != "" {
		result.ResourceUID = normalizeExtractedResourceUID(uid)
	} else if uid := result.Captures["resource_uid"]; uid != "" {
		result.ResourceUID = normalizeExtractedResourceUID(uid)
	}
	result.SeverityCode = mappedSeverity(rule.SeverityMappingJSON, message, rawLine)
	return result
}

func mergeNamedCaptures(dst map[string]string, re *regexp.Regexp, matches []string) {
	names := re.SubexpNames()
	for i, name := range names {
		if i == 0 || name == "" || i >= len(matches) {
			continue
		}
		dst[name] = matches[i]
	}
}

func extractWithNamedRegex(pattern, groupName, message, rawLine string) string {
	pattern = strings.TrimSpace(pattern)
	if pattern == "" {
		return ""
	}
	re, err := regexp.Compile(pattern)
	if err != nil {
		return ""
	}
	for _, text := range []string{message, rawLine} {
		matches := re.FindStringSubmatch(text)
		if matches == nil {
			continue
		}
		names := re.SubexpNames()
		for i, name := range names {
			if i > 0 && name == groupName && i < len(matches) {
				return strings.TrimSpace(matches[i])
			}
		}
		for i := 1; i < len(matches); i++ {
			if strings.TrimSpace(matches[i]) != "" {
				return strings.TrimSpace(matches[i])
			}
		}
	}
	return ""
}

func normalizeExtractedResourceUID(uid string) string {
	uid = strings.TrimSpace(uid)
	if uid == "" || strings.Contains(uid, ":") {
		return uid
	}
	return "network:" + uid
}

func mappedSeverity(mappingJSON, message, rawLine string) string {
	mappingJSON = strings.TrimSpace(mappingJSON)
	if mappingJSON == "" {
		return ""
	}
	var mapping map[string]string
	if err := json.Unmarshal([]byte(mappingJSON), &mapping); err != nil {
		return ""
	}
	for pattern, severity := range mapping {
		re, err := regexp.Compile(pattern)
		if err != nil {
			continue
		}
		if re.MatchString(message) || re.MatchString(rawLine) {
			return severity
		}
	}
	return ""
}

func trapShielded(e *Engine, addr *net.UDPAddr, trapOID string, pkt *gosnmp.SnmpPacket) bool {
	ip := addr.IP
	fp := varbindFingerprint(pkt)
	now := time.Now()
	e.mu.RLock()
	shields := e.shields
	e.mu.RUnlock()
	for i := range shields {
		s := &shields[i]
		if !s.Enabled {
			continue
		}
		if cidr := strings.TrimSpace(s.SourceIPCIDR); cidr != "" && !ipMatchesCIDR(ip, cidr) {
			continue
		}
		if p := strings.TrimSpace(s.OIDPrefix); p != "" && !strings.HasPrefix(normOID(trapOID), normOID(p)) {
			continue
		}
		if h := strings.TrimSpace(s.InterfaceHint); h != "" && !strings.Contains(fp, h) {
			continue
		}
		if !inTimeWindows(now, s.TimeWindowsJSON) {
			continue
		}
		return true
	}
	return false
}

func lookupTrapDict(e *Engine, trapOID string) *models.TrapDictionaryEntry {
	t := normOID(trapOID)
	e.mu.RLock()
	dict := e.trapDict
	e.mu.RUnlock()
	for i := range dict {
		if oid := normOID(dict[i].OID); oid != "" && t == oid {
			return &dict[i]
		}
		if prefix := normOID(dict[i].OIDPrefix); prefix != "" && strings.HasPrefix(t, prefix) {
			return &dict[i]
		}
	}
	return nil
}

func (e *Engine) HandleTrap(addr *net.UDPAddr, pkt *gosnmp.SnmpPacket) {
	trapOID := extractTrapOID(pkt)
	if trapShielded(e, addr, trapOID, pkt) {
		return
	}

	dict := lookupTrapDict(e, trapOID)
	fp := varbindFingerprint(pkt)
	vbJSON, _ := json.Marshal(trapVarbinds(pkt))

	readable := buildTrapReadable(trapOID, dict, fp)
	detailObj := map[string]interface{}{
		"trap_oid":         trapOID,
		"varbinds":         trapVarbinds(pkt),
		"dict_title":       "",
		"dict_description": "",
		"recovery":         "",
	}
	sev := "warning"
	if dict != nil {
		detailObj["dict_title"] = firstNonEmpty(dict.Name, dict.Title)
		detailObj["dict_description"] = dict.Description
		detailObj["recovery"] = dict.RecoveryMessage
		if dict.SeverityCode != "" {
			sev = dict.SeverityCode
		}
	}
	detailBytes, _ := json.Marshal(detailObj)
	ref, method := e.resolveResource(addr.IP.String(), addr.IP.String())

	ev := models.LogEvent{
		SourceKind:        "snmp_trap",
		RemoteAddr:        addr.String(),
		SourceIP:          addr.IP.String(),
		RawPayload:        fp,
		NormalizedSummary: readable,
		NormalizedDetail:  string(detailBytes),
		DeviceName:        addr.IP.String(),
		ResourceType:      ref.ResourceType,
		ResourceID:        ref.ResourceID,
		ResourceName:      ref.ResourceName,
		MatchMethod:       method,
		DispatchStatus:    "not_applicable",
		SeverityCode:      sev,
		TrapOID:           trapOID,
	}
	if err := impl.DBService.Create(&ev).Error; err != nil {
		return
	}

	e.mu.RLock()
	rules := e.trapRules
	e.mu.RUnlock()

	var matched *models.TrapRule
	for i := range rules {
		if trapRuleMatches(&rules[i], trapOID, fp) {
			matched = &rules[i]
			break
		}
	}

	if matched == nil && dict != nil && strings.TrimSpace(dict.SeverityCode) != "" {
		matched = &models.TrapRule{
			AlertName:    firstNonEmpty(firstNonEmpty(dict.Name, dict.Title), "SNMP Trap"),
			SeverityCode: dict.SeverityCode,
			PolicyID:     0,
		}
	}
	if matched == nil {
		rawBytes, mErr := json.Marshal(fp)
		if mErr != nil {
			return
		}
		body := AlertReceiveBody{
			AlertName:    "未解析 SNMP Trap",
			Summary:      readable,
			Description:  fp,
			SeverityCode: sev,
			Value:        string(vbJSON),
			Labels: map[string]string{
				"source_type":    "log",
				"source_subtype": "snmp_trap",
				"trap_oid":       trapOID,
				"remote_addr":    addr.String(),
				"ip":             addr.IP.String(),
				"instance":       addr.IP.String(),
				"job":            "logs-trap",
			},
			Agent:   "logs-trap",
			RawData: rawBytes,
		}
		if err := enqueueRawEvent(ev.ID, body, "unparsed"); err == nil {
			_ = impl.DBService.Model(&ev).Update("dispatch_status", "pending").Error
		}
		return
	}

	desc := readable
	if dict != nil && dict.RecoveryMessage != "" {
		desc = readable + "\n恢复建议: " + dict.RecoveryMessage
	}
	labels := map[string]string{
		"source_type":    "log",
		"source_subtype": "snmp_trap",
		"trap_oid":       trapOID,
		"remote_addr":    addr.String(),
		"ip":             addr.IP.String(),
		"instance":       addr.IP.String(),
		"job":            "logs-trap",
	}
	if matched.ID != 0 {
		labels["resource_type"] = "trap_rule"
		labels["resource_id"] = strconv.FormatUint(uint64(matched.ID), 10)
		labels["rule_name"] = matched.Name
	} else {
		labels["resource_type"] = "trap_dictionary"
		if trapOID != "" {
			labels["resource_id"] = trapOID
		}
	}
	resolved := map[string]interface{}{}
	if dict != nil {
		resolved["vendor"] = dict.Vendor
		resolved["oid"] = firstNonEmpty(dict.OID, dict.OIDPrefix)
		resolved["title"] = firstNonEmpty(dict.Name, dict.Title)
		resolved["description"] = dict.Description
		resolved["recovery"] = dict.RecoveryMessage
		resolved["severity_mapping"] = dict.SeverityMappingJSON
		resolved["parse_expression"] = dict.ParseExpression
	}
	rawObj := map[string]interface{}{
		"source":       "snmp_trap",
		"received_at":  time.Now().UTC().Format(time.RFC3339),
		"source_ip":    addr.IP.String(),
		"log_entry_id": ev.ID,
		"trap_oid":     trapOID,
		"varbinds":     trapVarbinds(pkt),
		"resolved":     resolved,
		"pdu_summary":  fp,
	}
	if matched.ID != 0 {
		rawObj["rule_id"] = matched.ID
	}
	rawBytes, mErr := json.Marshal(rawObj)
	if mErr != nil {
		return
	}
	body := AlertReceiveBody{
		AlertName:    firstNonEmpty(matched.AlertName, "SNMP Trap"),
		Summary:      readable,
		Description:  desc,
		SeverityCode: firstNonEmpty(matched.SeverityCode, sev),
		Value:        string(vbJSON),
		Labels:       labels,
		Agent:        "logs-trap",
		PolicyID:     matched.PolicyID,
		RawData:      rawBytes,
	}
	if err := enqueueAlert(ev.ID, body); err == nil {
		_ = impl.DBService.Model(&ev).Update("dispatch_status", "pending").Error
	}
}

func extractTrapOID(pkt *gosnmp.SnmpPacket) string {
	const snmpTrapOID = "1.3.6.1.6.3.1.1.4.1.0"
	for _, v := range pkt.Variables {
		if v.Name == snmpTrapOID || strings.HasSuffix(v.Name, ".1.3.6.1.6.3.1.1.4.1.0") {
			return oidToString(v.Value)
		}
	}
	for _, v := range pkt.Variables {
		if strings.Contains(v.Name, "1.3.6.1.6.3.1.1.4.1") {
			return oidToString(v.Value)
		}
	}
	return ""
}

func oidToString(val interface{}) string {
	switch x := val.(type) {
	case string:
		return x
	case []byte:
		return string(x)
	default:
		return fmt.Sprintf("%v", x)
	}
}

func trapVarbinds(pkt *gosnmp.SnmpPacket) []map[string]string {
	out := make([]map[string]string, 0, len(pkt.Variables))
	for _, v := range pkt.Variables {
		out = append(out, map[string]string{
			"oid":   v.Name,
			"type":  fmt.Sprintf("%v", v.Type),
			"value": fmtVarbindValue(v),
		})
	}
	return out
}

func buildTrapReadable(trapOID string, dict *models.TrapDictionaryEntry, varbindSummary string) string {
	if dict != nil && firstNonEmpty(dict.Name, dict.Title) != "" {
		return firstNonEmpty(dict.Name, dict.Title) + " (" + trapOID + ")"
	}
	if trapOID != "" {
		return "Trap " + trapOID
	}
	return truncate(varbindSummary, 256)
}

func trapRuleMatches(rule *models.TrapRule, trapOID, varbindFP string) bool {
	hasOID := strings.TrimSpace(rule.OIDPrefix) != ""
	hasRE := strings.TrimSpace(rule.VarbindMatchRegex) != ""
	if !hasOID && !hasRE {
		return false
	}
	if hasOID && !strings.HasPrefix(normOID(trapOID), normOID(rule.OIDPrefix)) {
		return false
	}
	if rule.VarbindMatchRegex != "" {
		re, err := regexp.Compile(rule.VarbindMatchRegex)
		if err != nil {
			return false
		}
		if !re.MatchString(varbindFP) {
			return false
		}
	}
	return true
}

func firstNonEmpty(a, b string) string {
	if strings.TrimSpace(a) != "" {
		return a
	}
	return b
}

func (e *Engine) resolveResource(sourceIP, hostname string) (resourceRef, string) {
	e.mu.RLock()
	ipMap := e.resourceByIP
	hnMap := e.resourceByHN
	e.mu.RUnlock()

	if ref, ok := ipMap[strings.TrimSpace(sourceIP)]; ok {
		return ref, "ip"
	}
	if ref, ok := hnMap[strings.ToLower(strings.TrimSpace(hostname))]; ok {
		return ref, "hostname"
	}
	return resourceRef{}, "none"
}