任务执行1-19

This commit is contained in:
zxr
2026-06-26 12:51:50 +08:00
parent 175d9f8f94
commit 19908230f2
19 changed files with 2615 additions and 1260 deletions

View File

@@ -0,0 +1,87 @@
# Syslog-Trap 接入与重放
## 接入目标
`logs` 服务负责接收 Syslog 与 SNMP Trap按字典和规则解析后写入 `logs_events`,并通过 `logs_alert_outbox` 异步转发到 `alert` 的原始事件池:
```text
Syslog / Trap -> logs_events -> logs_alert_outbox -> Alert/v1/raw-events/ingest
```
转发使用 `X-Internal-Key`,配置来自 `AlertForward.internal_key`。解析成功的事件 `parse_status=parsed`,未命中字典或规则的事件仍保存原始报文,并以 `parse_status=unparsed` 入队,便于规则调整后重放。
## 部署配置
`logs` 当前内置 UDP 接收器:
```yaml
Ingest:
syslog_listen_addr: "0.0.0.0:5140"
trap_listen_addr: "0.0.0.0:1620"
rule_refresh_secs: 30
AlertForward:
enabled: true
base_url: "http://127.0.0.1:18080"
internal_key: "change-me"
default_policy_id: 1
```
生产环境如需标准端口 `514/162`,建议由 systemd socket、firewalld rich rule、iptables REDIRECT 或外层采集网关转发到非特权端口。TCP Syslog 接入建议在网关层启用 TCP listener再转发到 UDP 或调用后续 HTTP ingest 入口;开启 TCP 时必须保留原始来源 IP 和 trace ID。
## 字典与规则
Trap 字典字段:
- `vendor`:厂商,例如 `H3C`
- `oid`:精确 Trap OID。
- `oid_prefix`OID 前缀,兼容旧字典。
- `name` / `title`:展示名称。
- `severity_mapping_json`:级别映射 JSON。
- `parse_expression`:解析 varbind 的表达式或正则。
Syslog 规则字段:
- `source_match`:来源 IP、主机名或原始行子串。
- `message_regex`:消息正文正则。
- `severity_mapping_json`:按正则映射平台级别。
- `resource_uid_extract_regex`:提取 `resource_uid`,优先使用命名分组 `resource_uid`
示例 Syslog
```text
<189>Jun 24 10:00:01 h3c-core-01 IFNET/4/LINK_DOWN: Interface GigabitEthernet1/0/1 is down, resource_uid=network:h3c-core-01
```
示例 H3C Trap OID
```text
1.3.6.1.6.3.1.1.5.3
```
## 未解析队列与重放
未解析事件仍写入 `logs_events`,并创建 outbox payload
- `source_type=syslog``trap`
- `parse_status=unparsed`
- `raw_payload` 保存原始报文或 varbind 摘要
重放接口:
```http
POST /Logs/v1/entries/{id}/replay
Authorization: Bearer <jwt>
```
成功响应会返回新的 `outbox_id`。重放 payload 使用 `parse_status=replayed`,并带上 `labels.replay_of_log_event_id`,前端可在“日志查询 -> 重放结果”中查看发送结果,失败任务可人工重试。
## Smoke 样例
输出 H3C Syslog 与 Trap 示例载荷:
```powershell
C:\Users\27105\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\python.exe scripts\test_alert_receive_smoke.py --print-log-samples
```
这些样例用于准备 UDP/TCP 接收器 smoke 数据,也可作为联调 alert 原始事件池时的期望字段参考。

View File

@@ -5,6 +5,7 @@ import (
"encoding/json" "encoding/json"
"fmt" "fmt"
"net/http" "net/http"
"strings"
"time" "time"
"git.apinb.com/ops/logs/internal/config" "git.apinb.com/ops/logs/internal/config"
@@ -24,6 +25,20 @@ type AlertReceiveBody struct {
RawData json.RawMessage `json:"raw_data"` RawData json.RawMessage `json:"raw_data"`
} }
type RawEventIngestBody struct {
SourceType string `json:"source_type"`
ResourceUID string `json:"resource_uid,omitempty"`
EventTime time.Time `json:"event_time"`
Severity string `json:"severity"`
Title string `json:"title"`
Message string `json:"message"`
Labels map[string]string `json:"labels,omitempty"`
Annotations map[string]string `json:"annotations,omitempty"`
ParseStatus string `json:"parse_status"`
RawPayload json.RawMessage `json:"raw_payload"`
TraceID string `json:"trace_id,omitempty"`
}
func forwardAlert(body AlertReceiveBody) error { func forwardAlert(body AlertReceiveBody) error {
cfg := config.Spec.AlertForward cfg := config.Spec.AlertForward
if cfg == nil || !cfg.Enabled || cfg.BaseURL == "" { if cfg == nil || !cfg.Enabled || cfg.BaseURL == "" {
@@ -38,11 +53,12 @@ func forwardAlert(body AlertReceiveBody) error {
if body.PolicyID == 0 && cfg.DefaultPolicyID > 0 { if body.PolicyID == 0 && cfg.DefaultPolicyID > 0 {
body.PolicyID = cfg.DefaultPolicyID body.PolicyID = cfg.DefaultPolicyID
} }
raw, err := json.Marshal(body) rawEvent := buildRawEventIngestBody(body, "parsed")
raw, err := json.Marshal(rawEvent)
if err != nil { if err != nil {
return err return err
} }
url := cfg.BaseURL + "/Alert/v1/alerts/receive" url := cfg.BaseURL + "/Alert/v1/raw-events/ingest"
req, err := http.NewRequest(http.MethodPost, url, bytes.NewReader(raw)) req, err := http.NewRequest(http.MethodPost, url, bytes.NewReader(raw))
if err != nil { if err != nil {
return err return err
@@ -62,3 +78,62 @@ func forwardAlert(body AlertReceiveBody) error {
} }
return nil return nil
} }
func buildRawEventIngestBody(body AlertReceiveBody, parseStatus string) RawEventIngestBody {
sourceType := rawEventSourceType(body)
if parseStatus == "" {
parseStatus = "parsed"
}
annotations := map[string]string{
"description": body.Description,
"value": body.Value,
"threshold": body.Threshold,
"agent": body.Agent,
}
return RawEventIngestBody{
SourceType: sourceType,
ResourceUID: rawEventResourceUID(body.Labels),
EventTime: time.Now().UTC(),
Severity: body.SeverityCode,
Title: firstNonEmpty(body.AlertName, "日志事件"),
Message: firstNonEmpty(body.Summary, body.Description),
Labels: body.Labels,
Annotations: annotations,
ParseStatus: parseStatus,
RawPayload: body.RawData,
}
}
func rawEventSourceType(body AlertReceiveBody) string {
if body.Labels != nil {
switch strings.TrimSpace(body.Labels["source_subtype"]) {
case "syslog":
return "syslog"
case "snmp_trap":
return "trap"
}
}
switch body.Agent {
case "logs-syslog":
return "syslog"
case "logs-trap":
return "trap"
default:
return "syslog"
}
}
func rawEventResourceUID(labels map[string]string) string {
if labels == nil {
return ""
}
if uid := strings.TrimSpace(labels["resource_uid"]); uid != "" {
return uid
}
category := strings.TrimSpace(labels["resource_category"])
identity := strings.TrimSpace(labels["service_identity"])
if category != "" && identity != "" {
return category + ":" + identity
}
return ""
}

View File

@@ -0,0 +1,59 @@
package ingest
import (
"encoding/json"
"net/http"
"net/http/httptest"
"testing"
"git.apinb.com/ops/logs/internal/config"
)
func TestForwardAlertPostsRawEventIngestPayload(t *testing.T) {
var gotPath string
var gotBody RawEventIngestBody
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
gotPath = r.URL.Path
if err := json.NewDecoder(r.Body).Decode(&gotBody); err != nil {
t.Fatalf("decode body: %v", err)
}
w.WriteHeader(http.StatusOK)
}))
defer server.Close()
config.Spec.AlertForward = &config.AlertForwardConf{
Enabled: true,
BaseURL: server.URL,
InternalKey: "internal-key",
}
err := forwardAlert(AlertReceiveBody{
AlertName: "H3C Trap",
Summary: "Interface down",
Description: "Interface down",
SeverityCode: "major",
Labels: map[string]string{
"source_subtype": "snmp_trap",
"ip": "192.168.1.10",
},
Agent: "logs-trap",
RawData: json.RawMessage(`{"trap_oid":"1.3.6.1.4.1"}`),
})
if err != nil {
t.Fatalf("forwardAlert returned error: %v", err)
}
if gotPath != "/Alert/v1/raw-events/ingest" {
t.Fatalf("expected raw event ingest path, got %q", gotPath)
}
if gotBody.SourceType != "trap" {
t.Fatalf("expected trap source type, got %q", gotBody.SourceType)
}
if gotBody.ParseStatus != "parsed" {
t.Fatalf("expected parsed status, got %q", gotBody.ParseStatus)
}
if string(gotBody.RawPayload) != `{"trap_oid":"1.3.6.1.4.1"}` {
t.Fatalf("raw payload changed: %s", string(gotBody.RawPayload))
}
}

View File

@@ -1,10 +1,14 @@
package ingest package ingest
import ( import (
"bytes"
"encoding/json" "encoding/json"
"fmt"
"net/http"
"strings" "strings"
"time" "time"
"git.apinb.com/ops/logs/internal/config"
"git.apinb.com/ops/logs/internal/impl" "git.apinb.com/ops/logs/internal/impl"
"git.apinb.com/ops/logs/internal/models" "git.apinb.com/ops/logs/internal/models"
) )
@@ -21,9 +25,21 @@ func enqueueAlert(logEventID uint, body AlertReceiveBody) error {
if err != nil { if err != nil {
return err return err
} }
return enqueuePayload(logEventID, string(payload))
}
func enqueueRawEvent(logEventID uint, body AlertReceiveBody, parseStatus string) error {
payload, err := json.Marshal(buildRawEventIngestBody(body, parseStatus))
if err != nil {
return err
}
return enqueuePayload(logEventID, string(payload))
}
func enqueuePayload(logEventID uint, payloadJSON string) error {
row := models.AlertOutbox{ row := models.AlertOutbox{
LogEventID: logEventID, LogEventID: logEventID,
PayloadJSON: string(payload), PayloadJSON: payloadJSON,
Status: outboxStatusPending, Status: outboxStatusPending,
RetryCount: 0, RetryCount: 0,
NextRetryAt: time.Now(), NextRetryAt: time.Now(),
@@ -67,7 +83,7 @@ func processOneOutbox(row models.AlertOutbox) {
markOutboxDead(row.ID, row.RetryCount, "invalid_payload: "+err.Error()) markOutboxDead(row.ID, row.RetryCount, "invalid_payload: "+err.Error())
return return
} }
if err := forwardAlert(body); err != nil { if err := forwardOutboxPayload(row.PayloadJSON, body); err != nil {
markOutboxRetry(row, err.Error()) markOutboxRetry(row, err.Error())
return return
} }
@@ -82,6 +98,47 @@ func processOneOutbox(row models.AlertOutbox) {
}).Error }).Error
} }
func forwardOutboxPayload(payloadJSON string, legacyBody AlertReceiveBody) error {
var rawEvent RawEventIngestBody
if err := json.Unmarshal([]byte(payloadJSON), &rawEvent); err == nil && rawEvent.SourceType != "" && len(rawEvent.RawPayload) > 0 {
return forwardRawEvent(rawEvent)
}
return forwardAlert(legacyBody)
}
func forwardRawEvent(body RawEventIngestBody) error {
cfg := config.Spec.AlertForward
if cfg == nil || !cfg.Enabled || cfg.BaseURL == "" {
return nil
}
if len(body.RawPayload) == 0 {
return fmt.Errorf("raw_payload 不能为空")
}
raw, err := json.Marshal(body)
if err != nil {
return err
}
url := cfg.BaseURL + "/Alert/v1/raw-events/ingest"
req, err := http.NewRequest(http.MethodPost, url, bytes.NewReader(raw))
if err != nil {
return err
}
req.Header.Set("Content-Type", "application/json")
if cfg.InternalKey != "" {
req.Header.Set("X-Internal-Key", cfg.InternalKey)
}
client := &http.Client{Timeout: 10 * time.Second}
resp, err := client.Do(req)
if err != nil {
return err
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return fmt.Errorf("alert returned HTTP %d", resp.StatusCode)
}
return nil
}
func markOutboxRetry(row models.AlertOutbox, msg string) { func markOutboxRetry(row models.AlertOutbox, msg string) {
retry := row.RetryCount + 1 retry := row.RetryCount + 1
const maxRetry = 5 const maxRetry = 5
@@ -122,4 +179,3 @@ func truncateError(s string, n int) string {
} }
return s[:n] return s[:n]
} }

View File

@@ -183,9 +183,12 @@ func (e *Engine) HandleSyslog(addr *net.UDPAddr, payload []byte) {
e.mu.RUnlock() e.mu.RUnlock()
var matched *models.SyslogRule var matched *models.SyslogRule
var matchDetails syslogRuleMatch
for i := range rules { for i := range rules {
if syslogRuleMatches(&rules[i], device, parsed.Message, parsed.RawLine) { details := syslogRuleMatchDetails(&rules[i], device, parsed.Message, parsed.RawLine)
if details.Matched {
matched = &rules[i] matched = &rules[i]
matchDetails = details
break break
} }
} }
@@ -195,6 +198,31 @@ func (e *Engine) HandleSyslog(addr *net.UDPAddr, payload []byte) {
} }
if matched == nil { if matched == nil {
rawBytes, mErr := json.Marshal(string(payload))
if mErr != nil {
return
}
body := AlertReceiveBody{
AlertName: "未解析 Syslog",
Summary: summary,
Description: parsed.RawLine,
SeverityCode: sev,
Value: parsed.Message,
Labels: map[string]string{
"source_type": "log",
"source_subtype": "syslog",
"device": device,
"remote_addr": addr.String(),
"ip": addr.IP.String(),
"instance": firstNonEmpty(device, addr.String()),
"job": "logs-syslog",
},
Agent: "logs-syslog",
RawData: rawBytes,
}
if err := enqueueRawEvent(ev.ID, body, "unparsed"); err == nil {
_ = impl.DBService.Model(&ev).Update("dispatch_status", "pending").Error
}
return return
} }
// 与 alert/doc/17-resource-correlation 约定一致(字段映射) // 与 alert/doc/17-resource-correlation 约定一致(字段映射)
@@ -210,6 +238,9 @@ func (e *Engine) HandleSyslog(addr *net.UDPAddr, payload []byte) {
"instance": firstNonEmpty(device, addr.String()), "instance": firstNonEmpty(device, addr.String()),
"job": "logs-syslog", "job": "logs-syslog",
} }
if matchDetails.ResourceUID != "" {
labels["resource_uid"] = matchDetails.ResourceUID
}
rawObj := map[string]interface{}{ rawObj := map[string]interface{}{
"source": "syslog", "source": "syslog",
"received_at": time.Now().UTC().Format(time.RFC3339), "received_at": time.Now().UTC().Format(time.RFC3339),
@@ -218,6 +249,7 @@ func (e *Engine) HandleSyslog(addr *net.UDPAddr, payload []byte) {
"log_entry_id": ev.ID, "log_entry_id": ev.ID,
"raw_packet": string(payload), "raw_packet": string(payload),
"parsed": detailObj, "parsed": detailObj,
"match": matchDetails.Captures,
} }
rawBytes, mErr := json.Marshal(rawObj) rawBytes, mErr := json.Marshal(rawObj)
if mErr != nil { if mErr != nil {
@@ -227,7 +259,7 @@ func (e *Engine) HandleSyslog(addr *net.UDPAddr, payload []byte) {
AlertName: matched.AlertName, AlertName: matched.AlertName,
Summary: summary, Summary: summary,
Description: summary, Description: summary,
SeverityCode: firstNonEmpty(matched.SeverityCode, sev), SeverityCode: firstNonEmpty(matchDetails.SeverityCode, firstNonEmpty(matched.SeverityCode, sev)),
Value: parsed.Message, Value: parsed.Message,
Labels: labels, Labels: labels,
Agent: "logs-syslog", Agent: "logs-syslog",
@@ -239,25 +271,132 @@ func (e *Engine) HandleSyslog(addr *net.UDPAddr, payload []byte) {
} }
} }
type syslogRuleMatch struct {
Matched bool
ResourceUID string
SeverityCode string
Captures map[string]string
}
func syslogRuleMatches(rule *models.SyslogRule, device, message, rawLine string) bool { func syslogRuleMatches(rule *models.SyslogRule, device, message, rawLine string) bool {
if strings.TrimSpace(rule.DeviceNameContains) == "" && strings.TrimSpace(rule.KeywordRegex) == "" { return syslogRuleMatchDetails(rule, device, message, rawLine).Matched
return false }
func syslogRuleMatchDetails(rule *models.SyslogRule, device, message, rawLine string) syslogRuleMatch {
result := syslogRuleMatch{Captures: map[string]string{}}
deviceContains := strings.TrimSpace(rule.DeviceNameContains)
sourceMatch := strings.TrimSpace(rule.SourceMatch)
keywordRegex := strings.TrimSpace(rule.KeywordRegex)
messageRegex := strings.TrimSpace(rule.MessageRegex)
if deviceContains == "" && sourceMatch == "" && keywordRegex == "" && messageRegex == "" {
return result
} }
deviceName := strings.ToLower(device) deviceName := strings.ToLower(device)
contains := strings.ToLower(rule.DeviceNameContains) contains := strings.ToLower(deviceContains)
if contains != "" && !strings.Contains(deviceName, contains) { if contains != "" && !strings.Contains(deviceName, contains) {
return false return result
} }
if rule.KeywordRegex != "" { if sourceMatch != "" {
re, err := regexp.Compile(rule.KeywordRegex) source := strings.ToLower(sourceMatch)
rawLower := strings.ToLower(rawLine)
msgLower := strings.ToLower(message)
if !strings.Contains(deviceName, source) && !strings.Contains(rawLower, source) && !strings.Contains(msgLower, source) {
return result
}
}
for _, pattern := range []string{keywordRegex, messageRegex} {
if pattern == "" {
continue
}
re, err := regexp.Compile(pattern)
if err != nil { if err != nil {
return false return result
} }
if !re.MatchString(message) && !re.MatchString(rawLine) { matches := re.FindStringSubmatch(message)
return false if matches == nil {
matches = re.FindStringSubmatch(rawLine)
}
if matches == nil {
return result
}
mergeNamedCaptures(result.Captures, re, matches)
}
result.Matched = true
if uid := extractWithNamedRegex(rule.ResourceUIDExtractRegex, "resource_uid", message, rawLine); uid != "" {
result.ResourceUID = normalizeExtractedResourceUID(uid)
} else if uid := result.Captures["resource_uid"]; uid != "" {
result.ResourceUID = normalizeExtractedResourceUID(uid)
}
result.SeverityCode = mappedSeverity(rule.SeverityMappingJSON, message, rawLine)
return result
}
func mergeNamedCaptures(dst map[string]string, re *regexp.Regexp, matches []string) {
names := re.SubexpNames()
for i, name := range names {
if i == 0 || name == "" || i >= len(matches) {
continue
}
dst[name] = matches[i]
} }
} }
return true
func extractWithNamedRegex(pattern, groupName, message, rawLine string) string {
pattern = strings.TrimSpace(pattern)
if pattern == "" {
return ""
}
re, err := regexp.Compile(pattern)
if err != nil {
return ""
}
for _, text := range []string{message, rawLine} {
matches := re.FindStringSubmatch(text)
if matches == nil {
continue
}
names := re.SubexpNames()
for i, name := range names {
if i > 0 && name == groupName && i < len(matches) {
return strings.TrimSpace(matches[i])
}
}
for i := 1; i < len(matches); i++ {
if strings.TrimSpace(matches[i]) != "" {
return strings.TrimSpace(matches[i])
}
}
}
return ""
}
func normalizeExtractedResourceUID(uid string) string {
uid = strings.TrimSpace(uid)
if uid == "" || strings.Contains(uid, ":") {
return uid
}
return "network:" + uid
}
func mappedSeverity(mappingJSON, message, rawLine string) string {
mappingJSON = strings.TrimSpace(mappingJSON)
if mappingJSON == "" {
return ""
}
var mapping map[string]string
if err := json.Unmarshal([]byte(mappingJSON), &mapping); err != nil {
return ""
}
for pattern, severity := range mapping {
re, err := regexp.Compile(pattern)
if err != nil {
continue
}
if re.MatchString(message) || re.MatchString(rawLine) {
return severity
}
}
return ""
} }
func trapShielded(e *Engine, addr *net.UDPAddr, trapOID string, pkt *gosnmp.SnmpPacket) bool { func trapShielded(e *Engine, addr *net.UDPAddr, trapOID string, pkt *gosnmp.SnmpPacket) bool {
@@ -295,7 +434,10 @@ func lookupTrapDict(e *Engine, trapOID string) *models.TrapDictionaryEntry {
dict := e.trapDict dict := e.trapDict
e.mu.RUnlock() e.mu.RUnlock()
for i := range dict { for i := range dict {
if strings.HasPrefix(t, normOID(dict[i].OIDPrefix)) { if oid := normOID(dict[i].OID); oid != "" && t == oid {
return &dict[i]
}
if prefix := normOID(dict[i].OIDPrefix); prefix != "" && strings.HasPrefix(t, prefix) {
return &dict[i] return &dict[i]
} }
} }
@@ -322,7 +464,7 @@ func (e *Engine) HandleTrap(addr *net.UDPAddr, pkt *gosnmp.SnmpPacket) {
} }
sev := "warning" sev := "warning"
if dict != nil { if dict != nil {
detailObj["dict_title"] = dict.Title detailObj["dict_title"] = firstNonEmpty(dict.Name, dict.Title)
detailObj["dict_description"] = dict.Description detailObj["dict_description"] = dict.Description
detailObj["recovery"] = dict.RecoveryMessage detailObj["recovery"] = dict.RecoveryMessage
if dict.SeverityCode != "" { if dict.SeverityCode != "" {
@@ -366,12 +508,37 @@ func (e *Engine) HandleTrap(addr *net.UDPAddr, pkt *gosnmp.SnmpPacket) {
if matched == nil && dict != nil && strings.TrimSpace(dict.SeverityCode) != "" { if matched == nil && dict != nil && strings.TrimSpace(dict.SeverityCode) != "" {
matched = &models.TrapRule{ matched = &models.TrapRule{
AlertName: firstNonEmpty(dict.Title, "SNMP Trap"), AlertName: firstNonEmpty(firstNonEmpty(dict.Name, dict.Title), "SNMP Trap"),
SeverityCode: dict.SeverityCode, SeverityCode: dict.SeverityCode,
PolicyID: 0, PolicyID: 0,
} }
} }
if matched == nil { if matched == nil {
rawBytes, mErr := json.Marshal(fp)
if mErr != nil {
return
}
body := AlertReceiveBody{
AlertName: "未解析 SNMP Trap",
Summary: readable,
Description: fp,
SeverityCode: sev,
Value: string(vbJSON),
Labels: map[string]string{
"source_type": "log",
"source_subtype": "snmp_trap",
"trap_oid": trapOID,
"remote_addr": addr.String(),
"ip": addr.IP.String(),
"instance": addr.IP.String(),
"job": "logs-trap",
},
Agent: "logs-trap",
RawData: rawBytes,
}
if err := enqueueRawEvent(ev.ID, body, "unparsed"); err == nil {
_ = impl.DBService.Model(&ev).Update("dispatch_status", "pending").Error
}
return return
} }
@@ -400,9 +567,13 @@ func (e *Engine) HandleTrap(addr *net.UDPAddr, pkt *gosnmp.SnmpPacket) {
} }
resolved := map[string]interface{}{} resolved := map[string]interface{}{}
if dict != nil { if dict != nil {
resolved["title"] = dict.Title resolved["vendor"] = dict.Vendor
resolved["oid"] = firstNonEmpty(dict.OID, dict.OIDPrefix)
resolved["title"] = firstNonEmpty(dict.Name, dict.Title)
resolved["description"] = dict.Description resolved["description"] = dict.Description
resolved["recovery"] = dict.RecoveryMessage resolved["recovery"] = dict.RecoveryMessage
resolved["severity_mapping"] = dict.SeverityMappingJSON
resolved["parse_expression"] = dict.ParseExpression
} }
rawObj := map[string]interface{}{ rawObj := map[string]interface{}{
"source": "snmp_trap", "source": "snmp_trap",
@@ -476,8 +647,8 @@ func trapVarbinds(pkt *gosnmp.SnmpPacket) []map[string]string {
} }
func buildTrapReadable(trapOID string, dict *models.TrapDictionaryEntry, varbindSummary string) string { func buildTrapReadable(trapOID string, dict *models.TrapDictionaryEntry, varbindSummary string) string {
if dict != nil && dict.Title != "" { if dict != nil && firstNonEmpty(dict.Name, dict.Title) != "" {
return dict.Title + " (" + trapOID + ")" return firstNonEmpty(dict.Name, dict.Title) + " (" + trapOID + ")"
} }
if trapOID != "" { if trapOID != "" {
return "Trap " + trapOID return "Trap " + trapOID

131
internal/ingest/replay.go Normal file
View File

@@ -0,0 +1,131 @@
package ingest
import (
"encoding/json"
"fmt"
"strconv"
"strings"
"time"
"git.apinb.com/ops/logs/internal/impl"
"git.apinb.com/ops/logs/internal/models"
)
func BuildReplayRawEventPayload(ev models.LogEvent) (RawEventIngestBody, error) {
sourceType := replaySourceType(ev.SourceKind)
if sourceType == "" {
return RawEventIngestBody{}, fmt.Errorf("unsupported source kind %q", ev.SourceKind)
}
rawObj := map[string]interface{}{
"source": sourceType,
"replayed_at": time.Now().UTC().Format(time.RFC3339),
"log_entry_id": ev.ID,
"source_ip": ev.SourceIP,
"remote_addr": ev.RemoteAddr,
"raw_packet": ev.RawPayload,
}
if ev.TrapOID != "" {
rawObj["trap_oid"] = ev.TrapOID
}
if ev.NormalizedDetail != "" {
var detail interface{}
if err := json.Unmarshal([]byte(ev.NormalizedDetail), &detail); err == nil {
rawObj["parsed"] = detail
}
}
rawBytes, err := json.Marshal(rawObj)
if err != nil {
return RawEventIngestBody{}, err
}
labels := map[string]string{
"source_type": "log",
"source_subtype": replaySubtype(ev.SourceKind),
"replay_of_log_event_id": strconv.FormatUint(uint64(ev.ID), 10),
"ip": ev.SourceIP,
"remote_addr": ev.RemoteAddr,
"device": ev.DeviceName,
"job": "logs-replay",
}
if uid := replayResourceUID(ev); uid != "" {
labels["resource_uid"] = uid
}
return RawEventIngestBody{
SourceType: sourceType,
ResourceUID: replayResourceUID(ev),
EventTime: time.Now().UTC(),
Severity: firstNonEmpty(ev.SeverityCode, "warning"),
Title: replayTitle(ev),
Message: firstNonEmpty(ev.NormalizedSummary, ev.RawPayload),
Labels: labels,
Annotations: map[string]string{
"replay": "true",
"dispatch_status": ev.DispatchStatus,
},
ParseStatus: "replayed",
RawPayload: rawBytes,
}, nil
}
func EnqueueReplayLogEvent(ev models.LogEvent) (uint, error) {
body, err := BuildReplayRawEventPayload(ev)
if err != nil {
return 0, err
}
payload, err := json.Marshal(body)
if err != nil {
return 0, err
}
row := models.AlertOutbox{
LogEventID: ev.ID,
PayloadJSON: string(payload),
Status: outboxStatusPending,
RetryCount: 0,
NextRetryAt: time.Now(),
}
if err := enqueueOutboxRow(&row); err != nil {
return 0, err
}
return row.ID, nil
}
func enqueueOutboxRow(row *models.AlertOutbox) error {
return impl.DBService.Create(row).Error
}
func replaySourceType(kind string) string {
switch strings.TrimSpace(kind) {
case "syslog":
return "syslog"
case "snmp_trap", "trap":
return "trap"
default:
return ""
}
}
func replaySubtype(kind string) string {
if kind == "snmp_trap" {
return "snmp_trap"
}
return replaySourceType(kind)
}
func replayResourceUID(ev models.LogEvent) string {
if strings.Contains(ev.ResourceID, ":") {
return ev.ResourceID
}
if ev.ResourceType != "" && ev.ResourceID != "" {
return ev.ResourceType + ":" + ev.ResourceID
}
return ""
}
func replayTitle(ev models.LogEvent) string {
if ev.SourceKind == "snmp_trap" && ev.TrapOID != "" {
return "重放 SNMP Trap " + ev.TrapOID
}
if ev.SourceKind == "syslog" {
return "重放 Syslog"
}
return "重放日志事件"
}

View File

@@ -0,0 +1,61 @@
package ingest
import (
"encoding/json"
"testing"
"git.apinb.com/ops/logs/internal/models"
)
func TestSyslogRuleMatchDetailsExtractsResourceUID(t *testing.T) {
rule := models.SyslogRule{
Name: "H3C link down",
Enabled: true,
SourceMatch: "h3c-core",
MessageRegex: `Interface (?P<iface>GigabitEthernet[0-9/]+) is down`,
ResourceUIDExtractRegex: `Interface (?P<resource_uid>GigabitEthernet[0-9/]+) is down`,
}
match := syslogRuleMatchDetails(&rule, "h3c-core-01", "Interface GigabitEthernet1/0/1 is down", "")
if !match.Matched {
t.Fatal("expected rule to match")
}
if match.ResourceUID != "network:GigabitEthernet1/0/1" {
t.Fatalf("unexpected resource uid: %q", match.ResourceUID)
}
}
func TestBuildReplayRawEventPayloadMarksReplayed(t *testing.T) {
ev := models.LogEvent{
ID: 12,
SourceKind: "syslog",
SourceIP: "10.1.2.3",
RemoteAddr: "10.1.2.3:514",
DeviceName: "h3c-core-01",
RawPayload: "<189>Jun 24 10:00:01 h3c-core-01 IFNET/4/LINK_DOWN: Interface GigabitEthernet1/0/1 is down",
NormalizedSummary: "h3c-core-01: Interface GigabitEthernet1/0/1 is down",
SeverityCode: "warning",
DispatchStatus: "pending",
}
body, err := BuildReplayRawEventPayload(ev)
if err != nil {
t.Fatalf("BuildReplayRawEventPayload returned error: %v", err)
}
if body.SourceType != "syslog" {
t.Fatalf("unexpected source type: %q", body.SourceType)
}
if body.ParseStatus != "replayed" {
t.Fatalf("unexpected parse status: %q", body.ParseStatus)
}
if body.Labels["replay_of_log_event_id"] != "12" {
t.Fatalf("missing replay label: %#v", body.Labels)
}
var raw map[string]any
if err := json.Unmarshal(body.RawPayload, &raw); err != nil {
t.Fatalf("raw payload should be json: %v", err)
}
if raw["raw_packet"] == "" {
t.Fatalf("raw packet missing: %#v", raw)
}
}

View File

@@ -0,0 +1,219 @@
package audit
import (
"errors"
"strings"
"time"
"git.apinb.com/ops/logs/internal/impl"
"git.apinb.com/ops/logs/internal/models"
)
const (
RiskNormal = "normal"
RiskDangerous = "dangerous"
ApprovalPending = "pending"
ApprovalApproved = "approved"
ApprovalRejected = "rejected"
)
type Record struct {
TraceID string `json:"trace_id,omitempty"`
SourceService string `json:"source_service,omitempty"`
ActorID string `json:"actor_id,omitempty"`
ActorName string `json:"actor_name,omitempty"`
Action string `json:"action,omitempty"`
ObjectType string `json:"object_type,omitempty"`
ObjectID string `json:"object_id,omitempty"`
OperationRisk string `json:"operation_risk,omitempty"`
ApprovalID string `json:"approval_id,omitempty"`
RequestMethod string `json:"request_method,omitempty"`
RequestPath string `json:"request_path,omitempty"`
ClientIP string `json:"client_ip,omitempty"`
BeforeJSON string `json:"before_json,omitempty"`
AfterJSON string `json:"after_json,omitempty"`
Result string `json:"result,omitempty"`
ErrorMessage string `json:"error_message,omitempty"`
}
type ApprovalRequest struct {
RequestID string `json:"request_id,omitempty"`
SourceService string `json:"source_service,omitempty"`
Action string `json:"action,omitempty"`
ObjectType string `json:"object_type,omitempty"`
ObjectID string `json:"object_id,omitempty"`
RequesterID string `json:"requester_id,omitempty"`
RequesterName string `json:"requester_name,omitempty"`
Reason string `json:"reason,omitempty"`
BeforeJSON string `json:"before_json,omitempty"`
AfterJSON string `json:"after_json,omitempty"`
Status string `json:"status,omitempty"`
ReviewerID string `json:"reviewer_id,omitempty"`
ReviewerName string `json:"reviewer_name,omitempty"`
ReviewComment string `json:"review_comment,omitempty"`
ReviewedAt *time.Time `json:"reviewed_at,omitempty"`
}
func NormalizeRecord(record Record) Record {
record.TraceID = strings.TrimSpace(record.TraceID)
record.SourceService = strings.TrimSpace(record.SourceService)
record.ActorID = strings.TrimSpace(record.ActorID)
record.ActorName = strings.TrimSpace(record.ActorName)
record.Action = strings.TrimSpace(record.Action)
record.ObjectType = strings.TrimSpace(record.ObjectType)
record.ObjectID = strings.TrimSpace(record.ObjectID)
record.OperationRisk = strings.TrimSpace(strings.ToLower(record.OperationRisk))
record.ApprovalID = strings.TrimSpace(record.ApprovalID)
record.RequestMethod = strings.TrimSpace(strings.ToUpper(record.RequestMethod))
record.RequestPath = strings.TrimSpace(record.RequestPath)
record.ClientIP = strings.TrimSpace(record.ClientIP)
record.Result = strings.TrimSpace(record.Result)
if record.Result == "" {
record.Result = "success"
}
if record.OperationRisk == "" {
if IsDangerousOperation(record.Action, record.ObjectType) {
record.OperationRisk = RiskDangerous
} else {
record.OperationRisk = RiskNormal
}
}
return record
}
func ValidateRecord(record Record) error {
record = NormalizeRecord(record)
if record.SourceService == "" {
return errors.New("source_service is required")
}
if record.ActorID == "" {
return errors.New("actor_id is required")
}
if record.Action == "" {
return errors.New("action is required")
}
if record.ObjectType == "" {
return errors.New("object_type is required")
}
if record.ObjectID == "" {
return errors.New("object_id is required")
}
if record.OperationRisk != RiskNormal && record.OperationRisk != RiskDangerous {
return errors.New("operation_risk must be normal or dangerous")
}
if record.OperationRisk == RiskDangerous && record.ApprovalID == "" {
return errors.New("approval_id is required for dangerous operation")
}
return nil
}
func IsDangerousOperation(action, objectType string) bool {
key := strings.ToLower(strings.TrimSpace(action) + " " + strings.TrimSpace(objectType))
dangerWords := []string{
"notification_policy",
"notification policy",
"silence_policy",
"suppression",
"escalation_policy",
"automation_script",
"script.execute",
"script.rollback",
}
for _, word := range dangerWords {
if strings.Contains(key, word) {
return true
}
}
return false
}
func SaveRecord(record Record) (models.AuditLog, error) {
record = NormalizeRecord(record)
if err := ValidateRecord(record); err != nil {
return models.AuditLog{}, err
}
row := models.AuditLog{
TraceID: record.TraceID,
SourceService: record.SourceService,
ActorID: record.ActorID,
ActorName: record.ActorName,
Action: record.Action,
ObjectType: record.ObjectType,
ObjectID: record.ObjectID,
OperationRisk: record.OperationRisk,
ApprovalID: record.ApprovalID,
RequestMethod: record.RequestMethod,
RequestPath: record.RequestPath,
ClientIP: record.ClientIP,
BeforeJSON: record.BeforeJSON,
AfterJSON: record.AfterJSON,
Result: record.Result,
ErrorMessage: record.ErrorMessage,
}
if err := impl.DBService.Create(&row).Error; err != nil {
return models.AuditLog{}, err
}
return row, nil
}
func NormalizeApproval(req ApprovalRequest) ApprovalRequest {
req.RequestID = strings.TrimSpace(req.RequestID)
req.SourceService = strings.TrimSpace(req.SourceService)
req.Action = strings.TrimSpace(req.Action)
req.ObjectType = strings.TrimSpace(req.ObjectType)
req.ObjectID = strings.TrimSpace(req.ObjectID)
req.RequesterID = strings.TrimSpace(req.RequesterID)
req.RequesterName = strings.TrimSpace(req.RequesterName)
req.Status = strings.TrimSpace(strings.ToLower(req.Status))
req.ReviewerID = strings.TrimSpace(req.ReviewerID)
req.ReviewerName = strings.TrimSpace(req.ReviewerName)
if req.Status == "" {
req.Status = ApprovalPending
}
return req
}
func ValidateApprovalRequest(req ApprovalRequest) error {
req = NormalizeApproval(req)
if req.SourceService == "" {
return errors.New("source_service is required")
}
if req.Action == "" {
return errors.New("action is required")
}
if req.ObjectType == "" {
return errors.New("object_type is required")
}
if req.ObjectID == "" {
return errors.New("object_id is required")
}
if req.RequesterID == "" {
return errors.New("requester_id is required")
}
if !IsDangerousOperation(req.Action, req.ObjectType) {
return errors.New("operation is not classified as dangerous")
}
return nil
}
func Transition(req ApprovalRequest, nextStatus, reviewerID, comment string) (ApprovalRequest, error) {
req = NormalizeApproval(req)
nextStatus = strings.TrimSpace(strings.ToLower(nextStatus))
reviewerID = strings.TrimSpace(reviewerID)
if req.Status != ApprovalPending {
return ApprovalRequest{}, errors.New("only pending approval can be reviewed")
}
if nextStatus != ApprovalApproved && nextStatus != ApprovalRejected {
return ApprovalRequest{}, errors.New("next status must be approved or rejected")
}
if reviewerID == "" {
return ApprovalRequest{}, errors.New("reviewer_id is required")
}
now := time.Now()
req.Status = nextStatus
req.ReviewerID = reviewerID
req.ReviewComment = strings.TrimSpace(comment)
req.ReviewedAt = &now
return req, nil
}

View File

@@ -0,0 +1,61 @@
package audit
import (
"testing"
)
func TestValidateRecordRequiresDangerousOperationsToCarryReviewID(t *testing.T) {
record := Record{
SourceService: "alert",
ActorID: "u-1",
Action: "policy.update",
ObjectType: "notification_policy",
ObjectID: "np-1",
OperationRisk: RiskDangerous,
}
if err := ValidateRecord(record); err == nil {
t.Fatal("expected dangerous operation without approval id to fail")
}
record.ApprovalID = "apr-1"
if err := ValidateRecord(record); err != nil {
t.Fatalf("expected valid dangerous audit record, got %v", err)
}
}
func TestNormalizeRecordClassifiesDangerousActions(t *testing.T) {
record := NormalizeRecord(Record{
SourceService: " alert ",
Action: "notification_policy.update",
ObjectType: " notification_policy ",
ObjectID: " np-1 ",
ActorID: " u-1 ",
})
if record.SourceService != "alert" || record.ObjectType != "notification_policy" || record.ObjectID != "np-1" {
t.Fatalf("record was not normalized: %#v", record)
}
if record.OperationRisk != RiskDangerous {
t.Fatalf("notification policy changes must be dangerous, got %q", record.OperationRisk)
}
}
func TestApprovalTransitionAllowsApproveOnlyFromPending(t *testing.T) {
req := ApprovalRequest{Status: ApprovalPending}
approved, err := Transition(req, ApprovalApproved, "reviewer-1", "ok")
if err != nil {
t.Fatalf("expected pending approval to approve: %v", err)
}
if approved.Status != ApprovalApproved {
t.Fatalf("unexpected status: %s", approved.Status)
}
if approved.ReviewerID != "reviewer-1" || approved.ReviewComment != "ok" {
t.Fatalf("review metadata not stored: %#v", approved)
}
if _, err := Transition(approved, ApprovalRejected, "reviewer-2", "late"); err == nil {
t.Fatal("expected approved request to reject further transition")
}
}

View File

@@ -0,0 +1,207 @@
package audit
import (
"errors"
"fmt"
"strconv"
"strings"
"time"
"git.apinb.com/bsm-sdk/core/infra"
"git.apinb.com/ops/logs/internal/impl"
"git.apinb.com/ops/logs/internal/models"
"github.com/gin-gonic/gin"
)
func ListAuditLogs(ctx *gin.Context) {
page, size := pageAndSize(ctx.DefaultQuery("page", "1"), ctx.DefaultQuery("page_size", "50"))
q := impl.DBService.Model(&models.AuditLog{})
if v := strings.TrimSpace(ctx.Query("source_service")); v != "" {
q = q.Where("source_service = ?", v)
}
if v := strings.TrimSpace(ctx.Query("actor_id")); v != "" {
q = q.Where("actor_id = ?", v)
}
if v := strings.TrimSpace(ctx.Query("action")); v != "" {
q = q.Where("action = ?", v)
}
if v := strings.TrimSpace(ctx.Query("object_type")); v != "" {
q = q.Where("object_type = ?", v)
}
if v := strings.TrimSpace(ctx.Query("object_id")); v != "" {
q = q.Where("object_id = ?", v)
}
if v := strings.TrimSpace(ctx.Query("operation_risk")); v != "" {
q = q.Where("operation_risk = ?", v)
}
if v := strings.TrimSpace(ctx.Query("result")); v != "" {
q = q.Where("result = ?", v)
}
var total int64
_ = q.Count(&total).Error
var rows []models.AuditLog
if err := q.Order("id desc").Offset((page - 1) * size).Limit(size).Find(&rows).Error; err != nil {
infra.Response.Error(ctx, err)
return
}
infra.Response.Success(ctx, gin.H{"total": total, "page": page, "page_size": size, "items": rows})
}
func CreateAuditLog(ctx *gin.Context) {
var req Record
if err := ctx.ShouldBindJSON(&req); err != nil {
infra.Response.Error(ctx, err)
return
}
req.ClientIP = firstNonEmpty(req.ClientIP, ctx.ClientIP())
req.RequestMethod = firstNonEmpty(req.RequestMethod, ctx.Request.Method)
req.RequestPath = firstNonEmpty(req.RequestPath, ctx.FullPath())
row, err := SaveRecord(req)
if err != nil {
infra.Response.Error(ctx, err)
return
}
infra.Response.Success(ctx, row)
}
func ListApprovals(ctx *gin.Context) {
page, size := pageAndSize(ctx.DefaultQuery("page", "1"), ctx.DefaultQuery("page_size", "50"))
q := impl.DBService.Model(&models.DangerousOperationApproval{})
if v := strings.TrimSpace(ctx.Query("source_service")); v != "" {
q = q.Where("source_service = ?", v)
}
if v := strings.TrimSpace(ctx.Query("status")); v != "" {
q = q.Where("status = ?", v)
}
if v := strings.TrimSpace(ctx.Query("requester_id")); v != "" {
q = q.Where("requester_id = ?", v)
}
if v := strings.TrimSpace(ctx.Query("object_type")); v != "" {
q = q.Where("object_type = ?", v)
}
var total int64
_ = q.Count(&total).Error
var rows []models.DangerousOperationApproval
if err := q.Order("id desc").Offset((page - 1) * size).Limit(size).Find(&rows).Error; err != nil {
infra.Response.Error(ctx, err)
return
}
infra.Response.Success(ctx, gin.H{"total": total, "page": page, "page_size": size, "items": rows})
}
func CreateApproval(ctx *gin.Context) {
var req ApprovalRequest
if err := ctx.ShouldBindJSON(&req); err != nil {
infra.Response.Error(ctx, err)
return
}
req = NormalizeApproval(req)
if req.RequestID == "" {
req.RequestID = fmt.Sprintf("apr-%d", time.Now().UnixNano())
}
if err := ValidateApprovalRequest(req); err != nil {
infra.Response.Error(ctx, err)
return
}
row := models.DangerousOperationApproval{
RequestID: req.RequestID,
SourceService: req.SourceService,
Action: req.Action,
ObjectType: req.ObjectType,
ObjectID: req.ObjectID,
RequesterID: req.RequesterID,
RequesterName: req.RequesterName,
Reason: req.Reason,
BeforeJSON: req.BeforeJSON,
AfterJSON: req.AfterJSON,
Status: ApprovalPending,
}
if err := impl.DBService.Create(&row).Error; err != nil {
infra.Response.Error(ctx, err)
return
}
infra.Response.Success(ctx, row)
}
func ApproveApproval(ctx *gin.Context) {
reviewApproval(ctx, ApprovalApproved)
}
func RejectApproval(ctx *gin.Context) {
reviewApproval(ctx, ApprovalRejected)
}
func reviewApproval(ctx *gin.Context, next string) {
id, err := parseUintParam(ctx, "id")
if err != nil {
infra.Response.Error(ctx, err)
return
}
var body struct {
ReviewerID string `json:"reviewer_id"`
ReviewerName string `json:"reviewer_name"`
ReviewComment string `json:"review_comment"`
}
if err := ctx.ShouldBindJSON(&body); err != nil {
infra.Response.Error(ctx, err)
return
}
var row models.DangerousOperationApproval
if err := impl.DBService.First(&row, id).Error; err != nil {
infra.Response.Error(ctx, err)
return
}
req := ApprovalRequest{
RequestID: row.RequestID,
SourceService: row.SourceService,
Action: row.Action,
ObjectType: row.ObjectType,
ObjectID: row.ObjectID,
RequesterID: row.RequesterID,
Status: row.Status,
}
nextReq, err := Transition(req, next, body.ReviewerID, body.ReviewComment)
if err != nil {
infra.Response.Error(ctx, err)
return
}
row.Status = nextReq.Status
row.ReviewerID = nextReq.ReviewerID
row.ReviewerName = strings.TrimSpace(body.ReviewerName)
row.ReviewComment = nextReq.ReviewComment
row.ReviewedAt = nextReq.ReviewedAt
if err := impl.DBService.Save(&row).Error; err != nil {
infra.Response.Error(ctx, err)
return
}
infra.Response.Success(ctx, row)
}
func parseUintParam(ctx *gin.Context, name string) (uint, error) {
v, err := strconv.ParseUint(ctx.Param(name), 10, 32)
if err != nil || v == 0 {
return 0, errors.New("invalid id")
}
return uint(v), nil
}
func pageAndSize(pageText, sizeText string) (int, int) {
page, _ := strconv.Atoi(pageText)
size, _ := strconv.Atoi(sizeText)
if page < 1 {
page = 1
}
if size < 1 || size > 500 {
size = 50
}
return page, size
}
func firstNonEmpty(values ...string) string {
for _, value := range values {
if strings.TrimSpace(value) != "" {
return strings.TrimSpace(value)
}
}
return ""
}

View File

@@ -0,0 +1,66 @@
package audit
import (
"bytes"
"io"
"strings"
"github.com/gin-gonic/gin"
)
type ActorResolver func(*gin.Context) (id string, name string)
func Middleware(sourceService string, resolveActor ActorResolver) gin.HandlerFunc {
sourceService = strings.TrimSpace(sourceService)
return func(ctx *gin.Context) {
if ctx.Request.Method == "GET" || ctx.Request.Method == "HEAD" || ctx.Request.Method == "OPTIONS" {
ctx.Next()
return
}
var body []byte
if ctx.Request.Body != nil {
body, _ = io.ReadAll(ctx.Request.Body)
ctx.Request.Body = io.NopCloser(bytes.NewBuffer(body))
}
ctx.Next()
actorID, actorName := "", ""
if resolveActor != nil {
actorID, actorName = resolveActor(ctx)
}
if actorID == "" {
actorID = firstNonEmpty(ctx.GetHeader("X-User-Id"), ctx.GetHeader("X-Actor-Id"), "unknown")
}
if actorName == "" {
actorName = firstNonEmpty(ctx.GetHeader("X-User-Name"), ctx.GetHeader("X-Actor-Name"))
}
result := "success"
if len(ctx.Errors) > 0 || ctx.Writer.Status() >= 400 {
result = "failed"
}
_, _ = SaveRecord(Record{
TraceID: firstNonEmpty(ctx.GetHeader("X-Trace-Id"), ctx.GetHeader("Request-Id")),
SourceService: sourceService,
ActorID: actorID,
ActorName: actorName,
Action: ctx.Request.Method + " " + ctx.FullPath(),
ObjectType: routeObjectType(ctx.FullPath()),
ObjectID: firstNonEmpty(ctx.Param("id"), ctx.Query("id"), ctx.FullPath()),
RequestMethod: ctx.Request.Method,
RequestPath: ctx.FullPath(),
ClientIP: ctx.ClientIP(),
AfterJSON: string(body),
Result: result,
})
}
}
func routeObjectType(path string) string {
path = strings.Trim(path, "/")
if path == "" {
return "unknown"
}
parts := strings.Split(path, "/")
return parts[len(parts)-1]
}

View File

@@ -311,3 +311,30 @@ func ListLogEvents(ctx *gin.Context) {
} }
infra.Response.Success(ctx, gin.H{"total": total, "page": page, "page_size": size, "items": rows}) infra.Response.Success(ctx, gin.H{"total": total, "page": page, "page_size": size, "items": rows})
} }
func ReplayLogEvent(ctx *gin.Context) {
id, err := parseID(ctx)
if err != nil {
infra.Response.Error(ctx, errors.New("invalid id"))
return
}
var row models.LogEvent
if err := impl.DBService.First(&row, id).Error; err != nil {
infra.Response.Error(ctx, err)
return
}
outboxID, err := ingest.EnqueueReplayLogEvent(row)
if err != nil {
infra.Response.Error(ctx, err)
return
}
if err := impl.DBService.Model(&models.LogEvent{}).Where("id = ?", id).Update("dispatch_status", "pending").Error; err != nil {
infra.Response.Error(ctx, err)
return
}
infra.Response.Success(ctx, gin.H{
"log_event_id": id,
"outbox_id": outboxID,
"status": "pending",
})
}

View File

@@ -0,0 +1,28 @@
package models
import "time"
type AuditLog struct {
ID uint `gorm:"primaryKey" json:"id"`
CreatedAt time.Time `json:"created_at"`
TraceID string `gorm:"size:96;index" json:"trace_id"`
SourceService string `gorm:"size:64;index" json:"source_service"`
ActorID string `gorm:"size:128;index" json:"actor_id"`
ActorName string `gorm:"size:128" json:"actor_name"`
Action string `gorm:"size:128;index" json:"action"`
ObjectType string `gorm:"size:128;index" json:"object_type"`
ObjectID string `gorm:"size:128;index" json:"object_id"`
OperationRisk string `gorm:"size:32;index" json:"operation_risk"`
ApprovalID string `gorm:"size:128;index" json:"approval_id"`
RequestMethod string `gorm:"size:16" json:"request_method"`
RequestPath string `gorm:"size:512" json:"request_path"`
ClientIP string `gorm:"size:64" json:"client_ip"`
BeforeJSON string `gorm:"type:text" json:"before_json"`
AfterJSON string `gorm:"type:text" json:"after_json"`
Result string `gorm:"size:32;index" json:"result"`
ErrorMessage string `gorm:"type:text" json:"error_message"`
}
func (AuditLog) TableName() string {
return "logs_audit_logs"
}

View File

@@ -0,0 +1,24 @@
package models
import (
"reflect"
"testing"
)
func TestTrapDictionaryEntryHasBlueprintFields(t *testing.T) {
typ := reflect.TypeOf(TrapDictionaryEntry{})
for _, name := range []string{"Vendor", "OID", "Name", "SeverityMappingJSON", "ParseExpression"} {
if _, ok := typ.FieldByName(name); !ok {
t.Fatalf("TrapDictionaryEntry missing blueprint field %s", name)
}
}
}
func TestSyslogRuleHasBlueprintFields(t *testing.T) {
typ := reflect.TypeOf(SyslogRule{})
for _, name := range []string{"SourceMatch", "MessageRegex", "SeverityMappingJSON", "ResourceUIDExtractRegex"} {
if _, ok := typ.FieldByName(name); !ok {
t.Fatalf("SyslogRule missing blueprint field %s", name)
}
}
}

View File

@@ -0,0 +1,28 @@
package models
import "time"
type DangerousOperationApproval struct {
ID uint `gorm:"primaryKey" json:"id"`
CreatedAt time.Time `json:"created_at"`
UpdatedAt time.Time `json:"updated_at"`
RequestID string `gorm:"size:128;uniqueIndex" json:"request_id"`
SourceService string `gorm:"size:64;index" json:"source_service"`
Action string `gorm:"size:128;index" json:"action"`
ObjectType string `gorm:"size:128;index" json:"object_type"`
ObjectID string `gorm:"size:128;index" json:"object_id"`
RequesterID string `gorm:"size:128;index" json:"requester_id"`
RequesterName string `gorm:"size:128" json:"requester_name"`
Reason string `gorm:"type:text" json:"reason"`
BeforeJSON string `gorm:"type:text" json:"before_json"`
AfterJSON string `gorm:"type:text" json:"after_json"`
Status string `gorm:"size:32;index" json:"status"`
ReviewerID string `gorm:"size:128;index" json:"reviewer_id"`
ReviewerName string `gorm:"size:128" json:"reviewer_name"`
ReviewComment string `gorm:"type:text" json:"review_comment"`
ReviewedAt *time.Time `json:"reviewed_at"`
}
func (DangerousOperationApproval) TableName() string {
return "logs_dangerous_operation_approvals"
}

View File

@@ -13,6 +13,8 @@ func GetAllModels() []interface{} {
&SyslogRule{}, &SyslogRule{},
&TrapRule{}, &TrapRule{},
&TrapShield{}, &TrapShield{},
&AuditLog{},
&DangerousOperationApproval{},
} }
} }
@@ -58,8 +60,24 @@ func seedDefaultSyslogRules(db *gorm.DB) error {
Priority: 90, Priority: 90,
DeviceNameContains: "", DeviceNameContains: "",
KeywordRegex: "(?i)(link down|interface .* down|port .* down)", KeywordRegex: "(?i)(link down|interface .* down|port .* down)",
SourceMatch: "",
MessageRegex: "(?i)(link down|interface .* down|port .* down|LINK_DOWN)",
AlertName: "Syslog链路中断", AlertName: "Syslog链路中断",
SeverityCode: "major", SeverityCode: "major",
SeverityMappingJSON: `{"(?i)(critical|fatal|emergency)":"critical","(?i)(error|LINK_DOWN|down)":"major","(?i)(warning|warn)":"warning"}`,
ResourceUIDExtractRegex: `(?i)(?:resource_uid=|resource=)(?P<resource_uid>[a-z0-9_-]+:[a-z0-9_.:/-]+)|Interface (?P<iface>[A-Za-z0-9/._-]+)`,
PolicyID: 0,
},
{
Name: "H3C-Syslog-接口中断",
Enabled: true,
Priority: 120,
SourceMatch: "h3c",
MessageRegex: `(?i)(LINK_DOWN|Interface .* down|port .* down)`,
AlertName: "H3C Syslog接口中断",
SeverityCode: "major",
SeverityMappingJSON: `{"(?i)(LINK_DOWN|down)":"major","(?i)(LINK_UP|up)":"info"}`,
ResourceUIDExtractRegex: `(?i)(?:resource_uid=|resource=)(?P<resource_uid>network:[a-z0-9_.:/-]+)|Interface (?P<iface>[A-Za-z0-9/._-]+)`,
PolicyID: 0, PolicyID: 0,
}, },
} }
@@ -99,18 +117,28 @@ func seedDefaultTrapDictionary(db *gorm.DB) error {
} }
rows := []TrapDictionaryEntry{ rows := []TrapDictionaryEntry{
{ {
Vendor: "H3C",
OID: "1.3.6.1.6.3.1.1.5.3",
OIDPrefix: "1.3.6.1.6.3.1.1.5.3", OIDPrefix: "1.3.6.1.6.3.1.1.5.3",
Name: "H3C ifDown 接口中断",
Title: "ifDown 接口中断", Title: "ifDown 接口中断",
Description: "检测到设备接口状态变为 down。", Description: "检测到设备接口状态变为 down。",
SeverityCode: "major", SeverityCode: "major",
SeverityMappingJSON: `{"down":"major","up":"info"}`,
ParseExpression: `(?i)(ifName|interface)=?(?P<interface>[A-Za-z0-9/._-]+)`,
RecoveryMessage: "请检查链路、端口状态和对端设备。", RecoveryMessage: "请检查链路、端口状态和对端设备。",
Enabled: true, Enabled: true,
}, },
{ {
Vendor: "H3C",
OID: "1.3.6.1.6.3.1.1.5.4",
OIDPrefix: "1.3.6.1.6.3.1.1.5.4", OIDPrefix: "1.3.6.1.6.3.1.1.5.4",
Name: "H3C ifUp 接口恢复",
Title: "ifUp 接口恢复", Title: "ifUp 接口恢复",
Description: "检测到设备接口状态恢复为 up。", Description: "检测到设备接口状态恢复为 up。",
SeverityCode: "info", SeverityCode: "info",
SeverityMappingJSON: `{"up":"info"}`,
ParseExpression: `(?i)(ifName|interface)=?(?P<interface>[A-Za-z0-9/._-]+)`,
RecoveryMessage: "接口已恢复,请确认业务连通性。", RecoveryMessage: "接口已恢复,请确认业务连通性。",
Enabled: true, Enabled: true,
}, },

View File

@@ -18,12 +18,20 @@ type SyslogRule struct {
Priority int `gorm:"index" json:"priority"` Priority int `gorm:"index" json:"priority"`
// DeviceNameContains 表示设备名称包含条件。 // DeviceNameContains 表示设备名称包含条件。
DeviceNameContains string `gorm:"size:512" json:"device_name_contains"` DeviceNameContains string `gorm:"size:512" json:"device_name_contains"`
// SourceMatch 表示来源匹配条件,可匹配来源 IP、主机名或原始行。
SourceMatch string `gorm:"size:512" json:"source_match"`
// KeywordRegex 表示关键字/内容匹配的正则表达式。 // KeywordRegex 表示关键字/内容匹配的正则表达式。
KeywordRegex string `gorm:"size:512" json:"keyword_regex"` KeywordRegex string `gorm:"size:512" json:"keyword_regex"`
// MessageRegex 表示消息正文匹配的正则表达式。
MessageRegex string `gorm:"size:1024" json:"message_regex"`
// AlertName 表示告警名称。 // AlertName 表示告警名称。
AlertName string `gorm:"size:256" json:"alert_name"` AlertName string `gorm:"size:256" json:"alert_name"`
// SeverityCode 表示严重级别编码。 // SeverityCode 表示严重级别编码。
SeverityCode string `gorm:"size:32" json:"severity_code"` SeverityCode string `gorm:"size:32" json:"severity_code"`
// SeverityMappingJSON 保存按正则分组或厂商级别映射到平台级别的 JSON。
SeverityMappingJSON string `gorm:"type:text" json:"severity_mapping_json"`
// ResourceUIDExtractRegex 表示从消息中提取 resource_uid 的正则。
ResourceUIDExtractRegex string `gorm:"size:1024" json:"resource_uid_extract_regex"`
// PolicyID 表示关联的告警/处理策略 ID。 // PolicyID 表示关联的告警/处理策略 ID。
PolicyID uint `json:"policy_id"` PolicyID uint `json:"policy_id"`
} }

View File

@@ -12,12 +12,22 @@ type TrapDictionaryEntry struct {
UpdatedAt time.Time `json:"updated_at"` UpdatedAt time.Time `json:"updated_at"`
// OIDPrefix 表示该字典条目对应的 OID 前缀(唯一)。 // OIDPrefix 表示该字典条目对应的 OID 前缀(唯一)。
OIDPrefix string `gorm:"size:512;uniqueIndex" json:"oid_prefix"` OIDPrefix string `gorm:"size:512;uniqueIndex" json:"oid_prefix"`
// Vendor 表示设备厂商,例如 H3C、Huawei、Cisco。
Vendor string `gorm:"size:128;index" json:"vendor"`
// OID 表示精确 Trap OID。为空时继续使用 OIDPrefix 做前缀匹配。
OID string `gorm:"size:512;index" json:"oid"`
// Name 表示 Trap 字典名称;保留 Title 作为旧页面兼容字段。
Name string `gorm:"size:512" json:"name"`
// Title 表示字典条目的标题。 // Title 表示字典条目的标题。
Title string `gorm:"size:512" json:"title"` Title string `gorm:"size:512" json:"title"`
// Description 表示字典条目的说明文本。 // Description 表示字典条目的说明文本。
Description string `gorm:"type:text" json:"description"` Description string `gorm:"type:text" json:"description"`
// SeverityCode 表示默认严重级别编码。 // SeverityCode 表示默认严重级别编码。
SeverityCode string `gorm:"size:32" json:"severity_code"` SeverityCode string `gorm:"size:32" json:"severity_code"`
// SeverityMappingJSON 保存按 varbind 或厂商级别映射到平台级别的 JSON。
SeverityMappingJSON string `gorm:"type:text" json:"severity_mapping_json"`
// ParseExpression 表示解析 varbind 的表达式或正则模板。
ParseExpression string `gorm:"type:text" json:"parse_expression"`
// RecoveryMessage 表示恢复/消警时的消息模板内容。 // RecoveryMessage 表示恢复/消警时的消息模板内容。
RecoveryMessage string `gorm:"type:text" json:"recovery_message"` RecoveryMessage string `gorm:"type:text" json:"recovery_message"`
// Enabled 表示该字典条目是否启用。 // Enabled 表示该字典条目是否启用。

View File

@@ -4,6 +4,7 @@ import (
"fmt" "fmt"
"git.apinb.com/bsm-sdk/core/middleware" "git.apinb.com/bsm-sdk/core/middleware"
"git.apinb.com/ops/logs/internal/logic/audit"
"git.apinb.com/ops/logs/internal/logic/controllers" "git.apinb.com/ops/logs/internal/logic/controllers"
"git.apinb.com/ops/logs/internal/logic/ping" "git.apinb.com/ops/logs/internal/logic/ping"
"github.com/gin-gonic/gin" "github.com/gin-gonic/gin"
@@ -42,7 +43,15 @@ func Register(srvKey string, engine *gin.Engine) {
api.POST("/resource-events", controllers.ReceiveResourceEvent) api.POST("/resource-events", controllers.ReceiveResourceEvent)
api.GET("/entries", controllers.ListLogEvents) api.GET("/entries", controllers.ListLogEvents)
api.POST("/entries/:id/replay", controllers.ReplayLogEvent)
api.GET("/alert-outbox", controllers.ListAlertOutbox) api.GET("/alert-outbox", controllers.ListAlertOutbox)
api.POST("/alert-outbox/:id/retry", controllers.RetryAlertOutbox) api.POST("/alert-outbox/:id/retry", controllers.RetryAlertOutbox)
api.GET("/audit/logs", audit.ListAuditLogs)
api.POST("/audit/logs", audit.CreateAuditLog)
api.GET("/audit/approvals", audit.ListApprovals)
api.POST("/audit/approvals", audit.CreateApproval)
api.POST("/audit/approvals/:id/approve", audit.ApproveApproval)
api.POST("/audit/approvals/:id/reject", audit.RejectApproval)
} }
} }