fix: 初验针对修改
This commit is contained in:
502
internal/logic/otlp/ingest.go
Normal file
502
internal/logic/otlp/ingest.go
Normal file
@@ -0,0 +1,502 @@
|
||||
package otlp
|
||||
|
||||
import (
|
||||
"crypto/subtle"
|
||||
"encoding/base64"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
"unicode/utf8"
|
||||
|
||||
"git.apinb.com/ops/logs/internal/config"
|
||||
"git.apinb.com/ops/logs/internal/impl"
|
||||
"git.apinb.com/ops/logs/internal/models"
|
||||
"github.com/gin-gonic/gin"
|
||||
"gorm.io/gorm/clause"
|
||||
)
|
||||
|
||||
const (
|
||||
maxRecordsPerRequest = 10000
|
||||
maxRequestBytes = 32 * 1024 * 1024
|
||||
)
|
||||
|
||||
type anyValue struct {
|
||||
StringValue *string `json:"stringValue"`
|
||||
BoolValue *bool `json:"boolValue"`
|
||||
IntValue json.RawMessage `json:"intValue"`
|
||||
DoubleValue *float64 `json:"doubleValue"`
|
||||
BytesValue string `json:"bytesValue"`
|
||||
ArrayValue *arrayValue `json:"arrayValue"`
|
||||
KVListValue *keyValueList `json:"kvlistValue"`
|
||||
}
|
||||
|
||||
type arrayValue struct {
|
||||
Values []anyValue `json:"values"`
|
||||
}
|
||||
|
||||
type keyValueList struct {
|
||||
Values []keyValue `json:"values"`
|
||||
}
|
||||
|
||||
type keyValue struct {
|
||||
Key string `json:"key"`
|
||||
Value anyValue `json:"value"`
|
||||
}
|
||||
|
||||
type resource struct {
|
||||
Attributes []keyValue `json:"attributes"`
|
||||
}
|
||||
|
||||
type traceRequest struct {
|
||||
ResourceSpans []struct {
|
||||
Resource resource `json:"resource"`
|
||||
ScopeSpans []struct {
|
||||
Spans []spanPayload `json:"spans"`
|
||||
} `json:"scopeSpans"`
|
||||
} `json:"resourceSpans"`
|
||||
}
|
||||
|
||||
type spanPayload struct {
|
||||
TraceID string `json:"traceId"`
|
||||
SpanID string `json:"spanId"`
|
||||
ParentSpanID string `json:"parentSpanId"`
|
||||
Name string `json:"name"`
|
||||
Kind json.RawMessage `json:"kind"`
|
||||
StartTimeUnixNano json.RawMessage `json:"startTimeUnixNano"`
|
||||
EndTimeUnixNano json.RawMessage `json:"endTimeUnixNano"`
|
||||
Attributes []keyValue `json:"attributes"`
|
||||
Status struct {
|
||||
Code json.RawMessage `json:"code"`
|
||||
Message string `json:"message"`
|
||||
} `json:"status"`
|
||||
}
|
||||
|
||||
type logRequest struct {
|
||||
ResourceLogs []struct {
|
||||
Resource resource `json:"resource"`
|
||||
ScopeLogs []struct {
|
||||
LogRecords []logPayload `json:"logRecords"`
|
||||
} `json:"scopeLogs"`
|
||||
} `json:"resourceLogs"`
|
||||
}
|
||||
|
||||
type logPayload struct {
|
||||
TimeUnixNano json.RawMessage `json:"timeUnixNano"`
|
||||
ObservedTimeUnixNano json.RawMessage `json:"observedTimeUnixNano"`
|
||||
SeverityText string `json:"severityText"`
|
||||
SeverityNumber json.RawMessage `json:"severityNumber"`
|
||||
Body anyValue `json:"body"`
|
||||
Attributes []keyValue `json:"attributes"`
|
||||
TraceID string `json:"traceId"`
|
||||
SpanID string `json:"spanId"`
|
||||
}
|
||||
|
||||
func ReceiveTraces(ctx *gin.Context) {
|
||||
if !authorize(ctx) {
|
||||
return
|
||||
}
|
||||
ctx.Request.Body = http.MaxBytesReader(ctx.Writer, ctx.Request.Body, maxRequestBytes)
|
||||
var request traceRequest
|
||||
if err := ctx.ShouldBindJSON(&request); err != nil {
|
||||
ctx.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "OTLP Trace 请求无效"})
|
||||
return
|
||||
}
|
||||
rows, rejected, message := decodeSpans(request)
|
||||
if len(rows) > 0 {
|
||||
if err := impl.DBService.Clauses(clause.OnConflict{Columns: []clause.Column{{Name: "trace_id"}, {Name: "span_id"}}, DoNothing: true}).
|
||||
CreateInBatches(rows, 500).Error; err != nil {
|
||||
ctx.AbortWithStatusJSON(http.StatusInternalServerError, gin.H{"error": "Trace 入库失败"})
|
||||
return
|
||||
}
|
||||
}
|
||||
ctx.JSON(http.StatusOK, gin.H{"partialSuccess": gin.H{"rejectedSpans": rejected, "errorMessage": message}})
|
||||
}
|
||||
|
||||
func ReceiveLogs(ctx *gin.Context) {
|
||||
if !authorize(ctx) {
|
||||
return
|
||||
}
|
||||
ctx.Request.Body = http.MaxBytesReader(ctx.Writer, ctx.Request.Body, maxRequestBytes)
|
||||
var request logRequest
|
||||
if err := ctx.ShouldBindJSON(&request); err != nil {
|
||||
ctx.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "OTLP Log 请求无效"})
|
||||
return
|
||||
}
|
||||
rows, rejected, message := decodeLogs(request)
|
||||
if len(rows) > 0 {
|
||||
if err := impl.DBService.CreateInBatches(rows, 500).Error; err != nil {
|
||||
ctx.AbortWithStatusJSON(http.StatusInternalServerError, gin.H{"error": "Log 入库失败"})
|
||||
return
|
||||
}
|
||||
}
|
||||
ctx.JSON(http.StatusOK, gin.H{"partialSuccess": gin.H{"rejectedLogRecords": rejected, "errorMessage": message}})
|
||||
}
|
||||
|
||||
func authorize(ctx *gin.Context) bool {
|
||||
provided := strings.TrimSpace(ctx.GetHeader("X-OTLP-Internal-Key"))
|
||||
expected := config.Spec.OTLP.InternalKey
|
||||
if provided == "" || len(provided) != len(expected) || subtle.ConstantTimeCompare([]byte(provided), []byte(expected)) != 1 {
|
||||
ctx.AbortWithStatusJSON(http.StatusUnauthorized, gin.H{"error": "OTLP 接入密钥无效"})
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func decodeSpans(request traceRequest) ([]models.TraceSpan, int, string) {
|
||||
rows := make([]models.TraceSpan, 0)
|
||||
rejected := 0
|
||||
for _, group := range request.ResourceSpans {
|
||||
resourceAttrs := attributesMap(group.Resource.Attributes)
|
||||
resourceJSON, _ := json.Marshal(resourceAttrs)
|
||||
for _, scope := range group.ScopeSpans {
|
||||
for _, input := range scope.Spans {
|
||||
if len(rows)+rejected >= maxRecordsPerRequest {
|
||||
rejected++
|
||||
continue
|
||||
}
|
||||
row, err := spanModel(input, resourceAttrs, string(resourceJSON))
|
||||
if err != nil {
|
||||
rejected++
|
||||
continue
|
||||
}
|
||||
rows = append(rows, row)
|
||||
}
|
||||
}
|
||||
}
|
||||
message := ""
|
||||
if rejected > 0 {
|
||||
message = "部分 Span 因标识或时间字段无效被拒绝"
|
||||
}
|
||||
return rows, rejected, message
|
||||
}
|
||||
|
||||
func spanModel(input spanPayload, resourceAttrs map[string]interface{}, resourceJSON string) (models.TraceSpan, error) {
|
||||
traceID, err := normalizeID(input.TraceID, 16)
|
||||
if err != nil {
|
||||
return models.TraceSpan{}, err
|
||||
}
|
||||
spanID, err := normalizeID(input.SpanID, 8)
|
||||
if err != nil {
|
||||
return models.TraceSpan{}, err
|
||||
}
|
||||
parentID := ""
|
||||
if strings.TrimSpace(input.ParentSpanID) != "" {
|
||||
parentID, err = normalizeID(input.ParentSpanID, 8)
|
||||
if err != nil {
|
||||
return models.TraceSpan{}, err
|
||||
}
|
||||
}
|
||||
start, err := nanoTime(input.StartTimeUnixNano)
|
||||
if err != nil {
|
||||
return models.TraceSpan{}, err
|
||||
}
|
||||
end, err := nanoTime(input.EndTimeUnixNano)
|
||||
if err != nil || end.Before(start) {
|
||||
return models.TraceSpan{}, fmt.Errorf("Span 时间无效")
|
||||
}
|
||||
attrs := attributesMap(input.Attributes)
|
||||
attrsJSON, _ := json.Marshal(attrs)
|
||||
serviceName := textAttribute(resourceAttrs, "service.name")
|
||||
if serviceName == "" {
|
||||
serviceName = "unknown_service"
|
||||
}
|
||||
serviceName = limitCharacters(serviceName, 255)
|
||||
operation := strings.TrimSpace(input.Name)
|
||||
if operation == "" {
|
||||
operation = "unnamed"
|
||||
}
|
||||
operation = limitCharacters(operation, 512)
|
||||
statusCode := statusText(input.Status.Code)
|
||||
if httpStatus := intAttribute(attrs, "http.response.status_code", "http.status_code"); statusCode == "unset" && httpStatus >= 500 {
|
||||
statusCode = "error"
|
||||
}
|
||||
now := time.Now().UTC()
|
||||
return models.TraceSpan{
|
||||
TraceID: traceID, SpanID: spanID, ParentSpanID: parentID, ServiceName: serviceName,
|
||||
OperationName: operation, SpanKind: spanKindText(input.Kind), StartTime: start, EndTime: end,
|
||||
DurationMs: float64(end.Sub(start).Nanoseconds()) / 1e6, StatusCode: statusCode,
|
||||
StatusMessage: strings.TrimSpace(input.Status.Message), BusinessSystemID: uintAttribute(resourceAttrs, attrs, "business.system.id", "business_system_id"),
|
||||
ResourceUID: limitCharacters(firstTextAttribute(resourceAttrs, attrs, "resource.uid", "resource_uid"), 255),
|
||||
HTTPMethod: limitCharacters(firstTextAttribute(attrs, nil, "http.request.method", "http.method"), 16),
|
||||
HTTPRoute: limitCharacters(firstTextAttribute(attrs, nil, "http.route", "url.path", "http.target"), 1024),
|
||||
HTTPStatusCode: intAttribute(attrs, "http.response.status_code", "http.status_code"),
|
||||
Attributes: string(attrsJSON), ResourceAttrs: resourceJSON, IngestedAt: &now,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func decodeLogs(request logRequest) ([]models.LogEvent, int, string) {
|
||||
rows := make([]models.LogEvent, 0)
|
||||
rejected := 0
|
||||
for _, group := range request.ResourceLogs {
|
||||
resourceAttrs := attributesMap(group.Resource.Attributes)
|
||||
for _, scope := range group.ScopeLogs {
|
||||
for _, input := range scope.LogRecords {
|
||||
if len(rows)+rejected >= maxRecordsPerRequest {
|
||||
rejected++
|
||||
continue
|
||||
}
|
||||
row, err := logModel(input, resourceAttrs)
|
||||
if err != nil {
|
||||
rejected++
|
||||
continue
|
||||
}
|
||||
rows = append(rows, row)
|
||||
}
|
||||
}
|
||||
}
|
||||
message := ""
|
||||
if rejected > 0 {
|
||||
message = "部分 LogRecord 因标识或内容无效被拒绝"
|
||||
}
|
||||
return rows, rejected, message
|
||||
}
|
||||
|
||||
func logModel(input logPayload, resourceAttrs map[string]interface{}) (models.LogEvent, error) {
|
||||
attrs := attributesMap(input.Attributes)
|
||||
detailJSON, _ := json.Marshal(attrs)
|
||||
body := scalarText(anyValueToInterface(input.Body))
|
||||
if body == "" {
|
||||
return models.LogEvent{}, fmt.Errorf("日志正文为空")
|
||||
}
|
||||
if len(body) > 64*1024 {
|
||||
body = limitUTF8Bytes(body, 64*1024)
|
||||
}
|
||||
traceID := ""
|
||||
spanID := ""
|
||||
var err error
|
||||
if strings.TrimSpace(input.TraceID) != "" {
|
||||
traceID, err = normalizeID(input.TraceID, 16)
|
||||
if err != nil {
|
||||
return models.LogEvent{}, err
|
||||
}
|
||||
}
|
||||
if strings.TrimSpace(input.SpanID) != "" {
|
||||
spanID, err = normalizeID(input.SpanID, 8)
|
||||
if err != nil {
|
||||
return models.LogEvent{}, err
|
||||
}
|
||||
}
|
||||
serviceName := limitCharacters(textAttribute(resourceAttrs, "service.name"), 512)
|
||||
resourceUID := limitCharacters(firstTextAttribute(resourceAttrs, attrs, "resource.uid", "resource_uid"), 255)
|
||||
return models.LogEvent{
|
||||
CreatedAt: timeFromLog(input), SourceKind: "otlp", RawPayload: body, NormalizedSummary: body,
|
||||
NormalizedDetail: string(detailJSON), DeviceName: serviceName, ResourceType: "service", ResourceUID: resourceUID,
|
||||
ResourceID: resourceUID, ResourceName: serviceName, BusinessSystemID: uintAttribute(resourceAttrs, attrs, "business.system.id", "business_system_id"),
|
||||
TraceID: traceID, SpanID: spanID, MatchMethod: "otlp", DispatchStatus: "not_applicable",
|
||||
SeverityCode: limitCharacters(severityText(input.SeverityText, input.SeverityNumber), 32),
|
||||
}, nil
|
||||
}
|
||||
|
||||
func limitCharacters(value string, maximum int) string {
|
||||
characters := []rune(value)
|
||||
if len(characters) <= maximum {
|
||||
return value
|
||||
}
|
||||
return string(characters[:maximum])
|
||||
}
|
||||
|
||||
func limitUTF8Bytes(value string, maximum int) string {
|
||||
if len(value) <= maximum {
|
||||
return value
|
||||
}
|
||||
end := maximum
|
||||
for end > 0 && !utf8.ValidString(value[:end]) {
|
||||
end--
|
||||
}
|
||||
return value[:end]
|
||||
}
|
||||
|
||||
func attributesMap(values []keyValue) map[string]interface{} {
|
||||
result := make(map[string]interface{}, len(values))
|
||||
for _, item := range values {
|
||||
if key := strings.TrimSpace(item.Key); key != "" {
|
||||
result[key] = anyValueToInterface(item.Value)
|
||||
}
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
func anyValueToInterface(value anyValue) interface{} {
|
||||
if value.StringValue != nil {
|
||||
return *value.StringValue
|
||||
}
|
||||
if value.BoolValue != nil {
|
||||
return *value.BoolValue
|
||||
}
|
||||
if value.DoubleValue != nil {
|
||||
return *value.DoubleValue
|
||||
}
|
||||
if len(value.IntValue) > 0 {
|
||||
var number json.Number
|
||||
if json.Unmarshal(value.IntValue, &number) == nil {
|
||||
return number
|
||||
}
|
||||
var text string
|
||||
if json.Unmarshal(value.IntValue, &text) == nil {
|
||||
return text
|
||||
}
|
||||
}
|
||||
if value.BytesValue != "" {
|
||||
return value.BytesValue
|
||||
}
|
||||
if value.ArrayValue != nil {
|
||||
items := make([]interface{}, 0, len(value.ArrayValue.Values))
|
||||
for _, item := range value.ArrayValue.Values {
|
||||
items = append(items, anyValueToInterface(item))
|
||||
}
|
||||
return items
|
||||
}
|
||||
if value.KVListValue != nil {
|
||||
return attributesMap(value.KVListValue.Values)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func normalizeID(value string, size int) (string, error) {
|
||||
value = strings.TrimSpace(value)
|
||||
if raw, err := hex.DecodeString(value); err == nil && len(raw) == size {
|
||||
return strings.ToLower(value), nil
|
||||
}
|
||||
raw, err := base64.StdEncoding.DecodeString(value)
|
||||
if err != nil || len(raw) != size {
|
||||
return "", fmt.Errorf("OTLP 标识无效")
|
||||
}
|
||||
return hex.EncodeToString(raw), nil
|
||||
}
|
||||
|
||||
func nanoTime(raw json.RawMessage) (time.Time, error) {
|
||||
value := strings.Trim(strings.TrimSpace(string(raw)), `"`)
|
||||
nanos, err := strconv.ParseInt(value, 10, 64)
|
||||
if err != nil || nanos <= 0 {
|
||||
return time.Time{}, fmt.Errorf("OTLP 时间无效")
|
||||
}
|
||||
return time.Unix(0, nanos).UTC(), nil
|
||||
}
|
||||
|
||||
func enumValue(raw json.RawMessage) string {
|
||||
return strings.Trim(strings.TrimSpace(string(raw)), `"`)
|
||||
}
|
||||
|
||||
func spanKindText(raw json.RawMessage) string {
|
||||
switch strings.ToUpper(enumValue(raw)) {
|
||||
case "2", "SPAN_KIND_SERVER":
|
||||
return "server"
|
||||
case "3", "SPAN_KIND_CLIENT":
|
||||
return "client"
|
||||
case "4", "SPAN_KIND_PRODUCER":
|
||||
return "producer"
|
||||
case "5", "SPAN_KIND_CONSUMER":
|
||||
return "consumer"
|
||||
default:
|
||||
return "internal"
|
||||
}
|
||||
}
|
||||
|
||||
func statusText(raw json.RawMessage) string {
|
||||
switch strings.ToUpper(enumValue(raw)) {
|
||||
case "1", "STATUS_CODE_OK":
|
||||
return "ok"
|
||||
case "2", "STATUS_CODE_ERROR":
|
||||
return "error"
|
||||
default:
|
||||
return "unset"
|
||||
}
|
||||
}
|
||||
|
||||
func severityText(text string, raw json.RawMessage) string {
|
||||
text = strings.ToLower(strings.TrimSpace(text))
|
||||
if text != "" {
|
||||
if strings.Contains(text, "fatal") {
|
||||
return "critical"
|
||||
}
|
||||
if strings.Contains(text, "error") {
|
||||
return "major"
|
||||
}
|
||||
if strings.Contains(text, "warn") {
|
||||
return "warning"
|
||||
}
|
||||
return "info"
|
||||
}
|
||||
value, _ := strconv.Atoi(enumValue(raw))
|
||||
if value >= 21 {
|
||||
return "critical"
|
||||
}
|
||||
if value >= 17 {
|
||||
return "major"
|
||||
}
|
||||
if value >= 13 {
|
||||
return "warning"
|
||||
}
|
||||
return "info"
|
||||
}
|
||||
|
||||
func timeFromLog(input logPayload) time.Time {
|
||||
if value, err := nanoTime(input.TimeUnixNano); err == nil {
|
||||
return value
|
||||
}
|
||||
if value, err := nanoTime(input.ObservedTimeUnixNano); err == nil {
|
||||
return value
|
||||
}
|
||||
return time.Now().UTC()
|
||||
}
|
||||
|
||||
func textAttribute(values map[string]interface{}, key string) string {
|
||||
return scalarText(values[key])
|
||||
}
|
||||
|
||||
func firstTextAttribute(primary, secondary map[string]interface{}, keys ...string) string {
|
||||
for _, key := range keys {
|
||||
if primary != nil {
|
||||
if value := scalarText(primary[key]); value != "" {
|
||||
return value
|
||||
}
|
||||
}
|
||||
if secondary != nil {
|
||||
if value := scalarText(secondary[key]); value != "" {
|
||||
return value
|
||||
}
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func intAttribute(values map[string]interface{}, keys ...string) int {
|
||||
for _, key := range keys {
|
||||
value, err := strconv.Atoi(scalarText(values[key]))
|
||||
if err == nil && value != 0 {
|
||||
return value
|
||||
}
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
func uintAttribute(primary, secondary map[string]interface{}, keys ...string) *uint {
|
||||
value := firstTextAttribute(primary, secondary, keys...)
|
||||
parsed, err := strconv.ParseUint(value, 10, 32)
|
||||
if err != nil || parsed == 0 {
|
||||
return nil
|
||||
}
|
||||
result := uint(parsed)
|
||||
return &result
|
||||
}
|
||||
|
||||
func scalarText(value interface{}) string {
|
||||
switch typed := value.(type) {
|
||||
case string:
|
||||
return typed
|
||||
case json.Number:
|
||||
return typed.String()
|
||||
case float64, bool:
|
||||
return fmt.Sprint(typed)
|
||||
default:
|
||||
if typed == nil {
|
||||
return ""
|
||||
}
|
||||
encoded, _ := json.Marshal(typed)
|
||||
return string(encoded)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user