监控埋点与可观测性

# docker-compose.yaml
services:
  prometheus:
    image: prom/prometheus
    ports:
      - "9090:9090"
    volumes:
      - ./prometheus.yml:/etc/prometheus/prometheus.yml

# prometheus.yml
global:
  scrape_interval: 15s       # 全局采集间隔，高并发应用不宜太频繁
  evaluation_interval: 15s   # 告警规则评估间隔

scrape_configs:
  - job_name: "webook"
    static_configs:
      - targets: ["host.docker.internal:8081"]  # 应用 /metrics 端口

package main

import (
    "github.com/prometheus/client_golang/prometheus/promhttp"
    "net/http"
)

func main() {
    // ...业务 server 启动 :8080
    // 单独起一个 server 暴露 metrics，避免业务请求干扰
    go func() {
        http.Handle("/metrics", promhttp.Handler())
        http.ListenAndServe(":8081", nil)
    }()
}

import "github.com/prometheus/client_golang/prometheus"

orderCounter := prometheus.NewCounter(prometheus.CounterOpts{
    Namespace: "shop",          // 部门 / 大业务
    Subsystem: "order",         // 子系统 / 模块
    Name:      "created_total", // 指标名，Counter 习惯加 _total 后缀
    Help:      "累计创建的订单数",
})
prometheus.MustRegister(orderCounter)

// 业务调用
orderCounter.Add(1) // 或 Inc()

activeReqGauge := prometheus.NewGauge(prometheus.GaugeOpts{
    Namespace: "webook",
    Subsystem: "http",
    Name:      "active_requests",
    Help:      "当前正在处理的 HTTP 请求数",
})

// middleware 里：
activeReqGauge.Inc()  // 请求进来 +1
defer activeReqGauge.Dec() // 请求结束 -1

// Buckets 是「上界」集合，例如 0.01, 0.05, 0.1, 0.5, 1, 5 表示
// <= 10ms 的个数、<= 50ms 的个数、...
// 最后一个隐藏桶是 +Inf
hist := prometheus.NewHistogram(prometheus.HistogramOpts{
    Namespace: "webook",
    Subsystem: "db",
    Name:      "query_duration_seconds",
    Help:      "数据库查询耗时分布",
    Buckets:   []float64{0.01, 0.05, 0.1, 0.5, 1, 5},
})

hist.Observe(0.12) // 记录一次 120ms 的查询

summary := prometheus.NewSummary(prometheus.SummaryOpts{
    Namespace:  "webook",
    Subsystem:  "http",
    Name:       "request_duration_seconds",
    Help:       "HTTP 请求耗时",
    Objectives: map[float64]float64{
        0.5:    0.01,   // P50，误差 1%
        0.9:    0.005,  // P90，误差 0.5%
        0.99:   0.001,  // P99，误差 0.1%
        0.999:  0.0001, // P999，误差 0.01%
    },
})

summary.Observe(0.128) // 记录一次 128ms 请求

// SummaryVec：带 label 的 Summary
httpDuration := prometheus.NewSummaryVec(prometheus.SummaryOpts{
    Namespace: "webook",
    Subsystem: "http",
    Name:      "request_duration_seconds",
    Objectives: map[float64]float64{0.99: 0.001},
}, []string{"pattern", "method", "status"}) // 三个动态 label

// 记录：当次请求 pattern=/user/:id, method=POST, status=200, 耗时 128ms
httpDuration.With(prometheus.Labels{
    "pattern": "/user/:id",
    "method":  "POST",
    "status":  "200",
}).Observe(0.128)

package middleware

import (
    "github.com/gin-gonic/gin"
    "github.com/prometheus/client_golang/prometheus"
    "time"
)

var httpDuration = prometheus.NewSummaryVec(prometheus.SummaryOpts{
    Namespace:  "webook",
    Subsystem:  "http",
    Name:       "request_duration_seconds",
    Objectives: map[float64]float64{0.99: 0.001},
}, []string{"pattern", "method", "status"})

func init() { prometheus.MustRegister(httpDuration) }

func Prometheus() gin.HandlerFunc {
    return func(c *gin.Context) {
        // 步骤 1：记录请求开始时间
        start := time.Now()
        // 步骤 2：放行，让后续 middleware 和 Handler 先执行
        c.Next()
        duration := time.Since(start).Seconds()
        // c.FullPath() 拿到路由模板，如 /users/:id，避免按 id 分裂指标
        httpDuration.With(prometheus.Labels{
            "pattern": c.FullPath(),
            "method":  c.Request.Method,
            "status":  strconv.Itoa(c.Writer.Status()),
        }).Observe(duration)
    }
}

var activeReq = prometheus.NewGauge(prometheus.GaugeOpts{
    Namespace: "webook",
    Subsystem: "http",
    Name:      "active_requests",
})

func init() { prometheus.MustRegister(activeReq) }

func ActiveRequests() gin.HandlerFunc {
    return func(c *gin.Context) {
        activeReq.Inc()
        defer activeReq.Dec()
        c.Next()
    }
}

import "gorm.io/plugin/prometheus"

db.Use(prometheus.New(prometheus.Config{
    DBName:          "webook",
    StartServer:     false, // 不另起 server，复用应用的 :8081
    PushAddr:        "",    // 不推送，让 Prometheus 来拉
}))

type queryMetrics struct {
    duration *prometheus.HistogramVec
}

func (q *queryMetrics) Register(db *gorm.DB) {
    // 在所有 CRUD 之前插入 Before
    db.Callback().Create().Before("*").Register("metrics_before", q.before)
    db.Callback().Query().Before("*").Register("metrics_before", q.before)
    db.Callback().Update().Before("*").Register("metrics_before", q.before)
    db.Callback().Delete().Before("*").Register("metrics_before", q.before)
    // 在所有 CRUD 之后插入 After
    db.Callback().Create().After("*").Register("metrics_after", q.after)
    db.Callback().Query().After("*").Register("metrics_after", q.after)
    db.Callback().Update().After("*").Register("metrics_after", q.after)
    db.Callback().Delete().After("*").Register("metrics_after", q.after)
}

func (q *queryMetrics) before(tx *gorm.DB) {
    start := time.Now()
    // 通过 InstanceSet / InstanceGet 把 start 传递到 after
    tx.InstanceSet("metrics_start", start)
}

func (q *queryMetrics) after(tx *gorm.DB) {
    val, ok := tx.InstanceGet("metrics_start")
    if !ok { return }
    start := val.(time.Time)
    duration := time.Since(start).Seconds()
    q.duration.WithLabelValues(tx.Statement.Table).Observe(duration)
}

func WrapResp[T any](c *gin.Context, code int, msg string, data T) {
    // 业务返回前统一埋点
    errCodeCounter.WithLabelValues(strconv.Itoa(code)).Inc()
    c.JSON(http.StatusOK, Result{
        Code: code, Msg: msg, Data: data,
    })
}

// sms 服务装饰器
type promoSmsService struct {
    svc    sms.Service
    cnt    *prometheus.CounterVec   // 调用次数
    dur    *prometheus.SummaryVec   // 调用耗时
}

func (p *promoSmsService) Send(ctx context.Context, tplId string, args []string, numbers ...string) error {
    start := time.Now()
    defer func() {
        p.dur.WithLabelValues(tplId).Observe(time.Since(start).Seconds())
    }()
    err := p.svc.Send(ctx, tplId, args, numbers...)
    if err != nil {
        p.cnt.WithLabelValues(tplId, "error").Inc()
    } else {
        p.cnt.WithLabelValues(tplId, "ok").Inc()
    }
    return err
}

import "github.com/redis/go-redis/v9"

var redisCmd = prometheus.NewCounterVec(prometheus.CounterOpts{
    Namespace: "webook",
    Subsystem: "redis",
    Name:      "commands_total",
}, []string{"cmd", "hit"})

func RedisHook() redis.Hook {
    return redis.Hook{
        ProcessHook: func(ctx context.Context, cmd redis.Cmder) error {
            err := cmd.Process(ctx)
            hit := "yes"
            if errors.Is(err, redis.Nil) {
                hit = "no" // 缓存未命中
            }
            redisCmd.WithLabelValues(cmd.Name(), hit).Inc()
            return err
        },
    }
}

// 注册：rdb.AddHook(RedisHook())

import (
    "context"
    "go.opentelemetry.io/otel"
    "go.opentelemetry.io/otel/exporters/zipkin"
    "go.opentelemetry.io/otel/propagation"
    "go.opentelemetry.io/otel/sdk/resource"
    sdktrace "go.opentelemetry.io/otel/sdk/trace"
    semconv "go.opentelemetry.io/otel/semconv/v1.4.0"
)

func InitTracer(ctx context.Context, endpoint string) (func(), error) {
    // 1. 创建 Exporter（这里用 Zipkin）
    exp, err := zipkin.New(endpoint) // endpoint 形如 http://localhost:9411/api/v2/spans
    if err != nil { return nil, err }

    // 2. 创建 TracerProvider
    res, _ := resource.New(ctx, resource.WithAttributes(
        semconv.ServiceNameKey.String("webook"), // 服务名
    ))
    tp := sdktrace.NewTracerProvider(
        sdktrace.WithBatcher(exp),     // 批量上报
        sdktrace.WithResource(res),
    )

    // 3. 全局注册
    otel.SetTracerProvider(tp)
    // 4. 设置 Propagator：让链路信息跨服务传递
    otel.SetTextMapPropagator(propagation.NewCompositeTextMapPropagator(
        propagation.TraceContext{},
        propagation.Baggage{},
    ))

    // 5. 返回清理函数，进程退出前调用
    return func() {
        _ = tp.Shutdown(ctx) // 必须 Shutdown，否则可能丢数据
    }, nil
}

> ⚠️ **新手必踩的坑：OTel Provider 不 `Shutdown` 会丢数据**。上面用 `WithBatcher` 批量上报，Span 是先攒在内存里再发的。如果进程退出前不调用 `tp.Shutdown(ctx)`，那些还没发出去的 Span 就丢了，链路里出现缺口。所以 `InitTracer` 返回的清理函数一定要在 `main` 退出前 `defer` 调用。
}

func doSomething(ctx context.Context) error {
    // 步骤 1：从 ctx 里找父 Span，没有就新建根 Span
    ctx, span := otel.Tracer("webook").Start(ctx, "doSomething")
    defer span.End() // 原则：谁创建谁 End
    // ... 业务逻辑
    return nil
}

// 1. 传值（类似其他语言的 thread local）
ctx = context.WithValue(ctx, "uid", 123)
uid, _ := ctx.Value("uid").(int)

// 2. 超时/取消控制
ctx, cancel := context.WithTimeout(ctx, 5*time.Second)
defer cancel()
// 所有派生的子 ctx 会一起被取消

import "go.opentelemetry.io/contrib/instrumentation/github.com/gin-gonic/gin/otelgin"

r.Use(otelgin.Middleware("webook"))

import "gorm.io/plugin/opentelemetry/tracing"

db.Use(tracing.NewPlugin())

// sms.Service 的 otel 装饰器
type otelSmsService struct {
    svc sms.Service
}

func (o *otelSmsService) Send(ctx context.Context, tplId string, args []string, numbers ...string) error {
    ctx, span := otel.Tracer("webook").Start(ctx, "sms_send")
    defer span.End()
    // 给 Span 加属性，方便筛选
    span.SetAttributes(attribute.String("tpl_id", tplId))
    err := o.svc.Send(ctx, tplId, args, numbers...)
    if err != nil {
        // 记录错误
        span.RecordError(err)
        span.SetStatus(codes.Error, err.Error())
    }
    return err
}