成本与资源账单模块

cost-module/
├── go.mod                          # Go module 依赖（gin + prometheus client）
├── cmd/cost-server/
│   └── main.go                    # gin HTTP 服务：成本查询/账单/预算 + /metrics
├── internal/cost/
│   ├── metrics/federated.go       # 联邦 Prometheus 查询客户端（HTTP /api/v1/query）
│   ├── engine/allocate.go         # 按 tenant/app/env/cluster 分摊（用量×单价×时长）
│   ├── engine/billing.go          # 月度账单生成与 CSV 导出、对账复核
│   └── budget/check.go            # 预算校验、超阈值告警（导出 budget_limit_cents）
└── deploy/prometheus/
    ├── servicemonitor.yaml        # 采集成本服务指标的 ServiceMonitor
    ├── cost-recording-rules.yaml  # 标签注入 + 成本 recording rules（带租户标签）
    └── budget-alerts.yaml         # 预算超阈值 / 预测超阈值告警规则

// file: internal/cost/metrics/federated.go
package metrics

import (
	"context"
	"encoding/json"
	"fmt"
	"net/http"
	"net/url"
	"strconv"
	"strings"
	"time"
)

// PrometheusClient 联邦 Prometheus HTTP 查询客户端。
type PrometheusClient struct {
	baseURL    string
	httpClient *http.Client
}

func NewPrometheusClient(baseURL string) *PrometheusClient {
	return &PrometheusClient{
		baseURL:    strings.TrimRight(baseURL, "/"),
		httpClient: &http.Client{Timeout: 30 * time.Second},
	}
}

type promResponse struct {
	Status string `json:"status"`
	Data   struct {
		ResultType string          `json:"resultType"`
		Result     json.RawMessage `json:"result"`
	} `json:"data"`
}

// Vector 单条即时向量样本。
type Vector struct {
	Metric map[string]string `json:"metric"`
	Value  []interface{}     `json:"value"`
}

// Matrix 区间样本（用于趋势分析）。
type Matrix struct {
	Metric map[string]string `json:"metric"`
	Values [][]interface{}   `json:"values"`
}

// Query 即时查询，返回向量。
func (c *PrometheusClient) Query(ctx context.Context, expr string) ([]Vector, error) {
	q := url.Values{}
	q.Set("query", expr)
	req, err := http.NewRequestWithContext(ctx, http.MethodGet, c.baseURL+"/api/v1/query?"+q.Encode(), nil)
	if err != nil {
		return nil, err
	}
	resp, err := c.httpClient.Do(req)
	if err != nil {
		return nil, fmt.Errorf("query prometheus: %w", err)
	}
	defer resp.Body.Close()
	var pr promResponse
	if err := json.NewDecoder(resp.Body).Decode(&pr); err != nil {
		return nil, err
	}
	if pr.Status != "success" {
		return nil, fmt.Errorf("prometheus query status: %s", pr.Status)
	}
	var vectors []Vector
	if err := json.Unmarshal(pr.Data.Result, &vectors); err != nil {
		return nil, err
	}
	return vectors, nil
}

// QueryRange 区间查询，返回矩阵。
func (c *PrometheusClient) QueryRange(ctx context.Context, expr, start, end, step string) ([]Matrix, error) {
	q := url.Values{}
	q.Set("query", expr)
	q.Set("start", start)
	q.Set("end", end)
	q.Set("step", step)
	req, err := http.NewRequestWithContext(ctx, http.MethodGet, c.baseURL+"/api/v1/query_range?"+q.Encode(), nil)
	if err != nil {
		return nil, err
	}
	resp, err := c.httpClient.Do(req)
	if err != nil {
		return nil, fmt.Errorf("query_range prometheus: %w", err)
	}
	defer resp.Body.Close()
	var pr promResponse
	if err := json.NewDecoder(resp.Body).Decode(&pr); err != nil {
		return nil, err
	}
	var m []Matrix
	if err := json.Unmarshal(pr.Data.Result, &m); err != nil {
		return nil, err
	}
	return m, nil
}

// ValueToFloat 把 PromQL 返回的 [ts, "val"] 中数值解析为 float64。
func ValueToFloat(v []interface{}) (float64, error) {
	if len(v) < 2 {
		return 0, fmt.Errorf("invalid value")
	}
	switch val := v[1].(type) {
	case string:
		return strconv.ParseFloat(val, 64)
	case float64:
		return val, nil
	default:
		return 0, fmt.Errorf("unexpected value type")
	}
}

// file: internal/cost/engine/allocate.go
package engine

import (
	"context"
	"fmt"
	"strconv"
	"strings"

	"github.com/your-org/paas-cost/internal/cost/metrics"
)

// Price 单价配置（单位：分）。
type Price struct {
	CPUCentsPerCoreHour   float64
	MemCentsPerGiBHour    float64
	StorageCentsPerGiBDay float64
	NetCentsPerGiB        float64
}

// CostCell 单个 tenant/app/env/cluster 维度的费用明细（分）。
type CostCell struct {
	Tenant       string
	App          string
	Env          string
	Cluster      string
	CPUCents     float64
	MemCents     float64
	StorageCents float64
	NetCents     float64
	TotalCents   float64
}

// parseDays 从 "30d" 解析天数（存储/网络按天计费用）。
func parseDays(window string) int {
	d, err := strconv.Atoi(strings.TrimSuffix(window, "d"))
	if err != nil {
		return 30
	}
	return d
}

// Allocate 从联邦 Prometheus 拉取原始用量并按 tenant/app/env/cluster 分摊成费用。
func Allocate(ctx context.Context, pc *metrics.PrometheusClient, p Price, window string) ([]CostCell, error) {
	cells := map[string]*CostCell{}
	days := parseDays(window)

	collect := func(expr string, factor float64, set func(*CostCell, float64)) error {
		vs, err := pc.Query(ctx, expr)
		if err != nil {
			return err
		}
		for _, v := range vs {
			t, a, e, cl := v.Metric["tenant"], v.Metric["app"], v.Metric["env"], v.Metric["cluster"]
			if t == "" || a == "" || e == "" || cl == "" {
				// 缺标签用量记入平台公共池（见 runbook），此处跳过避免错记到未知租户
				continue
			}
			val, err := metrics.ValueToFloat(v.Value)
			if err != nil {
				continue
			}
			k := strings.Join([]string{t, a, e, cl}, "|")
			cell, ok := cells[k]
			if !ok {
				cell = &CostCell{Tenant: t, App: a, Env: e, Cluster: cl}
				cells[k] = cell
			}
			set(cell, val*factor)
		}
		return nil
	}

	cpuExpr := fmt.Sprintf(`sum by (tenant,app,env,cluster) (increase(container_cpu_usage_seconds_total{container!=""}[%s])) / 3600`, window)
	memExpr := fmt.Sprintf(`sum by (tenant,app,env,cluster) (avg_over_time(container_memory_working_set_bytes{container!=""}[%s])) / 1073741824`, window)
	stoExpr := `sum by (tenant,app,env,cluster) (kube_persistentvolumeclaim_resource_requests_storage_bytes) / 1073741824`
	netExpr := fmt.Sprintf(`sum by (tenant,app,env,cluster) (increase(node_network_transmit_bytes_total{device!~"lo"}[%s])) / 1073741824`, window)

	if err := collect(cpuExpr, p.CPUCentsPerCoreHour, func(c *CostCell, v float64) { c.CPUCents = v }); err != nil {
		return nil, fmt.Errorf("cpu: %w", err)
	}
	if err := collect(memExpr, p.MemCentsPerGiBHour, func(c *CostCell, v float64) { c.MemCents = v }); err != nil {
		return nil, fmt.Errorf("mem: %w", err)
	}
	// 存储为瞬时请求量(GiB)，按月需乘以天数
	if err := collect(stoExpr, p.StorageCentsPerGiBDay*float64(days), func(c *CostCell, v float64) { c.StorageCents = v }); err != nil {
		return nil, fmt.Errorf("storage: %w", err)
	}
	if err := collect(netExpr, p.NetCentsPerGiB, func(c *CostCell, v float64) { c.NetCents = v }); err != nil {
		return nil, fmt.Errorf("net: %w", err)
	}

	out := make([]CostCell, 0, len(cells))
	for _, c := range cells {
		c.TotalCents = c.CPUCents + c.MemCents + c.StorageCents + c.NetCents
		out = append(out, *c)
	}
	return out, nil
}

// file: internal/cost/engine/billing.go
package engine

import (
	"encoding/csv"
	"sort"
	"strconv"
	"strings"
)

// GenerateBillCSV 按租户/应用/环境/集群明细生成可导出 CSV（含审计字段 month）。
func GenerateBillCSV(cells []CostCell, month string) (string, error) {
	var sb strings.Builder
	w := csv.NewWriter(&sb)
	header := []string{"month", "tenant", "app", "env", "cluster",
		"cpu_cents", "mem_cents", "storage_cents", "net_cents", "total_cents", "currency"}
	if err := w.Write(header); err != nil {
		return "", err
	}
	sort.Slice(cells, func(i, j int) bool {
		if cells[i].Tenant != cells[j].Tenant {
			return cells[i].Tenant < cells[j].Tenant
		}
		return cells[i].App < cells[j].App
	})
	for _, c := range cells {
		row := []string{
			month, c.Tenant, c.App, c.Env, c.Cluster,
			strconv.FormatFloat(c.CPUCents, 'f', 2, 64),
			strconv.FormatFloat(c.MemCents, 'f', 2, 64),
			strconv.FormatFloat(c.StorageCents, 'f', 2, 64),
			strconv.FormatFloat(c.NetCents, 'f', 2, 64),
			strconv.FormatFloat(c.TotalCents, 'f', 2, 64),
			"CNY",
		}
		if err := w.Write(row); err != nil {
			return "", err
		}
	}
	w.Flush()
	if err := w.Error(); err != nil {
		return "", err
	}
	return sb.String(), nil
}

// TenantRollup 按租户汇总（用于月度账单总额与对账）。
func TenantRollup(cells []CostCell) map[string]float64 {
	out := map[string]float64{}
	for _, c := range cells {
		out[c.Tenant] += c.TotalCents
	}
	return out
}

// TotalCents 返回账单合计（分），供对账复核。
func TotalCents(cells []CostCell) float64 {
	var sum float64
	for _, c := range cells {
		sum += c.TotalCents
	}
	return sum
}

# file: deploy/prometheus/cost-recording-rules.yaml
apiVersion: monitoring.coreos.com/v1
kind: PrometheusRule
metadata:
  name: cost-recording-rules
  namespace: paas-system
  labels:
    role: recording-rules
spec:
  groups:
    - name: cost-enrich
      interval: 1m
      rules:
        # 将 cAdvisor 指标按 kube_pod_labels 注入 tenant/app/env，cluster 由 Agent 外部标签提供
        - record: paas:container_cpu_usage_seconds_total:by_tenant_app_env_cluster
          expr: |
            sum by (tenant, app, env, cluster) (
              container_cpu_usage_seconds_total{container!=""}
              * on (pod) group_left(tenant, app, env)
                max by (pod, tenant, app, env) (
                  kube_pod_labels{label_tenant!=""} * 0 + 1
                )
            )            
        - record: paas:container_memory_working_set_bytes:by_tenant_app_env_cluster
          expr: |
            sum by (tenant, app, env, cluster) (
              container_memory_working_set_bytes{container!=""}
              * on (pod) group_left(tenant, app, env)
                max by (pod, tenant, app, env) (
                  kube_pod_labels{label_tenant!=""} * 0 + 1
                )
            )            
    - name: cost-price
      interval: 5m
      rules:
        - record: paas:cpu_cost_cents:hourly
          expr: |
            (sum by (tenant, app, env, cluster) (increase(paas:container_cpu_usage_seconds_total:by_tenant_app_env_cluster[1h])) / 3600) * 3.0            
        - record: paas:mem_cost_cents:hourly
          expr: |
            (sum by (tenant, app, env, cluster) (avg_over_time(paas:container_memory_working_set_bytes:by_tenant_app_env_cluster[1h])) / 1073741824) * 1.5            
        - record: paas:storage_cost_cents:daily
          expr: |
            (sum by (tenant, app, env, cluster) (kube_persistentvolumeclaim_resource_requests_storage_bytes) / 1073741824) * 0.1            
        - record: paas:network_cost_cents:daily
          expr: |
            (sum by (tenant, app, env, cluster) (increase(node_network_transmit_bytes_total{device!~"lo"}[1d])) / 1073741824) * 0.5            
        - record: paas:total_cost_cents:by_tenant_app_env_cluster
          expr: |
            paas:cpu_cost_cents:hourly + paas:mem_cost_cents:hourly + paas:storage_cost_cents:daily + paas:network_cost_cents:daily            

// file: internal/cost/budget/check.go
package budget

import (
	"context"
	"fmt"
	"net/http"
	"strings"

	"github.com/prometheus/client_golang/prometheus"
	"github.com/your-org/paas-cost/internal/cost/engine"
	"github.com/your-org/paas-cost/internal/cost/metrics"
)

var (
	budgetLimit = prometheus.NewGaugeVec(prometheus.GaugeOpts{
		Name: "budget_limit_cents",
		Help: "租户月度预算上限（分）",
	}, []string{"tenant"})
	budgetExceeded = prometheus.NewGaugeVec(prometheus.GaugeOpts{
		Name: "budget_exceeded",
		Help: "租户是否超预算 1=超 0=未超",
	}, []string{"tenant"})
)

func init() {
	prometheus.MustRegister(budgetLimit, budgetExceeded)
}

// Budget 单租户预算。
type Budget struct {
	Tenant     string
	LimitCents float64
}

// SetBudget 设置租户预算（写审计日志后生效，并导出 budget_limit_cents 指标供 Prometheus 规则引用）。
func SetBudget(b Budget) {
	budgetLimit.WithLabelValues(b.Tenant).Set(b.LimitCents)
}

// Result 单租户预算校验结果。
type Result struct {
	Tenant     string
	CostCents  float64
	LimitCents float64
	Exceeded   bool
}

// Check 当前窗口内各租户费用与预算比对，超阈值返回结果、上报指标并推送 Alertmanager。
func Check(ctx context.Context, pc *metrics.PrometheusClient, price engine.Price, budgets []Budget, window string) ([]Result, error) {
	cells, err := engine.Allocate(ctx, pc, price, window)
	if err != nil {
		return nil, fmt.Errorf("allocate: %w", err)
	}
	rollup := engine.TenantRollup(cells)
	out := make([]Result, 0, len(budgets))
	for _, b := range budgets {
		cost := rollup[b.Tenant]
		exceeded := cost > b.LimitCents
		if exceeded {
			budgetExceeded.WithLabelValues(b.Tenant).Set(1)
		} else {
			budgetExceeded.WithLabelValues(b.Tenant).Set(0)
		}
		out = append(out, Result{Tenant: b.Tenant, CostCents: cost, LimitCents: b.LimitCents, Exceeded: exceeded})
		if exceeded {
			if err := alertToAlertmanager(b.Tenant, cost, b.LimitCents); err != nil {
				// 告警发送失败不阻断校验
				_ = err
			}
		}
	}
	return out, nil
}

// alertToAlertmanager 向 Alertmanager 推送预算超阈值告警。
func alertToAlertmanager(tenant string, cost, limit float64) error {
	payload := fmt.Sprintf(
		`[{"labels":{"alertname":"BudgetExceeded","tenant":%q,"severity":"warning"},"annotations":{"summary":"租户 %s 超预算"}}]`,
		tenant, tenant)
	req, err := http.NewRequest(http.MethodPost, "http://alertmanager.paas.local:9093/api/v2/alerts", strings.NewReader(payload))
	if err != nil {
		return err
	}
	req.Header.Set("Content-Type", "application/json")
	resp, err := http.DefaultClient.Do(req)
	if err != nil {
		return err
	}
	defer resp.Body.Close()
	if resp.StatusCode >= 300 {
		return fmt.Errorf("alertmanager status %d", resp.StatusCode)
	}
	return nil
}

// file: cmd/cost-server/main.go
package main

import (
	"log"
	"net/http"
	"os"
	"strconv"
	"strings"

	"github.com/gin-gonic/gin"
	"github.com/prometheus/client_golang/prometheus/promhttp"

	"github.com/your-org/paas-cost/internal/cost/budget"
	"github.com/your-org/paas-cost/internal/cost/engine"
	"github.com/your-org/paas-cost/internal/cost/metrics"
)

func main() {
	pc := metrics.NewPrometheusClient(os.Getenv("PROMETHEUS_URL"))
	price := engine.Price{
		CPUCentsPerCoreHour:   3.0,
		MemCentsPerGiBHour:    1.5,
		StorageCentsPerGiBDay: 0.1,
		NetCentsPerGiB:        0.5,
	}

	r := gin.Default()

	r.GET("/api/v1/cost", func(c *gin.Context) {
		window := c.DefaultQuery("window", "30d")
		cells, err := engine.Allocate(c.Request.Context(), pc, price, window)
		if err != nil {
			c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
			return
		}
		c.JSON(http.StatusOK, cells)
	})

	r.GET("/api/v1/bill", func(c *gin.Context) {
		month := c.DefaultQuery("month", "2026-08")
		window := c.DefaultQuery("window", "30d")
		cells, err := engine.Allocate(c.Request.Context(), pc, price, window)
		if err != nil {
			c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
			return
		}
		csvStr, err := engine.GenerateBillCSV(cells, month)
		if err != nil {
			c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
			return
		}
		c.Header("Content-Type", "text/csv")
		c.Header("Content-Disposition", "attachment; filename=bill-"+month+".csv")
		c.String(http.StatusOK, csvStr)
	})

	r.POST("/api/v1/budget", func(c *gin.Context) {
		var req struct {
			Tenant     string  `json:"tenant"`
			LimitCents float64 `json:"limit_cents"`
		}
		if err := c.ShouldBindJSON(&req); err != nil {
			c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
			return
		}
		budget.SetBudget(budget.Budget{Tenant: req.Tenant, LimitCents: req.LimitCents})
		c.JSON(http.StatusOK, gin.H{"ok": true})
	})

	r.GET("/api/v1/budget/check", func(c *gin.Context) {
		window := c.DefaultQuery("window", "30d")
		budgets := loadBudgets()
		results, err := budget.Check(c.Request.Context(), pc, price, budgets, window)
		if err != nil {
			c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
			return
		}
		c.JSON(http.StatusOK, results)
	})

	r.GET("/metrics", gin.WrapH(promhttp.Handler()))

	log.Println("cost-server listening :8081")
	if err := r.Run(":8081"); err != nil {
		log.Fatal(err)
	}
}

func loadBudgets() []budget.Budget {
	raw := os.Getenv("BUDGETS") // 形如 tenantA:5000000,tenantB:3000000
	out := []budget.Budget{}
	for _, pair := range strings.Split(raw, ",") {
		kv := strings.SplitN(strings.TrimSpace(pair), ":", 2)
		if len(kv) != 2 {
			continue
		}
		v, err := strconv.ParseFloat(kv[1], 64)
		if err != nil {
			continue
		}
		out = append(out, budget.Budget{Tenant: kv[0], LimitCents: v})
	}
	return out
}

# file: go.mod
module github.com/your-org/paas-cost

go 1.22

require (
	github.com/gin-gonic/gin v1.10.0
	github.com/prometheus/client_golang v1.19.0
)

# file: deploy/prometheus/servicemonitor.yaml
apiVersion: monitoring.coreos.com/v1
kind: ServiceMonitor
metadata:
  name: paas-cost-server
  namespace: paas-system
  labels:
    team: finops
spec:
  selector:
    matchLabels:
      app: paas-cost-server
  endpoints:
    - port: http
      path: /metrics
      interval: 30s
  namespaceSelector:
    any: true

# 运维 runbook：设置预算 + 生成月度账单 CSV（对应 budget.SetBudget / engine.GenerateBillCSV）
curl -X POST http://cost-server.paas.local:8081/api/v1/budget \
  -H 'Content-Type: application/json' -d '{"tenant":"A","limit_cents":5000000}'

curl -s 'http://cost-server.paas.local:8081/api/v1/bill?month=2026-08&window=30d' -o bill-2026-08.csv
echo "账单总额(分):"
curl -s 'http://cost-server.paas.local:8081/api/v1/cost?window=30d' | jq 'map(.TotalCents) | add'

# file: deploy/prometheus/budget-alerts.yaml
apiVersion: monitoring.coreos.com/v1
kind: PrometheusRule
metadata:
  name: cost-budget-alerts
  namespace: paas-system
  labels:
    role: alert-rules
spec:
  groups:
    - name: cost-budget
      rules:
        - alert: BudgetExceeded
          expr: sum by (tenant) (paas:total_cost_cents:by_tenant_app_env_cluster) > on (tenant) group_left() budget_limit_cents
          for: 10m
          labels:
            severity: warning
          annotations:
            summary: "租户 {{ $labels.tenant }} 本月费用已超预算"
        - alert: BudgetProjectedExceeded
          expr: predict_linear(paas:total_cost_cents:by_tenant_app_env_cluster[6h], 86400*24) > on (tenant) group_left() budget_limit_cents
          for: 10m
          labels:
            severity: warning
          annotations:
            summary: "预测租户 {{ $labels.tenant }} 月末将超预算"

# 运维 runbook：确认缺标签用量（应被记入公共池并发告警）
curl -s 'http://prometheus-fed.paas.local:9090/api/v1/query?query=sum(container_cpu_usage_seconds_total{container!=""})%20by%20(pod)' \
  | jq '.data.result[] | select(.metric.tenant==null)'

# 运维 runbook：复核账单总额 == 联邦指标积分 × 单价
curl -s 'http://cost-server.paas.local:8081/api/v1/cost?window=30d' | jq 'map(.TotalCents) | add'