运维自动化与定时任务模块

paas-automation/
├── go.mod                                  # Go 依赖（gin / client-go / argo / cron / prometheus）
├── cmd/
│   └── scheduler/
│       └── main.go                         # gin 服务入口：任务提交/查询 API
├── internal/
│   ├── auth/
│   │   └── tenant.go                       # 复用模块05：租户/审批身份中间件
│   ├── scheduler/
│   │   └── cron.go                         # Cron 定时调度器
│   ├── workflow/
│   │   └── argo.go                         # Argo Workflows 提交（client-go）
│   ├── gateway/
│   │   └── executor.go                     # 经 API 网关跨集群执行（client-go）
│   ├── approval/
│   │   └── hook.go                         # 多级审批钩子
│   └── metrics/
│       └── reporter.go                     # 任务指标上报（tenant/app/env/cluster）
└── deploy/                                 # K8s IaC（见第三、四章）
    ├── 00-namespace-rbac.yaml              # automation ns + 跨集群执行 SA/ClusterRole
    ├── 01-cronjob-test.yaml                # 测试环境 CronJob（清闲置）
    ├── 02-cronjob-prod.yaml                # 生产环境 CronJob（差异化）
    ├── 03-job-template-scaleout.yaml       # 跨集群批量扩容 Job 模板
    ├── 04-argo-workflow-batch.yaml         # Argo Workflows 批量编排
    ├── 05-workflow-crd.yaml                 # Workflow CRD 引用说明
    └── 06-task-metrics.yaml                # 任务指标 ServiceMonitor + recording rules

# file: deploy/00-namespace-rbac.yaml
apiVersion: v1
kind: Namespace
metadata:
  name: automation
  labels:
    tenant: platform
    env: production
    cluster: central
---
apiVersion: v1
kind: ServiceAccount
metadata:
  name: automation-runner
  namespace: automation
---
# 仅允许操作 tn-acme 命名空间内的 deployments（限副本数变更/只读），禁止 secrets/删除
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
  name: automation-runner-tenant-acme
  namespace: tn-acme
rules:
  - apiGroups: ["apps"]
    resources: ["deployments"]
    verbs: ["get", "list", "patch", "update"]
  - apiGroups: [""]
    resources: ["pods", "pods/log"]
    verbs: ["get", "list"]
---
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
  name: automation-runner-tenant-acme
  namespace: tn-acme
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: Role
  name: automation-runner-tenant-acme
subjects:
  - kind: ServiceAccount
    name: automation-runner
    namespace: automation
---
# 平台控制面调用网关的权限（仅向网关发起，不直接触达集群）
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
  name: gateway-invoker
rules:
  - apiGroups: [""]
    resources: ["services"]
    resourceNames: ["k8s-api-gateway"]
    verbs: ["get", "create"]

# file: deploy/01-cronjob-test.yaml
# 【测试环境】低风险，免审批，直接执行；并发宽松
apiVersion: batch/v1
kind: CronJob
metadata:
  name: cleanup-evicted-test
  namespace: automation
  labels:
    tenant: platform
    env: testing
    cluster: testing-cluster
spec:
  schedule: "0 2 * * *"
  timeZone: "Asia/Shanghai"
  concurrencyPolicy: Forbid
  startingDeadlineSeconds: 300
  jobTemplate:
    spec:
      activeDeadlineSeconds: 600
      backoffLimit: 2
      template:
        spec:
          serviceAccountName: automation-runner
          containers:
            - name: cleanup
              image: registry.internal/kubectl:1.29
              command: ["sh", "-c", "kubectl delete pod -n tn-acme --field-selector status.phase=Failed"]
          restartPolicy: Never

# file: deploy/02-cronjob-prod.yaml
# 【生产环境】差异化：限流、严格时窗、失败即告警；提交需审批
apiVersion: batch/v1
kind: CronJob
metadata:
  name: cleanup-evicted-prod
  namespace: automation
  labels:
    tenant: platform
    env: production
    cluster: prod-cluster-a
spec:
  schedule: "0 3 * * *"
  timeZone: "Asia/Shanghai"
  concurrencyPolicy: Forbid
  startingDeadlineSeconds: 200
  successfulJobsHistoryLimit: 3
  failedJobsHistoryLimit: 3
  jobTemplate:
    spec:
      activeDeadlineSeconds: 900
      backoffLimit: 1          # 生产失败不盲目重试，转人工
      template:
        metadata:
          labels:
            tenant: platform
            env: production
            cluster: prod-cluster-a
        spec:
          serviceAccountName: automation-runner
          containers:
            - name: cleanup
              image: registry.internal/kubectl:1.29
              command: ["sh", "-c", "kubectl delete pod -n tn-acme --field-selector status.phase=Failed"]
          restartPolicy: Never

# file: deploy/03-job-template-scaleout.yaml
# 跨集群批量扩容：由 Go 调度器按目标集群分别渲染并提交（见 4.8）
apiVersion: batch/v1
kind: Job
metadata:
  name: scale-out-tn-acme          # 实际由调度器加集群后缀：scale-out-tn-acme-prod-a
  namespace: automation
  labels:
    tenant: acme
    app: web
    env: production
    cluster: prod-cluster-a
    task-template: scale-out
spec:
  backoffLimit: 1
  activeDeadlineSeconds: 300
  template:
    spec:
      serviceAccountName: automation-runner
      restartPolicy: Never
      containers:
        - name: scale
          image: registry.internal/kubectl:1.29
          command:
            - sh
            - -c
            - |
              kubectl -n tn-acme patch deployment web --type merge \
                -p "{\"spec\":{\"replicas\":${REPLICAS}}}"              
          env:
            - name: REPLICAS
              value: "6"

# file: deploy/04-argo-workflow-batch.yaml
# Argo Workflows：跨集群巡检 DAG（先 prod-a，再 prod-b，最后汇总共识）
apiVersion: argoproj.io/v1alpha1
kind: Workflow
metadata:
  name: batch-inspect-acme
  namespace: automation
  labels:
    tenant: acme
    env: production
    task-template: batch-inspect
spec:
  entrypoint: main
  templates:
    - name: main
      dag:
        tasks:
          - name: inspect-prod-a
            template: inspect
            arguments: {parameters: [{name: cluster, value: prod-cluster-a}]}
          - name: inspect-prod-b
            template: inspect
            dependencies: [inspect-prod-a]
            arguments: {parameters: [{name: cluster, value: prod-cluster-b}]}
    - name: inspect
      inputs:
        parameters:
          - name: cluster
      container:
        image: registry.internal/kubectl:1.29
        command: ["sh", "-c", "kubectl --context {{inputs.parameters.cluster}} get pods -n tn-acme"]

# file: deploy/05-workflow-crd.yaml
# Workflow CRD 由 argo 安装提供，此处为平台纳管说明（不重复定义，仅引用）
# 安装：kubectl apply -n argo -f https://github.com/argoproj/argo-workflows/releases/download/v3.5.4/install.yaml
# 平台侧仅创建 Workflow 实例（见 04），并绑定 RBAC 到 automation-runner。
apiVersion: apiextensions.k8s.io/v1
kind: CustomResourceDefinition
metadata:
  name: workflows.argoproj.io
spec:
  group: argoproj.io
  names:
    kind: Workflow
    plural: workflows
    singular: workflow
  scope: Namespaced
  versions:
    - name: v1alpha1
      served: true
      storage: true

# file: deploy/06-task-metrics.yaml
apiVersion: v1
kind: ConfigMap
metadata:
  name: task-recording-rules
  namespace: monitoring
data:
  task_recording.yml: |
    groups:
      - name: task_recording
        interval: 30s
        rules:
          # 近 7 天成功率（供 Grafana / 告警复用）
          - record: task:success_rate:rate7d
            expr: |
              sum by (tenant, env, cluster) (rate(task_executions_total{result="success"}[7d]))
              /
              sum by (tenant, env, cluster) (rate(task_executions_total[7d]))
          # 平均耗时
          - record: task:duration_avg:5m
            expr: sum by (tenant, env, cluster) (rate(task_duration_seconds_sum[5m])) / sum by (tenant, env, cluster) (rate(task_duration_seconds_count[5m]))
      - name: task_alerts
        rules:
          # 跨集群执行失败率超阈值 → 触发分级告警（复用模块05 Alertmanager）
          - alert: TaskFailureRateHigh
            expr: |
              sum by (tenant, env, cluster) (rate(task_executions_total{result="failed"}[1h]))
              /
              sum by (tenant, env, cluster) (rate(task_executions_total[1h])) > 0.2
            for: 10m
            labels:
              severity: warning
            annotations:
              summary: "租户 {{ $labels.tenant }} 任务失败率超 20%"    
---
# 任务执行 Pod 暴露 /metrics（pushgateway 或 sidecar 暴露），由 Agent 抓取
apiVersion: monitoring.coreos.com/v1
kind: ServiceMonitor
metadata:
  name: task-executor
  namespace: monitoring
  labels:
    tenant: platform
    cluster: central
spec:
  selector:
    matchLabels:
      app: paas-automation-scheduler
  endpoints:
    - port: metrics
      interval: 30s

# 提交一个跨集群批量扩容任务，经网关下发（生产必走审批）
paas-cli task submit \
  --template scale-out \
  --clusters prod-a,prod-b,test-b \
  --namespace tn-acme \
  --params replicas=6 \
  --approve-required true   # 生产必走审批

// file: cmd/scheduler/main.go
package main

import (
	"context"
	"fmt"
	"log"
	"net/http"
	"os"
	"strings"
	"time"

	"paas-automation/internal/approval"
	"paas-automation/internal/auth"
	"paas-automation/internal/gateway"
	"paas-automation/internal/metrics"
	"paas-automation/internal/scheduler"

	"github.com/gin-gonic/gin"
)

func main() {
	gwAddr := os.Getenv("API_GATEWAY_ADDR")
	if gwAddr == "" {
		gwAddr = "http://k8s-api-gateway.platform.svc:8080"
	}
	approvalAddr := os.Getenv("APPROVAL_ADDR")
	if approvalAddr == "" {
		approvalAddr = "http://approval-svc.platform.svc:8080"
	}

	exec := gateway.NewExecutor(gwAddr)
	approver := approval.NewHook(approvalAddr)
	cron := scheduler.NewCronScheduler()
	store := scheduler.NewMemoryStore()

	api := &apiHandler{
		exec:      exec,
		approver:  approver,
		cron:      cron,
		store:     store,
	}

	r := gin.Default()
	r.Use(auth.AuthTenant()) // 复用模块05 风格：解析 X-Tenant / X-Role

	v1 := r.Group("/api/v1")
	{
		// 提交一次性跨集群任务
		v1.POST("/tasks", api.submitTask)
		// 注册定时任务
		v1.POST("/schedules", api.addSchedule)
		// 查询任务状态
		v1.GET("/tasks/:id", api.getTask)
		// 指标自上报端点（供联邦抓取，标签含 tenant/app/env/cluster）
		v1.GET("/metrics", gin.WrapH(metrics.PromHandler()))
	}

	cron.Start()
	if err := r.Run(getenv("LISTEN_ADDR", ":8090")); err != nil {
		log.Fatalf("scheduler exit: %v", err)
	}
}

type apiHandler struct {
	exec      *gateway.Executor
	approver  *approval.Hook
	cron      *scheduler.CronScheduler
	store     *scheduler.MemoryStore
}

// getenv 读取环境变量，缺失时返回默认值。
func getenv(k, def string) string {
	if v := os.Getenv(k); v != "" {
		return v
	}
	return def
}

// runTask 执行一次跨集群任务：审批 → 网关下发 → 指标上报。
func (h *apiHandler) runTask(ctx context.Context, t scheduler.TaskRequest) error {
	approved, err := h.approver.RequireApproval(ctx, approval.Request{
		Tenant: t.Tenant, Cluster: t.Cluster, Namespace: t.Namespace, Action: t.Action, Env: t.Env,
	})
	if err != nil {
		return err
	}
	if !approved {
		return fmt.Errorf("approval required or rejected")
	}
	start := time.Now()
	_, err = h.exec.ExecuteOnCluster(ctx, gateway.ExecSpec{
		Cluster: t.Cluster, Namespace: t.Namespace, Tenant: t.Tenant, Env: t.Env, App: t.App, Manifest: t.Manifest,
	})
	elapsed := time.Since(start).Seconds()
	outcome := "success"
	if err != nil {
		outcome = "failed"
	}
	metrics.Report(t.Tenant, t.App, t.Env, t.Cluster, t.Template, outcome, elapsed)
	return err
}

// submitTask 提交一次性任务：先经审批钩子，再经网关逐集群下发。
func (h *apiHandler) submitTask(c *gin.Context) {
	var req scheduler.TaskRequest
	if err := c.ShouldBindJSON(&req); err != nil {
		c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
		return
	}
	if err := h.runTask(c.Request.Context(), req); err != nil {
		if strings.Contains(err.Error(), "approval") {
			c.JSON(http.StatusForbidden, gin.H{"error": err.Error()})
			return
		}
		c.JSON(http.StatusBadGateway, gin.H{"error": err.Error()})
		return
	}
	c.JSON(http.StatusOK, gin.H{"status": "dispatched", "task_id": req.ID})
}

// addSchedule 注册定时任务，到点在 runTask 中执行（同样走审批/网关/指标）。
func (h *apiHandler) addSchedule(c *gin.Context) {
	var req scheduler.TaskRequest
	if err := c.ShouldBindJSON(&req); err != nil {
		c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
		return
	}
	if _, err := h.cron.Add(req, func(ctx context.Context, t scheduler.TaskRequest) error {
		return h.runTask(ctx, t)
	}); err != nil {
		c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
		return
	}
	h.store.Add(req)
	c.JSON(http.StatusOK, gin.H{"status": "scheduled"})
}

// getTask 查询任务详情（示例存储）。
func (h *apiHandler) getTask(c *gin.Context) {
	t, ok := h.store.Get(c.Param("id"))
	if !ok {
		c.JSON(http.StatusNotFound, gin.H{"error": "task not found"})
		return
	}
	c.JSON(http.StatusOK, t)
}

// file: internal/scheduler/cron.go
package scheduler

import (
	"context"
	"sync"
	"time"

	"github.com/robfig/cron/v3"
)

// TaskRequest 为提交任务的请求体；字段与 3.8 指标标签一致。
type TaskRequest struct {
	ID        string `json:"id"`
	Tenant    string `json:"tenant"`
	App       string `json:"app"`
	Env       string `json:"env"`
	Cluster   string `json:"cluster"`
	Namespace string `json:"namespace"`
	Template  string `json:"template"`
	Action    string `json:"action"`
	Manifest  string `json:"manifest"`
	CronExpr  string `json:"cron_expr"`
}

// MemoryStore 为示例存储（生产替换为 etcd/数据库）。
type MemoryStore struct {
	mu    sync.RWMutex
	tasks map[string]TaskRequest
}

func NewMemoryStore() *MemoryStore {
	return &MemoryStore{tasks: map[string]TaskRequest{}}
}

// Add 写入一条任务（生产替换为 etcd/数据库）。
func (m *MemoryStore) Add(t TaskRequest) {
	m.mu.Lock()
	defer m.mu.Unlock()
	m.tasks[t.ID] = t
}

// Get 按 ID 读取任务。
func (m *MemoryStore) Get(id string) (TaskRequest, bool) {
	m.mu.RLock()
	defer m.mu.RUnlock()
	t, ok := m.tasks[id]
	return t, ok
}

// CronScheduler 封装 robfig/cron 调度。
type CronScheduler struct {
	c *cron.Cron
}

func NewCronScheduler() *CronScheduler {
	return &CronScheduler{c: cron.New(cron.WithLocation(time.Local))}
}

// Add 注册一个定时任务；回调里触发跨集群执行（通过 ExecuteFunc 注入执行器）。
func (s *CronScheduler) Add(t TaskRequest, run func(ctx context.Context, t TaskRequest) error) (cron.EntryID, error) {
	return s.c.AddFunc(t.CronExpr, func() {
		ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute)
		defer cancel()
		_ = run(ctx, t) // 实际执行由 main 注入，含审批/网关/指标
	})
}

func (s *CronScheduler) Start() {
	s.c.Start()
}

func (s *CronScheduler) Stop() {
	s.c.Stop()
}

// file: internal/workflow/argo.go
package workflow

import (
	"context"

	wfv1alpha1 "github.com/argoproj/argo-workflows/v3/pkg/apis/workflow/v1alpha1"
	wfclientset "github.com/argoproj/argo-workflows/v3/pkg/client/clientset/versioned"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)

// Submit 通过 Argo client-go 提交一个 Workflow 实例到 automation 命名空间。
// 注意：真正实现中该 client 的 rest.Config 由 API 网关经 KMS 解密后注入（见 gateway.Executor）。
func Submit(ctx context.Context, client wfclientset.Interface, ns string, wf *wfv1alpha1.Workflow) (*wfv1alpha1.Workflow, error) {
	return client.ArgoprojV1alpha1().Workflows(ns).Create(ctx, wf, metav1.CreateOptions{})
}

// NewBatchInspect 构造一个跨集群巡检 Workflow（与 deploy/04 对应）。
func NewBatchInspect(name, tenant string) *wfv1alpha1.Workflow {
	return &wfv1alpha1.Workflow{
		ObjectMeta: metav1.ObjectMeta{
			Name:   name,
			Namespace: "automation",
			Labels: map[string]string{
				"tenant":          tenant,
				"env":             "production",
				"task-template":   "batch-inspect",
			},
		},
		Spec: wfv1alpha1.WorkflowSpec{
			Entrypoint: "main",
			Templates: []wfv1alpha1.Template{
				{
					Name: "main",
					DAG: &wfv1alpha1.DAGTemplate{
						Tasks: []wfv1alpha1.DAGTask{
							{Name: "inspect-prod-a", Template: "inspect",
								Arguments: wfv1alpha1.Arguments{Parameters: []wfv1alpha1.Parameter{{Name: "cluster", Value: strPtr("prod-cluster-a")}}}},
							{Name: "inspect-prod-b", Template: "inspect", Dependencies: []string{"inspect-prod-a"},
								Arguments: wfv1alpha1.Arguments{Parameters: []wfv1alpha1.Parameter{{Name: "cluster", Value: strPtr("prod-cluster-b")}}}},
						},
					},
				},
				{
					Name: "inspect",
					Inputs: wfv1alpha1.Inputs{Parameters: []wfv1alpha1.Parameter{{Name: "cluster"}}},
					Container: &wfv1alpha1.Container{
						Image:   "registry.internal/kubectl:1.29",
						Command: []string{"sh", "-c", "kubectl --context {{inputs.parameters.cluster}} get pods -n tn-acme"},
					},
				},
			},
		},
	}
}

func strPtr(s string) *string { return &s }

// file: internal/approval/hook.go
package approval

import (
	"bytes"
	"context"
	"encoding/json"
	"fmt"
	"io"
	"net/http"
	"time"
)

// Hook 调用平台审批服务，实现多级审批闸口。
type Hook struct {
	BaseURL string
	HTTP    *http.Client
}

// NewHook 构造审批钩子客户端。
func NewHook(baseURL string) *Hook {
	return &Hook{BaseURL: baseURL, HTTP: &http.Client{Timeout: 15 * time.Second}}
}

// Request 与任务提交请求对齐。
type Request struct {
	Tenant    string `json:"tenant"`
	Cluster   string `json:"cluster"`
	Namespace string `json:"namespace"`
	Action    string `json:"action"`
	Env       string `json:"env"`
}

// RequireApproval 向审批服务询问是否放行。
// 测试环境低风险可由审批服务直接自动通过；生产高风险须人工多级审批。
func (h *Hook) RequireApproval(ctx context.Context, req Request) (bool, error) {
	body, err := json.Marshal(req)
	if err != nil {
		return false, err
	}
	httpReq, err := http.NewRequestWithContext(ctx, http.MethodPost,
		h.BaseURL+"/v1/approve", bytes.NewReader(body))
	if err != nil {
		return false, err
	}
	httpReq.Header.Set("Content-Type", "application/json")

	resp, err := h.HTTP.Do(httpReq)
	if err != nil {
		return false, err
	}
	defer resp.Body.Close()
	if resp.StatusCode >= 300 {
		b, _ := io.ReadAll(resp.Body)
		return false, fmt.Errorf("approval http %d: %s", resp.StatusCode, string(b))
	}
	var out struct {
		Approved bool `json:"approved"`
	}
	if err := json.NewDecoder(resp.Body).Decode(&out); err != nil {
		return false, err
	}
	return out.Approved, nil
}

// file: internal/gateway/executor.go
package gateway

import (
	"bytes"
	"context"
	"encoding/json"
	"fmt"
	"io"
	"net/http"
	"time"
)

// ExecSpec 描述一次跨集群执行；标签 tenant/app/env/cluster 用于指标与审计。
type ExecSpec struct {
	Tenant    string `json:"tenant"`
	App       string `json:"app"`
	Env       string `json:"env"`
	Cluster   string `json:"cluster"`
	Namespace string `json:"namespace"`
	Manifest  string `json:"manifest"`
}

// Result 为网关返回的执行结果。
type Result struct {
	TaskID string `json:"task_id"`
	Status string `json:"status"`
}

// Executor 通过统一 API 网关下发执行（不直接连接目标集群 apiserver）。
type Executor struct {
	GatewayBaseURL string
	HTTP           *http.Client
}

// NewExecutor 构造网关执行器。
func NewExecutor(gatewayBaseURL string) *Executor {
	return &Executor{
		GatewayBaseURL: gatewayBaseURL,
		HTTP:           &http.Client{Timeout: 60 * time.Second},
	}
}

// ExecuteOnCluster 把 manifest 经网关下发到指定集群的目标租户 Namespace。
// 网关侧会：① RBAC 校验 Namespace 租户权限；② KMS 解密该集群 kubeconfig；
// ③ 在 tn-<tenant> 内 apply manifest；④ 返回 task_id。
// 权限隔离在此闸口强制生效：A 租户任务无法触达 B 租户 Namespace。
func (e *Executor) ExecuteOnCluster(ctx context.Context, spec ExecSpec) (*Result, error) {
	body, err := json.Marshal(spec)
	if err != nil {
		return nil, err
	}
	req, err := http.NewRequestWithContext(ctx, http.MethodPost,
		e.GatewayBaseURL+"/v1/cluster/"+spec.Cluster+"/apply", bytes.NewReader(body))
	if err != nil {
		return nil, err
	}
	req.Header.Set("Content-Type", "application/json")
	req.Header.Set("X-Tenant", spec.Tenant)   // 网关据此做租户级 RBAC
	req.Header.Set("X-Env", spec.Env)

	resp, err := e.HTTP.Do(req)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()
	if resp.StatusCode >= 300 {
		b, _ := io.ReadAll(resp.Body)
		return nil, fmt.Errorf("gateway apply http %d: %s", resp.StatusCode, string(b))
	}
	var out Result
	if err := json.NewDecoder(resp.Body).Decode(&out); err != nil {
		return nil, err
	}
	return &out, nil
}

// file: internal/metrics/reporter.go
package metrics

import (
	"net/http"

	"github.com/prometheus/client_golang/prometheus"
	"github.com/prometheus/client_golang/prometheus/promauto"
	"github.com/prometheus/client_golang/prometheus/promhttp"
)

var (
	// 执行次数计数器：标签严格为 tenant/app/env/cluster/template/result
	executions = promauto.NewCounterVec(prometheus.CounterOpts{
		Name: "task_executions_total",
		Help: "Total automation task executions, labeled by tenant/app/env/cluster.",
	}, []string{"tenant", "app", "env", "cluster", "template", "result"})

	// 执行耗时直方图（秒）
	duration = promauto.NewHistogramVec(prometheus.HistogramOpts{
		Name:    "task_duration_seconds",
		Help:    "Automation task duration in seconds.",
		Buckets: []float64{1, 5, 15, 30, 60, 120, 300},
	}, []string{"tenant", "app", "env", "cluster", "template"})
)

// Report 上报一次任务执行结果（供联邦抓取，标签与 recording rules 一致）。
func Report(tenant, app, env, cluster, template, result string, durSeconds float64) {
	executions.WithLabelValues(tenant, app, env, cluster, template, result).Inc()
	duration.WithLabelValues(tenant, app, env, cluster, template).Observe(durSeconds)
}

// PromHandler 暴露 /metrics 端点，供 Prometheus Agent 抓取并联邦上报。
func PromHandler() http.Handler {
	return promhttp.Handler()
}

// file: internal/auth/tenant.go
package auth

import (
	"github.com/gin-gonic/gin"
)

// AuthTenant 解析 X-Tenant / X-Role，供调度器鉴权与审计使用。
func AuthTenant() gin.HandlerFunc {
	return func(c *gin.Context) {
		tenant := c.GetHeader("X-Tenant")
		if tenant == "" {
			tenant = "unknown"
		}
		role := c.GetHeader("X-Role") // platform-admin / tenant
		c.Set("tenant", tenant)
		c.Set("role", role)
		c.Next()
	}
}

// file: go.mod
module paas-automation

go 1.21

require (
	github.com/gin-gonic/gin v1.9.1
	github.com/argoproj/argo-workflows/v3 v3.5.4
	github.com/robfig/cron/v3 v3.0.1
	github.com/prometheus/client_golang v1.18.0
	k8s.io/client-go v0.29.0
	k8s.io/apimachinery v0.29.0
)

cd paas-automation
go mod tidy
go build ./...
go run ./cmd/scheduler

# 提交一个生产跨集群扩容任务（审批通过后才下发）
curl -XPOST http://localhost:8090/api/v1/tasks -H 'Content-Type: application/json' -d '{
  "tenant":"acme","app":"web","env":"production","cluster":"prod-cluster-a",
  "namespace":"tn-acme","template":"scale-out","action":"patch",
  "manifest":"apiVersion: batch/v1\nkind: Job\n..."
}'

# 1. 校验 automation-runner 对 tn-acme 的真实权限（绝不应含 secrets/delete 全局）
kubectl -n tn-acme auth can-i patch deployments --as=system:serviceaccount:automation:automation-runner
kubectl -n tn-acme auth can-i delete secrets     --as=system:serviceaccount:automation:automation-runner
# 期望：第一条 yes，第二条 no

# 2. 经网关下发一个 Job，观察 task_id（网关侧做 RBAC+KMS）
curl -XPOST http://k8s-api-gateway.platform.svc:8080/v1/cluster/prod-cluster-a/apply \
  -H "X-Tenant: acme" -H "X-Env: production" -H "Content-Type: application/json" \
  -d @job-scaleout.json

# 调度器 /metrics 暴露指标，确认命名空间 tn-acme 的 CronJob 是否上报
kubectl -n automation port-forward svc/paas-automation 8090:8090 &
curl -s 'http://localhost:8090/metrics' | grep '^task_executions_total'

# 在中心 Thanos Query 跨集群查该租户成功率（复用模块05）
curl -s 'http://thanos-query.monitoring.svc:9090/api/v1/query?query=task:success_rate:rate7d{tenant="acme"}' | jq .

# 注册 ServiceMonitor 让 Agent 抓取调度器指标（见 deploy/06）
kubectl -n monitoring apply -f deploy/06-task-metrics.yaml

# 直接提交生产高危任务应被拒（未审批）
curl -XPOST http://localhost:8090/api/v1/tasks -H 'Content-Type: application/json' -d '{
  "tenant":"acme","env":"production","cluster":"prod-cluster-a",
  "namespace":"tn-acme","template":"scale-out","action":"patch","manifest":"..."}'
# 期望返回 403 approval required or rejected

# 走审批通过后重试应成功（审批服务返回 approved=true）

# 查审计中该租户的任务执行记录（模块09 审计视图）
paas-cli audit query --tenant acme --last 7d

# 失败率超阈值时，Alertmanager 应产生 TaskFailureRateHigh（复用模块05 分级路由）
kubectl -n monitoring port-forward svc/alertmanager 9093:9093 &
curl 'http://localhost:9093/api/v2/alerts' | jq '.[] | select(.labels.alertname=="TaskFailureRateHigh")'