租户与组织权限中心

# file: deploy/crd-tenant.yaml
apiVersion: apiextensions.k8s.io/v1
kind: CustomResourceDefinition
metadata:
  name: tenants.paas.io
spec:
  group: paas.io
  names:
    kind: Tenant
    listKind: TenantList
    plural: tenants
    singular: tenant
  scope: Cluster
  versions:
    - name: v1
      served: true
      storage: true
      schema:
        openAPIV3Schema:
          type: object
          properties:
            spec:
              type: object
              required: ["env"]
              properties:
                env:
                  type: string
                  description: "production / staging / test / dev"
                quota:
                  type: object
                  properties:
                    cpu:
                      type: string
                    memory:
                      type: string
                    namespaceCount:
                      type: integer
                      minimum: 1
            status:
              type: object
              properties:
                phase:
                  type: string
      subresources:
        status: {}

# file: deploy/tenant-rbac.yaml
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
  name: order-readonly
  namespace: subcompany-a-order
  labels:
    tenant: subcompany-a
    project: order
rules:
- apiGroups: [""]
  resources: ["pods", "configmaps", "services"]
  verbs: ["get", "list", "watch"]
---
# 集群级只读 ClusterRole（供租户管理员查看节点/PV 等共享资源）
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
  name: tenant-cluster-readonly
  labels:
    paas.io/managed: "true"
rules:
- apiGroups: [""]
  resources: ["nodes", "persistentvolumes"]
  verbs: ["get", "list", "watch"]
---
# 把项目只读 Role 绑定到开发者用户（数据级：仅 project=order 的 namespace）
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
  name: order-ro-dev-lisi
  namespace: subcompany-a-order
  labels:
    tenant: subcompany-a
subjects:
- kind: User
  name: dev-lisi
  apiGroup: rbac.authorization.k8s.io
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: Role
  name: order-readonly

// file: platform/k8s/registry.go
package k8s

import (
	"fmt"
	"strings"
	"sync"

	"k8s.io/client-go/dynamic"
	"k8s.io/client-go/kubernetes"
	"k8s.io/client-go/rest"
	"k8s.io/client-go/tools/clientcmd"
)

// ClusterEntry 保存网关下发的单集群连接（来自统一 K8s API 网关）。
// 实际生产中 Config 由网关从 KMS 信封加密托管的 kubeconfig 在内存解密注入，
// 客户端不读取明文 kubeconfig。
type ClusterEntry struct {
	ClusterID string
	Env       string
	Config    *rest.Config
	Client    kubernetes.Interface
	Dyn       dynamic.Interface
}

type ClusterRegistry struct {
	mu       sync.RWMutex
	clusters map[string]*ClusterEntry
}

// NewClusterRegistry 从网关下发的多集群 kubeconfig 初始化所有集群连接。
// kubeconfigPath 为空时使用 in-cluster 配置（中心控制面运行于网关 Pod 内）。
func NewClusterRegistry(kubeconfigPath string) (*ClusterRegistry, error) {
	reg := &ClusterRegistry{clusters: make(map[string]*ClusterEntry)}

	var baseCfg *rest.Config
	var err error
	if kubeconfigPath == "" {
		baseCfg, err = rest.InClusterConfig()
	} else {
		loadingRules := clientcmd.NewDefaultClientConfigLoadingRules()
		loadingRules.ExplicitPath = kubeconfigPath
		baseCfg, err = clientcmd.NewNonInteractiveDeferredLoadingClientConfig(
			loadingRules, &clientcmd.ConfigOverrides{}).ClientConfig()
	}
	if err != nil {
		return nil, fmt.Errorf("build base rest.Config: %w", err)
	}

	rawCfg, err := clientcmd.LoadFromFile(kubeconfigPath)
	if err != nil {
		return nil, fmt.Errorf("load kubeconfig: %w", err)
	}
	for name, ctx := range rawCfg.Contexts {
		cfg := rest.CopyConfig(baseCfg)
		cfg.Host = rawCfg.Clusters[ctx.Cluster].Server
		// 真实场景注入 CA + 客户端证书；此处复用 baseCfg 的 TLS。
		cli, err := kubernetes.NewForConfig(cfg)
		if err != nil {
			return nil, fmt.Errorf("new clientset for %s: %w", name, err)
		}
		dyn, err := dynamic.NewForConfig(cfg)
		if err != nil {
			return nil, fmt.Errorf("new dynamic client for %s: %w", name, err)
		}
		reg.clusters[name] = &ClusterEntry{
			ClusterID: name,
			Env:       envFromContextName(name),
			Config:    cfg,
			Client:    cli,
			Dyn:       dyn,
		}
	}
	return reg, nil
}

// envFromContextName 由网关下发的 context 名推导环境（生产实践中来自平台 CMDB 元数据）。
func envFromContextName(name string) string {
	switch {
	case strings.Contains(name, "prod"):
		return "production"
	case strings.Contains(name, "staging"):
		return "staging"
	case strings.Contains(name, "test"):
		return "test"
	default:
		return "dev"
	}
}

// ClientAndDynForTenant 按租户环境路由到对应物理集群（生产租户只能命中 production 集群，跨环境默认拒绝）。
func (r *ClusterRegistry) ClientAndDynForTenant(tenant, env string) (kubernetes.Interface, dynamic.Interface, error) {
	r.mu.RLock()
	defer r.mu.RUnlock()
	for _, c := range r.clusters {
		if c.Env == env {
			return c.Client, c.Dyn, nil
		}
	}
	return nil, nil, fmt.Errorf("no cluster routed for tenant=%s env=%s", tenant, env)
}

// ClientForCluster 按集群 ID 取连接（排障/跨集群查询用）。
func (r *ClusterRegistry) ClientForCluster(id string) (*ClusterEntry, error) {
	r.mu.RLock()
	defer r.mu.RUnlock()
	c, ok := r.clusters[id]
	if !ok {
		return nil, fmt.Errorf("cluster %s not registered", id)
	}
	return c, nil
}

// file: platform/k8s/tenant.go
package k8s

import (
	"context"
	"fmt"

	corev1 "k8s.io/api/core/v1"
	netv1 "k8s.io/api/networking/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/api/resource"
	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
	"k8s.io/apimachinery/pkg/runtime/schema"
	"k8s.io/client-go/dynamic"
	"k8s.io/client-go/kubernetes"
)

var tenantGVR = schema.GroupVersionResource{Group: "paas.io", Version: "v1", Resource: "tenants"}

func mustQ(s string) resource.Quantity {
	q, _ := resource.ParseQuantity(s)
	return q
}

// ReconcileTenant 在目标集群创建 Tenant CR + Namespace + 配额 + 默认 Deny-All 网络策略。
func ReconcileTenant(ctx context.Context, cli kubernetes.Interface, dyn dynamic.Interface,
	tenant, env, cpu, mem string, nsCount int) error {

	// 1) 平台侧 CRD 落一条 Tenant 记录（DynamicClient 演示）
	t := &unstructured.Unstructured{Object: map[string]interface{}{
		"apiVersion": "paas.io/v1",
		"kind":       "Tenant",
		"metadata": map[string]interface{}{
			"name":  tenant,
			"labels": map[string]interface{}{"tenant": tenant, "env": env},
		},
		"spec": map[string]interface{}{
			"env": env,
			"quota": map[string]interface{}{
				"cpu":            cpu,
				"memory":         mem,
				"namespaceCount": nsCount,
			},
		},
		"status": map[string]interface{}{"phase": "Provisioning"},
	}}
	if _, err := dyn.Resource(tenantGVR).Create(ctx, t, metav1.CreateOptions{}); err != nil {
		return fmt.Errorf("create Tenant CR: %w", err)
	}

	// 2) 创建租户 Namespace（带 tenant/env 标签，用于指标隔离）
	ns := &corev1.Namespace{
		ObjectMeta: metav1.ObjectMeta{
			Name:   tenant,
			Labels: map[string]string{"tenant": tenant, "env": env},
		},
	}
	if _, err := cli.CoreV1().Namespaces().Create(ctx, ns, metav1.CreateOptions{}); err != nil {
		return fmt.Errorf("create namespace: %w", err)
	}

	// 3) 配额 + 限额范围 + 默认拒绝网络策略
	if err := applyResourceQuota(ctx, cli, tenant, cpu, mem); err != nil {
		return err
	}
	if err := applyLimitRange(ctx, cli, tenant); err != nil {
		return err
	}
	if err := applyDefaultDenyNetworkPolicy(ctx, cli, tenant); err != nil {
		return err
	}
	return nil
}

func applyResourceQuota(ctx context.Context, cli kubernetes.Interface, ns, cpu, mem string) error {
	q := &corev1.ResourceQuota{
		ObjectMeta: metav1.ObjectMeta{
			Name: "tenant-quota", Namespace: ns,
			Labels: map[string]string{"tenant": ns},
		},
		Spec: corev1.ResourceQuotaSpec{
			Hard: corev1.ResourceList{
				corev1.ResourceRequestsCPU:    mustQ(cpu),
				corev1.ResourceRequestsMemory: mustQ(mem),
			},
		},
	}
	_, err := cli.CoreV1().ResourceQuotas(ns).Create(ctx, q, metav1.CreateOptions{})
	return err
}

func applyLimitRange(ctx context.Context, cli kubernetes.Interface, ns string) error {
	lr := &corev1.LimitRange{
		ObjectMeta: metav1.ObjectMeta{Name: "tenant-limitrange", Namespace: ns},
		Spec: corev1.LimitRangeSpec{
			Limits: []corev1.LimitRangeItem{
				{
					Type: corev1.LimitTypeContainer,
					Default: corev1.ResourceList{
						corev1.ResourceCPU:    mustQ("500m"),
						corev1.ResourceMemory: mustQ("512Mi"),
					},
					DefaultRequest: corev1.ResourceList{
						corev1.ResourceCPU:    mustQ("100m"),
						corev1.ResourceMemory: mustQ("128Mi"),
					},
					Max: corev1.ResourceList{
						corev1.ResourceCPU:    mustQ("4"),
						corev1.ResourceMemory: mustQ("4Gi"),
					},
				},
			},
		},
	}
	_, err := cli.CoreV1().LimitRanges(ns).Create(ctx, lr, metav1.CreateOptions{})
	return err
}

func applyDefaultDenyNetworkPolicy(ctx context.Context, cli kubernetes.Interface, ns string) error {
	ingress := netv1.PolicyTypeIngress
	egress := netv1.PolicyTypeEgress
	np := &netv1.NetworkPolicy{
		ObjectMeta: metav1.ObjectMeta{
			Name: "default-deny-all", Namespace: ns,
			Labels: map[string]string{"tenant": ns},
		},
		Spec: netv1.NetworkPolicySpec{
			PodSelector: metav1.LabelSelector{},
			PolicyTypes: []netv1.PolicyType{ingress, egress},
		},
	}
	_, err := cli.NetworkingV1().NetworkPolicies(ns).Create(ctx, np, metav1.CreateOptions{})
	return err
}

// file: platform/k8s/rbac.go
package k8s

import (
	"context"
	"fmt"

	rbacv1 "k8s.io/api/rbac/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/client-go/kubernetes"
)

// CreateProjectReadonlyRole 创建「仅 project=<project> 只读」的 Role（数据级用标签选择器约束）。
func CreateProjectReadonlyRole(ctx context.Context, cli kubernetes.Interface,
	namespace, project, roleName string) error {
	role := &rbacv1.Role{
		ObjectMeta: metav1.ObjectMeta{
			Name:      roleName,
			Namespace: namespace,
			Labels:    map[string]string{"tenant": namespace, "project": project},
		},
		Rules: []rbacv1.PolicyRule{
			{
				APIGroups: []string{""},
				Resources: []string{"pods", "configmaps", "services"},
				Verbs:     []string{"get", "list", "watch"},
			},
		},
	}
	_, err := cli.RbacV1().Roles(namespace).Create(ctx, role, metav1.CreateOptions{})
	if err != nil {
		return fmt.Errorf("create role: %w", err)
	}
	return nil
}

// BindRoleToUser 把 Role 绑定到用户（或 ServiceAccount）。
func BindRoleToUser(ctx context.Context, cli kubernetes.Interface,
	namespace, roleName, user, bindingName string) error {
	rb := &rbacv1.RoleBinding{
		ObjectMeta: metav1.ObjectMeta{Name: bindingName, Namespace: namespace},
		Subjects: []rbacv1.Subject{
			{Kind: "User", Name: user, APIGroup: "rbac.authorization.k8s.io"},
		},
		RoleRef: rbacv1.RoleRef{
			APIGroup: "rbac.authorization.k8s.io",
			Kind:     "Role",
			Name:     roleName,
		},
	}
	_, err := cli.RbacV1().RoleBindings(namespace).Create(ctx, rb, metav1.CreateOptions{})
	if err != nil {
		return fmt.Errorf("create rolebinding: %w", err)
	}
	return nil
}

// CreateClusterReadOnlyRole 创建集群级 ClusterRole（租户管理员查看节点/PV）。
func CreateClusterReadOnlyRole(ctx context.Context, cli kubernetes.Interface, name string) error {
	cr := &rbacv1.ClusterRole{
		ObjectMeta: metav1.ObjectMeta{Name: name, Labels: map[string]string{"paas.io/managed": "true"}},
		Rules: []rbacv1.PolicyRule{
			{APIGroups: []string{""}, Resources: []string{"nodes", "persistentvolumes"}, Verbs: []string{"get", "list", "watch"}},
		},
	}
	_, err := cli.RbacV1().ClusterRoles().Create(ctx, cr, metav1.CreateOptions{})
	if err != nil {
		return fmt.Errorf("create clusterrole: %w", err)
	}
	return nil
}

// file: platform/k8s/metrics.go
package k8s

import (
	"github.com/prometheus/client_golang/prometheus"
	"github.com/prometheus/client_golang/prometheus/promauto"
)

var (
	rbacRoleBindingTotal = promauto.NewCounterVec(
		prometheus.CounterOpts{
			Name: "paas_rbac_rolebinding_total",
			Help: "租户 RoleBinding 创建总数（标签需与 Prometheus 规则一致）",
		},
		[]string{"tenant", "env", "app"},
	)
	tenantProvisionSeconds = promauto.NewHistogramVec(
		prometheus.HistogramOpts{
			Name:    "paas_tenant_provision_duration_seconds",
			Help:    "租户 Namespace/配额下发耗时",
			Buckets: prometheus.DefBuckets,
		},
		[]string{"tenant", "env"},
	)
	rbacSyncErrorsTotal = promauto.NewCounterVec(
		prometheus.CounterOpts{
			Name: "paas_rbac_sync_errors_total",
			Help: "RBAC 经网关同步失败次数",
		},
		[]string{"tenant", "cluster"},
	)
)

func IncRoleBinding(tenant, env, app string) {
	rbacRoleBindingTotal.WithLabelValues(tenant, env, app).Inc()
}
func ObserveProvision(tenant, env string, secs float64) {
	tenantProvisionSeconds.WithLabelValues(tenant, env).Observe(secs)
}
func IncSyncError(tenant, cluster string) {
	rbacSyncErrorsTotal.WithLabelValues(tenant, cluster).Inc()
}

// file: platform/k8s/audit.go
package k8s

import (
	"context"
	"encoding/json"
	"os"
	"sync"
	"time"
)

// AuditRecord 审计记录（字段含操作人/对象/动作/租户/时间/来源IP/审批单号）。
type AuditRecord struct {
	Timestamp time.Time `json:"ts"`
	Operator  string    `json:"operator"`
	Action    string    `json:"action"`
	Tenant    string    `json:"tenant"`
	Target    string    `json:"target"`
	SrcIP     string    `json:"src_ip"`
	Ticket    string    `json:"ticket"`
}

// AuditSink 审计落库（演示写 JSONL 文件，生产写入 Elasticsearch/对象存储，留存 ≥90 天）。
type AuditSink struct {
	mu   sync.Mutex
	path string
}

func NewAuditSink(path string) *AuditSink { return &AuditSink{path: path} }

func (a *AuditSink) Write(ctx context.Context, rec AuditRecord) error {
	rec.Timestamp = time.Now()
	b, err := json.Marshal(rec)
	if err != nil {
		return err
	}
	a.mu.Lock()
	defer a.mu.Unlock()
	f, err := os.OpenFile(a.path, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)
	if err != nil {
		return err
	}
	defer f.Close()
	_, err = f.Write(append(b, '\n'))
	return err
}

// file: platform/auth/sso.go
package auth

import (
	"crypto/hmac"
	"crypto/sha256"
	"encoding/hex"
	"net/http"
	"strconv"
	"strings"
	"time"

	"github.com/gin-gonic/gin"
)

// IssueToken 为通过 SSO/LDAP 校验的用户签发平台令牌（演示：HMAC 签名 tenant|user|ts）。
// 生产应换成 OIDC ID Token / 企业微信 code2session，并带短过期与吊销。
func IssueToken(tenant, user, secret string) string {
	ts := time.Now().Unix()
	body := tenant + "|" + user + "|" + strconv.FormatInt(ts, 10)
	mac := hmac.New(sha256.New, []byte(secret))
	mac.Write([]byte(body))
	return body + "|" + hex.EncodeToString(mac.Sum(nil))
}

// AuthMiddleware 校验 Authorization: Bearer <token>，并把 tenant/user 注入上下文。
func AuthMiddleware(secret string) gin.HandlerFunc {
	return func(c *gin.Context) {
		h := c.GetHeader("Authorization")
		parts := strings.SplitN(h, " ", 2)
		if len(parts) != 2 || parts[0] != "Bearer" {
			c.AbortWithStatusJSON(http.StatusUnauthorized, gin.H{"error": "missing bearer token"})
			return
		}
		fields := strings.Split(parts[1], "|")
		if len(fields) != 4 {
			c.AbortWithStatusJSON(http.StatusUnauthorized, gin.H{"error": "bad token"})
			return
		}
		body := fields[0] + "|" + fields[1] + "|" + fields[2]
		mac := hmac.New(sha256.New, []byte(secret))
		mac.Write([]byte(body))
		if hex.EncodeToString(mac.Sum(nil)) != fields[3] {
			c.AbortWithStatusJSON(http.StatusUnauthorized, gin.H{"error": "invalid signature"})
			return
		}
		c.Set("tenant", fields[0])
		c.Set("user", fields[1])
		c.Next()
	}
}

// file: platform/go.mod
module paas.example.com

go 1.22

require (
	github.com/gin-gonic/gin v1.9.1
	github.com/prometheus/client_golang v1.18.0
	k8s.io/api v0.29.0
	k8s.io/apimachinery v0.29.0
	k8s.io/client-go v0.29.0
)

// file: platform/main.go
package main

import (
	"log"
	"net/http"
	"time"

	"github.com/gin-gonic/gin"
	"github.com/prometheus/client_golang/prometheus/promhttp"

	"paas.example.com/platform/auth"
	"paas.example.com/platform/k8s"
)

func main() {
	reg, err := k8s.NewClusterRegistry("/etc/paas/gateway-kubeconfig")
	if err != nil {
		log.Fatalf("init cluster registry: %v", err)
	}
	audit := k8s.NewAuditSink("/var/log/paas/audit.jsonl")

	r := gin.Default()
	r.GET("/metrics", gin.WrapH(promhttp.Handler()))
	r.Use(auth.AuthMiddleware("change-me-in-prod"))

	// 新建租户（平台管理员）
	r.POST("/api/v1/tenants", func(c *gin.Context) {
		var req struct {
			Name string `json:"name"`
			Env  string `json:"env"`
			CPU  string `json:"cpu"`
			Mem  string `json:"memory"`
		}
		if err := c.ShouldBindJSON(&req); err != nil {
			c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
			return
		}
		// 按租户 env 路由到对应物理集群（网关多集群路由）
		cli, dyn, err := reg.ClientAndDynForTenant(req.Name, req.Env)
		if err != nil {
			c.JSON(http.StatusForbidden, gin.H{"error": err.Error()})
			return
		}
		start := time.Now()
		if err := k8s.ReconcileTenant(c.Request.Context(), cli, dyn, req.Name, req.Env, req.CPU, req.Mem, 20); err != nil {
			k8s.IncSyncError(req.Name, req.Env)
			c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
			return
		}
		k8s.ObserveProvision(req.Name, req.Env, time.Since(start).Seconds())
		k8s.IncRoleBinding(req.Name, req.Env, "tenant-provisioner")
		_ = audit.Write(c.Request.Context(), k8s.AuditRecord{
			Operator: c.GetString("user"), Action: "create-tenant",
			Tenant: req.Name, Target: "namespace/" + req.Name,
			SrcIP: c.ClientIP(), Ticket: "AUTO-" + req.Name,
		})
		c.JSON(http.StatusOK, gin.H{"status": "provisioned"})
	})

	// 给项目分配只读角色（子公司管理员）
	r.POST("/api/v1/projects/:ns/readonly", func(c *gin.Context) {
		ns := c.Param("ns")
		var req struct {
			User    string `json:"user"`
			Project string `json:"project"`
			Env     string `json:"env"`
		}
		if err := c.ShouldBindJSON(&req); err != nil {
			c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
			return
		}
		cli, _, err := reg.ClientAndDynForTenant(c.GetString("tenant"), req.Env)
		if err != nil {
			c.JSON(http.StatusForbidden, gin.H{"error": err.Error()})
			return
		}
		role := "order-readonly"
		if err := k8s.CreateProjectReadonlyRole(c.Request.Context(), cli, ns, req.Project, role); err != nil {
			c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
			return
		}
		if err := k8s.BindRoleToUser(c.Request.Context(), cli, ns, role, req.User, "ro-"+req.User); err != nil {
			c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
			return
		}
		k8s.IncRoleBinding(ns, c.GetString("tenant"), req.Project)
		_ = audit.Write(c.Request.Context(), k8s.AuditRecord{
			Operator: c.GetString("user"), Action: "grant-readonly",
			Tenant: ns, Target: "rolebinding/ro-" + req.User,
			SrcIP: c.ClientIP(), Ticket: "AUTO",
		})
		c.JSON(http.StatusOK, gin.H{"status": "bound"})
	})

	log.Println("tenant & org RBAC platform listening on :8080")
	if err := r.Run(":8080"); err != nil {
		log.Fatal(err)
	}
}

# file: deploy/tenant-namespace.yaml
apiVersion: v1
kind: Namespace
metadata:
  name: subcompany-a
  labels:
    tenant: subcompany-a
    env: production
    istio-injection: enabled

# file: deploy/tenant-quota.yaml
apiVersion: v1
kind: ResourceQuota
metadata:
  name: tenant-quota
  namespace: subcompany-a
  labels:
    tenant: subcompany-a
spec:
  hard:
    requests.cpu: "200"
    requests.memory: 800Gi
    limits.cpu: "400"
    limits.memory: 1600Gi
    pods: "200"
---
apiVersion: v1
kind: LimitRange
metadata:
  name: tenant-limitrange
  namespace: subcompany-a
spec:
  limits:
  - type: Container
    default:
      cpu: 500m
      memory: 512Mi
    defaultRequest:
      cpu: 100m
      memory: 128Mi
    max:
      cpu: 4
      memory: 4Gi

# 步骤 2：平台自动在纳管的生产集群创建专属 Namespace（管理员无需手敲 kubectl）
# 等价 REST 调用（token 由 SSO 签发）
curl -X POST https://paas-gateway/api/v1/tenants \
  -H "Authorization: Bearer $(paas-cli token issue --user zhangsan --tenant platform)" \
  -d '{"name":"subcompany-a","env":"production","cpu":"200","memory":"800Gi"}'

# 步骤 3：分配租户管理员角色（Namespace 级 + API 级）
paas-cli role assign --user zhangsan --role tenant-admin --tenant subcompany-a

# 步骤 1：用平台 API 分配（等效于 kubectl apply -f deploy/tenant-rbac.yaml）
curl -X POST https://paas-gateway/api/v1/projects/subcompany-a-order/readonly \
  -H "Authorization: Bearer $TOKEN" \
  -d '{"user":"dev-lisi","project":"order","env":"production"}'

# 步骤 2（排障）：直接核对 RoleBinding 是否落到正确 namespace
kubectl --context prod-cluster-01 -n subcompany-a-order get rolebinding order-ro-dev-lisi -o yaml

# file: deploy/tenant-networkpolicy.yaml
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
  name: default-deny-all
  namespace: subcompany-a
  labels:
    tenant: subcompany-a
spec:
  podSelector: {}
  policyTypes:
  - Ingress
  - Egress
---
# 仅放通 DNS 出向（否则 Pod 无法解析服务发现）
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
  name: allow-dns-egress
  namespace: subcompany-a
spec:
  podSelector: {}
  policyTypes:
  - Egress
  egress:
  - to:
    - namespaceSelector:
        matchLabels:
          kubernetes.io/metadata.name: kube-system
    ports:
    - protocol: UDP
      port: 53
    - protocol: TCP
      port: 53

# file: observability/prometheus-agent-scrape.yaml
# 各物理集群内的 Prometheus Agent：采集本集群指标，并强制注入 tenant/app/env/cluster 标签
global:
  scrape_interval: 30s
  external_labels:
    cluster: prod-cluster-01
scrape_configs:
  - job_name: kube-pods
    kubernetes_sd_configs:
      - role: pod
    relabel_configs:
      - source_labels: [__meta_kubernetes_namespace]
        target_label: namespace
      - source_labels: [__meta_kubernetes_pod_label_app]
        target_label: app
      # 命名空间名 == tenant，平台保证一致
      - source_labels: [__meta_kubernetes_namespace]
        target_label: tenant
      - target_label: env
        replacement: production
      - target_label: cluster
        replacement: prod-cluster-01

# file: observability/prometheus-federation.yaml
# 中心 Prometheus 联邦拉取各集群 Agent，honor_labels 保留 tenant/env/cluster 等源标签
global:
  scrape_interval: 30s
scrape_configs:
  - job_name: federate-prod
    honor_labels: true
    metrics_path: /federate
    params:
      match[]:
        - '{__name__=~"paas_.*|kube_.*"}'
    static_configs:
      - targets: ['prom-agent-prod.paas-monitoring.svc:9090']
  - job_name: federate-test
    honor_labels: true
    metrics_path: /federate
    params:
      match[]:
        - '{__name__=~"paas_.*|kube_.*"}'
    static_configs:
      - targets: ['prom-agent-test.paas-monitoring.svc:9090']

# file: observability/recording-rules.yaml
groups:
  - name: paas_tenant_rbac_recording
    rules:
      - record: paas:rbac:rolebinding_count_by_tenant
        expr: sum by (tenant, cluster) (paas_rbac_rolebinding_total)
      - record: paas:tenant:provision_p99_seconds
        expr: histogram_quantile(0.99, sum by (tenant, le) (rate(paas_tenant_provision_duration_seconds_bucket[5m])))

# file: observability/alert-rules.yaml
groups:
  - name: paas_tenant_rbac_alerts
    rules:
      - alert: TenantRBACSyncFailed
        expr: increase(paas_rbac_sync_errors_total[5m]) > 0
        for: 2m
        labels:
          severity: critical
          team: paas-platform
        annotations:
          summary: "租户 {{ $labels.tenant }} 在集群 {{ $labels.cluster }} RBAC 同步失败"
          description: "5 分钟内出现 RBAC 经网关同步失败，请检查 K8s API 网关鉴权。"
      - alert: TenantQuotaNearLimit
        expr: |
          kube_resourcequota{resource="requests.cpu", type="used"}
            / kube_resourcequota{resource="requests.cpu", type="hard"} > 0.9          
        for: 10m
        labels:
          severity: warning
        annotations:
          summary: "租户 {{ $labels.tenant }} CPU 配额使用超过 90%"

# file: observability/alertmanager.yaml
global:
  resolve_timeout: 5m
route:
  receiver: default
  group_by: ['cluster', 'env', 'tenant']
  routes:
    - matchers: ['env="production"']
      receiver: prod-oncall
      continue: true
    - matchers: ['severity="critical"']
      receiver: crit-page
receivers:
  - name: default
    webhook_configs:
      - url: http://paas-notify.svc/api/v1/alert
  - name: prod-oncall
    webhook_configs:
      - url: http://paas-notify.svc/api/v1/wechat
  - name: crit-page
    pagerduty_configs:
      - routing_key: ${PAGERDUTY_KEY}

# file: runbook/tenant-ops.sh（节选：故障1 排查）
kubectl --context prod-cluster-01 -n subcompany-a-order get rolebinding -l tenant=subcompany-a -o wide
kubectl --context prod-cluster-01 auth can-i get pods -n subcompany-a-order --as=dev-lisi
# 期望 yes；跨 namespace 应 no

# 查某用户全部 RoleBinding（跨租户审查）
kubectl --context prod-cluster-01 get rolebindings -A -o json \
  | jq -r '.items[] | select(.subjects[]?.name=="dev-lisi") | .metadata.namespace + "/" + .metadata.name'
# 查看审计中该用户最近动作
grep '"operator":"dev-lisi"' /var/log/paas/audit.jsonl | tail -5

# 校验联邦视图里 tenant 标签是否存在
curl -s 'http://prom-center:9090/api/v1/query?query=paas_rbac_rolebinding_total' \
  | jq '.data.result[].metric.tenant'

#!/usr/bin/env bash
# file: runbook/tenant-ops.sh
set -euo pipefail

# ---- 章节四 SOP：平台管理员创建租户 ----
paas-cli tenant create --name subcompany-a --env production
# 校验 Namespace 与标签、配额已自动生成
kubectl --context prod-cluster-01 get ns subcompany-a --show-labels
kubectl --context prod-cluster-01 get resourcequota,limitrange -n subcompany-a

# ---- 子公司管理员分配「仅 project=order 只读」角色 ----
kubectl --context prod-cluster-01 apply -f deploy/tenant-rbac.yaml
kubectl --context prod-cluster-01 create rolebinding order-ro \
  --role=order-readonly --user=dev-lisi -n subcompany-a-order

# ---- 验证越权被拒（使用只读用户身份）----
kubectl --context prod-cluster-01 auth can-i delete pods -n subcompany-a-order --as=dev-lisi
# 期望：no

# ---- 安全合规员：季度复审回收闲置账号（生产需人工确认）----
paas-cli rbac review --env production --auto-recycle=false

# ---- 查询审计（≥90 天留存）----
grep '"action":"grant-readonly"' /var/log/paas/audit.jsonl | tail -5

# ---- Grafana 多租户过滤：仅看自己 tenant ----
# PromQL: paas_rbac_rolebinding_total{tenant="subcompany-a"}

# ---- 故障3：校验联邦指标 tenant 标签 ----
curl -s 'http://prom-center:9090/api/v1/query?query=paas_rbac_rolebinding_total' \
  | jq '.data.result[].metric.tenant'