kubectl api-resources | grep coreos.com

NAME                              SHORTNAMES   APIVERSION                          NAMESPACED   KIND
alertmanagerconfigs               amcfg        monitoring.coreos.com/v1alpha1      true         AlertmanagerConfig
alertmanagers                     am           monitoring.coreos.com/v1            true         Alertmanager
podmonitors                       pmon         monitoring.coreos.com/v1            true         PodMonitor
probes                                         monitoring.coreos.com/v1            true         Probe
prometheusagents                  promagent    monitoring.coreos.com/v1            true         PrometheusAgent
prometheuses                      prom         monitoring.coreos.com/v1            true         Prometheus
prometheusrules                   promrule     monitoring.coreos.com/v1            true         PrometheusRule
scrapeconfigs                                  monitoring.coreos.com/v1            true         ScrapeConfig
servicemonitors                   smon         monitoring.coreos.com/v1            true         ServiceMonitor
thanosrulers                                   monitoring.coreos.com/v1            true         ThanosRuler

kubectl explain servicemonitor.spec --recursive

GROUP:      monitoring.coreos.com
KIND:       ServiceMonitor
VERSION:    v1

FIELD: spec <Object>


DESCRIPTION:
    spec defines the specification of desired Service selection for target
    discovery by
    Prometheus.

FIELDS:
  attachMetadata        <Object>
    node        <boolean>
  bodySizeLimit <string>
  convertClassicHistogramsToNHCB        <boolean>
  endpoints     <[]Object> -required-
    authorization       <Object>
      credentials       <Object>
        key     <string> -required-
        name    <string>
        optional        <boolean>
      type      <string>
    basicAuth   <Object>
      password  <Object>
        key     <string> -required-
        name    <string>
        optional        <boolean>
      username  <Object>
        key     <string> -required-
        name    <string>
        optional        <boolean>
    bearerTokenFile     <string>
    bearerTokenSecret   <Object>
      key       <string> -required-
      name      <string>
      optional  <boolean>
    enableHttp2 <boolean>
    filterRunning       <boolean>
    followRedirects     <boolean>
    honorLabels <boolean>
    honorTimestamps     <boolean>
    interval    <string>
    metricRelabelings   <[]Object>
      action    <string>
      enum: replace, Replace, keep, Keep, ....
      modulus   <integer>
      regex     <string>
      replacement       <string>
      separator <string>
      sourceLabels      <[]string>
      targetLabel       <string>
    noProxy     <string>
    oauth2      <Object>
      ...（oauth2/tlsConfig 子树结构较长，此处省略中段）
    params      <map[string][]string>
    path        <string>
    port        <string>
    proxyConnectHeader  <map[string][]Object>
    proxyFromEnvironment        <boolean>
    proxyUrl    <string>
    relabelings <[]Object>
      action    <string>
      enum: replace, Replace, keep, Keep, ....
      modulus   <integer>
      regex     <string>
      replacement       <string>
      separator <string>
      sourceLabels      <[]string>
      targetLabel       <string>
    scheme      <string>
    enum: http, https, HTTP, HTTPS
    scrapeTimeout       <string>
    targetPort  <Object>
    tlsConfig   <Object>
      ...（CA/cert/keySecret 子树，含 insecureSkipVerify、minVersion/maxVersion 枚举 TLS10~TLS13）
  ...（spec 一级字段续，见 3.3）

  enum: PrometheusProto, OpenMetricsText0.0.1, OpenMetricsText1.0.0, PrometheusText0.0.4, ....
  jobLabel      <string>
  keepDroppedTargets    <integer>
  labelLimit    <integer>
  labelNameLengthLimit  <integer>
  labelValueLengthLimit <integer>
  namespaceSelector     <Object>
    any <boolean>
    matchNames  <[]string>
  nativeHistogramBucketLimit    <integer>
  nativeHistogramMinBucketFactor        <Object>
  podTargetLabels       <[]string>
  sampleLimit   <integer>
  scrapeClass   <string>
  scrapeClassicHistograms       <boolean>
  scrapeNativeHistograms        <boolean>
  scrapeProtocols       <[]string>
  selector      <Object> -required-
    matchExpressions    <[]Object>
      key       <string> -required-
      operator  <string> -required-
      values    <[]string>
    matchLabels <map[string]string>
  selectorMechanism     <string>
  enum: RelabelConfig, RoleSelector
  serviceDiscoveryRole  <string>
  enum: Endpoints, EndpointSlice
  targetLabels  <[]string>
  targetLimit   <integer>

kubectl explain servicemonitor.spec.namespaceSelector

GROUP:      monitoring.coreos.com
KIND:       ServiceMonitor
VERSION:    v1

FIELD: namespaceSelector <Object>


DESCRIPTION:
    namespaceSelector defines in which namespace(s) Prometheus should discover
    the services.
    By default, the services are discovered in the same namespace as the
    `ServiceMonitor` object but it is possible to select pods across
    different/all namespaces.

FIELDS:
  any   <boolean>
    any defines the boolean describing whether all namespaces are selected in
    contrast to a
    list restricting them.

  matchNames    <[]string>
    matchNames defines the list of namespace names to select from.

apiVersion: monitoring.coreos.com/v1
kind: ServiceMonitor
metadata:
  labels:
    team: frontend          # 命中 main 实例的 serviceMonitorSelector
  name: example-app
  namespace: default
spec:
  endpoints:
  - interval: 15s
    path: /metrics
    port: web
  selector:
    matchLabels:
      app: example-app      # 选中有这个标签的 Service

serviceMonitor/default/example-app/0
podMonitor/default/example-app-pods/0

example-app http://10.42.0.5:8080/metrics up

spec:
  namespaceSelector:
    matchNames:
    - monitoring
    - default

apiVersion: monitoring.coreos.com/v1
kind: PodMonitor
metadata:
  name: example-app-pods
  namespace: default
  labels:
    team: frontend
spec:
  selector:
    matchLabels:
      app: example-app
  podMetricsEndpoints:
  - port: web
    path: /metrics
    interval: 15s

kubectl apply -f example-app-pods.yaml

podmonitor.monitoring.coreos.com/example-app-pods created

default/example-app-pods http://10.42.0.5:8080/metrics unknown
example-app http://10.42.0.5:8080/metrics up

kubectl patch prometheus main -n default --type merge \
  -p '{"spec":{"podMonitorSelector":{"matchLabels":{"team":"frontend"}}}}'

prometheus.monitoring.coreos.com/main patched

apiVersion: monitoring.coreos.com/v1
kind: PrometheusRule
metadata:
  name: example-app-rules
  namespace: default
  labels:
    team: frontend              # 命中 main 实例的 ruleSelector
spec:
  groups:
  - name: example-app.rules     # 规则组名（Prometheus 内即 group）
    interval: 30s               # 组评估周期，缺省跟随 Prometheus evaluationInterval
    rules:
    - record: job:http_requests:rate5m    # 记录规则：把表达式结果存成新指标
      expr: sum(rate(http_requests_total[5m]))
    - alert: ExampleAppAlwaysFiringDemo   # 告警规则：条件满足且持续 for 后进入 firing
      expr: vector(1) == 1
      for: 1m
      labels:
        severity: demo
      annotations:
        summary: "演示告警：恒定触发的表达式 vector(1) == 1"
        description: "用于验证 PrometheusRule 的加载与告警评估链路。"

kubectl apply -f example-app-rules.yaml

prometheusrule.monitoring.coreos.com/example-app-rules created

group: example-app.rules
  recording job:http_requests:rate5m health=ok
  alerting ExampleAppAlwaysFiringDemo health=ok state=firing

ExampleAppAlwaysFiringDemo firing demo

kubectl apply -f bad-rule.yaml

The  "prometheusrules" is invalid: : 5:11: group "bad.rules", rule 1, "BadPromQL": could not parse expression: 1:32: parse error: unexpected ")" in subquery or range, expected ":" or "]"

kubectl get alertmanager -n monitoring

NAME               VERSION   REPLICAS   READY   RECONCILED   AVAILABLE   AGE
kps-alertmanager   v0.34.1   1          1       True         True        9h

apiVersion: v1
kind: Secret
metadata:
  name: am-webhook-url
  namespace: monitoring
type: Opaque
stringData:
  url: http://example-app.default.svc.cluster.local:8080/webhook
---
apiVersion: monitoring.coreos.com/v1alpha1    # 注意：v1alpha1，不是 v1
kind: AlertmanagerConfig
metadata:
  name: demo-webhook
  namespace: monitoring
  labels:
    release: kps      # 命中 kps Alertmanager 的 alertmanagerConfigSelector（helm 形态默认按 release 标签圈定）
spec:
  route:
    receiver: demo-webhook
    groupBy: ["alertname"]
    groupWait: 30s
    groupInterval: 5m
    repeatInterval: 12h
  receivers:
  - name: demo-webhook
    webhookConfigs:
    - urlSecret:
        name: am-webhook-url
        key: url
      sendResolved: true

kubectl apply -f demo-webhook-amcfg.yaml

secret/am-webhook-url created
alertmanagerconfig.monitoring.coreos.com/demo-webhook created

curl -s http://localhost:30093/api/v2/status

  group_by:
  - receiver: monitoring/demo-webhook/demo-webhook
    group_by:
- name: monitoring/demo-webhook/demo-webhook
    name: monitoring/demo-webhook/demo-webhook

apiVersion: monitoring.coreos.com/v1
kind: Prometheus
metadata:
  name: main
  namespace: monitoring
spec:
  version: v3.15.0
  replicas: 2                     # 双副本抓取冗余
  shards: 2                       # 目标数量大时启用分片
  retention: 15d
  retentionPercentage: 80         # 卷占用 80% 时按比例清理（0.94.0+，需 PV 支持）
  scrapeInterval: 30s
  evaluationInterval: 30s
  resources:
    requests:
      cpu: "1"
      memory: 4Gi
    limits:
      memory: 8Gi                 # 防单实例 OOM 拖垮节点
  serviceMonitorSelector: {}      # 平台实例：接受全部命名空间的监控声明
  podMonitorSelector: {}
  probeSelector: {}
  scrapeConfigSelector: {}
  ruleSelector:
    matchLabels:
      rule-tier: platform         # 高危规则收归平台管理
  serviceMonitorNamespaceSelector: {}
  alerting:
    alertmanagers:
    - namespace: monitoring
      name: kps-alertmanager
  remoteWrite:
  - url: http://vminsert.monitoring.svc:8480/insert/0/prometheus/api/v1/write

example-app http://10.42.0.5:8080/metrics up

kubectl get prometheus main -n default -o jsonpath='{.spec.podMonitorSelector}'

（输出为空——main 实例从未配置 podMonitorSelector）

kubectl patch prometheus main -n default --type merge \
  -p '{"spec":{"podMonitorSelector":{"matchLabels":{"team":"frontend"}}}}'

prometheus.monitoring.coreos.com/main patched

serviceMonitor/default/example-app/0
podMonitor/default/example-app-pods/0

default/example-app-pods http://10.42.0.5:8080/metrics unknown
example-app http://10.42.0.5:8080/metrics up

error: resource mapping not found for name: "demo-webhook" namespace: "monitoring" from "STDIN": no matches for kind "AlertmanagerConfig" in version "monitoring.coreos.com/v1"
ensure CRDs are installed first

alertmanagerconfig.monitoring.coreos.com/demo-webhook created