K8s服务发布Ingress

# 1. Ingress Nginx Controller 安装

# 1.1 Helm 安装 Ingress Nginx Controller

1. 安装 Helm

# wget https://get.helm.sh/helm-v3.6.3-linux-amd64.tar.gz
# tar xf helm-v3.6.3-linux-amd64.tar.gz
# mv linux-amd64/helm /usr/local/bin/helm
# helm version

2. 下载 Ingress Nginx Controller 安装包

官方文档：https://github.com/kubernetes/ingress-nginx/tree/helm-chart-4.8.2         #根据自己k8s版本下载
# helm repo add ingress-nginx https://kubernetes.github.io/ingress-nginx
# helm repo update
# helm repo list
# helm pull ingress-nginx/ingress-nginx --version 4.8.2

3. 配置 Ingress Nginx Controller

# tar xf ingress-nginx-4.8.2.tgz
# cd ingress-nginx
# vim values.yaml
...
 16 controller:
 17   name: controller
 18   enableAnnotationValidations: false
 19   image:
 20     ## Keep false as default for now!
 21     chroot: false
 22     registry: registry.cn-hangzhou.aliyuncs.com
 23     image: kubernetes_public/ingress-nginx-controller
 24     ## for backwards compatibility consider setting the full image url via the repository value below
 25     ## use *either* current default registry/image or repository format or installing chart by providing the values.yaml wil    l fail
 26     ## repository:
 27     tag: "v1.9.3"
 28     #digest: sha256:8fd21d59428507671ce0fb47f818b1d859c92d2ad07bb7c947268d433030ba98
...
 42   # -- Will add custom configuration options to Nginx https://kubernetes.github.io/ingress-nginx/user-guide/nginx-configurat    ion/configmap/
 43   config:
 44     allow-snippet-annotations: true          #开启server snippet的配置
...
 67   dnsPolicy: ClusterFirstWithHostNet
...
 88   hostNetwork: true
...
107   ingressClassResource:
108     # -- Name of the ingressClass
109     name: nginx
110     # -- Is this ingressClass enabled or not
111     enabled: true
112     # -- Is this the default ingressClass for the cluster
113     default: true
...
184   kind: DaemonSet
...
287   nodeSelector:
288     kubernetes.io/os: linux
289     ingress: "true"
...
638       image:
639         registry: registry.cn-hangzhou.aliyuncs.com
640         image: kubernetes_public/kube-webhook-certgen
641         ## for backwards compatibility consider setting the full image url via the repository value below
642         ## use *either* current default registry/image or repository format or installing chart by providing the values.yaml     will fail
643         ## repository:
644         tag: v20231011-8b53cabe0
645         #digest: sha256:a7943503b45d552785aa3b5e457f169a5661fb94d82b8a3373bcd9ebaf9aac80
...

4. 给需要部署 ingress 的节点上打标签

# kubectl label node k8s-node02 ingress=true
# kubectl label node k8s-node01 ingress=true
# kubectl create ns ingress-nginx
# helm install ingress-nginx -n ingress-nginx .     #安装
# helm upgrade ingress-nginx -n ingress-nginx .     #更新
# kubectl get pods -n ingress-nginx 
NAME                             READY   STATUS    RESTARTS   AGE
ingress-nginx-controller-7nfqn   1/1     Running   0          27s
ingress-nginx-controller-k4p2n   1/1     Running   0          17m
ingress-nginx-controller-kw5jk   1/1     Running   0          24s

# 1.2 Bare metal 安装 Ingress Nginx Controller

1. 下载 Ingress 部署文件，链接地址：https://kubernetes.github.io/ingress-nginx/deploy/#bare-metal-clusters

[root@k8s-master01 ~]# wget https://raw.githubusercontent.com/kubernetes/ingress-nginx/controller-v1.12.1/deploy/static/provider/baremetal/deploy.yaml

2. 配置 Ingress

[root@k8s-master01 ingress-master]# cat deploy.yaml 
apiVersion: v1
kind: Namespace
metadata:
  labels:
    app.kubernetes.io/instance: ingress-nginx
    app.kubernetes.io/name: ingress-nginx
  name: ingress-nginx
---
apiVersion: v1
automountServiceAccountToken: true
kind: ServiceAccount
metadata:
  labels:
    app.kubernetes.io/component: controller
    app.kubernetes.io/instance: ingress-nginx
    app.kubernetes.io/name: ingress-nginx
    app.kubernetes.io/part-of: ingress-nginx
    app.kubernetes.io/version: 1.12.1
  name: ingress-nginx
  namespace: ingress-nginx
---
apiVersion: v1
automountServiceAccountToken: true
kind: ServiceAccount
metadata:
  labels:
    app.kubernetes.io/component: admission-webhook
    app.kubernetes.io/instance: ingress-nginx
    app.kubernetes.io/name: ingress-nginx
    app.kubernetes.io/part-of: ingress-nginx
    app.kubernetes.io/version: 1.12.1
  name: ingress-nginx-admission
  namespace: ingress-nginx
---
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
  labels:
    app.kubernetes.io/component: controller
    app.kubernetes.io/instance: ingress-nginx
    app.kubernetes.io/name: ingress-nginx
    app.kubernetes.io/part-of: ingress-nginx
    app.kubernetes.io/version: 1.12.1
  name: ingress-nginx
  namespace: ingress-nginx
rules:
- apiGroups:
  - ""
  resources:
  - namespaces
  verbs:
  - get
- apiGroups:
  - ""
  resources:
  - configmaps
  - pods
  - secrets
  - endpoints
  verbs:
  - get
  - list
  - watch
- apiGroups:
  - ""
  resources:
  - services
  verbs:
  - get
  - list
  - watch
- apiGroups:
  - networking.k8s.io
  resources:
  - ingresses
  verbs:
  - get
  - list
  - watch
- apiGroups:
  - networking.k8s.io
  resources:
  - ingresses/status
  verbs:
  - update
- apiGroups:
  - networking.k8s.io
  resources:
  - ingressclasses
  verbs:
  - get
  - list
  - watch
- apiGroups:
  - coordination.k8s.io
  resourceNames:
  - ingress-nginx-leader
  resources:
  - leases
  verbs:
  - get
  - update
- apiGroups:
  - coordination.k8s.io
  resources:
  - leases
  verbs:
  - create
- apiGroups:
  - ""
  resources:
  - events
  verbs:
  - create
  - patch
- apiGroups:
  - discovery.k8s.io
  resources:
  - endpointslices
  verbs:
  - list
  - watch
  - get
---
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
  labels:
    app.kubernetes.io/component: admission-webhook
    app.kubernetes.io/instance: ingress-nginx
    app.kubernetes.io/name: ingress-nginx
    app.kubernetes.io/part-of: ingress-nginx
    app.kubernetes.io/version: 1.12.1
  name: ingress-nginx-admission
  namespace: ingress-nginx
rules:
- apiGroups:
  - ""
  resources:
  - secrets
  verbs:
  - get
  - create
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
  labels:
    app.kubernetes.io/instance: ingress-nginx
    app.kubernetes.io/name: ingress-nginx
    app.kubernetes.io/part-of: ingress-nginx
    app.kubernetes.io/version: 1.12.1
  name: ingress-nginx
rules:
- apiGroups:
  - ""
  resources:
  - configmaps
  - endpoints
  - nodes
  - pods
  - secrets
  - namespaces
  verbs:
  - list
  - watch
- apiGroups:
  - coordination.k8s.io
  resources:
  - leases
  verbs:
  - list
  - watch
- apiGroups:
  - ""
  resources:
  - nodes
  verbs:
  - get
- apiGroups:
  - ""
  resources:
  - services
  verbs:
  - get
  - list
  - watch
- apiGroups:
  - networking.k8s.io
  resources:
  - ingresses
  verbs:
  - get
  - list
  - watch
- apiGroups:
  - ""
  resources:
  - events
  verbs:
  - create
  - patch
- apiGroups:
  - networking.k8s.io
  resources:
  - ingresses/status
  verbs:
  - update
- apiGroups:
  - networking.k8s.io
  resources:
  - ingressclasses
  verbs:
  - get
  - list
  - watch
- apiGroups:
  - discovery.k8s.io
  resources:
  - endpointslices
  verbs:
  - list
  - watch
  - get
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
  labels:
    app.kubernetes.io/component: admission-webhook
    app.kubernetes.io/instance: ingress-nginx
    app.kubernetes.io/name: ingress-nginx
    app.kubernetes.io/part-of: ingress-nginx
    app.kubernetes.io/version: 1.12.1
  name: ingress-nginx-admission
rules:
- apiGroups:
  - admissionregistration.k8s.io
  resources:
  - validatingwebhookconfigurations
  verbs:
  - get
  - update
---
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
  labels:
    app.kubernetes.io/component: controller
    app.kubernetes.io/instance: ingress-nginx
    app.kubernetes.io/name: ingress-nginx
    app.kubernetes.io/part-of: ingress-nginx
    app.kubernetes.io/version: 1.12.1
  name: ingress-nginx
  namespace: ingress-nginx
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: Role
  name: ingress-nginx
subjects:
- kind: ServiceAccount
  name: ingress-nginx
  namespace: ingress-nginx
---
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
  labels:
    app.kubernetes.io/component: admission-webhook
    app.kubernetes.io/instance: ingress-nginx
    app.kubernetes.io/name: ingress-nginx
    app.kubernetes.io/part-of: ingress-nginx
    app.kubernetes.io/version: 1.12.1
  name: ingress-nginx-admission
  namespace: ingress-nginx
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: Role
  name: ingress-nginx-admission
subjects:
- kind: ServiceAccount
  name: ingress-nginx-admission
  namespace: ingress-nginx
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
  labels:
    app.kubernetes.io/instance: ingress-nginx
    app.kubernetes.io/name: ingress-nginx
    app.kubernetes.io/part-of: ingress-nginx
    app.kubernetes.io/version: 1.12.1
  name: ingress-nginx
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: ingress-nginx
subjects:
- kind: ServiceAccount
  name: ingress-nginx
  namespace: ingress-nginx
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
  labels:
    app.kubernetes.io/component: admission-webhook
    app.kubernetes.io/instance: ingress-nginx
    app.kubernetes.io/name: ingress-nginx
    app.kubernetes.io/part-of: ingress-nginx
    app.kubernetes.io/version: 1.12.1
  name: ingress-nginx-admission
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: ingress-nginx-admission
subjects:
- kind: ServiceAccount
  name: ingress-nginx-admission
  namespace: ingress-nginx
---
apiVersion: v1
data: null
kind: ConfigMap
metadata:
  labels:
    app.kubernetes.io/component: controller
    app.kubernetes.io/instance: ingress-nginx
    app.kubernetes.io/name: ingress-nginx
    app.kubernetes.io/part-of: ingress-nginx
    app.kubernetes.io/version: 1.12.1
  name: ingress-nginx-controller
  namespace: ingress-nginx
---
apiVersion: v1
kind: Service
metadata:
  labels:
    app.kubernetes.io/component: controller
    app.kubernetes.io/instance: ingress-nginx
    app.kubernetes.io/name: ingress-nginx
    app.kubernetes.io/part-of: ingress-nginx
    app.kubernetes.io/version: 1.12.1
  name: ingress-nginx-controller
  namespace: ingress-nginx
spec:
  ipFamilies:
  - IPv4
  ipFamilyPolicy: SingleStack
  ports:
  - appProtocol: http
    name: http
    port: 80
    protocol: TCP
    targetPort: http
  - appProtocol: https
    name: https
    port: 443
    protocol: TCP
    targetPort: https
  selector:
    app.kubernetes.io/component: controller
    app.kubernetes.io/instance: ingress-nginx
    app.kubernetes.io/name: ingress-nginx
  #type: NodePort
  type: ClusterIP
---
apiVersion: v1
kind: Service
metadata:
  labels:
    app.kubernetes.io/component: controller
    app.kubernetes.io/instance: ingress-nginx
    app.kubernetes.io/name: ingress-nginx
    app.kubernetes.io/part-of: ingress-nginx
    app.kubernetes.io/version: 1.12.1
  name: ingress-nginx-controller-admission
  namespace: ingress-nginx
spec:
  ports:
  - appProtocol: https
    name: https-webhook
    port: 443
    targetPort: webhook
  selector:
    app.kubernetes.io/component: controller
    app.kubernetes.io/instance: ingress-nginx
    app.kubernetes.io/name: ingress-nginx
  type: ClusterIP
---
apiVersion: apps/v1
#kind: Deployment
kind: DaemonSet
metadata:
  labels:
    app.kubernetes.io/component: controller
    app.kubernetes.io/instance: ingress-nginx
    app.kubernetes.io/name: ingress-nginx
    app.kubernetes.io/part-of: ingress-nginx
    app.kubernetes.io/version: 1.12.1
  name: ingress-nginx-controller
  namespace: ingress-nginx
spec:
  minReadySeconds: 0
  revisionHistoryLimit: 10
  selector:
    matchLabels:
      app.kubernetes.io/component: controller
      app.kubernetes.io/instance: ingress-nginx
      app.kubernetes.io/name: ingress-nginx
  template:
    metadata:
      labels:
        app.kubernetes.io/component: controller
        app.kubernetes.io/instance: ingress-nginx
        app.kubernetes.io/name: ingress-nginx
        app.kubernetes.io/part-of: ingress-nginx
        app.kubernetes.io/version: 1.12.1
    spec:
      containers:
      - args:
        - /nginx-ingress-controller
        - --election-id=ingress-nginx-leader
        - --controller-class=k8s.io/ingress-nginx
        - --ingress-class=nginx
        - --configmap=$(POD_NAMESPACE)/ingress-nginx-controller
        - --validating-webhook=:8443
        - --validating-webhook-certificate=/usr/local/certificates/cert
        - --validating-webhook-key=/usr/local/certificates/key
        env:
        - name: POD_NAME
          valueFrom:
            fieldRef:
              fieldPath: metadata.name
        - name: POD_NAMESPACE
          valueFrom:
            fieldRef:
              fieldPath: metadata.namespace
        - name: LD_PRELOAD
          value: /usr/local/lib/libmimalloc.so
        image: registry.cn-hangzhou.aliyuncs.com/kubernetes_public/ingress-nginx-controller-v1.12.1:v1.12.1 
        imagePullPolicy: IfNotPresent
        lifecycle:
          preStop:
            exec:
              command:
              - /wait-shutdown
        livenessProbe:
          failureThreshold: 5
          httpGet:
            path: /healthz
            port: 10254
            scheme: HTTP
          initialDelaySeconds: 10
          periodSeconds: 10
          successThreshold: 1
          timeoutSeconds: 1
        name: controller
        ports:
        - containerPort: 80
          name: http
          protocol: TCP
        - containerPort: 443
          name: https
          protocol: TCP
        - containerPort: 8443
          name: webhook
          protocol: TCP
        readinessProbe:
          failureThreshold: 3
          httpGet:
            path: /healthz
            port: 10254
            scheme: HTTP
          initialDelaySeconds: 10
          periodSeconds: 10
          successThreshold: 1
          timeoutSeconds: 1
        resources:
          requests:
            cpu: 100m
            memory: 90Mi
        securityContext:
          allowPrivilegeEscalation: false
          capabilities:
            add:
            - NET_BIND_SERVICE
            drop:
            - ALL
          readOnlyRootFilesystem: false
          runAsGroup: 82
          runAsNonRoot: true
          runAsUser: 101
          seccompProfile:
            type: RuntimeDefault
        volumeMounts:
        - mountPath: /usr/local/certificates/
          name: webhook-cert
          readOnly: true
      hostNetwork: true                         # 与节点共享网络名称空间
      #dnsPolicy: ClusterFirst
      dnsPolicy: ClusterFirstWithHostNet        # dns 策略
      nodeSelector:                             # 节点选择器
        kubernetes.io/os: linux
        ingress: "true"
      serviceAccountName: ingress-nginx
      terminationGracePeriodSeconds: 300
      volumes:
      - name: webhook-cert
        secret:
          secretName: ingress-nginx-admission
---
apiVersion: batch/v1
kind: Job
metadata:
  labels:
    app.kubernetes.io/component: admission-webhook
    app.kubernetes.io/instance: ingress-nginx
    app.kubernetes.io/name: ingress-nginx
    app.kubernetes.io/part-of: ingress-nginx
    app.kubernetes.io/version: 1.12.1
  name: ingress-nginx-admission-create
  namespace: ingress-nginx
spec:
  template:
    metadata:
      labels:
        app.kubernetes.io/component: admission-webhook
        app.kubernetes.io/instance: ingress-nginx
        app.kubernetes.io/name: ingress-nginx
        app.kubernetes.io/part-of: ingress-nginx
        app.kubernetes.io/version: 1.12.1
      name: ingress-nginx-admission-create
    spec:
      containers:
      - args:
        - create
        - --host=ingress-nginx-controller-admission,ingress-nginx-controller-admission.$(POD_NAMESPACE).svc
        - --namespace=$(POD_NAMESPACE)
        - --secret-name=ingress-nginx-admission
        env:
        - name: POD_NAMESPACE
          valueFrom:
            fieldRef:
              fieldPath: metadata.namespace
        image: registry.cn-hangzhou.aliyuncs.com/kubernetes_public/kube-webhook-certgen-v1.5.2:v1.5.2 
        imagePullPolicy: IfNotPresent
        name: create
        securityContext:
          allowPrivilegeEscalation: false
          capabilities:
            drop:
            - ALL
          readOnlyRootFilesystem: true
          runAsGroup: 65532
          runAsNonRoot: true
          runAsUser: 65532
          seccompProfile:
            type: RuntimeDefault
      nodeSelector:
        kubernetes.io/os: linux
      restartPolicy: OnFailure
      serviceAccountName: ingress-nginx-admission
---
apiVersion: batch/v1
kind: Job
metadata:
  labels:
    app.kubernetes.io/component: admission-webhook
    app.kubernetes.io/instance: ingress-nginx
    app.kubernetes.io/name: ingress-nginx
    app.kubernetes.io/part-of: ingress-nginx
    app.kubernetes.io/version: 1.12.1
  name: ingress-nginx-admission-patch
  namespace: ingress-nginx
spec:
  template:
    metadata:
      labels:
        app.kubernetes.io/component: admission-webhook
        app.kubernetes.io/instance: ingress-nginx
        app.kubernetes.io/name: ingress-nginx
        app.kubernetes.io/part-of: ingress-nginx
        app.kubernetes.io/version: 1.12.1
      name: ingress-nginx-admission-patch
    spec:
      containers:
      - args:
        - patch
        - --webhook-name=ingress-nginx-admission
        - --namespace=$(POD_NAMESPACE)
        - --patch-mutating=false
        - --secret-name=ingress-nginx-admission
        - --patch-failure-policy=Fail
        env:
        - name: POD_NAMESPACE
          valueFrom:
            fieldRef:
              fieldPath: metadata.namespace
        image: registry.cn-hangzhou.aliyuncs.com/kubernetes_public/kube-webhook-certgen-v1.5.2:v1.5.2 
        imagePullPolicy: IfNotPresent
        name: patch
        securityContext:
          allowPrivilegeEscalation: false
          capabilities:
            drop:
            - ALL
          readOnlyRootFilesystem: true
          runAsGroup: 65532
          runAsNonRoot: true
          runAsUser: 65532
          seccompProfile:
            type: RuntimeDefault
      nodeSelector:
        kubernetes.io/os: linux
      restartPolicy: OnFailure
      serviceAccountName: ingress-nginx-admission
---
apiVersion: networking.k8s.io/v1
kind: IngressClass
metadata:
  labels:
    app.kubernetes.io/component: controller
    app.kubernetes.io/instance: ingress-nginx
    app.kubernetes.io/name: ingress-nginx
    app.kubernetes.io/part-of: ingress-nginx
    app.kubernetes.io/version: 1.12.1
  name: nginx
spec:
  controller: k8s.io/ingress-nginx
---
apiVersion: admissionregistration.k8s.io/v1
kind: ValidatingWebhookConfiguration
metadata:
  labels:
    app.kubernetes.io/component: admission-webhook
    app.kubernetes.io/instance: ingress-nginx
    app.kubernetes.io/name: ingress-nginx
    app.kubernetes.io/part-of: ingress-nginx
    app.kubernetes.io/version: 1.12.1
  name: ingress-nginx-admission
webhooks:
- admissionReviewVersions:
  - v1
  clientConfig:
    service:
      name: ingress-nginx-controller-admission
      namespace: ingress-nginx
      path: /networking/v1/ingresses
      port: 443
  failurePolicy: Fail
  matchPolicy: Equivalent
  name: validate.nginx.ingress.kubernetes.io
  rules:
  - apiGroups:
    - networking.k8s.io
    apiVersions:
    - v1
    operations:
    - CREATE
    - UPDATE
    resources:
    - ingresses
  sideEffects: None

• type: ClusterIP #service 类型改为 ClusterIP
• hostNetwork: true # 与节点共享网络名称空间
• dnsPolicy: ClusterFirstWithHostNet # dns 策略
• nodeSelector: # 节点选择器
• kind: DaemonSet # 资源类型 DaemonSet

3. 在指定节点部署 Ingress-Controller

[root@k8s-master01 ingress-master]# kubectl apply -f deploy.yaml -n ingress-nginx

[root@k8s-master01 ingress-master]# kubectl label node k8s-node01 ingress=true
[root@k8s-master01 ingress-master]# kubectl label node k8s-node02 ingress=true
[root@k8s-master01 ingress-master]# kubectl label node k8s-master03 ingress-     #取消节点部署

[root@k8s-master01 ingress-master]# kubectl get pods -n ingress-nginx 
NAME                                   READY   STATUS      RESTARTS   AGE
ingress-nginx-admission-create-zp6mh   0/1     Completed   0          12m
ingress-nginx-admission-patch-f2bpd    0/1     Completed   0          12m
ingress-nginx-controller-rgtkc         1/1     Running     0          3m59s
ingress-nginx-controller-trmn8         1/1     Running     0          3m59s

# 2. Ingress Nginx 入门使用

# cat web-ingress.yaml 
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  name: web-ingress
spec:
  ingressClassName: "nginx"
  rules:
  - host: test.hmallleasing.com
    http:
      paths:
      - backend:
          service:
            name: nginx
            port:
              number: 80
        path: /
        pathType: ImplementationSpecific

# 3. Ingress Nginx 域名重定向 Redirect

# cat redirect-ingress.yaml 
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  name: redirect-ingress
  annotations:
    nginx.ingress.kubernetes.io/permanent-redirect: https://www.baidu.com
spec:
  ingressClassName: "nginx"
  rules:
  - host: redirect.hmallleasing.com
    http:
      paths:
      - backend:
          service:
            name: nginx
            port:
              number: 80
        path: /
        pathType: ImplementationSpecific

# 4. Ingress Nginx 前后端分离 Rewrite

# cat rewrite-ingress.yaml 
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  name: rewrite-ingress
  annotations:
    nginx.ingress.kubernetes.io/rewrite-target: /$2
spec:
  ingressClassName: "nginx"
  rules:
  - host: rewrite.hmallleasing.com
    http:
      paths:
      - backend:
          service:
            name: nginx
            port:
              number: 80
        path: /api(/|$)(.*)
        pathType: ImplementationSpecif

# 5. Ingress Nginx 错误代码重定向

# 6. Ingress Nginx SSL

1.生成证书
# openssl req -x509 -nodes -days 365 -newkey rsa:2048 -keyout tls.key -out tls.cert -subj "/CN=s.hmallleasing.com/O=tls.hmallleasing.com"

2.创建证书
# kubectl create secret tls tls.hmallleasig.com --key tls.key --cert tls.cert

3.ingress配置
# kubectl create secret tls tls.hmallleasig.com --cert=tls.crt --key=tls.key
# cat tls-ingress.yaml 
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  name: tls-ingress
  annotations:
    nginx.ingress.kubernetes.io/ssl-redirect: "false"    #禁用https强制跳转
spec:
  ingressClassName: "nginx"
  rules:
  - host: tls.hmallleasing.com
    http:
      paths:
      - backend:
          service:
            name: nginx
            port:
              number: 80
        path: /
        pathType: ImplementationSpecific
  tls:                  #https
  - hosts:
    - tls.hmallleasing.com
    secretName: "tls.hmallleasig.com"	

# 7. Ingress Nginx 匹配请求头

1.部署移动端应用
# kubectl create deploy phone --image=registry.cn-beijing.aliyuncs.com/dotbalo/nginx:phone
# kubectl expose deploy phone --port 80
# vim m-ingress.yaml
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  name: m-ingress
spec:
  ingressClassName: "nginx"
  rules:
  - host: m.hmallleasing.com
    http:
      paths:
      - backend:
          service:
            name: phone
            port:
              number: 80
        path: /
        pathType: ImplementationSpecific

2.部署PC端应用		
# kubectl create deploy laptop --image=registry.cn-beijing.aliyuncs.com/dotbalo/nginx:laptop	
# kubectl expose deploy laptop --port 80
# vim laptop-ingress.yaml
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  annotations:
    kubernetes.io/ingress.class: nginx
    nginx.ingress.kubernetes.io/server-snippet: |
      set $agentflag 0;
          if ($http_user_agent ~* "(Android|iPhone|Windows Phone|UC|Kindle)" ){
              set $agentflag 1;
          }
          if ( $agentflag = 1 ) {
              return 301 http://m.hmallleaing.com;
          }
  name: laptop-ingress
spec:
  ingressClassName: "nginx"
  rules:
  - host: hmallleasing.com
    http:
      paths:
      - backend:
          service:
            name: laptop
            port:
              number: 80
        path: /
        pathType: ImplementationSpecific	

# 8.Ingress Nginx 基本认证

# yum install httpd -y
# htpasswd -c auth superman
# cat auth 
superman:$apr1$AC1pc3dK$RJyWnyDJFNKY6twneGVrA1		

# kubectl create secret generic basic-auth --from-file=auth
# cat basic-ingress.yaml 
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  name: basic-ingress
  annotations:
    nginx.ingress.kubernetes.io/auth-type: basic  # 认证类型
    nginx.ingress.kubernetes.io/auth-secret: basic-auth  # 包含用户和密码的 secret 资源名称
    nginx.ingress.kubernetes.io/auth-realm: 'Please User password'  # 要显示的信息
spec:
  ingressClassName: "nginx"
  rules:
  - host: basic.hmallleasing.com
    http:
      paths:
      - backend:
          service:
            name: nginx
            port:
              number: 80
        path: /
        pathType: ImplementationSpecific

# 9. Ingress Nginx 黑 / 白名单

写法一：
# cat white-ingress.yaml 
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  name: white-ingress
  annotations:
    nginx.ingress.kubernetes.io/whitelist-source-range: "192.168.40.101"
spec:
  ingressClassName: "nginx"
  rules:
  - host: white.hmallleasing.com
    http:
      paths:
      - backend:
          service:
            name: nginx
            port:
              number: 80
        path: /
        pathType: ImplementationSpecific	

写法二：		
[root@k8s-master01 ingress]# cat white-ingress.yaml 
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  name: white-ingress
  annotations:
    nginx.ingress.kubernetes.io/whitelist-source-range: "192.168.40.0/24"
spec:
  ingressClassName: "nginx"
  rules:
  - host: white.hmallleasing.com
    http:
      paths:
      - backend:
          service:
            name: nginx
            port:
              number: 80
        path: /
        pathType: ImplementationSpecific


写法三：
# cat white-ingress.yaml 
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  name: white-ingress
  annotations:
    nginx.ingress.kubernetes.io/server-snippet: |
      allow 192.168.40.0/24;
      deny all;
spec:
  ingressClassName: "nginx"
  rules:
  - host: white.hmallleasing.com
    http:
      paths:
      - backend:
          service:
            name: nginx
            port:
              number: 80
        path: /
        pathType: ImplementationSpecific
		

#Master01测试		
# curl -H "Host:white.hmallleasing.com" http://192.168.40.103 -I
HTTP/1.1 200 OK
Date: Sat, 14 Oct 2023 13:12:03 GMT
Content-Type: text/html
Content-Length: 612
Connection: keep-alive
Last-Modified: Tue, 16 Apr 2019 13:08:19 GMT
ETag: "5cb5d3c3-264"
Accept-Ranges: bytes		

#Master02测试
# curl -H "Host:white.hmallleasing.com" http://192.168.40.103 -I
HTTP/1.1 403 Forbidden
Date: Sat, 14 Oct 2023 13:13:34 GMT
Content-Type: text/html
Content-Length: 146
Connection: keep-alive

# 10. Ingress Nginx 速率限制

[root@k8s-master01 ingress]# cat limit-rate-ingress.yaml 
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  name: rate-limit-ingress
  annotations:
    nginx.ingress.kubernetes.io/limit-rps: "50"
spec:
  ingressClassName: "nginx"
  rules:
  - host: rate-limit.hmallleasing.com
    http:
      paths:
      - backend:
          service:
            name: nginx
            port:
              number: 80
        path: /
        pathType: ImplementationSpecific

# ab -c 20 -n 1000 http://rate-limit.hmallleasing.com/ |grep request
Complete requests:      1000
Failed requests:        724
Time per request:       10.301 [ms] (mean)
Time per request:       0.515 [ms] (mean, across all concurrent requests)
Percentage of the requests served within a certain time (ms)

# 11. 使用 Nginx 实现灰度 / 金丝雀发布

1.创建 v1 版本
# kubectl create deploy canary-v1 --image=registry.cn-beijing.aliyuncs.com/dotbalo/canary:v1	
# kubectl expose deploy canary-v1 --port 8080
# cat canary-v1-ingress.yaml 
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  name: canary-v1-ingress
spec:
  ingressClassName: "nginx"
  rules:
  - host: canary.hmallleasing.com
    http:
      paths:
      - backend:
          service:
            name: canary-v1
            port:
              number: 8080
        path: /
        pathType: ImplementationSpecific
		
# curl -H "Host:canary.hmallleasing.com" http://192.168.40.103 	

2.创建 v2 版本
# kubectl create deploy canary-v2 --image=registry.cn-beijing.aliyuncs.com/dotbalo/canary:v2
# kubectl expose deploy canary-v2 --port 8080
# cat canary-v2-ingress.yaml 
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  name: canary-v2-ingress
  annotations:
    nginx.ingress.kubernetes.io/canary: "true"    #启动灰度发布
    nginx.ingress.kubernetes.io/canary-weight: "20"  #基于权重,50%流量调度到这个灰度的版本上
spec:
  ingressClassName: "nginx"
  rules:
  - host: canary.hmallleasing.com
    http:
      paths:
      - path: /
        pathType: Prefix
        backend:
          service:
            name: canary-v2
            port:
              number: 8080

#测试灰度发布
[root@k8s-master01 ingress]# cat canary.sh 
#!/bin/bash

while true
do
	curl -H "Host:canary.hmallleasing.com" http://192.168.40.103
	sleep 0.5
done

# 12. kubernetes-dashboard 配置证书

1.创建证书
kubectl create secret tls kubernetes-dashboard-certs --key *.hmallleasing.com_key.key --cert *.hmallleasing.com_chain.crt -n kubernetes-dashboard

2.修改kubernetes-dashboard资源清单
kubectl edit deployment -n kubernetes-dashboard kubernetes-dashboard
...
      - args:
        - --auto-generate-certificates=false
        - --tls-key-file=_.hmallleasing.com_key.key
        - --tls-cert-file=_.hmallleasing.com_chain.crt
        - --token-ttl=21600
        - --authentication-mode=basic,token
        - --namespace=kubernetes-dashboard
...

3.创建ingress
#cat dashboard-ingress.yaml 
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  name: dashboard-ingress
  namespace: kubernetes-dashboard
  annotations:
    nginx.ingress.kubernetes.io/ssl-passthrough: "true"    
    nginx.ingress.kubernetes.io/backend-protocol: "HTTPS"    
spec:
  ingressClassName: "nginx"
  rules:
  - host: dashboard.hmallleasing.com
    http:
      paths:
      - path: /
        pathType: Prefix
        backend:
          service:
            name: kubernetes-dashboard
            port:
              number: 443

# kubectl apply -f dashboard-ingress.yaml 

# 13. 入口 LB 配置

[root@lb nginx]# cat /etc/nginx/conf.d/ingress.conf 
upstream ingress {
	server 192.168.40.103:80 max_conns=2000 max_fails=2 fail_timeout=5s;
	server 192.168.40.104:80 max_conns=2000 max_fails=2 fail_timeout=5s;
	server 192.168.40.105:80 max_conns=2000 max_fails=2 fail_timeout=5s;
}

server {
    listen 443 ssl;
    server_name test.hmallleasing.com;
    client_max_body_size 1G; 
    ssl_prefer_server_ciphers on;
    ssl_certificate  /etc/nginx/sslkey/*.hmallleasing.com_chain.crt;
    ssl_certificate_key  /etc/nginx/sslkey/*.hmallleasing.com_key.key;

    location / {
        proxy_pass http://ingress;
        include proxy_params;
	    proxy_next_upstream error timeout http_500 http_502 http_503 http_504;
	    proxy_next_upstream_tries 2;
	    proxy_next_upstream_timeout 3s;
    }
}

server {
    listen 80;
    server_name test.hmallleasing.com;
    return 302 https://$server_name$request_uri;
}

[root@lb ~]# mkdir /etc/nginx/sslkey -p


[root@lb ~]# cat proxy_params 
proxy_http_version 1.1;
proxy_set_header Connectin "";

proxy_set_header Host $http_host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;

proxy_connect_timeout 60;
proxy_send_timeout 120;
proxy_read_timeout 120;

proxy_buffering on;
proxy_buffer_size 32k;
proxy_buffers 4 128k;
proxy_temp_file_write_size 10240k;
proxy_max_temp_file_size 10240k;