二进制方式部署kubernetes 1.16.0

环境

操作系统：CentOS Linux release 7.7.1908 (Core)Kernel version：3.10.0-1062.1.1.el7.x86_64工作系统：win10 on Ubuntu 19.04ETCD 部署IP： 192.168.30.50ETCD 版本： v3.4.1kube-apiserver，kube-scheduler，kube-controller-manager 部署IP: 192.168.30.52kubelet部署IP：192.168.30.52，192.168.30.51flannel版本：v0.11.0cni版本：v0.8.2kubernetes版本： 1.16.0工作目录：/root/work远程服务器工作目录：/apps/业务名称kubernetes 集群通信cidr： 10.66.0.0/16POD 集群通信cidr：10.67.0.0/16

准备工作

# 创建工作目录mkdir /root/workcd /root/work# 下载二进制包wget https://storage.googleapis.com/kubernetes-release/release/v1.16.0/kubernetes-server-linux-amd64.tar.gzwget https://github.com/etcd-io/etcd/releases/download/v3.4.1/etcd-v3.4.1-linux-amd64.tar.gzwget https://github.com/containernetworking/plugins/releases/download/v0.8.2/cni-plugins-linux-amd64-v0.8.2.tgz# 解压压缩包tar -xvf kubernetes-server-linux-amd64.tar.gztar -xvf etcd-v3.4.1-linux-amd64.tar.gz# cni-plugins 压缩包不带子文件夹所以我们创建cni 文件夹mkdir cnicd cnimv ../cni-plugins-linux-amd64-v0.8.2.tgz ./tar -xvf cni-plugins-linux-amd64-v0.8.2.tgz# 清理解压缩无用的文件创建目录结构方便分发rm cni-plugins-linux-amd64-v0.8.2.tgzcd ../etcd-v3.4.1-linux-amd64rm -rf Documentation  README-etcdctl.md  README.md  READMEv2-etcdctl.mdmkdir -p {bin,ssl,conf,data}mv etcd* ./bincd ../kubernetes/server/bin/rm -rf *.tarrm -rf *_tagrm -rf apiextensions-apiserver  hyperkube  kubeadm mounter# 备份旧kubectlmv /bin/kubectl /bin/kubectl1.14cp kubectl /bin/kubectl

ETCD 部署

# 回到顶级工作目录/root/workcd /root/work# 创建ssl 配置文件目录mkdir -p cfssl/# 创建ca 证书jsoncat << EOF | tee ./cfssl/ca-config.json{"signing": {"default": {"expiry": "87600h"},"profiles": {"kubernetes": {"usages": ["signing","key encipherment","server auth","client auth"],"expiry": "87600h"}}}}EOF# 创建etcd ca证书配置mkdir -p ./cfssl/etcdcat << EOF | tee ./cfssl/etcd/etcd-ca-csr.json{"CN": "etcd","key": {"algo": "rsa","size": 2048},"names": [{"C": "CN","ST": "GuangDong","L": "GuangZhou","O": "cluster","OU": "cluster"}]}EOF# 生成 ETCD CA 证书和私钥mkdir -p ./cfssl/pki/etcdcfssl gencert -initca ./cfssl/etcd/etcd-ca-csr.json | \cfssljson -bare ./cfssl/pki/etcd/etcd-ca# 创建 ETCD Server 证书 export ETCD_SERVER_IPS=" \\"192.168.30.50\" \" && \export ETCD_SERVER_HOSTNAMES=" \\"etcd\" \" && \cat << EOF | tee ./cfssl/etcd/etcd_server.json{"CN": "etcd","hosts": ["127.0.0.1",${ETCD_SERVER_IPS},${ETCD_SERVER_HOSTNAMES}],"key": {"algo": "rsa","size": 2048},"names": [{"C": "CN","ST": "GuangDong","L": "GuangZhou","O": "cluster","OU": "cluster"}]}EOF# 生成 ETCD Server 证书和私钥cfssl gencert \-ca=./cfssl/pki/etcd/etcd-ca.pem \-ca-key=./cfssl/pki/etcd/etcd-ca-key.pem \-config=./cfssl/ca-config.json \-profile=kubernetes \./cfssl/etcd/etcd_server.json | \cfssljson -bare ./cfssl/pki/etcd/etcd_server# 创建 ETCD Member 证书export ETCD_MEMBER_1_IP=" \    \"192.168.30.50\" \" && \export ETCD_MEMBER_1_HOSTNAMES="etcd\" && \cat << EOF | tee ./cfssl/etcd/${ETCD_MEMBER_1_HOSTNAMES}.json{  "CN": "etcd",  "hosts": [    "127.0.0.1",    ${ETCD_MEMBER_1_IP},    "${ETCD_MEMBER_1_HOSTNAMES}"  ],  "key": {    "algo": "rsa",    "size": 2048  },  "names": [    {      "C": "CN",      "ST": "GuangDong",      "L": "GuangZhou",      "O": "cluster",      "OU": "cluster"    }  ]}EOF##### 生成 ETCD Member 1 证书和私钥cfssl gencert \    -ca=./cfssl/pki/etcd/etcd-ca.pem \    -ca-key=./cfssl/pki/etcd/etcd-ca-key.pem \    -config=./cfssl/ca-config.json \    -profile=kubernetes \    ./cfssl/etcd/${ETCD_MEMBER_1_HOSTNAMES}.json | \    cfssljson -bare ./cfssl/pki/etcd/etcd_member_${ETCD_MEMBER_1_HOSTNAMES}# 创建 ETCD Client 配置文件cat << EOF | tee ./cfssl/etcd/etcd_client.json{"CN": "client","hosts": [""], "key": {"algo": "rsa","size": 2048},"names": [{"C": "CN","ST": "GuangDong","L": "GuangZhou","O": "cluster","OU": "cluster"}]}EOF#生成 ETCD Client 证书和私钥cfssl gencert \-ca=./cfssl/pki/etcd/etcd-ca.pem \-ca-key=./cfssl/pki/etcd/etcd-ca-key.pem \-config=./cfssl/ca-config.json \-profile=kubernetes \./cfssl/etcd/etcd_client.json | \cfssljson -bare ./cfssl/pki/etcd/etcd_client# 复制证书到etcd 分发目录cp -pdr ./cfssl/pki/etcd/*  ./etcd-v3.4.1-linux-amd64/ssl# 创建etcd 启动配置文件cd ./etcd-v3.4.1-linux-amd64/confvi etcdETCD_OPTS="--name=etcd \           --data-dir=/apps/etcd/data/default.etcd \           --listen-peer-urls=https://192.168.30.50:2380 \           --listen-client-urls=https://192.168.30.50:2379,https://127.0.0.1:2379 \           --advertise-client-urls=https://192.168.30.50:2379 \           --initial-advertise-peer-urls=https://192.168.30.50:2380 \           --initial-cluster=etcd=https://192.168.30.50:2380\           --initial-cluster-token=node4=etcd=https://192.168.30.50:2380 \           --initial-cluster-state=new \           --heartbeat-interval=6000 \           --election-timeout=30000 \           --snapshot-count=5000 \           --auto-compaction-retention=1 \           --max-request-bytes=33554432 \           --quota-backend-bytes=17179869184 \           --trusted-ca-file=/apps/etcd/ssl/etcd-ca.pem \           --cert-file=/apps/etcd/ssl/etcd_server.pem \           --key-file=/apps/etcd/ssl/etcd_server-key.pem \           --peer-cert-file=/apps/etcd/ssl/etcd_member_etcd.pem \           --peer-key-file=/apps/etcd/ssl/etcd_member_etcd-key.pem \           --peer-client-cert-auth \           --peer-trusted-ca-file=/apps/etcd/ssl/etcd-ca.pem"# 远程服务器创建目录ssh 192.168.30.50 mkdir -p /apps/etcd# 分发ETCD 到远程运行服务器#回到etcd-v3.4.1-linux-amd64cd ../scp -r * 192.168.30.50:/apps/etcd# 远程服务器创建etcd 账号ssh 192.168.30.50 useradd etcd -s /sbin/nologin -M# 远程etcd 目录etcd 账号权限ssh 192.168.30.50 chown -R etcd.etcd /apps/etcd# 创建etcd.service vi etcd.service[Unit]Description=Etcd ServerAfter=network.targetAfter=network-online.targetWants=network-online.target[Service]Type=notifyLimitNOFILE=1024000LimitNPROC=1024000LimitCORE=infinityLimitMEMLOCK=infinityUser=etcdGroup=etcdEnvironmentFile=-/apps/etcd/conf/etcdExecStart=/bin/bash -c "GOMAXPROCS=$(nproc) /apps/etcd/bin/etcd $ETCD_OPTS"Restart=on-failure[Install]WantedBy=multi-user.target# 分发etcd.service 到远程服务器scp etcd.service 192.168.30.50:/usr/lib/systemd/system/etcd.service# 启动 etcd ssh 192.168.30.50 systemctl start etcd# 查看启动是否成功ssh 192.168.30.50  systemctl status etcd[root@]~/work/etcd-v3.4.1-linux-amd64]#ssh 192.168.30.50  systemctl status etcd● etcd.service - Etcd Server   Loaded: loaded (/usr/lib/systemd/system/etcd.service; enabled; vendor preset: disabled)   Active: active (running) since Thu 2019-09-19 16:54:34 CST; 17h ago Main PID: 9321 (etcd)   CGroup: /system.slice/etcd.service           └─9321 /apps/etcd/bin/etcd --name=etcd --data-dir=/apps/etcd/data/default.etcd --listen-peer-urls=https://192.168.30.50:2380 --listen-client-urls=https://192.168.30.50:2379,https://127.0.0.1:2379 --advertise-client-urls=https://192.168.30.50:2379 --initial-advertise-peer-urls=https://192.168.30.50:2380 --initial-cluster=etcd=https://192.168.30.50:2380 --initial-cluster-token=node4=etcd=https://192.168.30.50:2380 --initial-cluster-state=new --heartbeat-interval=6000 --election-timeout=30000 --snapshot-count=5000 --auto-compaction-retention=1 --max-request-bytes=33554432 --quota-backend-bytes=17179869184 --trusted-ca-file=/apps/etcd/ssl/etcd-ca.pem --cert-file=/apps/etcd/ssl/etcd_server.pem --key-file=/apps/etcd/ssl/etcd_server-key.pem --peer-cert-file=/apps/etcd/ssl/etcd_member_etcd.pem --peer-key-file=/apps/etcd/ssl/etcd_member_etcd-key.pem --peer-client-cert-auth --peer-trusted-ca-file=/apps/etcd/ssl/etcd-ca.pemSep 20 10:10:02 etcd etcd[9321]: store.index: compact 79182Sep 20 10:10:02 etcd etcd[9321]: finished scheduled compaction at 79182 (took 1.966939ms)# 设置开机启动ssh 192.168.30.50 systemctl enable etcd

kubernetes 证书准备

#创建kube-apiserver ca证书配置mkdir -p ./cfssl/k8scat << EOF | tee ./cfssl/k8s/k8s-ca-csr.json{"CN": "kubernetes","key": {"algo": "rsa","size": 2048},"names": [{"C": "CN","ST": "GuangDong","L": "GuangZhou","O": "cluster","OU": "cluster"}]}EOF#生成 Kubernetes CA 证书和私钥mkdir -p ./cfssl/pki/k8scfssl gencert -initca ./cfssl/k8s/k8s-ca-csr.json | \cfssljson -bare ./cfssl/pki/k8s/k8s-ca#创建 Kubernetes API Server 证书配置文件export K8S_APISERVER_VIP=" \\"192.168.30.52\" \" && \export K8S_APISERVER_SERVICE_CLUSTER_IP="10.66.0.1" && \export K8S_APISERVER_HOSTNAME="api.k8s.cluster.local" && \export K8S_CLUSTER_DOMAIN_SHORTNAME="cluster" && \export K8S_CLUSTER_DOMAIN_FULLNAME="cluster.local" && \cat << EOF | tee ./cfssl/k8s/k8s_apiserver.json{"CN": "kubernetes","hosts": ["127.0.0.1",${K8S_APISERVER_VIP},"${K8S_APISERVER_SERVICE_CLUSTER_IP}", "${K8S_APISERVER_HOSTNAME}","kubernetes","kubernetes.default","kubernetes.default.svc","kubernetes.default.svc.${K8S_CLUSTER_DOMAIN_SHORTNAME}","kubernetes.default.svc.${K8S_CLUSTER_DOMAIN_FULLNAME}" ],"key": {"algo": "rsa","size": 2048},"names": [{"C": "CN","ST": "GuangDong","L": "GuangZhou","O": "cluster","OU": "cluster"}]}EOF#生成 Kubernetes API Server 证书和私钥cfssl gencert \-ca=./cfssl/pki/k8s/k8s-ca.pem \-ca-key=./cfssl/pki/k8s/k8s-ca-key.pem \-config=./cfssl/ca-config.json \-profile=kubernetes \./cfssl/k8s/k8s_apiserver.json | \cfssljson -bare ./cfssl/pki/k8s/k8s_server# 创建 Kubernetes webhook 证书配置文件cat << EOF | tee ./cfssl/k8s/aggregator.json{  "CN": "aggregator",  "hosts": [""],   "key": {    "algo": "rsa",    "size": 2048  },  "names": [    {      "C": "CN",      "ST": "GuangDong",      "L": "GuangZhou",      "O": "cluster",      "OU": "cluster"    }  ]}EOF# 生成Kubernetes webhook 证书cfssl gencert \    -ca=./cfssl/pki/k8s/k8s-ca.pem \    -ca-key=./cfssl/pki/k8s/k8s-ca-key.pem \    -config=./cfssl/ca-config.json \    -profile=kubernetes \    ./cfssl/k8s/aggregator.json | \    cfssljson -bare ./cfssl/pki/k8s/aggregator    # 创建 Kubernetes admin 证书配置文件        cat << EOF | tee ./cfssl/k8s/k8s_apiserver_admin.json{  "CN": "admin",  "hosts": [""],   "key": {    "algo": "rsa",    "size": 2048  },  "names": [    {      "C": "CN",      "ST": "GuangDong",      "L": "GuangZhou",      "O": "system:masters",      "OU": "Kubernetes-manual"    }  ]}EOF# 生成Kubernetes admin 证书cfssl gencert -ca=./cfssl/pki/k8s/k8s-ca.pem \    -ca-key=./cfssl/pki/k8s/k8s-ca-key.pem \    -config=./cfssl/ca-config.json \    -profile=kubernetes \        ./cfssl/k8s/k8s_apiserver_admin.json | \        cfssljson -bare ./cfssl/pki/k8s/k8s_apiserver_admin# 创建kube-scheduler  证书配置文件  cat << EOF | tee ./cfssl/k8s/k8s_scheduler.json{  "CN": "system:kube-scheduler",  "hosts": [""],   "key": {    "algo": "rsa",    "size": 2048  },  "names": [    {      "C": "CN",      "ST": "GuangDong",      "L": "GuangZhou",      "O": "system:kube-scheduler",      "OU": "Kubernetes-manual"    }  ]}EOF#  生成 Kubernetes Scheduler 证书和私钥cfssl gencert \    -ca=./cfssl/pki/k8s/k8s-ca.pem \    -ca-key=./cfssl/pki/k8s/k8s-ca-key.pem \    -config=./cfssl/ca-config.json \    -profile=kubernetes \    ./cfssl/k8s/k8s_scheduler.json | \    cfssljson -bare ./cfssl/pki/k8s/k8s_scheduler# kube-controller-manager        证书配置文件 cat << EOF | tee ./cfssl/k8s/k8s_controller_manager.json{  "CN": "system:kube-controller-manager",  "hosts": [""],   "key": {    "algo": "rsa",    "size": 2048  },  "names": [    {      "C": "CN",      "ST": "GuangDong",      "L": "GuangZhou",      "O": "system:kube-controller-manager",      "OU": "Kubernetes-manual"    }  ]}EOF## 生成 Kubernetes Controller Manager 证书和私钥cfssl gencert \    -ca=./cfssl/pki/k8s/k8s-ca.pem \    -ca-key=./cfssl/pki/k8s/k8s-ca-key.pem \    -config=./cfssl/ca-config.json \    -profile=kubernetes \    ./cfssl/k8s/k8s_controller_manager.json | \    cfssljson -bare ./cfssl/pki/k8s/k8s_controller_manager# 创建flannel 证书配置cat << EOF | tee ./cfssl/k8s/flannel.json{  "CN": "flannel",  "hosts": [""],   "key": {    "algo": "rsa",    "size": 2048  },  "names": [    {      "C": "CN",      "ST": "GuangDong",      "L": "GuangZhou",      "O": "system:masters",      "OU": "Kubernetes-manual"    }  ]}EOF## 生成 flannel 证书和私钥cfssl gencert \        -ca=./cfssl/pki/k8s/k8s-ca.pem \        -ca-key=./cfssl/pki/k8s/k8s-ca-key.pem \        -config=./cfssl/ca-config.json \        -profile=kubernetes \         ./cfssl/k8s/flannel.json | \         cfssljson -bare ./cfssl/pki/k8s/flannel# 创建kube-proxy 证书配置cat << EOF | tee ./cfssl/k8s/kube-proxy.json{  "CN": "system:kube-proxy",  "hosts": [""],   "key": {    "algo": "rsa",    "size": 2048  },  "names": [    {      "C": "CN",      "ST": "GuangDong",      "L": "GuangZhou",      "O": "system:masters",      "OU": "Kubernetes-manual"    }  ]}EOF## 生成 kube-proxy 证书和私钥cfssl gencert \        -ca=./cfssl/pki/k8s/k8s-ca.pem \        -ca-key=./cfssl/pki/k8s/k8s-ca-key.pem \        -config=./cfssl/ca-config.json \        -profile=kubernetes \         ./cfssl/k8s/kube-proxy.json | \         cfssljson -bare ./cfssl/pki/k8s/kube-proxy# 创建 kubernetes-dashboard证书配置cat << EOF | tee ./cfssl/k8s/dashboard.json{  "CN": "dashboard",  "hosts": [""],   "key": {    "algo": "rsa",    "size": 2048  },  "names": [    {      "C": "CN",      "ST": "GuangDong",      "L": "GuangZhou",      "O": "cluster",      "OU": "cluster"    }  ]}EOF##### 生成kubernetes-dashboard 证书cfssl gencert \        -ca=./cfssl/pki/k8s/k8s-ca.pem \       -ca-key=./cfssl/pki/k8s/k8s-ca-key.pem \        -config=./k8s/cfssl/ca-config.json \        -profile=kubernetes \        ./cfssl/k8s/dashboard.json | \        cfssljson -bare ./dashboard         # 创建metrics-server 证书配置  cat << EOF | tee ./cfssl/k8s/metrics-server.json{  "CN": "metrics-server",  "key": {    "algo": "rsa",    "size": 2048  },  "names": [    {      "C": "CN",      "ST": "GuangDong",      "L": "GuangZhou",      "O": "cluster",      "OU": "cluster"    }  ]}EOF# 生成metrics-server证书cfssl gencert -ca=./cfssl/pki/k8s/k8s-ca.pem \    -ca-key=./cfssl/pki/k8s/k8s-ca-key.pem \    -config=./cfssl/ca-config.json \    -profile=kubernetes ./cfssl/k8s/metrics-server.json | \        cfssljson -bare ./metrics-server

创建kubernetes kubeconfig配置文件

设置环境变量export KUBE_APISERVER="https://192.168.30.52:5443"# 创建 admin kubeconfig# 设置集群参数kubectl config set-cluster kubernetes \--certificate-authority=./cfssl/pki/k8s/k8s-ca.pem \--embed-certs=true  \--server=${KUBE_APISERVER} \--kubeconfig=admin.kubeconfig# 设置客户端认证参数 kubectl config set-credentials admin \ --client-certificate=./cfssl/pki/k8s/k8s_apiserver_admin.pem \ --client-key=./cfssl/pki/k8s/k8s_apiserver_admin-key.pem \ --embed-certs=true \ --kubeconfig=admin.kubeconfig # 设置上下文参数kubectl config set-context kubernetes \--cluster=kubernetes \--user=admin \--namespace=kube-system \--kubeconfig=admin.kubeconfig# 设置默认上下文kubectl config use-context kubernetes --kubeconfig=admin.kubeconfig# 创建kube-scheduler kubeconfig 配置文件# 设置集群参数kubectl config set-cluster kubernetes \    --certificate-authority=./cfssl/pki/k8s/k8s-ca.pem \    --embed-certs=true \    --server=${KUBE_APISERVER} \    --kubeconfig=kube_scheduler.kubeconfig# 设置客户端认证参数kubectl config set-credentials system:kube-scheduler \    --client-certificate=./cfssl/pki/k8s/k8s_scheduler.pem \    --embed-certs=true \    --client-key=./cfssl/pki/k8s/k8s_scheduler-key.pem \    --kubeconfig=kube_scheduler.kubeconfig # 设置上下文参数kubectl config set-context kubernetes \    --cluster=kubernetes \    --user=system:kube-scheduler \    --kubeconfig=kube_scheduler.kubeconfig# 设置默认上下文kubectl config use-context kubernetes --kubeconfig=kube_scheduler.kubeconfig# 创建kube-controller-manager kubeconfig 配置文件# 设置集群参数kubectl config set-cluster kubernetes \   --certificate-authority=./cfssl/pki/k8s/k8s-ca.pem \   --embed-certs=true \   --server=${KUBE_APISERVER} \   --kubeconfig=kube_controller_manager.kubeconfig# 设置客户端认证参数kubectl config set-credentials system:kube-controller-manager \   --client-certificate=./cfssl/pki/k8s/k8s_controller_manager.pem \   --embed-certs=true \   --client-key=./cfssl/pki/k8s/k8s_controller_manager-key.pem \   --kubeconfig=kube_controller_manager.kubeconfig # 设置上下文参数kubectl config set-context kubernetes \   --cluster=kubernetes \   --user=system:kube-controller-manager \   --kubeconfig=kube_controller_manager.kubeconfig# 设置默认上下文kubectl config use-context kubernetes --kubeconfig=kube_controller_manager.kubeconfig# 创建bootstrap  kubeconfig 配置# 生成TOKENexport TOKEN_ID=$(head -c 6 /dev/urandom | md5sum | head -c 6)export TOKEN_SECRET=$(head -c 16 /dev/urandom | md5sum | head -c 16)export BOOTSTRAP_TOKEN=${TOKEN_ID}.${TOKEN_SECRET}# 设置集群参数kubectl config set-cluster kubernetes \  --certificate-authority=./cfssl/pki/k8s/k8s-ca.pem \  --embed-certs=true \  --server=${KUBE_APISERVER} \  --kubeconfig=bootstrap.kubeconfig# 设置客户端认证参数kubectl config set-credentials system:bootstrap:${TOKEN_ID} \  --token=${BOOTSTRAP_TOKEN} \  --kubeconfig=bootstrap.kubeconfig# 设置上下文参数kubectl config set-context default \  --cluster=kubernetes \  --user=system:bootstrap:${TOKEN_ID} \  --kubeconfig=bootstrap.kubeconfig# 设置默认上下文kubectl config use-context default --kubeconfig=bootstrap.kubeconfig# 创建flannel kubeconfig# 设置集群参数kubectl config set-cluster kubernetes \  --certificate-authority=./cfssl/pki/k8s/k8s-ca.pem \  --embed-certs=true \  --server=${KUBE_APISERVER} \  --kubeconfig=kubeconfig.conf# 设置客户端认证参数    kubectl config set-credentials flannel \  --client-certificate=./cfssl/pki/k8s/flannel.pem \  --client-key=./cfssl/pki/k8s/flannel-key.pem \  --embed-certs=true \  --kubeconfig=kubeconfig.conf# 设置上下文参数    kubectl config set-context default \  --cluster=kubernetes \  --user=flannel \  --kubeconfig=kubeconfig.conf# 设置默认上下文kubectl config use-context default --kubeconfig=kubeconfig.conf# 创建kube-proxy kubeconfig# 设置集群参数kubectl config set-cluster kubernetes \  --certificate-authority=./cfssl/pki/k8s/k8s-ca.pem \  --embed-certs=true \  --server=${KUBE_APISERVER} \  --kubeconfig=kube-proxy.kubeconfig # 设置客户端认证参数    kubectl config set-credentials system:kube-proxy \  --client-certificate=./cfssl/pki/k8s/kube-proxy.pem \  --client-key=./cfssl/pki/k8s/kube-proxy-key.pem \  --embed-certs=true \  --kubeconfig=kube-proxy.kubeconfig # 设置上下文参数    kubectl config set-context default \  --cluster=kubernetes \  --user=system:kube-proxy \  --kubeconfig=kube-proxy.kubeconfig # 设置默认上下文kubectl config use-context default --kubeconfig=kube-proxy.kubeconfig # k8s 所需要的kubeconfig 创建完成

创建kubernetes 启动配置文件

cd /root/work/kubernetes/server# 创建配置文件目录# 目录说明 conf 主要存放启动参数文件 config 存放其它配置文件 log 存放运行日志 kubelet-plugins 插件目录mkdir conf config log  kubelet-plugins# 创建启动配置文件cd conf#kube-apiserver 配置vi kube-apiserverKUBE_APISERVER_OPTS="--logtostderr=false \        --bind-address=192.168.30.52 \        --advertise-address=192.168.30.52 \        --secure-port=5443 \        --insecure-port=0 \        --service-cluster-ip-range=10.66.0.0/16 \        --service-node-port-range=30000-65000 \        --etcd-cafile=/apps/kubernetes/ssl/etcd/etcd-ca.pem \        --etcd-certfile=/apps/kubernetes/ssl/etcd/etcd_client.pem \        --etcd-keyfile=/apps/kubernetes/ssl/etcd/etcd_client-key.pem \        --etcd-prefix=/registry \        --etcd-servers=https://192.168.30.50:2379 \        --client-ca-file=/apps/kubernetes/ssl/k8s/k8s-ca.pem \        --tls-cert-file=/apps/kubernetes/ssl/k8s/k8s_server.pem \        --tls-private-key-file=/apps/kubernetes/ssl/k8s/k8s_server-key.pem \        --kubelet-client-certificate=/apps/kubernetes/ssl/k8s/k8s_server.pem \        --kubelet-client-key=/apps/kubernetes/ssl/k8s/k8s_server-key.pem \        --service-account-key-file=/apps/kubernetes/ssl/k8s/k8s-ca.pem \        --requestheader-client-ca-file=/apps/kubernetes/ssl/k8s/k8s-ca.pem \        --proxy-client-cert-file=/apps/kubernetes/ssl/k8s/aggregator.pem \        --proxy-client-key-file=/apps/kubernetes/ssl/k8s/aggregator-key.pem \        --requestheader-allowed-names=aggregator \        --requestheader-group-headers=X-Remote-Group \        --requestheader-extra-headers-prefix=X-Remote-Extra- \        --requestheader-username-headers=X-Remote-User \        --enable-aggregator-routing=true \        --anonymous-auth=false \        --allow-privileged=true \        --experimental-encryption-provider-config=/apps/kubernetes/config/encryption-config.yaml \        --enable-admission-plugins=DefaultStorageClass,DefaultTolerationSeconds,LimitRanger,NamespaceExists,NamespaceLifecycle,NodeRestriction,OwnerReferencesPermissionEnforcement,PodNodeSelector,PersistentVolumeClaimResize,PodPreset,PodTolerationRestriction,ResourceQuota,ServiceAccount,StorageObjectInUseProtection MutatingAdmissionWebhook ValidatingAdmissionWebhook \        --disable-admission-plugins=DenyEscalatingExec,ExtendedResourceToleration,ImagePolicyWebhook,LimitPodHardAntiAffinityTopology,NamespaceAutoProvision,Priority,EventRateLimit,PodSecurityPolicy \        --cors-allowed-origins=.* \        --enable-swagger-ui \        --runtime-config=api/all=true \        --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname \        --authorization-mode=Node,RBAC \        --apiserver-count=1 \        --audit-log-maxage=30 \        --audit-log-maxbackup=3 \        --audit-log-maxsize=100 \        --kubelet-https \        --event-ttl=1h \        --feature-gates=RotateKubeletServerCertificate=true,RotateKubeletClientCertificate=true \        --enable-bootstrap-token-auth=true \        --audit-log-path=/apps/kubernetes/log/api-server-audit.log \        --alsologtostderr=true \        --log-dir=/apps/kubernetes/log \        --v=2 \        --endpoint-reconciler-type=lease \        --max-mutating-requests-inflight=100 \        --max-requests-inflight=500 \        --target-ram-mb=6000"# 创建kube-controller-manager 配置文件vi kube-controller-managerKUBE_CONTROLLER_MANAGER_OPTS="--logtostderr=false \ --leader-elect=true \ --address=0.0.0.0 \ --service-cluster-ip-range=10.66.0.0/16 \ --cluster-cidr=10.67.0.0/16 \ --node-cidr-mask-size=24 \ --cluster-name=kubernetes \ --allocate-node-cidrs=true \ --kubeconfig=/apps/kubernetes/config/kube_controller_manager.kubeconfig \ --authentication-kubeconfig=/apps/kubernetes/config/kube_controller_manager.kubeconfig \ --authorization-kubeconfig=/apps/kubernetes/config/kube_controller_manager.kubeconfig \ --use-service-account-credentials=true \ --client-ca-file=/apps/kubernetes/ssl/k8s/k8s-ca.pem \ --requestheader-client-ca-file=/apps/kubernetes/ssl/k8s/k8s-ca.pem \ --node-monitor-grace-period=40s \ --node-monitor-period=5s \ --pod-eviction-timeout=5m0s \ --terminated-pod-gc-threshold=50 \ --alsologtostderr=true \ --cluster-signing-cert-file=/apps/kubernetes/ssl/k8s/k8s-ca.pem \ --cluster-signing-key-file=/apps/kubernetes/ssl/k8s/k8s-ca-key.pem  \ --deployment-controller-sync-period=10s \ --experimental-cluster-signing-duration=86700h0m0s \ --enable-garbage-collector=true \ --root-ca-file=/apps/kubernetes/ssl/k8s/k8s-ca.pem \ --service-account-private-key-file=/apps/kubernetes/ssl/k8s/k8s-ca-key.pem \ --feature-gates=RotateKubeletServerCertificate=true,RotateKubeletClientCertificate=true \ --controllers=*,bootstrapsigner,tokencleaner \ --horizontal-pod-autoscaler-use-rest-clients=true \ --horizontal-pod-autoscaler-sync-period=10s \ --flex-volume-plugin-dir=/apps/kubernetes/kubelet-plugins/volume \ --tls-cert-file=/apps/kubernetes/ssl/k8s/k8s_controller_manager.pem \ --tls-private-key-file=/apps/kubernetes/ssl/k8s/k8s_controller_manager-key.pem \ --kube-api-qps=100 \ --kube-api-burst=100 \ --log-dir=/apps/kubernetes/log \ --v=2" # 创建kube-scheduler 配置文件 vi kube-scheduler KUBE_SCHEDULER_OPTS=" \                   --logtostderr=false \                   --address=0.0.0.0 \                   --leader-elect=true \                   --kubeconfig=/apps/kubernetes/config/kube_scheduler.kubeconfig \                   --authentication-kubeconfig=/apps/kubernetes/config/kube_scheduler.kubeconfig \                   --authorization-kubeconfig=/apps/kubernetes/config/kube_scheduler.kubeconfig \                   --alsologtostderr=true \                   --kube-api-qps=100 \                   --kube-api-burst=100 \                   --log-dir=/apps/kubernetes/log \                   --v=2"# 创建kubelet 配置文件KUBELET_OPTS="--bootstrap-kubeconfig=/apps/kubernetes/conf/bootstrap.kubeconfig \              --fail-swap-on=false \              --network-plugin=cni --cni-conf-dir=/etc/cni/net.d --cni-bin-dir=/apps/cni/bin \              --kubeconfig=/apps/kubernetes/conf/kubelet.kubeconfig \              --address=192.168.30.52 \节点IP 一定要修改              --node-ip=192.168.30.52 \ 节点IP 一定要修改              --hostname-override=master \节点hostname 一定要修改              --cluster-dns=10.66.0.2 \ # dns IP              --cluster-domain=cluster.local \ # 集群域              --authorization-mode=Webhook \              --authentication-token-webhook=true \              --client-ca-file=/apps/kubernetes/ssl/k8s/k8s-ca.pem \              --rotate-certificates=true \              --cgroup-driver=cgroupfs \              --healthz-port=10248 \              --healthz-bind-address=192.168.30.52 \ # 节点IP 一定要修改              --cert-dir=/apps/kubernetes/ssl \              --feature-gates=RotateKubeletClientCertificate=true,RotateKubeletServerCertificate=true \              --node-labels=node.kubernetes.io/k8s-node=true \ # node-role.kubernetes.io 已经取消添加会报错              --serialize-image-pulls=false \              --enforce-node-allocatable=pods,kube-reserved,system-reserved \              --pod-manifest-path=/apps/work/kubernetes/manifests \              --runtime-cgroups=/systemd/system.slice/kubelet.service \              --kubelet-cgroups=/systemd/system.slice/kubelet.service \              --kube-reserved-cgroup=/systemd/system.slice/kubelet.service \              --system-reserved-cgroup=/systemd/system.slice \              --root-dir=/apps/work/kubernetes/kubelet \              --log-dir=/apps/kubernetes/log \              --alsologtostderr=true \              --logtostderr=false \              --anonymous-auth=true \              --image-gc-high-threshold=70 \              --image-gc-low-threshold=50 \              --kube-reserved=cpu=500m,memory=512Mi,ephemeral-storage=1Gi \              --system-reserved=cpu=1000m,memory=1024Mi,ephemeral-storage=1Gi \              --eviction-hard=memory.available<500Mi,nodefs.available<10% \              --serialize-image-pulls=false \              --sync-frequency=30s \              --resolv-conf=/etc/resolv.conf \              --pod-infra-container-image=registry.cn-hangzhou.aliyuncs.com/google-containers/pause-amd64:3.0 \              --image-pull-progress-deadline=30s \              --v=2 \              --event-burst=30 \              --event-qps=15 \              --kube-api-burst=30 \              --kube-api-qps=15 \              --max-pods=200 \              --pods-per-core=10 \              --read-only-port=0 \              --allowed-unsafe-sysctls 'kernel.msg*,kernel.shm*,kernel.sem,fs.mqueue.*,net.*' \              --volume-plugin-dir=/apps/kubernetes/kubelet-plugins/volume"# 创建 kube-proxy 配置文件vi  kube-proxyKUBE_PROXY_OPTS="--logtostderr=false \--v=2 \--feature-gates=SupportIPVSProxyMode=true \--masquerade-all=true \--proxy-mode=ipvs \--ipvs-min-sync-period=5s \--ipvs-sync-period=5s \--ipvs-scheduler=rr \--cluster-cidr=10.67.0.0/16 \  #pod  CIDR --log-dir=/apps/kubernetes/log \--kubeconfig=/apps/kubernetes/conf/kube-proxy.kubeconfig"# 创建kube-apiserver 其它配置 放到config目录cd ../config#创建 encryption-config.yaml export ENCRYPTION_KEY=$(head -c 32 /dev/urandom | base64) cat > encryption-config.yaml <

分发文件并启动kubernetes server

cd /root/work/kubernetes/server# 创建远程目录ssh 192.168.30.52 mkdir -p /apps/kubernetesscp -r bin conf config ssl 192.168.30.52:/apps/kubernetes# 分发启动文件scp -r *.service 192.168.30.52:/usr/lib/systemd/system/# 创建 k8s 用户ssh 192.168.30.52 useradd k8s -s /sbin/nologin -M# /apps/kubernetes 目录k8s 权限ssh 192.168.30.52 chown -R k8s.k8s /apps/kubernetes# 启动  kube-apiserver  kube-controller-manager kube-scheduler# 启动kube-apiserverssh 192.168.30.52 systemctl start kube-apiserver# 设置开机启动ssh 192.168.30.52 systemctl enable kube-apiserver# 启动状态ssh 192.168.30.52 systemctl status kube-apiserver# 验证api 是否启动成功# 备份旧config 文件mv  ~/.kube/config  ~/.kube/config.old# 复制kubeconfig 到~/.kube 目录cp ~/work/admin.kubeconfig ~/.kube/config# 验证kube-apiserver 是否正常kubectl cluster-info [root@]~/work]#kubectl cluster-infoKubernetes master is running at https://192.168.30.52:5443# 启动kube-controller-manager ssh 192.168.30.52 systemctl start kube-controller-manager # 设置开机启动ssh 192.168.30.52 systemctl enable kube-controller-manager # 启动状态ssh 192.168.30.52 systemctl status kube-controller-manager # 启动  kube-schedulerssh 192.168.30.52 systemctl startkube-scheduler# 设置开机启动ssh 192.168.30.52 systemctl enable kube-scheduler# 启动状态ssh 192.168.30.52 systemctl status kube-scheduler# 验证是否启动成功[root@]~/work]#kubectl get cs # 最新kubectl  返回NAME                 AGEcontroller-manager   scheduler            etcd-0               ../kubernetes-1.14.4/_output/bin/kubectl get cs[root@]~/work]#../kubernetes-1.14.4/_output/bin/kubectl get csNAME                 STATUS    MESSAGE             ERRORcontroller-manager   Healthy   okscheduler            Healthy   oketcd-0               Healthy   {"health":"true"}# 配置 kube-controller-manager，kubelet 、kube-scheduler 访问kube-api 用户授权授予 kubernetes API 的权限kubectl create clusterrolebinding controller-node-clusterrolebing --clusterrole=system:kube-controller-manager  --user=system:kube-controller-managerkubectl create clusterrolebinding scheduler-node-clusterrolebing  --clusterrole=system:kube-scheduler --user=system:kube-schedulerkubectl create clusterrolebinding controller-manager:system:auth-delegator --user system:kube-controller-manager --clusterrole system:auth-delegator授予 kubernetes 证书访问 kubelet API 的权限kubectl create clusterrolebinding --user system:serviceaccount:kube-system:default kube-system-cluster-admin --clusterrole cluster-adminkubectl create clusterrolebinding kubelet-node-clusterbinding --clusterrole=system:node --group=system:nodeskubectl create clusterrolebinding kube-apiserver:kubelet-apis --clusterrole=system:kubelet-api-admin --user kubernetes

配置启动node 节点

192.168.30.52 daemon 节点其它节点参考分发cnicd /root/workscp -r cni 192.168.30.52:/apps/# 创建cni 配置目录ssh 192.168.30.52 mkdir -p  /etc/cni/net.d# 安装docker # 创建 /etc/docker/daemon.jsonssh 192.168.30.52 mkdir -p /etc/dockervi daemon.json{    "max-concurrent-downloads": 20,    "data-root": "/apps/docker",    "exec-root": "/apps/docker",    "log-driver": "json-file",    "bridge": "none",    "oom-score-adjust": -1000,    "debug": false,    "log-opts": {        "max-size": "100M",        "max-file": "10"    },    "default-ulimits": {        "nofile": {            "Name": "nofile",            "Hard": 1024000,            "Soft": 1024000        },        "nproc": {            "Name": "nproc",            "Hard": 1024000,            "Soft": 1024000        },      "core": {            "Name": "core",            "Hard": -1,            "Soft": -1          }    }}scp -r daemon.json 192.168.30.52:/etc/dockerssh 192.168.30.52 sudo yum-config-manager --add-repo http://mirrors.aliyun.com/docker-ce/linux/centos/docker-ce.repossh 192.168.30.52 yum install -y  epel-releasessh 192.168.30.52 yum install -y   yum-utils  ipvsadm  telnet  wget  net-tools  conntrack  ipset  jq  iptables  curl  sysstat  libseccomp  socat  nfs-utils  fuse  fuse-devel # 安装docker依赖ssh 192.168.30.52 yum install -y    python-pip python-devel yum-utils device-mapper-persistent-data lvm2 # 安装dockerssh 192.168.30.52 yum install -y docker-ce# reload service 配置ssh 192.168.30.52 systemctl daemon-reload# 重启dockerssh 192.168.30.52 systemctl restart docker# 设置开机启动ssh 192.168.30.52 systemctl enable docker# bootstrap secret cat << EOF | tee bootstrap.secret.yamlapiVersion: v1kind: Secretmetadata:  # Name MUST be of form "bootstrap-token-"  name: bootstrap-token-${TOKEN_ID}  namespace: kube-system# Type MUST be 'bootstrap.kubernetes.io/token'type: bootstrap.kubernetes.io/tokenstringData:  # Human readable description. Optional.  description: "The default bootstrap token generated by 'kubelet '."  # Token ID and secret. Required.  token-id: ${TOKEN_ID}  token-secret: ${TOKEN_SECRET}  # Allowed usages.  usage-bootstrap-authentication: "true"  usage-bootstrap-signing: "true"  # Extra groups to authenticate the token as. Must start with "system:bootstrappers:"  auth-extra-groups: system:bootstrappers:worker,system:bootstrappers:ingress---# A ClusterRole which instructs the CSR approver to approve a node requesting a# serving cert matching its client cert.kind: ClusterRoleapiVersion: rbac.authorization.k8s.io/v1metadata:  name: system:certificates.k8s.io:certificatesigningrequests:selfnodeserverrules:- apiGroups: ["certificates.k8s.io"]  resources: ["certificatesigningrequests/selfnodeserver"]  verbs: ["create"]---apiVersion: rbac.authorization.k8s.io/v1kind: ClusterRolemetadata:  annotations:    rbac.authorization.kubernetes.io/autoupdate: "true"  labels:    kubernetes.io/bootstrapping: rbac-defaults  name: system:kubernetes-to-kubeletrules:  - apiGroups:      - ""    resources:      - nodes/proxy      - nodes/stats      - nodes/log      - nodes/spec      - nodes/metrics    verbs:      - "*"---apiVersion: rbac.authorization.k8s.io/v1kind: ClusterRoleBindingmetadata:  name: system:kubernetes  namespace: ""roleRef:  apiGroup: rbac.authorization.k8s.io  kind: ClusterRole  name: system:kubernetes-to-kubeletsubjects:  - apiGroup: rbac.authorization.k8s.io    kind: User    name: kubernetesEOF#  创建资源kubectl create -f bootstrap.secret.yaml### 查看创建的tokenkubeadm token list# 允许 system:bootstrappers 组用户创建 CSR 请求kubectl create clusterrolebinding kubelet-bootstrap --clusterrole=system:node-bootstrapper --group=system:bootstrappers# 自动批准 system:bootstrappers 组用户 TLS bootstrapping 首次申请证书的 CSR 请求kubectl create clusterrolebinding node-client-auto-approve-csr --clusterrole=system:certificates.k8s.io:certificatesigningrequests:nodeclient --group=system:bootstrappers# 自动批准 system:nodes 组用户更新 kubelet 自身与 apiserver 通讯证书的 CSR 请求kubectl create clusterrolebinding node-client-auto-renew-crt --clusterrole=system:certificates.k8s.io:certificatesigningrequests:selfnodeclient --group=system:nodes# 自动批准 system:nodes 组用户更新 kubelet 10250 api 端口证书的 CSR 请求kubectl create clusterrolebinding node-server-auto-renew-crt --clusterrole=system:certificates.k8s.io:certificatesigningrequests:selfnodeserver --group=system:nodes# kubernetes 工作目录ssh 192.168.30.52 mkdir -p /apps/work/kubernetes/{manifests,kubelet}# 启动kubeletssh 192.168.30.52 systemctl kubelet# 设置开机启动ssh 192.168.30.52 systemctl enable kubelet# 启动状态ssh 192.168.30.52 systemctl status kubelet# 启动kube-proxyssh 192.168.30.52 systemctl kube-proxy# 设置开机启动ssh 192.168.30.52 systemctl enable kube-proxy# 启动状态ssh 192.168.30.52 systemctl status kube-proxy[root@]~/work]#kubectl get nodeNAME      STATUS     ROLES    AGE    VERSIONmaster    NotReady      140m   v1.16.0master2   NotReady      34m    v1.16.0# 由于cni 一直没就绪所以一直存在这个状态

flannel 部署

# 创建flannel configmap  kubeconfigkubectl create configmap "kube-proxy" --from-file=kubeconfig.conf# 创建yamlvi kube-flannel.yml---kind: ClusterRoleapiVersion: rbac.authorization.k8s.io/v1beta1metadata:  name: flannelrules:  - apiGroups:      - ""    resources:      - pods    verbs:      - get  - apiGroups:      - ""    resources:      - nodes    verbs:      - list      - watch  - apiGroups:      - ""    resources:      - nodes/status    verbs:      - patch---kind: ClusterRoleBindingapiVersion: rbac.authorization.k8s.io/v1beta1metadata:  name: flannelroleRef:  apiGroup: rbac.authorization.k8s.io  kind: ClusterRole  name: flannelsubjects:- kind: ServiceAccount  name: flannel  namespace: kube-system---apiVersion: v1kind: ServiceAccountmetadata:  name: flannel  namespace: kube-system---kind: ConfigMapapiVersion: v1metadata:  name: kube-flannel-cfg  namespace: kube-system  labels:    tier: node    app: flanneldata:  cni-conf.json: |     {     "name":"cni0",     "cniVersion":"0.3.1", # 一定要添加不然Kubelet 一直出现NotReady 状态     "plugins":[       {         "type":"flannel",         "delegate":{          "hairpinMode": true,           "isDefaultGateway":true         }       },       {         "type":"portmap",         "capabilities":{           "portMappings":true         }       }     ]     }  net-conf.json: |    {      "Network": "10.67.0.0/16", # 记得修改POD cidr      "Backend": {        "Type": "vxlan"      }    }---apiVersion: apps/v1kind: DaemonSetmetadata:  name: kube-flannel-ds-amd64  namespace: kube-system  labels:    tier: node    app: flannelspec:  selector:    matchLabels:      app: flannel  template:    metadata:      labels:        tier: node        app: flannel    spec:      hostNetwork: true      nodeSelector:        beta.kubernetes.io/arch: amd64      tolerations:      - operator: Exists        effect: NoSchedule      serviceAccountName: flannel      initContainers:      - name: install-cni        image: quay.io/coreos/flannel:v0.11.0-amd64        command:        - cp        args:        - -f        - /etc/kube-flannel/cni-conf.json        - /etc/cni/net.d/10-flannel.conflist        volumeMounts:        - name: cni          mountPath: /etc/cni/net.d        - name: flannel-cfg          mountPath: /etc/kube-flannel/      containers:      - name: kube-flannel        image: quay.io/coreos/flannel:v0.11.0-amd64        command:        - /opt/bin/flanneld        args:        - --ip-masq        - --kube-subnet-mgr        - --healthz-port=10244        - --kubeconfig-file=/var/lib/flannel/kubeconfig        livenessProbe:          httpGet:            path: /healthz            port: 10244          initialDelaySeconds: 10          periodSeconds: 3        resources:          requests:            cpu: "100m"            memory: "50Mi"          limits:            cpu: "100m"            memory: "50Mi"        securityContext:          privileged: true        env:        - name: POD_NAME          valueFrom:            fieldRef:              fieldPath: metadata.name        - name: POD_NAMESPACE          valueFrom:            fieldRef:              fieldPath: metadata.namespace        volumeMounts:        - name: run          mountPath: /run        - name: flannel-cfg          mountPath: /etc/kube-flannel/        - name: kubeconfig          mountPath: /var/lib/flannel          readOnly: true      volumes:        - name: run          hostPath:            path: /run        - name: cni          hostPath:            path: /etc/cni/net.d        - name: flannel-cfg          configMap:            name: kube-flannel-cfg        - name: kubeconfig          configMap:            name: kube-proxy            items:            - key: kubeconfig.conf              path: kubeconfig  updateStrategy:    rollingUpdate:      maxUnavailable: 1    type: RollingUpdate# 创建 flannel 服务kubectl apply -f kube-flannel.yml# 查看POD 状态traefik-zs6h3                     1/1     Running   0          3h37m[root@]~/work]#kubectl get pod| grep flannelkube-flannel-ds-amd64-6bpf7       1/1     Running   0          3h57mkube-flannel-ds-amd64-6sxz2       1/1     Running   0          3h58m# 查看node 状态[root@]~/work]#kubectl get nodeNAME      STATUS   ROLES    AGE   VERSIONmaster    Ready       18h   v1.16.0master2   Ready       16h   v1.16.0# 已经正常状态 cni 也能正常分配ip[root@master2 ~]# ip a | grep cni7: cni0:  mtu 1450 qdisc noqueue state UP group default qlen 1000    inet 10.67.2.1/24 brd 10.67.2.255 scope global cni0

部署coredns

# __MACHINE_GENERATED_WARNING__apiVersion: v1kind: ServiceAccountmetadata:  name: coredns  namespace: kube-system  labels:      kubernetes.io/cluster-service: "true"      addonmanager.kubernetes.io/mode: Reconcile---apiVersion: rbac.authorization.k8s.io/v1kind: ClusterRolemetadata:  labels:    kubernetes.io/bootstrapping: rbac-defaults    addonmanager.kubernetes.io/mode: Reconcile  name: system:corednsrules:- apiGroups:  - ""  resources:  - endpoints  - services  - pods  - namespaces  verbs:  - list  - watch- apiGroups:  - ""  resources:  - nodes  verbs:  - get---apiVersion: rbac.authorization.k8s.io/v1kind: ClusterRoleBindingmetadata:  annotations:    rbac.authorization.kubernetes.io/autoupdate: "true"  labels:    kubernetes.io/bootstrapping: rbac-defaults    addonmanager.kubernetes.io/mode: EnsureExists  name: system:corednsroleRef:  apiGroup: rbac.authorization.k8s.io  kind: ClusterRole  name: system:corednssubjects:- kind: ServiceAccount  name: coredns  namespace: kube-system---apiVersion: v1kind: ConfigMapmetadata:  name: coredns  namespace: kube-system  labels:      addonmanager.kubernetes.io/mode: EnsureExistsdata:  Corefile: |    .:53 {        errors        health        kubernetes cluster.local in-addr.arpa ip6.arpa {            pods insecure            upstream /etc/resolv.conf            fallthrough in-addr.arpa ip6.arpa        }        prometheus :9153        forward . /etc/resolv.conf        cache 30        reload        loadbalance    }---apiVersion: apps/v1kind: Deploymentmetadata:  name: coredns  namespace: kube-system  labels:    k8s-app: kube-dns    kubernetes.io/cluster-service: "true"    addonmanager.kubernetes.io/mode: Reconcile    kubernetes.io/name: "CoreDNS"spec:  # replicas: not specified here:  # 1. In order to make Addon Manager do not reconcile this replicas parameter.  # 2. Default is 1.  # 3. Will be tuned in real time if DNS horizontal auto-scaling is turned on.  strategy:    type: RollingUpdate    rollingUpdate:      maxUnavailable: 1  selector:    matchLabels:      k8s-app: kube-dns  template:    metadata:      labels:        k8s-app: kube-dns      annotations:        seccomp.security.alpha.kubernetes.io/pod: 'docker/default'    spec:      priorityClassName: system-cluster-critical      serviceAccountName: coredns      tolerations:        - key: "CriticalAddonsOnly"          operator: "Exists"      nodeSelector:        beta.kubernetes.io/os: linux      containers:      - name: coredns        image: coredns/coredns        imagePullPolicy: Always        resources:          limits:            memory: 170Mi          requests:            cpu: 100m            memory: 70Mi        args: [ "-conf", "/etc/coredns/Corefile" ]        volumeMounts:        - name: config-volume          mountPath: /etc/coredns          readOnly: true        ports:        - containerPort: 53          name: dns          protocol: UDP        - containerPort: 53          name: dns-tcp          protocol: TCP        - containerPort: 9153          name: metrics          protocol: TCP        livenessProbe:          httpGet:            path: /health            port: 8080            scheme: HTTP          initialDelaySeconds: 60          timeoutSeconds: 5          successThreshold: 1          failureThreshold: 5        readinessProbe:          httpGet:            path: /health            port: 8080            scheme: HTTP        securityContext:          allowPrivilegeEscalation: false          capabilities:            add:            - NET_BIND_SERVICE            drop:            - all          readOnlyRootFilesystem: true      dnsPolicy: Default      volumes:        - name: config-volume          configMap:            name: coredns            items:            - key: Corefile              path: Corefile---apiVersion: v1kind: Servicemetadata:  name: kube-dns  namespace: kube-system  annotations:    prometheus.io/port: "9153"    prometheus.io/scrape: "true"  labels:    k8s-app: kube-dns    kubernetes.io/cluster-service: "true"    addonmanager.kubernetes.io/mode: Reconcile    kubernetes.io/name: "CoreDNS"spec:  selector:    k8s-app: kube-dns  clusterIP: 10.66.0.2  ports:  - name: dns    port: 53    protocol: UDP  - name: dns-tcp    port: 53    protocol: TCP  - name: metrics    port: 9153    protocol: TCP# 创建CoreDNS 服务kubectl apply -f coredns.yaml# 验证dns[root@master net.d]# dig @10.66.0.2 www.baidu.com; <<>> DiG 9.11.4-P2-RedHat-9.11.4-9.P2.el7 <<>> @10.66.0.2 www.baidu.com; (1 server found);; global options: +cmd;; Got answer:;; ->>HEADER<<- opcode: QUERY, status: NOERROR, id: 31727;; flags: qr rd ra; QUERY: 1, ANSWER: 3, AUTHORITY: 4, ADDITIONAL: 5;; OPT PSEUDOSECTION:; EDNS: version: 0, flags:; udp: 4096;; QUESTION SECTION:;www.baidu.com.                 IN      A;; ANSWER SECTION:www.baidu.com.          30      IN      CNAME   www.a.shifen.com.www.a.shifen.com.       30      IN      A       14.215.177.38www.a.shifen.com.       30      IN      A       14.215.177.39;; AUTHORITY SECTION:shifen.com.             30      IN      NS      ns3.baidu.com.shifen.com.             30      IN      NS      ns4.baidu.com.shifen.com.             30      IN      NS      ns2.baidu.com.shifen.com.             30      IN      NS      dns.baidu.com.;; ADDITIONAL SECTION:ns3.baidu.com.          30      IN      A       112.80.248.64ns2.baidu.com.          30      IN      A       220.181.33.31ns4.baidu.com.          30      IN      A       14.215.178.80dns.baidu.com.          30      IN      A       202.108.22.220;; Query time: 3 msec;; SERVER: 10.66.0.2#53(10.66.0.2);; WHEN: Fri Sep 20 13:07:01 CST 2019;; MSG SIZE  rcvd: 413返回正常

创建 traefik Ingress 启用https

# base64 加密cat tls.crt |base64 | tr -d '\n'cat tls.key|base64 | tr -d '\n'# 创建traefik-secretvi traefik-secret.yaml---apiVersion: v1kind: Secretmetadata:  labels:    k8s-app: tls-cert  name: tls-certtype: Opaquedata:  tls.crt: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUdYVENDQlVXZ0F3SUJBZ0lTQTBCSy82MUwvZVNHb2FjNmFjRnZLaDVOTUEwR0NTcUdTSWIzRFFFQkN3VUEKTUVveEN6QUpCZ05WQkFZVEFsVlRNUll3RkFZRFZRUUtFdzFNWlhRbmN5QkZibU55ZVhCME1TTXdJUVlEVlFRRApFeHBNWlhRbmN5QkZibU55ZVhCMElFRjFkR2h3Y21sMGVTQllNekFlRncweE9UQTVNRFl4TlRNNE1EVmFGdzB4Ck9URXlNRFV4TlRNNE1EVmFNQll4RkRBU0JnTlZCQU1UQzIxa1pHZGhiV1V1WTI5dE1JSUNJakFOQmdrcWhraUcKOXcwQkFRRUZBQU9DQWc4QU1JSUNDZ0tDQWdFQW9mSVdOdTE4YUp1T3Jzd0JjZE9lODN0dWpXZ2dpUXl0VVYxQwpqNVhYbzNjQTM1L2ZxQXNGVHpJRGNwUmxhTGJ6SHd1d1psOWNSKzJuRENaUzI4VlhZaXcrSkQvQXpna3FzTHFJCjZ3YlFhcHNCa1lYUzRuT1UrZzhSMVgwcm52ckpickE1eHFJSWJKM002ajVLTXZ4RktvMEV3YXNBY2NiYlVGOW4KMHQ2RzNreG4zWW1Sek5HeHh2bXZ4V2prNWNkSWMza0MyT1VuRktGOG5XemJab2JiNk9PUnZSaElEWW5YdjkxdgoyMUYwQnZ0Q21GY0FEaDRqZXUrLzNKVDVLcEJkdkFHOHI3aU1wbkhKaFU1alhqTXlPRytMbkcvcnJuRzJGaXpHCmx1UHQwKzRlK0ZRSXFZY1BUM1cyTUF2ZDlzQTNEMThsUW82M00vZlMyYjNIYVNidFY0b1pmNS9zTzJNeEVPVnoKVEd1M0NxYk40TkcrZE8ycXoxYWxMQmlGZlVjNEdmUVpYRmlLaDFzazl3Qm5zeWhqYUZmdUx6bHRxMDg3STJLYQorVlRaUzFQSlJFbGduM3UwY1FmaENjelF5ZTJ3Vjl6RE9lVmUxeTBjLzZ0RWJhNllCeGR2ZGcwOFpKL0QwYTBLCnJvWlVJMW5Rc2RKeE8rQ3N1OURLYjROZzJCYnZkWVpHVWJrSCtSUDU0UUdrS1VnYnVxNVIwbXI0U1I2VUwrRE4KZjNxem81a3ZiMXVRWXFpaDZYUFVDVUVPOTNOU1Y2MTNUSUVOTUpyYjVhbGRLUkhPZlpWL201QThlUy9ibFFYcgpOV3FCRy9OL2RtckZjMmcyNGJEY3d5OXIzL3FkNy9MTWxmMVRVdzJGczR3M2x2VHJFanlwWEZhQ3BRRGxkc0xJCkYwcWVKVnNDQXdFQUFhT0NBbTh4Z2dKck1BNEdBMVVkRHdFQi93UUVBd0lGb0RBZEJnTlZIU1VFRmpBVUJnZ3IKQmdFRkJRY0RBUVlJS3dZQkJRVUhBd0l3REFZRFZSMFRBUUgvQkFJd0FEQWRCZ05WSFE0RUZnUVVHUUNXOGNFbgpaNWhVWjBDa004QW03Wjh7NGJNd0h4WURWUjBqQkJnd0ZvQVVxRXBxWXdSOTNicm0wVG0zcGtWbDcvT283S0V3CmJ3WUlLd1lCQlFVSEFRRUVZekJoTUM0R0NDc0dBUVVGQnpBQmhpSm9kSFJ3T2k4dmIyTnpjQzVwYm5RdGVETXUKYkdWMGMyVnVZM0o1Y0hRdWIzSm5NQzhHQ0NzR0FRVUZCekFDaGlOb2RIUndPaTh3WTJWeWRDNXBiblF0ZURNdQpiR1YwYzJWdVkzSjVjSFF1YjNKbkx6QWxCZ05WSFJFRUhqQWNnZzBxTG0xa1pHZGhiV1V1WTI5dGdndHRaR1JuCllXMWxMbU52YlRCTUJnTlZIU0FFUlRCRE1BZ0dCbWVCREFFQ0FUQTNCZ3NyQmdFRUFZTGZFd0VCQVRBb01DWUcKQ0NzR0FRVUZCd0lCRmhwb2RIUndPaTh3WTNCekxteGxkSE5sYm1OeWVYQjBMbTl5WnpDQ0FRUUdDaXNHQVFRQgoxbmtDQkFJRWdmVUVnZklBOEFCM0FPSnBTNjRtNk9sQUNlaUdHN1k3ZzlRKzUvNTBpUHVranlpVEFaM2Q4ZHYrCkFBQUJiUWR3b2dNQUFBUURBRWd3UmdJaEFLWldRaVVPZkZDcGdjT0JPZ0xoTjFBQjgycHg3bUR2QXYxUnRKVmoKQU0zNEFpRUFtQWpPY012WTQ2Y0VwT2lKbW4vKzB4bnZsTmR0TlNoNExvWHJaUW9sUnJZQWRRQXBQRkdXVk1nNQpaYnFxVVB4WUI5UzNiNzlZZWlseTNLVEREUFRsUlVmMGVBQUFBVzBIY0tJbEFBQUVBd0JHTUVRQ0lEeGhFMThpCm14MjBySFFHS2RpYzVCVnQ3bFBiTzBRNy9KdGI3bkVvR1grSEFpQnRDTWxXbGxlMStNV3JrUXBKbXBaTHE3bWYKWXEyZjZXc2k1QVpmQmZFRndqQU5CZ2txaGtpRzl3MEJBUXNGQUFPQ0FRRUFXUS8ycG8wcHRXWXJkbk5ndkZIbgpMK2RyclBDT2xpUXNuaFJWajdiTlhFOGNWb0l6TmU3VGRjazJINE5CUTZUZkZicmkvdHdubkFXRThzcDNPNHVWClV1bVM1Y2FGYmFPdnJIa3ZLVTNUVGhLODNqcmpFZ1N6cEo0d3k2MUlkNGhPZ0FYODVpd2REUEhvL0o0YXkzVDEKanpyMGduY0x0N1R0Tjd3dzJ5Z1RZSXBPTTBVVWtjd05GUGZZYmFRYzVqVjdvcU1raGlMNUtiSGpYVDdRcXR4YwprY3J2VXZMdERDTTQvMGpWN01FNnd4enhCQ1N1ekZWTlVlSEVVS0dDci9qRHRXV0hFZ25JNEZ5MGhQT0F0RlZzCmpDVDhWSTVYMUVmeExTRUdONkxob2NoOHl1akJYWTVNSGlVUDc5REtHaXkzaXJTZ2xtU3BVZXpMSzkwdDVzb3MKd2c9PQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCgotLS0tLUJFR0lOIENFUlRJRklDQVRFLS0tLS0KTUlJRWtqQ0NBM3FnQXdJQkFnSVFDZ0ZCUWdBQUFWT0ZjMm9MaGV5bkNEQU5CZ2txaGtpRzl3MEJBUXNGQURBLwpNU1F3SWdZRFZRUUtFeHRFYVdkcGRHRnNJRk5wWjI1aGRIVnlaU0JVY25WemRDQkRieTR4RnpBVkJnTlZCQU1UCkRrUlRWQ0JTYjI5MElFTkJJRmd6TUI0WERURTJNRE14TnpFMk5EQTBObG9YRFRJeE1ETXhOekUyTkRBME5sb3cKU2pFTE1Ba0dBMVVFQmhNQ1ZWTXhGakFVQmdOVkJBb1REVXhsZENkeklFVnVZM0o1Y0hReEl6QWhCZ05WQkFNVApHa3hsZENkeklFVnVZM0o1Y0hRZ1FYVjBhRzl5YVhSNUlGZ3pNSUlCSWpBTkJna3Foa2lHOXcwQkFRRUZBQU9DCkFROEFNSUlCQ2dLQ0FRRUFuTk1NOEZybExrZTNjbDAzZzdOb1l6RHExelVtR1NYaHZiNDE4WENTTDdlNFMwRUYKcTZtZU5RaFk3TEVxeEdpSEM2UGpkZVRtODZkaWNicDVnV0FmMTVHYW4vUFFlR2R4eUdrT2xaSFAvdWFaNldBOApTTXgreWsxM0VpU2RSeHRhNjduc0hqY0FISnlzZTZjRjZzNUs2NzFCNVRhWXVjdjliVHlXYU44aktrS1FESVowClo4aC9wWnE0VW1FVUV6OWw2WUtIeTl2NkRsYjJob256aFQrWGhxK3czQnJ2YXcyVkZuM0VLNkJsc3BrRU5uV0EKYTZ4Szh5dVFTWGd2b3BaUEtpQWxLUVRHZE1EUU1jMlBNVGlWRnJxb003aEQ4YkVmd3pCL29ua3hFejB0TnZqagovUEl6YXJrNU1jV3Z4STBOSFdRV002cjZoQ20yMUF2QTJIM0Rrd0lEQVFBQm80SUJmVENDQVhrd0VnWURWUjBUCkFRSC9CQWd3QmdFQi93SUJBREFPQmdOVkhROEJBZjhFQkFNQ0FZWXdmd1lJS3dZQkJRVUhBUUVFY3pCeE1ESUcKQ0NzR0FRVUZCekFCaGlab2RIUndPaTh3YVhOeVp5NTBjblZ6ZEdsa0xtOWpjM0F1YVdSbGJuUnlkWE4wTG1OdgpiVEE3QmdnckJnRUZCUWN3QW9ZdmFIUjBjRG92TDJGd2NITXVhV1JsYm5SeWRYTjBMbU52YlM5eWIyOTBjeTlrCmMzUnliMjkwWTJGNE15NXdOMk13SHdZRFZSMGpCQmd3Rm9BVXhLZXhwSHNzY2ZyYjRVdVFkZi9FRldDRmlSQXcKVkFZRFZSMGdCRTB3U3pBSUJnWm5nUXdCQWdFd1B3WUxLd1lCQkFHQzN4TUJBUUV3TURBdUJnZ3JCZ0VGQlFjQwpBUllpYUhSMGNEb3ZMMk53Y3k1eWIyOTBMWGd4TG14bGRITmxibU55ZVhCMExtOXlaekE4QmdOVkhSOEVOVEF6Ck1ER2dMNkF0aGl0b2RIUndPaTh3WTNKc0xtbGtaVzUwY25WemRDNWpiMjB2UkZOVVVrOVBWRU5CV0RORFVrd3UKWTNKc01CMEdBMVVkRGdRV0JCU29TbXBqQkgzZHV1YlJPYmVtUldYdjg2anNvVEFOQmdrcWhraUc5dzBCQVFzRgpBQU9DQVFFQTNUUFhFZk5qV0RqZEdCWDdDVlcrZGxhNWNFaWxhVWNuZThJa0NKTHhXaDlLRWlrM0pIUlJIR0pvCnVNMlZjR2ZsOTZTOFRpaFJ6WnZvcm9lZDZ0aTZXcUVCbXR6dzNXb2RhdGcrVnlPZXBoNEVZcHIvMXdYS3R4OC8Kd0FwSXZKU3d0bVZpNE1GVTVhTXFyU0RFNmVhNzNNajJ0Y015bzVqTWQ2am1lV1VISzhzby9qb1dVb0hPVWd3dQpYNFBvMVFZeiszZHN6a0RxTXA0ZmtseEJ3WFJzVzEwS1h7UE1UWitzT1BBdmV5eGluZG1qa1c4bEd5K1FzUmxHClBmWitHNlo2aDdtamVtMFkraVdsa1ljVjRQSVdMMWl3Qmk4c2FDYkdTNWpOMnA4TStYK1E3VU5LRWtST2IzTjYKS09xa3FtNTdUSDJIM2VESkFrU25oNi9ETkZ1MFFnPT0KLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQo=  tls.key: LS0tLS1CRUdJTiBSU0EgUFJJVkFURSBLRVktLS0tLQpNSUlKS0FJQkFBS0NBZ0VBb2ZJV051MThhSnVPcnN3QmNkT2U4M3R1aldnZ2lReXRVVjFDajVYWG8zY0EzNS9mCnFBc0ZUeklEY3BSbGFMYnpId3V3Wmw5Y1IrMm5EQ1pTMjhWWFlpdytKRC9BemdrcXNMcUk2d2JRYXBzQmtZWFMKNG5PVStnOFIxWDBybnZySmJyQTV4cUlJYkozTTZqNUtNdnhGS28wRXdhc0FjY2JiVUY5bjB0Nkcza3huM1ltUgp6Tkd4eHVtdnhXams1Y2RJYzNrQzJPVW5GS0Y4bld6YlpvYmI2T09SdlJoSURZblh3OTF2MjFGMEJ2dENtRmNBCkRoNGpldSsvM0pUNUtwQmR2QUc4cjdpTXBuSEpoVTVqWGpNeU9HK0xuRy9ycm5HMkZpekdsdVB0MCs0ZStGUUkKcVljUFQzVzJNQXZkOXNBM0QxOGxRbzYzTS9mUzJiM0hhU2J0VjRvWmY1L3NPMk14RU9WelRHdTNDcWJONE5HKwpkTzJxejFhbExCaUZmVWM0R2ZRWlhGaUtoMXNrOXdCbnN5aGphRmZ1THpsdHEwODdJMkthK1ZUWlMxUEpSRWxnCm4zdTBjUWZoQ2N6UXllMndWOXpET2VWZTF5MGMvNnRFYmE2WUJ4ZHZkZzA4WkovRDBhMEtyb1pVSTFuUXNkSngKTytDc3U5REtiNE5nMkJidmRZWkdVYmtIK1JQNTRRR2tLVWdidXE1UjBtcjRTUjZVTCtETmYzcXpvNWt2YjF1UQpZcWloNlhQVUNVRU85M05TVjYxM1RJRU5NSnJiNWFsZEtSSE9mWlYvbTVBOGVTL2JsUVhyTldxQkcvTi9kbXJGCmMyZzI0YkRjd3k5cjMvcWQ3L0xNbGYxVFV3MkZzNHczbHZUckVqeXBYRmFDcFFEbGRzTElGMHFlSlZzQ0F3RUEKQVFLQ0FnQXY5Zk13UnpzTisrdlF4cWd5M3JwM1gzbkpOU3BWakVTVUVTdVNQSTFGWXd3R0xtSGRjWTRiK3pMYwpMeWl0VDJsSEszNE5nM1pmOHZrQzl5S1k1YVBRZGt2ZERtaDZYR3FoTmswd1ZhOUpzeWhPd2JSSHpuVXpiVjBaCnZkMDZVd2x1MTQvMHpLMzBCUFBYOTZTZjN1aFpCclIrNnJiUisxT2VSUE1KbDArWDdFYmliRWlhd1F1R1hsVHAKQVB5eE5FaTNzZ0h2M0VhcnJIdXNYNzNHYW5BY1U3RW9zRlUrZFRGSktEcGxXSVVsUUNwajFYZzF0aVZKMWxFYQo4Wit0UkY0T1BQRjFsUkZLaGU1cHBXSjJWbkVzRjVUZ09xRXc0NHBLbk80Zlo5ZGFhVzRRbTBxSmNtOU5XQTRoCndwSDA3czRmcGt6eG5qU1JsbmFDZDlyandGeVBsSkJzUXNhVlFFNzlpQzJZMTRnTk9KQ0xyMXRKSEQ2ODN3bW4KS3ZNOHZpOTdHTmIybXZHeWNtZnloNVpzTFBpTWNqOFFER3VWZU53dlNESXpybnhqVkZlc0liTWt5UlZRem9IVApTTHRQbXdVR3lwRHVrMDhaZytsT0lYOC85K3lqMER3MDRqenllTVptYlFVdkd2N2lNWjFUaHdaRHF1YkJXV3J4CmtYTmJwTG9BMGxrcHh5bjdGam9Ya20zM2ZKQURjd2xWSS82WFNrSm1FaFVlZmZnaFFSMGNyVGphQVd1Qkx2Qk0KT0s5aEEzT3RTN2F0S2FDb1lvSmRrYkpHQTdWdytNNzA4NEJOTGhxM1Fyckg4S3M3Z05pdC9NN3lxSnU1alBaZgo2SE1seHNyWU9NVUhuVlk4VDkwN0Q3cS9ORUNnRThzODhnZzAyQ3JNWTFqanE4UnBpUUtDQVFFQTE2UHJaMUEwClNISS83akdmS3BETkJzQ0xrVUFxRERKSzQ0dFdJYmJBUXFhRTN1eDh4bkFlU2NjSHozbS9ScEpPSGtteHZTZlgKbTJ1Wk8veGtNTWhYK2lwOHdFOHZibzR1enVNYitTSXE3bWpialJkK1JJczJ5NHJsZVQ2NGVjRWc4R2pZckExZgpiSEI0MmhQclVTcXpxUVIwOTZocm1Lb1diU0RDZDZwOUVNeWVzT3IwTjdtQmJYVVZPazJxZGtYRlZWbHBlUDdpClFxWGdRUUI0bHgzLzJJdlpBMlhJUXlQdGJ0RWVRbmgyQ3FNM2NDMzR0VEVjZ244K0VwNG9SWmkwTTBHaUY3bXgKOTEvZHY2THZlNTR5K1pON1lXd1NFQ09ubzd5bDlvTlBZVnVGMGRiMjh0elppMThCeHJTQ2JESE1XbExvUzhWNgpXTEo0OGlSODJDYkc1d0tDQVFFQXdFRjM4KzYyeDhDU2x0blZZNlJaN0J0NEdiNEJqVWhWYXZ0NFkxUGFlbXFNCjFidFVnR2JyUnBoNHFUSEFTckUwUUZLeVZKYnlCUkJyRHIxWHU4WWRSVXQzZC92VzlIR1dPd1BKdTN2M3pLbHMKQ2xsZnpFY3J5L1l2aHAzSzlEcGR6OE1icHdueW5xcGV6b0xMNlJpL3JnK0hyTzBueXd1RSt0T2xYVFo2eUtadApHWVdTSVBWaG00NUJkc2ZxUzhnYjVvbjA0bHh4bnhxVnJvN0c0TUR6cmVEYlFhaGdyS3VuRWxwajZ4eW1PVWpBCkdCZDR3QUVrUExxNUUrRWcreDY4TkRLVTYwK29ybFhLWVhDQm5HSFZOQ3BVcmswVXkrcHFZZmFEN3VuR2VzaHMKSEwra3lXbXl5a3ErTmNKbnRXMFNSNy9sU1IvZUFhVEZyVzZVaXV0RGJRS0NBUUVBemhRYU9PNmVPSW51N016QgpScVdCT3EyeDg4cjFKQmpBRnZzbkFpc3JTOGJsZmtGVTdXREdvVTB5K3FWb0ZhSm1RMjI4RFlCUS9YZnp4aTdxCjlPL1JuQU1VbTVoUlJQOWVYbHNPZGFXZ2o1em9ETXRoNFZHRnVUbHhHZERGN1oyU3hBMysyMVlnVm5xYUZCY3IKTUxOMVpOWWNqajJITGl1R0tSNUFtcW4wd2FRN0YrcENJQ3NKTkxqSzQ2QXJnc0lrMXU4TzdCSHgyeTI0eFlZVQp1SjV6emRmQU9nNEFONkhURzY5L2twaWFmb29DeGhNNDlyZ0xmZTdxUEZLbk8vTzJhckdUbmNiWi9BWEMzb3h4Ci81dHRMYlF6R2lSMGtyWHdWSHRKdys4elltQmIzL0RtcWF4RHZueTZMdEo5UGJiTmk1aGw1VnZCRTVqa0dzeWgKL3RQNEN3S0NBUUJ2R1dZb0lKcWZkRGxCMHovdEJOeXlCRzJ5OG9vVEN1blJtT0JKQmZ3TEllZWcyMUJKb3kveQo2OGxPZk9HU1NEVFp0dkEyMGNPcUNZTFVVYmFSWERzdUFCNVp4NzdBSTZPZEZ1Tk01S2FlTG9td3NWVWF4MFlYCjUzd3ZYcUFaNG1DejN4dnJ1MlBwTEtyOHk3anFTdEw1MHgra1hxZlFQaWZxaXNQVXlkYktmT0l2RFhFVWVyaWQKRytmWXJFNUkzS3JDM3BZVStUWmJ1eEVrZm4yUEEvSE5XVk5hN2VKdjVnSDJLU1gwaCtuRzBMT3hPRjhmRlluTApUbHdGa09OdU9xU254Vk1wYUM4aUQ1R1VIVi9JN3dBMTFRQjZlVEM3Wmd0ejhQRHM3MHN6U1A2dzNrNXIxaGpyCnJhV2RpMnBDL1hUQzRiR3VRQ3dhNXcwVTNBSWJCVGxCQW9JQkFEc1RONGhvclVHNWw3MXhLZk5ibVBTbDZ6RlIKYTJ4d2U2VVZPOVZzMFpHeEdLWWJSN1VuVDBDL1FqUiswS2JsbE9leDdFY3cyMklCcmFFVzBGbXpuVnoyUW9FNwpMUE5COXhyTTFEeE56UjZEbFBUeERMcEFGWVlUcm40SWY1cjFVdVdpc2lMdmd6T2xGTlVITnN5UFJIZWNGblhUCnNhTk9JWkgrQTJ5KzF3QWdpSFZIS2JPRGRHeVFQVlQ0TXFFWkJaY2pQcmRBekNKcnloSHlYdHBqRjFSdlFEYTMKTVM3U3JVTGM4djJGQWJ1VG1QZ2R1ZHBKd1Q4dENCa2VRKzZ4YmJWN3YrZzBEMG5EWFNIZFVwNXFyUzcrTnhtVwp4NWV4UHo1VENhYXcxSnkzWjRmT1MzMTV6eHJGdmRHTmhWRXhMMzRlUVlzOHRYN0N0VWxuWkNray9zYz0KLS0tLS1FTkQgUlNBIFBSSVZBVEUgS0VZLS0tLS0K#  创建traefik bacvi traefik-rbac.yaml---kind: ClusterRoleapiVersion: rbac.authorization.k8s.io/v1beta1metadata:  name: traefik  namespace: kube-systemrules:  - apiGroups:      - ""    resources:      - services      - endpoints      - secrets    verbs:      - get      - list      - watch  - apiGroups:      - extensions    resources:      - ingresses    verbs:      - get      - list      - watch  - apiGroups:    - extensions    resources:    - ingresses/status    verbs:    - update---kind: ClusterRoleBindingapiVersion: rbac.authorization.k8s.io/v1beta1metadata:  name: traefikroleRef:  apiGroup: rbac.authorization.k8s.io  kind: ClusterRole  name: traefiksubjects:- kind: ServiceAccount  name: traefik  namespace: kube-system# 创建 traefik-daemonset-httpsvi traefik-daemonset-https.yaml---apiVersion: v1kind: ServiceAccountmetadata:  name: traefik  namespace: kube-system---kind: DaemonSetapiVersion: apps/v1metadata:  name: traefik  namespace: kube-system  labels:    k8s-app: traefikspec:  selector:    matchLabels:      k8s-app: traefik  template:    metadata:      labels:        k8s-app: traefik        name: traefik    spec:      serviceAccountName: traefik      terminationGracePeriodSeconds: 60      volumes:      - name: ssl        secret:          secretName: tls-cert      hostNetwork: true      dnsPolicy: ClusterFirstWithHostNet            containers:      - image: traefik:v1.7.16        name: traefik        imagePullPolicy: Always        volumeMounts:        - mountPath: /certs                   name: "ssl"        ports:        - name: http          containerPort: 80          hostPort: 80        - name: https          containerPort: 443          hostPort: 443        - name: admin          containerPort: 8080        securityContext:          capabilities:            drop:            - ALL            add:            - NET_BIND_SERVICE        args:        - --api        - --web        - --api.dashboard        - --logLevel=INFO        - --web.metrics        - --metrics.prometheus        - --web.metrics.prometheus        - --kubernetes        - --traefiklog        - --traefiklog.format=json        - --accesslog        - --accesslog.format=json        - --accessLog.fields.headers.defaultMode=redact        - --insecureskipverify=true        - --defaultentrypoints=http,https        - --entrypoints=Name:https Address::443 TLS        - --entrypoints=Name:http Address::80            #nodeSelector:      #  ingress: "yes"      tolerations:      - effect: NoSchedule        key: node-role.kubernetes.io/ingress        operator: Equal  updateStrategy:    type: RollingUpdate    rollingUpdate:      maxUnavailable: 1---kind: ServiceapiVersion: v1metadata:  labels:    k8s-app: traefik  name: traefik  namespace: kube-systemspec:  selector:    k8s-app: traefik  clusterIP: None  ports:    - protocol: TCP      port: 80      name: http    - protocol: TCP      port: 443      name: https    - protocol: TCP      port: 8080      name: admin  type: ClusterIP# 创建traefik-dashboardvi traefik-dashboard.yamlapiVersion: extensions/v1beta1kind: Ingressmetadata:  name: traefik-dashboard  namespace: kube-system  annotations:    kubernetes.io/ingress.class: traefik    traefik.ingress.kubernetes.io/frontend-entry-points: http,httpsspec:  rules:  - host: trae.xxxx.com    http:      paths:        - backend:            serviceName: traefik            servicePort: 8080  tls:   - secretName: tls-cert#创建traefik 服务kubectl apply -f .# dns 解析 http://trae.xxxx.com

kubernetes-dashboard 部署

# base64 加密cat dashboard.pem|base64 | tr -d '\n'cat dashboard-key.pem|base64 | tr -d '\n'vi kubernetes-dashboard.yaml# Copyright 2017 The Kubernetes Authors.## Licensed under the Apache License, Version 2.0 (the "License");# you may not use this file except in compliance with the License.# You may obtain a copy of the License at##     http://www.apache.org/licenses/LICENSE-2.0## Unless required by applicable law or agreed to in writing, software# distributed under the License is distributed on an "AS IS" BASIS,# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.# See the License for the specific language governing permissions and# limitations under the License.apiVersion: v1kind: Namespacemetadata:  name: kubernetes-dashboard---apiVersion: v1kind: ServiceAccountmetadata:  labels:    k8s-app: kubernetes-dashboard  name: kubernetes-dashboard  namespace: kubernetes-dashboardkind: ServiceapiVersion: v1metadata:  labels:    k8s-app: kubernetes-dashboard  name: kubernetes-dashboard  namespace: kubernetes-dashboard  annotations:    kubernetes.io/ingress.class: traefikspec:  ports:    - port: 443      targetPort: 8443  selector:    k8s-app: kubernetes-dashboard---apiVersion: v1kind: Secretmetadata:  labels:    k8s-app: kubernetes-dashboard  name: kubernetes-dashboard-certs  namespace: kubernetes-dashboardtype: Opaquedata:  dashboard.key: LS0tLS1CRUdJTiBSU0EgUFJJVkFURSBLRVktLS0tLQpNSUlFcEFJQkFBS0NBUUVBeFlFV0MxbGlqcnFzNW5vcHBxTXF0YzZSY0pnSWFJSGhGemZZUWhRQm5pK0Vjam8vCkRTUkYvY3BUOFlkTTg2MVpEV1lSN1FEelFLNmJUTmRLWXJJYmpVWHJpRFVFU01EUW13Y1VteTMzWjFpeXR6K0wKUUVmTVFvWVNReGVIY2RqUHp3bUhFS0todk9vNmxQTHNFWkMwQ3ZCamw2VHlERjhuSDEzby9kRlRVbGJhWUlGaQpPeGVIWkxMMTZKbmNLK3RVaW9ncjdLekFKMUkxTjdwOVQ1blZ5YU9PbWNCVEFnU3RJM0ZwSzdMZG1zaVU0ZEZ0CkpSSFZ0eTh6Y3dCSU9wWnhqV29mM2ROVkRrVUFsYjVtV2psU0RaQ2lhYmFYQi91NmJ0R0k3RlY2cENaUzdDVG4KeWlpUFlFSXRPSGRCT0VycGpKZWQ0bHQ5K2MvNDE3UTRIaiswdndJREFRQUJBb0lCQVFDK1daSWdjQTZRRnhScQpzVlNST1BNQjlFdXlJNlQrN0NZL2xXQUZGM2tUdHlKRVlTVEJpck0yVFprbjBFbjNGSndlVU1CNEZwRmJScTJBCm1vSWpxeHJveG5taGRjOWlPd3NTVHZtcU1kd2ZLNXBiQ0pBeDdNRE5ZS0FiTDRNbjAxazlaaVpaZnhTNG1WcksKa1hHNTRDZlYzeWR0VU5qRDJiVkFBdWQ2TVJQSDV5QWJTVktsMG9ONkRCaFV4MlYyWEo0WnRUVHE0b3R6VGYxZwp3SjNJeVFjSXl3czE2V3dkeHpuYStqVmpOYU5OQ3ZCT1BMbm9TeXZBQXZGRG9UYmUrMG1tcnZLVmlSeDBDT1FzCkUwNjFtNHY2eUExL3locndkT1BDYXN6SkpjWlYzOThJTzFKb2QxUHk3OU9aT1FpY1FEOGhwQmxqb0FSQ2JlY3QKRFFPcG5CR0JBb0dCQVBhYlJSSGpPTkxIQ25JZWlFQU1EYXNwQXo2RGxRNkQvdWNNdzROdkVPRVNVa3dvQ0p4cApwK1hJeVVzT1B1d2swTzVCcHJRcHZjdGYyWXlLZTFtR25iVUpmUVNWNGpLdWpqb0M0OWhOWk9lSE8zd0xMcnNXCkl1SU1Qeko0TjhxSzl0dUpDQ3BVYUZFVzRiN1R2OGsyK1pJWHJwN3hzNklDd01EUnpTaW9wY0hCQW9HQkFNMEgKQVl1bmdzY3hTM2JnZ05idU5sQ3lIOHBLZFVPbi95cU9IQUdYcG9vZmJUbXJiUUlWN0ZOVSszUTlYc2ErVVE0QwpUbVdFbzhabVhrL3lIV2FDVWxpRkN0ckRhTzNUZVhvb2pia1JyaDcxakFXN0pjVDRVZ1ZwcG1RakFVUW8vOWtVCmxHMUNpOTFZZy94dlV5dHlYM1BnZHJ6SnU2aWNsM1pVZ1h4dzNoWi9Bb0dBZENmY2w3bFVLWXZSTXNHSTRjb0wKb2lRMlAvclFlYjdZa05IbFFZSk9EQVdLT0E3ZlIzVkl2U1lmRWpoS2tRWWlWeWNiTTE4NTQ1SnBNUmFGVlR6ZwpDY2JIV1NLVUlkVXdic2l2czFGNUJza2V6cVdoeEVOLytNTlYvUnE5QkswQjY1UVhBWUV5aFlkbW0zQzN0RG90CndZOWdFOE83SGNONE1ScGhMUmFLeE1FQ2dZRUFoS2E5eHorUUM1VEhRSmlzZzJNSVhWbUIyLzRrdEt0akdvTnIKZDFSSStpQ3ZLSnJUSW9CUXNQSFE1em8xc2R5ODBKV0paNEZUL1MrS1lhdENmbXBmSU1xalpUcjlEcksrYTkwRgpKUEpkZDhaaTIrcGoyM2JXaW8zNmk5dGlIRmx5ZjE4alVUVzNESFVTb0NiZTVzTlBJc2ZkeXZPeXFMcjMvQ1ZjCnlaOU1jYjBDZ1lBMVp2RVM3bU42Nm10T2JpSlR3a3hhaTVvS2tHbDdHTDJkZXJFUmxsc1YrNWRCSVY4dG5DTnAKT2tjMFlMbHV2TEg4cG4zd2VCNzg5dUFCQjNXYmNKcHg0L2NIRm9oZDNhdlR0RThRVjJod0tNS2RKQVBvTHNoMgprK2lEUWd1dmFxSzNmL1RYUW43bWU3dWFqSDk3SXZldXJtWWsvVmRJY0dicnd1SVRzd0FEYWc9PQotLS0tLUVORCBSU0EgUFJJVkFURSBLRVktLS0tLQo=  dashboard.crt: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUQ5ekNDQXQrZ0F3SUJBZ0lVUWRIVXdKS1JYc1ZRb2VYS1JDTjd0eVcwWU04d0RRWUpLb1pJaHZjTkFRRUwKQlFBd2JqRUxNQWtHQTFVRUJoTUNRMDR4RWpBUUJnTlZCQWdUQ1VkMVlXNW5SRzl1WnpFU01CQUdBMVVFQnhNSgpSM1ZoYm1kYWFHOTFNUkF3RGdZRFZRUUtFd2R0WkdSbllXMWxNUkF3RGdZRFZRUUxFd2R0WkdSbllXMWxNUk13CkVRWURWUVFERXdwcmRXSmxjbTVsZEdWek1CNFhEVEU1TURjd05ERXhNVE13TUZvWERUSTVNRGN3TVRFeE1UTXcKTUZvd2JURUxNQWtHQTFVRUJoTUNRMDR4RWpBUUJnTlZCQWdUQ1VkMVlXNW5SRzl1WnpFU01CQUdBMVVFQnhNSgpSM1ZoYm1kYWFHOTFNUkF3RGdZRFZRUUtFd2R0WkdSbllXMWxNUkF3RGdZRFZRUUxFd2R0WkdSbllXMWxNUkl3CkVBWURWUVFERXdsa1lYTm9ZbTloY21Rd2dnRWlNQTBHQ1NxR1NJYjNEUUVCQVFVQUE0SUJEd0F3Z2dFS0FvSUIKQVFERmdSWUxXV0tPdXF6bWVpbW1veXExenBGd21BaG9nZUVYTjloQ0ZBR2VMNFJ5T2o4TkpFWDl5bFB4aDB6egpyVmtOWmhIdEFQTkFycHRNMTBwaXNodU5SZXVJTlFSSXdOQ2JCeFNiTGZkbldMSzNQNHRBUjh5Q2hoSkRGNGR4CjJNL1BDWWNRb3FHODZqcVU4dXdSa0xRSzhHT1hwUElNWHljZlhlajkwVk5TVnRwZ2dXSTdGNGRrc3ZYb21kd3IKNjFTS2lDdnNyTUFuVWpVM3VuMVBtZFhKbzQ2WndGTUNCSzBqY1drcnN0MmF5SlRoMFcwbEVkVzNMekp6QUVnNgpsbkdOYWgvZDAxVU9SUUNWdm1aYU9WSU5rS0pwdHBjSCs3cHUwWWpzVlhxa0psTHNKT2ZLS0k5Z1FpMDRkMEU0ClN1bU1sNTNpVzMzNXovalh0RGdlUDdTL0FnTUJBQUdqZ1kwd2dZb3dEZ1lEVlIwUEFRSC9CQVFEQWdXZ01CMEcKQTFVZEpRUVdNQlFHQ0NzR0FRVUZCd01CQmdnckJnRUZCUWNEQWpBTUJnTlZIUk1CQWY4RUFqQUFNQjBHQTFVZApEZ1FXQkJURTl6cWx4dkErRXMrbE8zWlFEMlhubGFHRFpqQWZCZ05WSFNNRUdEQVdnQlJ4NEtjQVJjYWtSL2J4Cm13b1RCZURzK3hBb2FUQUxCZ05WSFJFRUJEQUNnZ0F3RFFZSktvWklodmNOQVFFTEJRQURnZ0VCQUJnWHZwTEMKQjIybXlQaURlZnhsWGNZRzAvY0R2RXlYcTlENWtKTnBxKzFZQ0EvMlp2RDIyN1Q5VjY3aHVyTlA3T2FvSG95Tgo0MHpkR3lZTGRNV3pyZTQwVksxdC84N3pDTENzamt1ZXRCRWEwNVRqUTJhbDRhSzJ6TXl5MkJLWEpYbjlvdkhzCjJwNndvL001eklEOXl2OEhyRkZqWHM3NitTUTFzNXpOdUxuaDBET0Z1SktiZUZxSUJyNmZRbXlsb0l1VURtZjYKcGtQYkJyRnJpNHFGS0lDcVZKRCt3Z01zRFBiclVMZXF5NWlBVjNqRzJKMFgxOE4zdklCeUFwdWhZbjNudlV0TwpLREVIWkFJcFpjRWdqQ2ZLVDNyaERLL3JLN0VFZkxLcGlCdGJya3pFbjVWV3FQUFJEK3ZPU2VySldETDl1K0xyCmhEazlvZ084cmNqQzZGdz0KLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQ==---apiVersion: v1kind: Secretmetadata:  labels:    k8s-app: kubernetes-dashboard  name: dashboard-tls-cert  namespace: kubernetes-dashboardtype: Opaquedata:  tls.crt: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUdYVENDQlVXZ0F3SUJBZ0lTQTBCSy82MUwvZVNHb2FjNmFjRnZLaDVOTUEwR0NTcUdTSWIzRFFFQkN3VUEKTUVveEN6QUpCZ05WQkFZVEFsVlRNUll3RkFZRFZRUUtFdzFNWlhRbmN5QkZibU55ZVhCME1TTXdJUVlEVlFRRApFeHBNWlhRbmN5QkZibU55ZVhCMElFRjFkR2h3Y21sMGVTQllNekFlRncweE9UQTVNRFl4TlRNNE1EVmFGdzB4Ck9URXlNRFV4TlRNNE1EVmFNQll4RkRBU0JnTlZCQU1UQzIxa1pHZGhiV1V1WTI5dE1JSUNJakFOQmdrcWhraUcKOXcwQkFRRUZBQU9DQWc4QU1JSUNDZ0tDQWdFQW9mSVdOdTE4YUp1T3Jzd0JjZE9lODN0dWpXZ2dpUXl0VVYxQwpqNVhYbzNjQTM1L2ZxQXNGVHpJRGNwUmxhTGJ6SHd1d1psOWNSKzJuRENaUzI4VlhZaXcrSkQvQXpna3FzTHFJCjZ3YlFhcHNCa1lYUzRuT1UrZzhSMVgwcm52ckpickE1eHFJSWJKM002ajVLTXZ4RktvMEV3YXNBY2NiYlVGOW4KMHQ2RzNreG4zWW1Sek5HeHh2bXZ4V2prNWNkSWMza0MyT1VuRktGOG5XemJab2JiNk9PUnZSaElEWW5YdjkxdgoyMUYwQnZ0Q21GY0FEaDRqZXUrLzNKVDVLcEJkdkFHOHI3aU1wbkhKaFU1alhqTXlPRytMbkcvcnJuRzJGaXpHCmx1UHQwKzRlK0ZRSXFZY1BUM1cyTUF2ZDlzQTNEMThsUW82M00vZlMyYjNIYVNidFY0b1pmNS9zTzJNeEVPVnoKVEd1M0NxYk40TkcrZE8ycXoxYWxMQmlGZlVjNEdmUVpYRmlLaDFzazl3Qm5zeWhqYUZmdUx6bHRxMDg3STJLYQorVlRaUzFQSlJFbGduM3UwY1FmaENjelF5ZTJ3Vjl6RE9lVmUxeTBjLzZ0RWJhNllCeGR2ZGcwOFpKL0QwYTBLCnJvWlVJMW5Rc2RKeE8rQ3N1OURLYjROZzJCYnZkWVpHVWJrSCtSUDU0UUdrS1VnYnVxNVIwbXI0U1I2VUwrRE4KZjNxem81a3ZiMXVRWXFpaDZYUFVDVUVPOTNOU1Y2MTNUSUVOTUpyYjVhbGRLUkhPZlpWL201QThlUy9ibFFYcgpOV3FCRy9OL2RtckZjMmcyNGJEY3d5OXIzL3FkNy9MTWxmMVRVdzJGczR3M2x2VHJFanlwWEZhQ3BRRGxkc0xJCkYwcWVKVnNDQXdFQUFhT0NBbTh4Z2dKck1BNEdBMVVkRHdFQi93UUVBd0lGb0RBZEJnTlZIU1VFRmpBVUJnZ3IKQmdFRkJRY0RBUVlJS3dZQkJRVUhBd0l3REFZRFZSMFRBUUgvQkFJd0FEQWRCZ05WSFE0RUZnUVVHUUNXOGNFbgpaNWhVWjBDa004QW03Wjh7NGJNd0h4WURWUjBqQkJnd0ZvQVVxRXBxWXdSOTNicm0wVG0zcGtWbDcvT283S0V3CmJ3WUlLd1lCQlFVSEFRRUVZekJoTUM0R0NDc0dBUVVGQnpBQmhpSm9kSFJ3T2k4dmIyTnpjQzVwYm5RdGVETXUKYkdWMGMyVnVZM0o1Y0hRdWIzSm5NQzhHQ0NzR0FRVUZCekFDaGlOb2RIUndPaTh3WTJWeWRDNXBiblF0ZURNdQpiR1YwYzJWdVkzSjVjSFF1YjNKbkx6QWxCZ05WSFJFRUhqQWNnZzBxTG0xa1pHZGhiV1V1WTI5dGdndHRaR1JuCllXMWxMbU52YlRCTUJnTlZIU0FFUlRCRE1BZ0dCbWVCREFFQ0FUQTNCZ3NyQmdFRUFZTGZFd0VCQVRBb01DWUcKQ0NzR0FRVUZCd0lCRmhwb2RIUndPaTh3WTNCekxteGxkSE5sYm1OeWVYQjBMbTl5WnpDQ0FRUUdDaXNHQVFRQgoxbmtDQkFJRWdmVUVnZklBOEFCM0FPSnBTNjRtNk9sQUNlaUdHN1k3ZzlRKzUvNTBpUHVranlpVEFaM2Q4ZHYrCkFBQUJiUWR3b2dNQUFBUURBRWd3UmdJaEFLWldRaVVPZkZDcGdjT0JPZ0xoTjFBQjgycHg3bUR2QXYxUnRKVmoKQU0zNEFpRUFtQWpPY012WTQ2Y0VwT2lKbW4vKzB4bnZsTmR0TlNoNExvWHJaUW9sUnJZQWRRQXBQRkdXVk1nNQpaYnFxVVB4WUI5UzNiNzlZZWlseTNLVEREUFRsUlVmMGVBQUFBVzBIY0tJbEFBQUVBd0JHTUVRQ0lEeGhFMThpCm14MjBySFFHS2RpYzVCVnQ3bFBiTzBRNy9KdGI3bkVvR1grSEFpQnRDTWxXbGxlMStNV3JrUXBKbXBaTHE3bWYKWXEyZjZXc2k1QVpmQmZFRndqQU5CZ2txaGtpRzl3MEJBUXNGQUFPQ0FRRUFXUS8ycG8wcHRXWXJkbk5ndkZIbgpMK2RyclBDT2xpUXNuaFJWajdiTlhFOGNWb0l6TmU3VGRjazJINE5CUTZUZkZicmkvdHdubkFXRThzcDNPNHVWClV1bVM1Y2FGYmFPdnJIa3ZLVTNUVGhLODNqcmpFZ1N6cEo0d3k2MUlkNGhPZ0FYODVpd2REUEhvL0o0YXkzVDEKanpyMGduY0x0N1R0Tjd3dzJ5Z1RZSXBPTTBVVWtjd05GUGZZYmFRYzVqVjdvcU1raGlMNUtiSGpYVDdRcXR4YwprY3J2VXZMdERDTTQvMGpWN01FNnd4enhCQ1N1ekZWTlVlSEVVS0dDci9qRHRXV0hFZ25JNEZ5MGhQT0F0RlZzCmpDVDhWSTVYMUVmeExTRUdONkxob2NoOHl1akJYWTVNSGlVUDc5REtHaXkzaXJTZ2xtU3BVZXpMSzkwdDVzb3MKd2c9PQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCgotLS0tLUJFR0lOIENFUlRJRklDQVRFLS0tLS0KTUlJRWtqQ0NBM3FnQXdJQkFnSVFDZ0ZCUWdBQUFWT0ZjMm9MaGV5bkNEQU5CZ2txaGtpRzl3MEJBUXNGQURBLwpNU1F3SWdZRFZRUUtFeHRFYVdkcGRHRnNJRk5wWjI1aGRIVnlaU0JVY25WemRDQkRieTR4RnpBVkJnTlZCQU1UCkRrUlRWQ0JTYjI5MElFTkJJRmd6TUI0WERURTJNRE14TnpFMk5EQTBObG9YRFRJeE1ETXhOekUyTkRBME5sb3cKU2pFTE1Ba0dBMVVFQmhNQ1ZWTXhGakFVQmdOVkJBb1REVXhsZENkeklFVnVZM0o1Y0hReEl6QWhCZ05WQkFNVApHa3hsZENkeklFVnVZM0o1Y0hRZ1FYVjBhRzl5YVhSNUlGZ3pNSUlCSWpBTkJna3Foa2lHOXcwQkFRRUZBQU9DCkFROEFNSUlCQ2dLQ0FRRUFuTk1NOEZybExrZTNjbDAzZzdOb1l6RHExelVtR1NYaHZiNDE4WENTTDdlNFMwRUYKcTZtZU5RaFk3TEVxeEdpSEM2UGpkZVRtODZkaWNicDVnV0FmMTVHYW4vUFFlR2R4eUdrT2xaSFAvdWFaNldBOApTTXgreWsxM0VpU2RSeHRhNjduc0hqY0FISnlzZTZjRjZzNUs2NzFCNVRhWXVjdjliVHlXYU44aktrS1FESVowClo4aC9wWnE0VW1FVUV6OWw2WUtIeTl2NkRsYjJob256aFQrWGhxK3czQnJ2YXcyVkZuM0VLNkJsc3BrRU5uV0EKYTZ4Szh5dVFTWGd2b3BaUEtpQWxLUVRHZE1EUU1jMlBNVGlWRnJxb003aEQ4YkVmd3pCL29ua3hFejB0TnZqagovUEl6YXJrNU1jV3Z4STBOSFdRV002cjZoQ20yMUF2QTJIM0Rrd0lEQVFBQm80SUJmVENDQVhrd0VnWURWUjBUCkFRSC9CQWd3QmdFQi93SUJBREFPQmdOVkhROEJBZjhFQkFNQ0FZWXdmd1lJS3dZQkJRVUhBUUVFY3pCeE1ESUcKQ0NzR0FRVUZCekFCaGlab2RIUndPaTh3YVhOeVp5NTBjblZ6ZEdsa0xtOWpjM0F1YVdSbGJuUnlkWE4wTG1OdgpiVEE3QmdnckJnRUZCUWN3QW9ZdmFIUjBjRG92TDJGd2NITXVhV1JsYm5SeWRYTjBMbU52YlM5eWIyOTBjeTlrCmMzUnliMjkwWTJGNE15NXdOMk13SHdZRFZSMGpCQmd3Rm9BVXhLZXhwSHNzY2ZyYjRVdVFkZi9FRldDRmlSQXcKVkFZRFZSMGdCRTB3U3pBSUJnWm5nUXdCQWdFd1B3WUxLd1lCQkFHQzN4TUJBUUV3TURBdUJnZ3JCZ0VGQlFjQwpBUllpYUhSMGNEb3ZMMk53Y3k1eWIyOTBMWGd4TG14bGRITmxibU55ZVhCMExtOXlaekE4QmdOVkhSOEVOVEF6Ck1ER2dMNkF0aGl0b2RIUndPaTh3WTNKc0xtbGtaVzUwY25WemRDNWpiMjB2UkZOVVVrOVBWRU5CV0RORFVrd3UKWTNKc01CMEdBMVVkRGdRV0JCU29TbXBqQkgzZHV1YlJPYmVtUldYdjg2anNvVEFOQmdrcWhraUc5dzBCQVFzRgpBQU9DQVFFQTNUUFhFZk5qV0RqZEdCWDdDVlcrZGxhNWNFaWxhVWNuZThJa0NKTHhXaDlLRWlrM0pIUlJIR0pvCnVNMlZjR2ZsOTZTOFRpaFJ6WnZvcm9lZDZ0aTZXcUVCbXR6dzNXb2RhdGcrVnlPZXBoNEVZcHIvMXdYS3R4OC8Kd0FwSXZKU3d0bVZpNE1GVTVhTXFyU0RFNmVhNzNNajJ0Y015bzVqTWQ2am1lV1VISzhzby9qb1dVb0hPVWd3dQpYNFBvMVFZeiszZHN6a0RxTXA0ZmtseEJ3WFJzVzEwS1h7UE1UWitzT1BBdmV5eGluZG1qa1c4bEd5K1FzUmxHClBmWitHNlo2aDdtamVtMFkraVdsa1ljVjRQSVdMMWl3Qmk4c2FDYkdTNWpOMnA4TStYK1E3VU5LRWtST2IzTjYKS09xa3FtNTdUSDJIM2VESkFrU25oNi9ETkZ1MFFnPT0KLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQo=  tls.key: LS0tLS1CRUdJTiBSU0EgUFJJVkFURSBLRVktLS0tLQpNSUlKS0FJQkFBS0NBZ0VBb2ZJV051MThhSnVPcnN3QmNkT2U4M3R1aldnZ2lReXRVVjFDajVYWG8zY0EzNS9mCnFBc0ZUeklEY3BSbGFMYnpId3V3Wmw5Y1IrMm5EQ1pTMjhWWFlpdytKRC9BemdrcXNMcUk2d2JRYXBzQmtZWFMKNG5PVStnOFIxWDBybnZySmJyQTV4cUlJYkozTTZqNUtNdnhGS28wRXdhc0FjY2JiVUY5bjB0Nkcza3huM1ltUgp6Tkd4eHVtdnhXams1Y2RJYzNrQzJPVW5GS0Y4bld6YlpvYmI2T09SdlJoSURZblh3OTF2MjFGMEJ2dENtRmNBCkRoNGpldSsvM0pUNUtwQmR2QUc4cjdpTXBuSEpoVTVqWGpNeU9HK0xuRy9ycm5HMkZpekdsdVB0MCs0ZStGUUkKcVljUFQzVzJNQXZkOXNBM0QxOGxRbzYzTS9mUzJiM0hhU2J0VjRvWmY1L3NPMk14RU9WelRHdTNDcWJONE5HKwpkTzJxejFhbExCaUZmVWM0R2ZRWlhGaUtoMXNrOXdCbnN5aGphRmZ1THpsdHEwODdJMkthK1ZUWlMxUEpSRWxnCm4zdTBjUWZoQ2N6UXllMndWOXpET2VWZTF5MGMvNnRFYmE2WUJ4ZHZkZzA4WkovRDBhMEtyb1pVSTFuUXNkSngKTytDc3U5REtiNE5nMkJidmRZWkdVYmtIK1JQNTRRR2tLVWdidXE1UjBtcjRTUjZVTCtETmYzcXpvNWt2YjF1UQpZcWloNlhQVUNVRU85M05TVjYxM1RJRU5NSnJiNWFsZEtSSE9mWlYvbTVBOGVTL2JsUVhyTldxQkcvTi9kbXJGCmMyZzI0YkRjd3k5cjMvcWQ3L0xNbGYxVFV3MkZzNHczbHZUckVqeXBYRmFDcFFEbGRzTElGMHFlSlZzQ0F3RUEKQVFLQ0FnQXY5Zk13UnpzTisrdlF4cWd5M3JwM1gzbkpOU3BWakVTVUVTdVNQSTFGWXd3R0xtSGRjWTRiK3pMYwpMeWl0VDJsSEszNE5nM1pmOHZrQzl5S1k1YVBRZGt2ZERtaDZYR3FoTmswd1ZhOUpzeWhPd2JSSHpuVXpiVjBaCnZkMDZVd2x1MTQvMHpLMzBCUFBYOTZTZjN1aFpCclIrNnJiUisxT2VSUE1KbDArWDdFYmliRWlhd1F1R1hsVHAKQVB5eE5FaTNzZ0h2M0VhcnJIdXNYNzNHYW5BY1U3RW9zRlUrZFRGSktEcGxXSVVsUUNwajFYZzF0aVZKMWxFYQo4Wit0UkY0T1BQRjFsUkZLaGU1cHBXSjJWbkVzRjVUZ09xRXc0NHBLbk80Zlo5ZGFhVzRRbTBxSmNtOU5XQTRoCndwSDA3czRmcGt6eG5qU1JsbmFDZDlyandGeVBsSkJzUXNhVlFFNzlpQzJZMTRnTk9KQ0xyMXRKSEQ2ODN3bW4KS3ZNOHZpOTdHTmIybXZHeWNtZnloNVpzTFBpTWNqOFFER3VWZU53dlNESXpybnhqVkZlc0liTWt5UlZRem9IVApTTHRQbXdVR3lwRHVrMDhaZytsT0lYOC85K3lqMER3MDRqenllTVptYlFVdkd2N2lNWjFUaHdaRHF1YkJXV3J4CmtYTmJwTG9BMGxrcHh5bjdGam9Ya20zM2ZKQURjd2xWSS82WFNrSm1FaFVlZmZnaFFSMGNyVGphQVd1Qkx2Qk0KT0s5aEEzT3RTN2F0S2FDb1lvSmRrYkpHQTdWdytNNzA4NEJOTGhxM1Fyckg4S3M3Z05pdC9NN3lxSnU1alBaZgo2SE1seHNyWU9NVUhuVlk4VDkwN0Q3cS9ORUNnRThzODhnZzAyQ3JNWTFqanE4UnBpUUtDQVFFQTE2UHJaMUEwClNISS83akdmS3BETkJzQ0xrVUFxRERKSzQ0dFdJYmJBUXFhRTN1eDh4bkFlU2NjSHozbS9ScEpPSGtteHZTZlgKbTJ1Wk8veGtNTWhYK2lwOHdFOHZibzR1enVNYitTSXE3bWpialJkK1JJczJ5NHJsZVQ2NGVjRWc4R2pZckExZgpiSEI0MmhQclVTcXpxUVIwOTZocm1Lb1diU0RDZDZwOUVNeWVzT3IwTjdtQmJYVVZPazJxZGtYRlZWbHBlUDdpClFxWGdRUUI0bHgzLzJJdlpBMlhJUXlQdGJ0RWVRbmgyQ3FNM2NDMzR0VEVjZ244K0VwNG9SWmkwTTBHaUY3bXgKOTEvZHY2THZlNTR5K1pON1lXd1NFQ09ubzd5bDlvTlBZVnVGMGRiMjh0elppMThCeHJTQ2JESE1XbExvUzhWNgpXTEo0OGlSODJDYkc1d0tDQVFFQXdFRjM4KzYyeDhDU2x0blZZNlJaN0J0NEdiNEJqVWhWYXZ0NFkxUGFlbXFNCjFidFVnR2JyUnBoNHFUSEFTckUwUUZLeVZKYnlCUkJyRHIxWHU4WWRSVXQzZC92VzlIR1dPd1BKdTN2M3pLbHMKQ2xsZnpFY3J5L1l2aHAzSzlEcGR6OE1icHdueW5xcGV6b0xMNlJpL3JnK0hyTzBueXd1RSt0T2xYVFo2eUtadApHWVdTSVBWaG00NUJkc2ZxUzhnYjVvbjA0bHh4bnhxVnJvN0c0TUR6cmVEYlFhaGdyS3VuRWxwajZ4eW1PVWpBCkdCZDR3QUVrUExxNUUrRWcreDY4TkRLVTYwK29ybFhLWVhDQm5HSFZOQ3BVcmswVXkrcHFZZmFEN3VuR2VzaHMKSEwra3lXbXl5a3ErTmNKbnRXMFNSNy9sU1IvZUFhVEZyVzZVaXV0RGJRS0NBUUVBemhRYU9PNmVPSW51N016QgpScVdCT3EyeDg4cjFKQmpBRnZzbkFpc3JTOGJsZmtGVTdXREdvVTB5K3FWb0ZhSm1RMjI4RFlCUS9YZnp4aTdxCjlPL1JuQU1VbTVoUlJQOWVYbHNPZGFXZ2o1em9ETXRoNFZHRnVUbHhHZERGN1oyU3hBMysyMVlnVm5xYUZCY3IKTUxOMVpOWWNqajJITGl1R0tSNUFtcW4wd2FRN0YrcENJQ3NKTkxqSzQ2QXJnc0lrMXU4TzdCSHgyeTI0eFlZVQp1SjV6emRmQU9nNEFONkhURzY5L2twaWFmb29DeGhNNDlyZ0xmZTdxUEZLbk8vTzJhckdUbmNiWi9BWEMzb3h4Ci81dHRMYlF6R2lSMGtyWHdWSHRKdys4elltQmIzL0RtcWF4RHZueTZMdEo5UGJiTmk1aGw1VnZCRTVqa0dzeWgKL3RQNEN3S0NBUUJ2R1dZb0lKcWZkRGxCMHovdEJOeXlCRzJ5OG9vVEN1blJtT0JKQmZ3TEllZWcyMUJKb3kveQo2OGxPZk9HU1NEVFp0dkEyMGNPcUNZTFVVYmFSWERzdUFCNVp4NzdBSTZPZEZ1Tk01S2FlTG9td3NWVWF4MFlYCjUzd3ZYcUFaNG1DejN4dnJ1MlBwTEtyOHk3anFTdEw1MHgra1hxZlFQaWZxaXNQVXlkYktmT0l2RFhFVWVyaWQKRytmWXJFNUkzS3JDM3BZVStUWmJ1eEVrZm4yUEEvSE5XVk5hN2VKdjVnSDJLU1gwaCtuRzBMT3hPRjhmRlluTApUbHdGa09OdU9xU254Vk1wYUM4aUQ1R1VIVi9JN3dBMTFRQjZlVEM3Wmd0ejhQRHM3MHN6U1A2dzNrNXIxaGpyCnJhV2RpMnBDL1hUQzRiR3VRQ3dhNXcwVTNBSWJCVGxCQW9JQkFEc1RONGhvclVHNWw3MXhLZk5ibVBTbDZ6RlIKYTJ4d2U2VVZPOVZzMFpHeEdLWWJSN1VuVDBDL1FqUiswS2JsbE9leDdFY3cyMklCcmFFVzBGbXpuVnoyUW9FNwpMUE5COXhyTTFEeE56UjZEbFBUeERMcEFGWVlUcm40SWY1cjFVdVdpc2lMdmd6T2xGTlVITnN5UFJIZWNGblhUCnNhTk9JWkgrQTJ5KzF3QWdpSFZIS2JPRGRHeVFQVlQ0TXFFWkJaY2pQcmRBekNKcnloSHlYdHBqRjFSdlFEYTMKTVM3U3JVTGM4djJGQWJ1VG1QZ2R1ZHBKd1Q4dENCa2VRKzZ4YmJWN3YrZzBEMG5EWFNIZFVwNXFyUzcrTnhtVwp4NWV4UHo1VENhYXcxSnkzWjRmT1MzMTV6eHJGdmRHTmhWRXhMMzRlUVlzOHRYN0N0VWxuWkNray9zYz0KLS0tLS1FTkQgUlNBIFBSSVZBVEUgS0VZLS0tLS0K---apiVersion: v1kind: Secretmetadata:  labels:    k8s-app: kubernetes-dashboard  name: kubernetes-dashboard-csrf  namespace: kubernetes-dashboardtype: Opaquedata:  csrf: ""---apiVersion: v1kind: Secretmetadata:  labels:    k8s-app: kubernetes-dashboard  name: kubernetes-dashboard-key-holder  namespace: kubernetes-dashboardtype: Opaque---kind: ConfigMapapiVersion: v1metadata:  labels:    k8s-app: kubernetes-dashboard  name: kubernetes-dashboard-settings  namespace: kubernetes-dashboard---kind: RoleapiVersion: rbac.authorization.k8s.io/v1metadata:  labels:    k8s-app: kubernetes-dashboard  name: kubernetes-dashboard  namespace: kubernetes-dashboardrules:  # Allow Dashboard to get, update and delete Dashboard exclusive secrets.  - apiGroups: [""]    resources: ["secrets"]    resourceNames: ["kubernetes-dashboard-key-holder", "kubernetes-dashboard-certs", "kubernetes-dashboard-csrf"]    verbs: ["get", "update", "delete"]    # Allow Dashboard to get and update 'kubernetes-dashboard-settings' config map.  - apiGroups: [""]    resources: ["configmaps"]    resourceNames: ["kubernetes-dashboard-settings"]    verbs: ["get", "update"]    # Allow Dashboard to get metrics.  - apiGroups: [""]    resources: ["services"]    resourceNames: ["heapster", "dashboard-metrics-scraper"]    verbs: ["proxy"]  - apiGroups: [""]    resources: ["services/proxy"]    resourceNames: ["heapster", "http:heapster:", "https:heapster:", "dashboard-metrics-scraper", "http:dashboard-metrics-scraper"]    verbs: ["get"]---kind: ClusterRoleapiVersion: rbac.authorization.k8s.io/v1metadata:  labels:    k8s-app: kubernetes-dashboard  name: kubernetes-dashboardrules:  # Allow Metrics Scraper to get metrics from the Metrics server  - apiGroups: ["metrics.k8s.io"]    resources: ["pods", "nodes"]    verbs: ["get", "list", "watch"]---apiVersion: rbac.authorization.k8s.io/v1kind: RoleBindingmetadata:  labels:    k8s-app: kubernetes-dashboard  name: kubernetes-dashboard  namespace: kubernetes-dashboardroleRef:  apiGroup: rbac.authorization.k8s.io  kind: Role  name: kubernetes-dashboardsubjects:  - kind: ServiceAccount    name: kubernetes-dashboard    namespace: kubernetes-dashboard---apiVersion: rbac.authorization.k8s.io/v1kind: ClusterRoleBindingmetadata:  name: kubernetes-dashboard  namespace: kubernetes-dashboardroleRef:  apiGroup: rbac.authorization.k8s.io  kind: ClusterRole  name: kubernetes-dashboardsubjects:  - kind: ServiceAccount    name: kubernetes-dashboard    namespace: kubernetes-dashboard---kind: DeploymentapiVersion: apps/v1metadata:  labels:    k8s-app: kubernetes-dashboard  name: kubernetes-dashboard  namespace: kubernetes-dashboardspec:  replicas: 1  revisionHistoryLimit: 10  selector:    matchLabels:      k8s-app: kubernetes-dashboard  template:    metadata:      labels:        k8s-app: kubernetes-dashboard    spec:      containers:        - name: kubernetes-dashboard          image: kubernetesui/dashboard:v2.0.0-beta4          imagePullPolicy: Always          ports:            - containerPort: 8443              protocol: TCP          args:            - --auto-generate-certificates            - --namespace=kubernetes-dashboard            - --token-ttl=43200            # Uncomment the following line to manually specify Kubernetes API server Host            # If not specified, Dashboard will attempt to auto discover the API server and connect            # to it. Uncomment only if the default does not work.            # - --apiserver-host=http://my-address:port          volumeMounts:            - name: kubernetes-dashboard-certs              mountPath: /certs              # Create on-disk volume to store exec logs            - mountPath: /tmp              name: tmp-volume          livenessProbe:            httpGet:              scheme: HTTPS              path: /              port: 8443            initialDelaySeconds: 30            timeoutSeconds: 30      volumes:        - name: kubernetes-dashboard-certs          secret:            secretName: kubernetes-dashboard-certs        - name: tmp-volume          emptyDir: {}      serviceAccountName: kubernetes-dashboard      # Comment the following tolerations if Dashboard must not be deployed on master      tolerations:        - key: node-role.kubernetes.io/master          effect: NoSchedule---kind: ServiceapiVersion: v1metadata:  labels:    k8s-app: dashboard-metrics-scraper  name: dashboard-metrics-scraper  namespace: kubernetes-dashboardspec:  ports:    - port: 8000      targetPort: 8000  selector:    k8s-app: dashboard-metrics-scraper---kind: DeploymentapiVersion: apps/v1metadata:  labels:    k8s-app: dashboard-metrics-scraper  name: dashboard-metrics-scraper  namespace: kubernetes-dashboardspec:  replicas: 1  revisionHistoryLimit: 10  selector:    matchLabels:      k8s-app: dashboard-metrics-scraper  template:    metadata:      labels:        k8s-app: dashboard-metrics-scraper    spec:      containers:        - name: dashboard-metrics-scraper          image: kubernetesui/metrics-scraper:v1.0.1          ports:            - containerPort: 8000              protocol: TCP          livenessProbe:            httpGet:              scheme: HTTP              path: /              port: 8000            initialDelaySeconds: 30            timeoutSeconds: 30          volumeMounts:          - mountPath: /tmp            name: tmp-volume      serviceAccountName: kubernetes-dashboard      # Comment the following tolerations if Dashboard must not be deployed on master      tolerations:        - key: node-role.kubernetes.io/master          effect: NoSchedule      volumes:        - name: tmp-volume          emptyDir: {}---apiVersion: extensions/v1beta1kind: Ingressmetadata:  labels:    k8s-app: kubernetes-dashboard  name: kubernetes-dashboard  namespace: kubernetes-dashboard  annotations:    kubernetes.io/ingress.class: traefik    traefik.ingress.kubernetes.io/frontend-entry-points: http,https    traefik.ingress.kubernetes.io/redirect-entry-point: httpsspec:  rules:  - host: csdd.xxxx.com    http:      paths:        - backend:            serviceName: kubernetes-dashboard            servicePort: 443  tls:   - secretName: dashboard-tls-cert # 创建kubernetes-dashboard 服务 kubectl apply -f kubernetes-dashboard.yaml # 创建kubernetes-dashboard token 登录#  生成tokenkubectl create sa dashboard-admin -n kube-system # 授权token 访问权限kubectl create clusterrolebinding dashboard-admin --clusterrole=cluster-admin --serviceaccount=kube-system:dashboard-admin# 获取token ADMIN_SECRET=$(kubectl get secrets -n kube-system | grep dashboard-admin | awk '{print $1}')# 获取dashboard.kubeconfig 使用token   值DASHBOARD_LOGIN_TOKEN=$(kubectl describe secret -n kube-system ${ADMIN_SECRET} | grep -E '^token' | awk '{print $2}')echo ${DASHBOARD_LOGIN_TOKEN}# 设置集群参数kubectl config set-cluster kubernetes \  --certificate-authority=/apps/work/k8s/cfssl/pki/k8s/k8s-ca.pem \  --embed-certs=true \  --server=${KUBE_APISERVER} \  --kubeconfig=dashboard.kubeconfig# 设置客户端认证参数，使用上面创建的 Tokenkubectl config set-credentials dashboard_user \  --token=${DASHBOARD_LOGIN_TOKEN} \  --kubeconfig=dashboard.kubeconfig# 设置上下文参数kubectl config set-context default \  --cluster=kubernetes \  --user=dashboard_user \  --kubeconfig=dashboard.kubeconfig# 设置默认上下文kubectl config use-context default --kubeconfig=dashboard.kubeconfig# 绑定hostshttps://csdd.xxxx.com/#/overview?namespace=default# kubernetes-dashboard 使用metrics 显示cpu内存资源 所有要部署metrics-server

metrics-server 部署

# 创建metrics-server-secrets.yaml# base64 加密cat metrics-server.pem|base64 | tr -d '\n'cat metrics-server-key.pem|base64 | tr -d '\n'vi metrics-server-secrets.yamlapiVersion: v1kind: Secretmetadata:  labels:    k8s-app: metrics-server  name: metrics-server-certs  namespace: kube-systemtype: Opaquedata:  metrics-server.pem: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUQ3VENDQXRXZ0F3SUJBZ0lVZmdGSjJSUTF6Y20ydndjazFQc1NzTXJtNnJJd0RRWUpLb1pJaHZjTkFRRUwKQlFBd2JqRUxNQWtHQTFVRUJoTUNRMDR4RWpBUUJnTlZCQWdUQ1VkMVlXNW5SRzl1WnpFU01CQUdBMVVFQnhNSgpSM1ZoYm1kYWFHOTFNUkF3RGdZRFZRUUtFd2RqYkhWemRHVnlNUkF3RGdZRFZRUUxFd2RqYkhWemRHVnlNUk13CkVRWURWUVFERXdwcmRXSmxjbTVsZEdWek1CNFhEVEU1TURreU1EQXhOREV3TUZvWERUSTVNRGt4TnpBeE5ERXcKTUZvd2NqRUxNQWtHQTFVRUJoTUNRMDR4RWpBUUJnTlZCQWdUQ1VkMVlXNW5SRzl1WnpFU01CQUdBMVVFQnhNSgpSM1ZoYm1kYWFHOTFNUkF3RGdZRFZRUUtFd2RqYkhWemRHVnlNUkF3RGdZRFZRUUxFd2RqYkhWemRHVnlNUmN3CkZRWURWUVFERXc1dFpYUnlhV056TFhObGNuWmxjakNDQVNJd0RRWUpLb1pJaHZjTkFRRUJCUUFEZ2dFUEFEQ0MKQVFvQ2dnRUJBTzIvZ1RTUERMOU94RUhmQTBtdmpFYWtLMUtrWmoxSDlMYXV0dFRQbS8xYlR5T2pSMFNxemFsZgpCSFJESlNkME5DOGlyZGRDcGtZWUVROVJBWWFIWldIb2ZLbDJ6eWpPSXZqc2JrVWJ0T2N4Q0R6dlNZcDBkaXRECnd4NGpuR2hmSGhrUUw1TWQwaEFQTm5rLzdkakxsQ0c2azlBN00wUHB6dEZZNWVOSWJmZG8wMFRMb2c1VjBIb3YKUTZ4TDRGQWwzbGhSRy9nRHYrdjBkSy9NVUJ3eWtaK2E1WlJJRk5hNE5sS2lac29QdVZ2WlpRek10VFJWLzBXbwp1NmlJZnVkcTB4eWk1dWIxSXFnbml0Wm5TN09ZeDBXWW0xV2VEZlIwbis1TU5oR3hTQ3c4bEM1QkVLU3I2a252Ckp6M0tHci95NzFrU2RsTWhHeVFxT09Ld2pkTmowQThDQXdFQUFhTi9NSDB3RGdZRFZSMFBBUUgvQkFRREFnV2cKTUIwR0ExVWRKUVFXTUJRR0NDc0dBUVVGQndNQkJnZ3JCZ0VGQlFjREFqQU1CZ05WSFJNQkFmOEVBakFBTUIwRwpBMVVkRGdRV0JCVHBBUm5VVmNOS1kzYWFOS0NNZXlNZnZ4aXFPREFmQmdOVkhTTUVHREFXZ0JRWkx0RG9rcm55CnJ4WUhEb2FZSlZQQVZKUDg0ekFOQmdrcWhraUc5dzBCQVFzRkFBT0NBUUVBT3AwWTg1ZjRCSXBscXQ2K3FDMFMKbm9DR2RiVVoyNUNpQ1hEUDBFTWNuQ1QzZzdiWTNGaUpSZDhkbGg1aVNncTRxUlR6N1hDeWhMb1VNVmJTRVVlZApvaEI2Vk90eGJ1azErKzhob2diUUtIQVZlbERjS1MyOGttclZHRXdsSGpjNG1pRUxIbHZ6K0xUYUZzdlYxMFlzCjROdmNMTEE3SVE5N3hMcFNvZURVek5LU2dzU3RQUmVSTW5DR1BMdE1MMFcxRHpjOHNFRWJsTm9sbEFYRWdKMU4KVkxmbHJxbWdxaTcyQmZHOTBFbEtmbVFMV3QwR0lKU0wwdmF2a0kxL0Z0NGFuV2RVT2Zqa3JmYzBpUnhxUVBVNQphRWN5cjF6MzhnZmR6c3ZtV1VuVnpMS0pNeGtPckV3NGZBWHJudmRFSm92YlhJeU9Rd2pqQjhNaklsWER2ZjRvCnN3PT0KLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQo=  metrics-server-key.pem: LS0tLS1CRUdJTiBSU0EgUFJJVkFURSBLRVktLS0tLQpNSUlFcFFJQkFBS0NBUUVBN2IrQk5JOE12MDdFUWQ4RFNhK01ScVFyVXFSbVBVZjB0cTYyMU0rYi9WdFBJNk5IClJLck5xVjhFZEVNbEozUTBMeUt0MTBLbVJoZ1JEMUVCaG9kbFllaDhxWGJQS000aStPeHVSUnUwNXpFSVBPOUoKaW5SMkswUERIaU9jYUY4ZUdSQXZreDNTRUE4MmVUL3QyTXVVSWJxVDBEc3pRK25PMFZqbDQwaHQ5MmpUUk11aQpEbFhRZWk5RHJFdmdVQ1hlV0ZFYitBTy82L1Iwcjh5UUhES1JuNXJsbEVnVTFyZzJVcUpteWcrNVc5bGxETXkxCk5GWC9SYWk3cUloKzUyclRIS0xtNXZVaXFDZUsxbWRMczVqSFJaaWJWWjROOUhTZjdrdzJFYkZJTER5VUxrRVEKcEt2cVNlOG5QY29hdi9MdldSSjJVeUViSkNvNDRyQ04wMlBRRHdJREFRQUJBb0lCQVFDV09RVW84cUo1VnduSApIV1QwY0VuUWNQYzIxczRMTnFZM3NCbXlTaVFrYUVlUEd5SnpEd0c0WFdOeEd1UWxFOVhOV3JwQlk4bXdUSkNxCi91Slo0TDk4cCt2dElEY3hiMTdGcm83V2QvVk1oN3pPMDl2QjhtaWdXY2EyQ29aUHBKcGQ5ODQzeFFYd1E4eUYKdkpGTEJRZHFjSHZwZlI2ZGNPVFBmcjV6YUZhamxvN3l4eWc5QTRpQ2xSRXUwNVAwNytTeWNiNndUbGJjVFlOeAo4ZU5DaUs5QTVCZXlYZVNmeEJJYmh4bHcrZ2kwdmV4RUxEOTJFWWZjVXNzZjlzT2sxZUhJVmpJNGZmQndodk5kCjhhbkVNRXJEd3pCc0ZGT2Z3Mkdya3hWSndoMXVEMkl4c253b3hjZ0RxWFI5dy9Wais3emhKZ2ZiWXFpT0FUVmUKQTZOeExHb3hBb0dCQVBhUk1ENHFqOGxwcEhnR3FjZWI0aHpaWkI4cG9maS9aUTIvNUl0aHlQV2FHMDlSRFkxZAowd1Z5dXFWb0RwalFWOENGU2U2RzZlQzNXSDRGVy9IMU9wKzVjSjBDT1B1MFJWMUV0eDlaL0I1U1Z2K3lQWU5RCm12Tm9qeWF2Mi9ZbnpuZDBGVkYrckJhT0lDZ0pxUVMvcnA0OGhmelVaWWsrNFlVVUdkeVBHUS85QW9HQkFQYlgKOGRKZjllcTgwN2lGTW9GTlFuODR4L1pUdVJ4N21obEJwaUhOa0tIUGZ1cHBCMFlMRDJQOWxhYkFGaWtZblY2SgpCQjZ1LzdMTm1mOFZHUjNVOEFiak0rZW43VkdVMkExcXIydTZ5ejRESWRTM040czRlVUw5QmZkYWQ4cGVkZis5Cks5SmQ3MmNSaFM2UUxYcmttZXZFOUpNYTF1VWpzeU1RaFQvK1g1LzdBb0dBWkt5ZGZSU1Z4eEJhZGlPS1dSVTkKK3JlTW1PaS8yTGdWUThyeFB6UDdBTVVlbDRFcHZtbnJ5cEt3d082KzN3aGFmQ0l3TUxObmRUaUhhbFUzMkpCZgprbTMrSEMyWEpMYlRoNlNSL0x3YUpDdE1tSFNuaHlGM1V5R0RLYkd1WjFDVGpkU1pDOEJqOVlXc2ZZeU1OWU1xCmdqT0dKZGgzYU5XQzhYcG1vTmJRemVVQ2dZRUF6M1VlWUZrV0xYc1YxZmJjUTUvVFMybEZaZGxuc25DUFNycksKRFk3ZkI1K0VZeTV5Vm9QbEkzeDAwZmlPcDJ0d2w0dEFVeWx3N2EydXg1dkx5QzYyckpNM2hISzJHZUttMGwvZgpud01XM2I5MEozcjB5NlZqQk5IeXViam5CTVh3RmtpL0U4YXU5a2piVGc4T3FrS0d1b2lGcFR6aGJ5Tlo0eFozClp4azY5UkVDZ1lFQW1qTVdrM0JlSy95ZVkrLytCcFVQVzRteWtlOXdHQnNIeUpPa0wyMllXT3lJYnJBMTdMY1QKZUFzNmtvRitMTjY3Q01UQk45Qm5zQnpYTURsQ3RhMjlCcDkyNWZraEpNU0VMajhWNUIwRmNVQXg2dlVEY2o4RAoxQlJBRWNpZjN5eFExd1ZkTUJnaFdEUkduZzFKREtRVXUwaWY1WkNDWkFyWnpCek95YUhreUE4PQotLS0tLUVORCBSU0EgUFJJVkFURSBLRVktLS0tLQo=# resource-reader.yamlvi resource-reader.yaml---apiVersion: rbac.authorization.k8s.io/v1kind: ClusterRolemetadata:  name: system:metrics-serverrules:- apiGroups:  - ""  resources:  - pods  - nodes  - nodes/stats  verbs:  - get  - list  - watch---apiVersion: rbac.authorization.k8s.io/v1kind: ClusterRoleBindingmetadata:  name: system:metrics-serverroleRef:  apiGroup: rbac.authorization.k8s.io  kind: ClusterRole  name: system:metrics-serversubjects:- kind: ServiceAccount  name: metrics-server  namespace: kube-system# metrics-server-service.yamlvi metrics-server-service.yaml---apiVersion: v1kind: Servicemetadata:  name: metrics-server  namespace: kube-system  labels:    kubernetes.io/name: "Metrics-server"spec:  selector:    k8s-app: metrics-server  ports:  - port: 443    protocol: TCP    targetPort: 443# metrics-apiservice.yamlvi metrics-apiservice.yaml---apiVersion: apiregistration.k8s.io/v1beta1kind: APIServicemetadata:  name: v1beta1.metrics.k8s.iospec:  service:    name: metrics-server    namespace: kube-system  group: metrics.k8s.io  version: v1beta1  insecureSkipTLSVerify: true  groupPriorityMinimum: 100  versionPriority: 100# auth-reader.yamlvi auth-reader.yaml---apiVersion: rbac.authorization.k8s.io/v1beta1kind: RoleBindingmetadata:  name: metrics-server-auth-reader  namespace: kube-systemroleRef:  apiGroup: rbac.authorization.k8s.io  kind: Role  name: extension-apiserver-authentication-readersubjects:- kind: ServiceAccount  name: metrics-server  namespace: kube-system#  auth-delegator.yamlvi auth-delegator.yaml---apiVersion: rbac.authorization.k8s.io/v1beta1kind: ClusterRoleBindingmetadata:  name: metrics-server:system:auth-delegatorroleRef:  apiGroup: rbac.authorization.k8s.io  kind: ClusterRole  name: system:auth-delegatorsubjects:- kind: ServiceAccount  name: metrics-server  namespace: kube-system# aggregated-metrics-reader.yamlvi aggregated-metrics-reader.yamlkind: ClusterRoleapiVersion: rbac.authorization.k8s.io/v1metadata:  name: system:aggregated-metrics-reader  labels:    rbac.authorization.k8s.io/aggregate-to-view: "true"    rbac.authorization.k8s.io/aggregate-to-edit: "true"    rbac.authorization.k8s.io/aggregate-to-admin: "true"rules:- apiGroups: ["metrics.k8s.io"]  resources: ["pods"]  verbs: ["get", "list", "watch"]# metrics-server-deployment.yamlvi  metrics-server-deployment.yaml---apiVersion: v1kind: ServiceAccountmetadata:  name: metrics-server  namespace: kube-system---apiVersion: apps/v1kind: Deploymentmetadata:  name: metrics-server  namespace: kube-system  labels:    k8s-app: metrics-serverspec:  selector:    matchLabels:      k8s-app: metrics-server  template:    metadata:      name: metrics-server      labels:        k8s-app: metrics-server    spec:      serviceAccountName: metrics-server      tolerations:        - effect: NoSchedule          key: node.kubernetes.io/unschedulable          operator: Exists        - key: NoSchedule          operator: Exists          effect: NoSchedule      volumes:      # mount in tmp so we can safely use from-scratch images and/or read-only containers      - name: tmp-dir        emptyDir: {}      - name: metrics-server-certs        secret:          secretName: metrics-server-certs      containers:      - name: metrics-server        image: juestnow/metrics-server-amd64:v0.3.4        imagePullPolicy: Always        command:        - /metrics-server        - --tls-cert-file=/certs/metrics-server.pem        - --tls-private-key-file=/certs/metrics-server-key.pem        - --kubelet-preferred-address-types=InternalIP,Hostname,InternalDNS,ExternalDNS,ExternalIP        - --kubelet-insecure-tls        volumeMounts:        - name: tmp-dir          mountPath: /tmp        - name: metrics-server-certs          mountPath: /certs# 创建metrics-server 服务# 创建metrics-server 服务kubectl apply -f .# 验证metrics-server kubectl top noderoot@]~/work]#kubectl top nodeNAME      CPU(cores)   CPU%   MEMORY(bytes)   MEMORY%master    162m         6%     957Mi           16%master2   155m         2%     591Mi           4%[root@]~/work]#kubectl top pods -ANAMESPACE              NAME                                         CPU(cores)   MEMORY(bytes)kube-system            coredns-9d5b6bdb6-phcwt                      6m           11Mikube-system            kube-flannel-ds-amd64-6bpf7                  3m           11Mikube-system            kube-flannel-ds-amd64-6sxz2                  4m           12Mikube-system            metrics-server-668c6bb96b-z7nfl              1m           14Mikube-system            traefik-zkhd2                                8m           23Mikube-system            traefik-zs6h3                                5m           21Mikubernetes-dashboard   dashboard-metrics-scraper-566cddb686-sft6k   7m           12Mikubernetes-dashboard   kubernetes-dashboard-6cd89cd7df-nlkzj        18m          29Mi