kubernetes 使用cri-o 替换docker

部署环境

# 操作系统：CentOS Linux release 8.0.1905 (Core)# kubelet 版本： v1.14.0# runc版本：v1.0.0-rc9# crictl 版本：v1.16.1#crio版本：v1.15.2#conmon版本：v2.0.1# cni版本：v0.8.2# 网络插件： kube-router#工作目录： /apps# 二进制文件目录： /usr/bin# cni 目录：/apps/cni# 工作节点：win10 on Ubuntu 19.10# 工作目录：/mnt/g/work/crio# 节点IP：192.168.2.196# 节点名：node05

准备所需二进制文件

mkdir -p /mnt/g/work/criocd /mnt/g/work/criowget https://github.com/opencontainers/runc/releases/download/v1.0.0-rc9/runc.amd64wget https://github.com/kubernetes-sigs/cri-tools/releases/download/v1.16.1/crictl-v1.16.1-linux-amd64.tar.gzwget https://github.com/cri-o/cri-o/releases/download/v1.15.2/crio-v1.15.2.tar.gzwget https://github.com/containernetworking/plugins/releases/download/v0.8.2/cni-plugins-linux-amd64-v0.8.2.tgzwget https://github.com/containers/conmon/archive/v2.0.1.zip

解压下载文件及编译源码

# conmon 官方不提供二进制下载所以要编译# 工作节点编译然后分发sudo apt-get install \  gcc \  git \  libc6-dev \  libglib2.0-dev \  pkg-config \  make \  unzip \  runc# 编译conmon# 解压conmon unzip  v2.0.1.zipcd conmon-2.0.1make# 二进制文件目录 ./bin/conmoncp -pdr ./bin/conmon ../rm -rf conmon-2.0.1  v2.0.1.zip # 解压cnimkdir ./cni/bincd ./cni/binmv ../../cni-plugins-linux-amd64-v0.8.2.tgz ./tar -xvf cni-plugins-linux-amd64-v0.8.2.tgzrm -rf cni-plugins-linux-amd64-v0.8.2.tgz# 回到最上级目录cd  ../../# 解压crictltar -xvf crictl-v1.16.1-linux-amd64.tar.gz rm -f crictl-v1.16.1-linux-amd64.tar.gz # 解压crio tar -xvf crio-v1.15.2.tar.gz cd crio-v1.15.2/bin/ mv crio-x86_64-static-glibc ../../crio rm -rf  crio-v1.15.2  crio-v1.15.2.tar.gz # 重命名runc.amd64 mv runc.amd64 runc chmod +x  runc

创建配置文件

cd /mnt/g/work/crio# 配置cni 测试配置文件 接入kubernetes  集群需要删除此cni 配置文件 mkdir -p ./etc/cni/net.d cat > ./etc/cni/net.d/87-bridge.conflist < ./etc/containers/policy.json < ./etc/containers/registries.conf < ./etc/crio/crio.conf# 修改生成配置vim  ./etc/crio/crio.conf# 取消注释修改存储目录root = "/apps/lib/containers/storage"# 取消注释修改容器运行存储目录runroot = "/apps/run/containers/storage"# 取消注释修改容器运行时文件打开数default_ulimits = [          "nofile=1024000:1024000",          "nproc=1024000:1024000",          "core=-1:-1",]#修改conmon 路径 默认conmon = "/usr/local/libexec/crio/conmon"conmon = "/usr/bin/conmon"# 修改pids_limit 默认1024pids_limit = 102400# pause_image 默认pause_image = "k8s.gcr.io/pause:3.1" 由于网络不通所有要修改pause_image = "docker.io/juestnow/pause-amd64:3.1"#修改cni 二进制路径plugin_dirs = [        "/apps/cni/bin/",# 创建crictl 配置cat >./etc/crictl.yaml <./crio.service <

分发文件

scp -r crio crictl 192.168.2.196:/usr/local/bin/scp conmon runc 192.168.2.196:/usr/bin/# 分发cniscp -r cni 192.168.2.196:/apps/# 分发配置文件scp -r etc 192.168.2.196:/# 分发启动文件scp  crio.service 192.168.2.196:/usr/lib/systemd/system/crio.service

启动crio 安装对应依赖

dnf -y install  epel-releasednf install   dnf-utils ipvsadm telnet wget net-tools conntrack ipset jq iptables curl sysstat libseccomp socat nfs-utils fuse fuse-devel  device-mapper# 启动 criosystemctl daemon-reloadsystemctl enable criosystemctl start crio[root@node05 ~]# systemctl status crio● crio.service - OCI-based implementation of Kubernetes Container Runtime Interface   Loaded: loaded (/usr/lib/systemd/system/crio.service; enabled; vendor preset: disabled)   Active: active (running) since Mon 2019-10-28 10:22:14 CST; 10h ago     Docs: https://github.com/github.com/cri-o/cri-o  Process: 714 ExecStartPre=/sbin/modprobe overlay (code=exited, status=0/SUCCESS)  Process: 659 ExecStartPre=/sbin/modprobe br_netfilter (code=exited, status=0/SUCCESS) Main PID: 726 (crio)    Tasks: 17 (limit: 49836)   Memory: 89.3M   CGroup: /system.slice/crio.service           └─726 /usr/local/bin/crio --log-level info# 创建测试应用cat > sandbox_config.json <

配置 kubelet以支持cri-o

vim /apps/kubernetes/conf/kubelet----------------------------------------------------------------------------------------------------------------------KUBELET_OPTS="--bootstrap-kubeconfig=/apps/kubernetes/conf/bootstrap.kubeconfig \              --fail-swap-on=false \              --network-plugin=cni --cni-conf-dir=/etc/cni/net.d --cni-bin-dir=/apps/cni/bin \              --kubeconfig=/apps/kubernetes/conf/kubelet.kubeconfig \              --address=192.168.2.196 \              --node-ip=192.168.2.196 \              --hostname-override=node05 \              --cluster-dns=10.64.0.2 \              --cluster-domain=cluster.local \              --authorization-mode=Webhook \              --authentication-token-webhook=true \              --client-ca-file=/apps/kubernetes/ssl/k8s/k8s-ca.pem \              --rotate-certificates=true \              --cgroup-driver=cgroupfs \              --allow-privileged=true \              --healthz-port=10248 \              --healthz-bind-address=192.168.2.196 \              --cert-dir=/apps/kubernetes/ssl \              --feature-gates=RotateKubeletClientCertificate=true,RotateKubeletServerCertificate=true \              --node-labels=node-role.kubernetes.io/k8s-node=true \              --serialize-image-pulls=false \              --enforce-node-allocatable=pods,kube-reserved,system-reserved \              --pod-manifest-path=/apps/work/kubernetes/manifests \              --runtime-cgroups=/systemd/system.slice/kubelet.service \              --kube-reserved-cgroup=/systemd/system.slice/kubelet.service \              --system-reserved-cgroup=/systemd/system.slice \              --root-dir=/apps/work/kubernetes/kubelet \              --log-dir=/apps/kubernetes/log \              --alsologtostderr=true \              --logtostderr=false \              --anonymous-auth=true \              --container-log-max-files=10 \              --container-log-max-size=100Mi \              --container-runtime=remote \              --container-runtime-endpoint=unix:///var/run/crio/crio.sock \              --containerd=unix:///var/run/crio/crio.sock \              --runtime-request-timeout=15m \              --image-gc-high-threshold=70 \              --image-gc-low-threshold=50 \              --kube-reserved=cpu=500m,memory=512Mi,ephemeral-storage=1Gi \              --system-reserved=cpu=1000m,memory=1024Mi,ephemeral-storage=1Gi \              --eviction-hard=memory.available<500Mi,nodefs.available<10% \              --serialize-image-pulls=false \              --sync-frequency=30s \              --resolv-conf=/etc/resolv.conf \              --pod-infra-container-image=docker.io/juestnow/pause-amd64:3.1 \              --image-pull-progress-deadline=30s \              --v=2 \              --event-burst=30 \              --event-qps=15 \              --kube-api-burst=30 \              --kube-api-qps=15 \              --max-pods=100 \              --pods-per-core=10 \              --read-only-port=0 \              --volume-plugin-dir=/apps/kubernetes/kubelet-plugins/volume"---------------------------------------------------------------------------------------------------------------------------------------------# 修改启动文件kubelet.servicevim /usr/lib/systemd/system/kubelet.service--------------------------------------------------------------------------------------------------------------------------------------------[Unit]Description=Kubernetes KubeletAfter=crio.serviceRequires=crio.service[Service]ExecStartPre=-/bin/mkdir -p /sys/fs/cgroup/hugetlb/systemd/system.slice/kubelet.serviceExecStartPre=-/bin/mkdir -p /sys/fs/cgroup/blkio/systemd/system.slice/kubelet.serviceExecStartPre=-/bin/mkdir -p /sys/fs/cgroup/cpuset/systemd/system.slice/kubelet.serviceExecStartPre=-/bin/mkdir -p /sys/fs/cgroup/devices/systemd/system.slice/kubelet.serviceExecStartPre=-/bin/mkdir -p /sys/fs/cgroup/net_cls,net_prio/systemd/system.slice/kubelet.serviceExecStartPre=-/bin/mkdir -p /sys/fs/cgroup/perf_event/systemd/system.slice/kubelet.serviceExecStartPre=-/bin/mkdir -p /sys/fs/cgroup/cpu,cpuacct/systemd/system.slice/kubelet.serviceExecStartPre=-/bin/mkdir -p /sys/fs/cgroup/freezer/systemd/system.slice/kubelet.serviceExecStartPre=-/bin/mkdir -p /sys/fs/cgroup/memory/systemd/system.slice/kubelet.serviceExecStartPre=-/bin/mkdir -p /sys/fs/cgroup/pids/systemd/system.slice/kubelet.serviceExecStartPre=-/bin/mkdir -p /sys/fs/cgroup/systemd/systemd/system.slice/kubelet.serviceEnvironmentFile=-/apps/kubernetes/conf/kubeletExecStart=/apps/kubernetes/bin/kubelet $KUBELET_OPTSRestart=on-failureKillMode=processLimitNOFILE=1024000LimitNPROC=1024000LimitCORE=infinityLimitMEMLOCK=infinity[Install]WantedBy=multi-user.target# 说明在使用docker 时可以不需要创建kubelet.service 目录# 使用crio 必须手动创建目录# 配置生效systemctl daemon-reload# 启动 crio.service systemctl start crio.service# 启动 kubelet systemctl restart kubelet.service# 创建cni 配置文件10-kuberouter.conflistcat > /etc/cni/net.d/10-kuberouter.conflist <        CentOS Linux 7 (Core)   3.10.0-1062.1.2.el7.x86_64    docker://19.3.3nginx-2   Ready    k8s-node     47d     v1.14.0   192.168.2.189           CentOS Linux 7 (Core)   5.1.0-1.el7.elrepo.x86_64     docker://18.9.6node01    Ready    k8s-master   194d    v1.14.0   192.168.2.253           CentOS Linux 7 (Core)   5.0.7-1.el7.elrepo.x86_64     docker://18.9.4node02    Ready    k8s-master   194d    v1.14.0   192.168.3.4             CentOS Linux 7 (Core)   5.0.7-1.el7.elrepo.x86_64     docker://18.9.4node03    Ready    k8s-node     47d     v1.14.0   192.168.2.165           CentOS Linux 7 (Core)   5.1.0-1.el7.elrepo.x86_64     docker://18.9.6node04    Ready    k8s-node     47d     v1.14.0   192.168.2.167           CentOS Linux 7 (Core)   5.1.14-1.el7.elrepo.x86_64    docker://18.9.6node05    Ready    k8s-node     5h34m   v1.14.0   192.168.2.196           CentOS Linux 8 (Core)   4.18.0-80.11.2.el8_0.x86_64   cri-o://1.15.2[root@node05 conf]#  crictl psCONTAINER           IMAGE                                                                                                         CREATED             STATE               NAME                ATTEMPT             POD ID5942e8c161029       docker.io/library/traefik@sha256:9a77d02ad23622cd85e38eec127a85110ad73ba8258d8b457cf17bd3ad1eeef0             3 hours ago         Running             traefik             0                   b03a2afb9dd919b23f9bc1d51a       docker.io/juestnow/process-exporter@sha256:8ef99f0488d8ea30770e048b4e4255a0ace7732422f243ab57fede0a003945bf   3 hours ago         Running             process-exporter    1                   7d133f4fba6c21e5a5a07676f1       docker.io/prom/node-exporter@sha256:b630fb29d99b3483c73a2a7db5fc01a967392a3d7ad754c8eccf9f4a67e7ee31          3 hours ago         Running             node-exporter       1                   cfa2a3d2b851c[root@node05 conf]#  crictl podsPOD ID              CREATED             STATE               NAME                      NAMESPACE           ATTEMPTb03a2afb9dd91       3 hours ago         Ready               traefik-f94cf4494-2n4x5   kube-system         0cfa2a3d2b851c       3 hours ago         Ready               node-exporter-rmv24       monitoring          17d133f4fba6c2       3 hours ago         Ready               process-exporter-hrbcv    monitoring          1# 容器能正常运行cri-o 替换docker 成功# kube-router 下载地址 https://github.com/cloudnativelabs/kube-router/releases/download/v0.3.2/kube-router_0.3.2_linux_amd64.tar.gz