Kubernetes部署

一.构建harbor镜像仓库

1.安装docker

[root@harbor ~]# cat > /etc/yum.repos.d/docker.repo <<EOF
[docker]
name = docker
baseurl = https://mirrors.aliyun.com/docker-ce/linux/rhel/9.6/x86_64/stable/
gpgcheck = 0
EOF


[root@harbor ~]# dnf  install docker-ce-3:28.5.2-1.el9  -y
[root@harbor ~]# echo br_netfilter > /etc/modules-load.d/docker_mod.conf
[root@harbor ~]# modprobe -a br_netfilter
[root@harbor ~]# vim /etc/sysctl.d/docker.conf
net.bridge.bridge-nf-call-iptables = 1
net.bridge.bridge-nf-call-ip6tables = 1
net.ipv4.ip_forward = 1
[root@harbor ~]# sysctl  --system

[root@harbor ~]# vim /lib/systemd/system/docker.service
ExecStart=/usr/bin/dockerd -H fd:// --containerd=/run/containerd/containerd.sock --iptables=true
[root@harbor ~]# systemctl daemon-reload
[root@harbor ~]# systemctl enable --now docker

2.生成key

[root@harbor ~]# mkdir /data/certs -p
[root@harbor ~]# mkdir /data/certs -p
[root@harbor ~]# openssl req -newkey  rsa:4096 \
-nodes -sha256 -keyout /data/certs/timinglee.org.key \
-addext "subjectAltName = DNS:reg.timinglee.org" \
-x509 -days 365 -out /data/certs/timinglee.org.crt

You are about to be asked to enter information that will be incorporated
into your certificate request.
What you are about to enter is what is called a Distinguished Name or a DN.
There are quite a few fields but you can leave some blank
For some fields there will be a default value,
If you enter '.', the field will be left blank.
-----
Country Name (2 letter code) [XX]:CN
State or Province Name (full name) []:Shannxi
Locality Name (eg, city) [Default City]:Xi'an
Organization Name (eg, company) [Default Company Ltd]:kubernetes
Organizational Unit Name (eg, section) []:harbor
Common Name (eg, your name or your server's hostname) []:reg.timinglee.org
Email Address []:admin@timinglee.org

3.编辑harbor配置文件

[root@harbor ~]# tar zxf  harbor-offline-installer-v2.5.4.tgz -C /opt/
[root@harbor ~]# cd /opt/harbor/
[root@harbor harbor]# ls
common.sh  harbor.v2.5.4.tar.gz  harbor.yml.tmpl  install.sh  LICENSE  prepare
[root@harbor harbor]# cp harbor.yml.tmpl harbor.yml
[root@harbor harbor]# vim harbor.yml
hostname: reg.timinglee.org
  certificate: /data/certs/timinglee.org.crt
  private_key: /data/certs/timinglee.org.key
harbor_admin_password: lee


[root@harbor harbor]# ./install.sh --with-chartmuseum

4.启动并验证

[root@harbor harbor]# mkdir  /etc/docker/certs.d/reg.timinglee.org/ -p
[root@harbor harbor]# cp /data/certs/timinglee.org.crt  /etc/docker/certs.d/reg.timinglee.org/ca.crt
[root@harbor harbor]# vim /etc/hosts
127.0.0.1   localhost localhost.localdomain localhost4 localhost4.localdomain4
::1         localhost localhost.localdomain localhost6 localhost6.localdomain6
172.25.254.200     harbor reg.timinglee.org


[root@harbor harbor]# systemctl restart docker
[root@harbor harbor]# docker compose up -d

[root@harbor harbor]# docker login  reg.timinglee.org -u admin
Password:

WARNING! Your credentials are stored unencrypted in '/root/.docker/config.json'.
Configure a credential helper to remove this warning. See
https://docs.docker.com/go/credential-store/

Login Succeeded

二.构建部署kubernetes所需主机

1.所有主机配置

关闭swap

systemctl disable --now  swap.target
systemctl mask swap.target
sed '/swap/s/^/#/g' -i /etc/fstab

安装docker

[root@harbor ~]# cat > /etc/yum.repos.d/docker.repo <<EOF
[docker]
name = docker
baseurl = https://mirrors.aliyun.com/docker-ce/linux/rhel/9.6/x86_64/stable/
gpgcheck = 0
EOF


[root@harbor ~]# dnf  install docker-ce-3:28.5.2-1.el9  -y
[root@harbor ~]# echo br_netfilter > /etc/modules-load.d/docker_mod.conf
[root@harbor ~]# modprobe -a br_netfilter
[root@harbor ~]# vim /etc/sysctl.d/docker.conf
net.bridge.bridge-nf-call-iptables = 1
net.bridge.bridge-nf-call-ip6tables = 1
net.ipv4.ip_forward = 1
[root@harbor ~]# sysctl  --system

[root@harbor ~]# vim /lib/systemd/system/docker.service
ExecStart=/usr/bin/dockerd -H fd:// --containerd=/run/containerd/containerd.sock --iptables=true
[root@harbor ~]# systemctl daemon-reload
[root@harbor ~]# systemctl enable --now docker

配置可以使用harbor仓库

mkdir  /etc/docker/certs.d/reg.timinglee.org/ -p
#在harbor主机中分发证书到所有主机
[root@harbor ~]# for i in 100 10 20
> do
> scp /data/certs/timinglee.org.crt root@172.25.254.$i:/etc/docker/certs.d/reg.timinglee.org/ca.crt
> done

systemctl enable  docker
systemctl restart docker

所有主机配置docker加速器

cat >/etc/docker/daemon.json <<EOF
{
  "registry-mirrors":["https://reg.timinglee.org"]
}
EOF
systemctl restart docker

docker info
可以看到
Registry Mirrors:
  https://reg.timinglee.org/

所有主机彼此建立解析

 vim /etc/hosts

127.0.0.1   localhost localhost.localdomain localhost4 localhost4.localdomain4
::1         localhost localhost.localdomain localhost6 localhost6.localdomain6
172.25.254.100     master
172.25.254.10      node1
172.25.254.20      node2
172.25.254.200     reg.timinglee.org

所有主机配置kubernetes安装源

 vim /etc/yum.repos.d/kubernetes.repo
[kubernetes]
name = kubernetes
baseurl = https://mirrors.aliyun.com/kubernetes-new/core/stable/v1.35/rpm/
gpgcheck = 0

#检测
dnf list kubelet

以上操作完成后重启主机检测swap分区

swapon  -s  没有任何输出表示ok

三.kubernetes的部署

1.安装cri-dockerd（所有主机中安装）

[root@master ~]# ls
anaconda-ks.cfg  cri-dockerd-0.3.14-3.el8.x86_64.rpm  libcgroup-0.41-19.el8.x86_64.rpm
[root@master ~]# rpm -ivh *.rpm
警告：libcgroup-0.41-19.el8.x86_64.rpm: 头V4 RSA/SHA256 Signature, 密钥 ID 6d745a60: NOKEY
Verifying...                          ################################# [100%]
准备中...                          ################################# [100%]
正在升级/安装...
   1:libcgroup-0.41-19.el8            ################################# [ 50%]
   2:cri-dockerd-3:0.3.14-3.el8       ################################# [100%]

systemctl daemon-reload

#修改这个文件
[root@master ~]# vim /usr/lib/systemd/system/docker.service
ExecStart=/usr/bin/dockerd -H fd:// -H unix:///var/run/docker.sock --containerd=/run/containerd/containerd.sock
#

vim /lib/systemd/system/cri-docker.service
ExecStart=/usr/bin/cri-dockerd --container-runtime-endpoint fd:// --network-plugin=cni --pod-infra-container-image=reg.timinglee.org/k8s/pause:3.10

systemctl enable --now cri-docker
ll /var/run/cri-dockerd.sock

2.安装构建kubernetes 集群所需软件

master节点

dnf install kubelet kubeadm kubectl -y
systemctl enable --now kubelet.service

node节点

dnf install kubelet kubeadm  -y
systemctl enable --now kubelet.service

master节点中 kubectl 和kubeadm 补齐

[root@master ~]# echo "source <(kubectl completion bash)" >> ~/.bashrc
[root@master ~]# echo "source <(kubeadm completion bash)" >> ~/.bashrc
[root@master ~]# source  ~/.bashrc

3.下载kubernetes集群所需镜像

下载镜像

# 先解锁文件（防止之前有锁）
chattr -i /etc/resolv.conf
# 写入114 DNS
cat > /etc/resolv.conf << EOF
nameserver 114.114.114.114
nameserver 114.114.115.115
EOF
# 给文件上锁，防止被NetworkManager覆盖
chattr +i /etc/resolv.conf


[root@master ~]# kubeadm config images pull \
--image-repository registry.aliyuncs.com/google_containers \
--kubernetes-version v1.35.3 \
--cri-socket=unix:///var/run/cri-dockerd.sock
[config/images] Pulled registry.aliyuncs.com/google_containers/kube-apiserver:v1.35.3
[config/images] Pulled registry.aliyuncs.com/google_containers/kube-controller-manager:v1.35.3
[config/images] Pulled registry.aliyuncs.com/google_containers/kube-scheduler:v1.35.3
[config/images] Pulled registry.aliyuncs.com/google_containers/kube-proxy:v1.35.3
[config/images] Pulled registry.aliyuncs.com/google_containers/coredns:v1.13.1
[config/images] Pulled registry.aliyuncs.com/google_containers/pause:3.10.1
[config/images] Pulled registry.aliyuncs.com/google_containers/etcd:3.6.6-0

上传镜像到本地harbor

[root@master ~]# vi /etc/docker/daemon.json
{
  "registry-mirrors": ["https://reg.timinglee.org"],
  "insecure-registries": ["reg.timinglee.org"]
}

[root@master ~]# docker  login  reg.timinglee.org -u admin
Password:

WARNING! Your credentials are stored unencrypted in '/root/.docker/config.json'.
Configure a credential helper to remove this warning. See
https://docs.docker.com/go/credential-store/

Login Succeeded


[root@master ~]# docker images  --format  "{{.Repository}}:{{.Tag}}" | awk -F "/" '/google/{system("docker tag "$0" reg.timinglee.org/k8s/"$3)}'

[root@master ~]# docker images  --format  "{{.Repository}}:{{.Tag}}" | awk -F "/" '/timinglee/{system("docker push "$0)}'

4.在master中初始化kubernetes集群

在master中完成集群初始化

[root@master ~]# kubeadm init --pod-network-cidr=10.244.0.0/16 \
--image-repository reg.timinglee.org/k8s \
--kubernetes-version v1.35.3 \
--cri-socket=unix:///var/run/cri-dockerd.sock
。。。

#每个人不一样，其他主机加入本集群的凭证
kubeadm join 172.25.254.100:6443 --token tdwjoc.8d1yw3wl4r4tm6c4 \
        --discovery-token-ca-cert-hash sha256:6b5950ef2cdba85d6dfdb564ee90d4187fa3d341767dc9852cbdd5c9dee4f927
 
#如果忘记
[root@master ~]#  kubeadm token create --print-join-command
kubeadm join 172.25.254.100:6443 --token fgdf84.c89sx9rd7919gwvl --discovery-token-ca-cert-hash sha256:4a8a55f3f52a432969c716284e5888e63ec240487d30fcf984e2a44888b661ba



#如果初始化出问题
[root@master ~]# kubeadm reset  --cri-socket=unix:///var/run/cri-dockerd.sock #可以重置集群设定

[root@master ~]#  kubeadm token create --print-join-command
kubeadm join 172.25.254.100:6443 --token fgdf84.c89sx9rd7919gwvl --discovery-token-ca-cert-hash sha256:4a8a55f3f52a432969c716284e5888e63ec240487d30fcf984e2a44888b661ba

添加kubernets环境变量到本机

[root@master ~]# echo "export KUBECONFIG=/etc/kubernetes/admin.conf" > ~/.bash_profile
[root@master ~]# source  ~/.bash_profile
[root@master ~]# kubectl get nodes
NAME     STATUS     ROLES           AGE    VERSION
master   NotReady   control-plane   102s   v1.35.3

添加node节点到本集群

[root@node1 ~]# kubeadm token create --print-join-command
kubeadm join 172.25.254.100:6443 --token fgdf84.c89sx9rd7919gwvl --discovery-token-ca-cert-hash sha256:4a8a55f3f52a432969c716284e5888e63ec240487d30fcf984e2a44888b661ba --cri-socket=unix:///var/run/cri-dockerd.sock

#测试
[root@master ~]# kubectl get nodes		#可以看到集群中主机但是因为网络插件问题状态是NotReady
NAME     STATUS     ROLES           AGE     VERSION
master   NotReady   control-plane   5m16s   v1.35.3
node1    NotReady   <none>          66s     v1.35.3
node2    NotReady   <none>          58s     v1.35.3

5.安装网络插件

[root@master ~]# ls
anaconda-ks.cfg                      flannel-0.28.1.tar  libcgroup-0.41-19.el8.x86_64.rpm
cri-dockerd-0.3.14-3.el8.x86_64.rpm  kube-flannel.yml
[root@master ~]# docker load  -i flannel-0.28.1.tar


[root@master ~]# docker tag ghcr.io/flannel-io/flannel-cni-plugin:v1.9.0-flannel1 reg.timinglee.org/flannel-io/flannel-cni-plugin:v1.9.0-flannel1
[root@master ~]# docker push  reg.timinglee.org/flannel-io/flannel-cni-plugin:v1.9.0-flannel1
The push refers to repository [reg.timinglee.org/flannel-io/flannel-cni-plugin]
2c8aa52d4746: Pushed
5aa68bbbc67e: Pushed
v1.9.0-flannel1: digest: sha256:b3d30c221113b30fea3e8a7fccb145e929b097d0319b9eeb6b5a591b10b5c671 size: 739

[root@master ~]# docker tag ghcr.io/flannel-io/flannel:v0.28.1 reg.timinglee.org/flannel-io/flannel:v0.28.1
[root@master ~]# docker push reg.timinglee.org/flannel-io/flannel:v0.28.1
The push refers to repository [reg.timinglee.org/flannel-io/flannel]
5668da16a30b: Pushed
5f70bf18a086: Pushed
00012e17b6cc: Pushed
9738bb9596cf: Pushed
b6233bc105d7: Pushed
03767449b95b: Pushed
9a7d8cf9ff51: Pushed
13a60c10faeb: Pushed
70dc5e033175: Pushed
256f393e029f: Pushed
v0.28.1: digest: sha256:e671adbc267460164555159210066d3304a43e3b5dd85cc0b5b6ad62e83aab52 size: 2414


[root@master ~]# vim kube-flannel.yml
        image: flannel-io/flannel:v0.28.1
        image: flannel-io/flannel-cni-plugin:v1.9.0-flannel1
        image: flannel-io/flannel:v0.28.1

[root@master ~]# kubectl apply -f kube-flannel.yml
namespace/kube-flannel created
serviceaccount/flannel created
clusterrole.rbac.authorization.k8s.io/flannel created
clusterrolebinding.rbac.authorization.k8s.io/flannel created
configmap/kube-flannel-cfg created
daemonset.apps/kube-flannel-ds created

#测试
[root@master ~]# kubectl get  nodes
NAME     STATUS   ROLES           AGE     VERSION
master   Ready    control-plane   13m     v1.35.3
node1    Ready    <none>          9m44s   v1.35.3
node2    Ready    <none>          9m36s   v1.35.3