源码构建

将客户端的证书由1年到10年并生成需要镜像

--构建环境

root@op:~# apt install curl make make-guile docker.io

root@op:~# docker version

Client: Version:           19.03.6 API version:       1.40 Go version:        go1.12.17 Git commit:        369ce74a3c Built:             Fri Feb 28 23:45:43 2020 OS/Arch:           linux/amd64 Experimental:      false Server: Engine:  Version:          19.03.6  API version:      1.40 (minimum version 1.12)  Go version:       go1.12.17  Git commit:       369ce74a3c  Built:            Wed Feb 19 01:06:16 2020  OS/Arch:          linux/amd64  Experimental:     false containerd:  Version:          1.3.3-0ubuntu1~18.04.2  GitCommit: runc:  Version:          spec: 1.0.1-dev  GitCommit: docker-init:  Version:          0.18.0  GitCommit:       --源码

root@op:/opt# wget

root@op:/opt# tar zxvf v1.18.8+k3s1.tar.gz

---修改证书的时间

root@op:/opt/k3s-1.18.8-k3s1/vendor/github.com/rancher/dynamiclistener/cert# vim cert.go

# vim cert.go

  NotAfter:     time.Now().Add(duration365d).UTC(),   改为  NotAfter:     time.Now().Add(duration365d * 10).UTC(), 

root@op:/opt/k3s-1.18.8-k3s1# git init

root@op:/opt/k3s-1.18.8-k3s1# git add .

root@op:/opt/k3s-1.18.8-k3s1# git config --global user.name root

root@op:/opt/k3s-1.18.8-k3s1# git config --global user.email root@yesnocom.com

root@op:/opt/k3s-1.18.8-k3s1# git commit -m "init"

---构建完整版本的二进制文件及需要的镜像

root@op:/opt/k3s-1.18.8-k3s1# SKIP_VALIDATE=true make

root@op:/opt/k3s-1.18.8-k3s1/dist/artifacts# ls -la

total 397292drwxr-xr-x 2 root root      4096 Sep  7 04:42 .drwxr-xr-x 3 root root      4096 Sep  7 04:41 ..-rwxr-xr-x 1 root root  53448704 Sep  7 04:41 k3s-rw------- 1 root root 352955392 Sep  7 04:42 k3s-airgap-images-amd64.tar-rw-r--r-- 1 root root       272 Sep  7 04:42 k3s-images.txt

root@op:/opt/k3s-1.18.8-k3s1/dist/artifacts# ./k3s -v

k3s version v1.18.8+k3s-c8d17880 (c8d17880)

在3个server(u1/u2/u3)节点上：

mkdir -p /data/rancher/logs_k3s/pods

mkdir -p /data/rancher/logs_k3s/containers

mkdir -p /data/rancher/kubelet_k3s/kubelet

mkdir -p /data/rancher/data_k3s/rancher/k3s/agent/images

ln -s /data/rancher/data_k3s/rancher /var/lib/

ln -s /data/rancher/kubelet_k3s/kubelet /var/lib/

ln -s /data/rancher/logs_k3s/pods /var/log/

ln -s /data/rancher/logs_k3s/containers /var/log/

将构建好的二进制软件（k3s)分别分发到3个server节点上（u1/u2/u3）的/usr/local/bin目录中

root@u1:~# ls -l /usr/local/bin/k3s

-rwxr-xr-x 1 root root 53448704 Sep  7 08:41 /usr/local/bin/k3s

root@u2:~# ls -l /usr/local/bin/k3s

-rwxr-xr-x 1 root root 53448704 Sep  7 08:41 /usr/local/bin/k3s

root@u3:~# ls -l /usr/local/bin/k3s

-rwxr-xr-x 1 root root 53448704 Sep  7 08:41 /usr/local/bin/k3s

将构建好的需要的镜像包（k3s-airgap-images-amd64.tar）分别分发到3个server节点上（u1/u2/u3）的/var/lib/rancher/k3s/agent/images目录中

root@u1:~# ls -l /var/lib/rancher/k3s/agent/images/k3s-airgap-images-amd64.tar

-rw------- 1 root root 352955392 Sep  7 08:42 /var/lib/rancher/k3s/agent/images/k3s-airgap-images-amd64.tar

root@u2:~# ls -l /var/lib/rancher/k3s/agent/images/k3s-airgap-images-amd64.tar

-rw------- 1 root root 352955392 Sep  7 08:42 /var/lib/rancher/k3s/agent/images/k3s-airgap-images-amd64.tar

root@u3:~# ls -l /var/lib/rancher/k3s/agent/images/k3s-airgap-images-amd64.tar

-rw------- 1 root root 352955392 Sep  7 08:42 /var/lib/rancher/k3s/agent/images/k3s-airgap-images-amd64.tar

部署server节点上初始集群（u1/u2/u3）

root@u1:~# curl -sfL  | \> INSTALL_K3S_SKIP_DOWNLOAD=true \> INSTALL_K3S_EXEC=" \> server \> --write-kubeconfig-mode 644 \> --datastore-endpoint '; \> --datastore-cafile '/srv/etcd/pki/ca.crt' \> --datastore-certfile '/srv/etcd/pki/client.crt' \> --datastore-keyfile '/srv/etcd/pki/client.key' \> -t agent-secret \> --tls-san vip.yesnocom.com" \> sh -[INFO]  Skipping k3s download and verify[INFO]  Creating /usr/local/bin/kubectl symlink to k3s[INFO]  Creating /usr/local/bin/crictl symlink to k3s[INFO]  Creating /usr/local/bin/ctr symlink to k3s[INFO]  Creating killall script /usr/local/bin/k3s-killall.sh[INFO]  Creating uninstall script /usr/local/bin/k3s-uninstall.sh[INFO]  env: Creating environment file /etc/systemd/system/k3s.service.env[INFO]  systemd: Creating service file /etc/systemd/system/k3s.service[INFO]  systemd: Enabling k3s unitCreated symlink /etc/systemd/system/multi-user.target.wants/k3s.service → /etc/systemd/system/k3s.service.[INFO]  systemd: Starting k3s

root@u2:~# curl -sfL  | \> INSTALL_K3S_SKIP_DOWNLOAD=true \> INSTALL_K3S_EXEC=" \> server \> --write-kubeconfig-mode 644 \> --datastore-endpoint '; \> --datastore-cafile '/srv/etcd/pki/ca.crt' \> --datastore-certfile '/srv/etcd/pki/client.crt' \> --datastore-keyfile '/srv/etcd/pki/client.key' \> -t agent-secret \> --tls-san vip.yesnocom.com" \> sh -[INFO]  Skipping k3s download and verify[INFO]  Creating /usr/local/bin/kubectl symlink to k3s[INFO]  Creating /usr/local/bin/crictl symlink to k3s[INFO]  Creating /usr/local/bin/ctr symlink to k3s[INFO]  Creating killall script /usr/local/bin/k3s-killall.sh[INFO]  Creating uninstall script /usr/local/bin/k3s-uninstall.sh[INFO]  env: Creating environment file /etc/systemd/system/k3s.service.env[INFO]  systemd: Creating service file /etc/systemd/system/k3s.service[INFO]  systemd: Enabling k3s unitCreated symlink /etc/systemd/system/multi-user.target.wants/k3s.service → /etc/systemd/system/k3s.service.[INFO]  systemd: Starting k3s

root@u3:~# curl -sfL  | \> INSTALL_K3S_SKIP_DOWNLOAD=true \> INSTALL_K3S_EXEC=" \> server \> --write-kubeconfig-mode 644 \> --datastore-endpoint '; \> --datastore-cafile '/srv/etcd/pki/ca.crt' \> --datastore-certfile '/srv/etcd/pki/client.crt' \> --datastore-keyfile '/srv/etcd/pki/client.key' \> -t agent-secret \> --tls-san vip.yesnocom.com" \> sh -[INFO]  Skipping k3s download and verify[INFO]  Creating /usr/local/bin/kubectl symlink to k3s[INFO]  Creating /usr/local/bin/crictl symlink to k3s[INFO]  Creating /usr/local/bin/ctr symlink to k3s[INFO]  Creating killall script /usr/local/bin/k3s-killall.sh[INFO]  Creating uninstall script /usr/local/bin/k3s-uninstall.sh[INFO]  env: Creating environment file /etc/systemd/system/k3s.service.env[INFO]  systemd: Creating service file /etc/systemd/system/k3s.service[INFO]  systemd: Enabling k3s unitCreated symlink /etc/systemd/system/multi-user.target.wants/k3s.service → /etc/systemd/system/k3s.service.[INFO]  systemd: Starting k3s

root@u1:~# kubectl get node -o wide

NAME   STATUS   ROLES    AGE     VERSION        INTERNAL-IP      EXTERNAL-IP   OS-IMAGE            KERNEL-VERSION       CONTAINER-RUNTIMEu1     Ready    master   2m31s   v1.18.8+k3s-   192.168.100.11   <none>        Ubuntu 18.04.5 LTS   4.15.0-115-generic   containerd://1.3.3-k3s2u2     Ready    master   83s     v1.18.8+k3s-   192.168.100.12   <none>        Ubuntu 18.04.5 LTS   4.15.0-115-generic   containerd://1.3.3-k3s2u3     Ready    master   52s     v1.18.8+k3s-   192.168.100.13   <none>        Ubuntu 18.04.5 LTS   4.15.0-115-generic   containerd://1.3.3-k3s2

root@u1:~# kubectl get pod -A

NAMESPACE     NAME                                     READY   STATUS      RESTARTS   AGEkube-system   coredns-7944c66d8d-l7jtq                 1/1     Running     0          6m59skube-system   helm-install-traefik-hctws               0/1     Completed   0          6m59skube-system   local-path-provisioner-6d59f47c7-n9qjd   1/1     Running     0          6m59skube-system   metrics-server-7566d596c8-9wzsz          1/1     Running     0          6m59skube-system   svclb-traefik-47xws                      2/2     Running     0          5m43skube-system   svclb-traefik-sz8b8                      2/2     Running     0          6m40skube-system   svclb-traefik-wcn7s                      2/2     Running     0          5m13skube-system   traefik-758cd5fc85-f242c                 1/1     Running     0          6m40s

root@u1:~# kubectl get apiservices |grep 'metrics'

v1beta1.metrics.k8s.io                 kube-system/metrics-server   True        6m44s

root@u1:~# kubectl top node

NAME   CPU(cores)   CPU%   MEMORY(bytes)   MEMORY%u1     148m         7%     968Mi           49%u2     94m          4%     726Mi           36%u3     93m          4%     728Mi           36% 

HA部署（u1/u2/u3）

# apt install haproxy -y

# apt install keepalived -y

haproxy 配置（3个节点上【u1/u2/u3】配置文件相同）

# cat /etc/haproxy/haproxy.cfg

global    log /dev/log    local0    log /dev/log    local1 notice    chroot /var/lib/haproxy    stats socket /var/run/haproxy-admin.sock mode 660 level admin    stats timeout 30s    user  haproxy    group haproxy    daemon    nbproc 1 defaults    log     global    timeout connect 5000    timeout client  10m    timeout server  10m listen  admin_stats    bind 0.0.0.0:10080    mode http    log 127.0.0.1 local0 err    stats refresh 30s    stats uri /status    stats realm welcome login\ Haproxy    stats auth admin:Jieshi11gR2.    stats hide-version    stats admin if TRUE listen kube-master    bind 0.0.0.0:8443    mode tcp    option tcplog    balance source    server 192.168.100.91 192.168.100.11:6443 check inter 2000 fall 2 rise 2 weight 1    server 192.168.100.92 192.168.100.12:6443 check inter 2000 fall 2 rise 2 weight 1    server 192.168.100.93 192.168.100.13:6443 check inter 2000 fall 2 rise 2 weight 1

# systemctl restart haproxy.service

# ss -tunpla|cat |grep 8443

tcp   LISTEN   0   128   0.0.0.0:8443    0.0.0.0:*  users:(("haproxy",pid=9399,fd=9))

keepalived 配置文件（采用一主多备）

u1节点上：

root@u1:~# cat /etc/keepalived/keepalived.conf

global_defs {    router_id lb-master-105} vrrp_script check-haproxy {    script "killall -0 haproxy"    interval 5    weight -30} vrrp_instance VI-kube-master {    state MASTER    priority 120    dont_track_primary    interface ens33    virtual_router_id 68    advert_int 3    track_script {        check-haproxy    }    virtual_ipaddress {        192.168.100.88 dev ens33 label ens33:1    }} 

root@u1:~# systemctl restart keepalived.service

root@u1:~# ifconfig |grep -A 3 ens33:1

ens33:1: flags=4163<UP,BROADCAST,RUNNING,MULTICAST>  mtu 1500        inet 192.168.100.88  netmask 255.255.255.255  broadcast 0.0.0.0        ether 00:0c:29:61:7d:9a  txqueuelen 1000  (Ethernet)

备节点上（u2/u3 配置文件一样）

# cat /etc/keepalived/keepalived.conf

global_defs {    router_id lb-backup-105} vrrp_script check-haproxy {    script "killall -0 haproxy"    interval 5    weight -30} vrrp_instance VI-kube-master {    state BACKUP    priority 110    dont_track_primary    interface ens33    virtual_router_id 68    advert_int 3    track_script {        check-haproxy    }    virtual_ipaddress {        192.168.100.88 dev ens33 label ens33:1    }} # systemctl restart keepalived.service

# ps -ef |grep keep

root       7883      1  0 09:48 ?        00:00:00 /usr/sbin/keepalivedroot       7893   7883  0 09:48 ?        00:00:00 /usr/sbin/keepalivedroot       7895   7883  0 09:48 ?        00:00:00 /usr/sbin/keepalived