$ vi config/cubectl.toml

[shared-storage]
install = true
storage-ip = "192.168.11.7"
private-ip = "172.16.11.7"
#volume-dir = "/data/storage"
#nfs_version = "4.1"

$ sudo bin/cubectl create -p ~/.ssh/id_rsa -u cocktail

$ sudo vi /var/lib/cubectl/cert/openssl.conf

# 예시
[ alt_names_registry ]
DNS.1 = localhost
DNS.2 = registry
DNS.3 = regi.acornsoft.io
IP.1 = 127.0.0.1
IP.2 = 192.168.77.154

$ sudo mkdir /etc/docker/certs.d/regi.acornsoft.io

$ sudo sed -i 's/192.168.77.154/regi.acornsoft.io/g' /var/lib/cubectl/harbor/harbor.yml

# 예시
hostname: regi.acornsoft.io

$ cd /var/lib/cubectl/harbor/
$ sudo ./install.sh

$ cd /var/lib/cubectl/harbor/common/config/core
$ vi env

# AS-IS
EXT_ENDPOINT=https://10.1.1.50

# TO-BE (접속될 접속주소, 예시:)
EXT_ENDPOINT=https://www.regi.acornsoft.io

$ sudo mkdir -p /etc/containerd/certs.d/regi.acornsoft.io
$ sudo cp -r /etc/containerd/certs.d/192.168.77.154/* /etc/containerd/certs.d/regi.acornsoft.io

$ sudo sed -i 's/192.168.77.154/regi.acornsoft.io/g' /etc/containerd/certs.d/regi.acornsoft.io/hosts.toml

$ sudo sed -i 's/192.168.77.154/regi.acornsoft.io/g' /etc/containerd/config.toml

$ sudo systemctl restart containerd

$ sudo sed -i '/ca =/d' /etc/containerd/certs.d/regi.acornsoft.io/hosts.toml

# 도메인주소로 변경하는 경우 ca 라인 삭제
server = "https://www.regi.acornsoft.io"

[host."https://www.regi.acornsoft.io"]

$ sudo find /etc/kubernetes -type f -exec sed -i 's/192.168.77.154/regi.acornsoft.io/g' {} +

# Control plane에서 IP가 변경되는 파일 목록

# Addon
/etc/kubernetes/addon/calico/calico.yaml
/etc/kubernetes/addon/metrics-server/metrics-server.yaml
# Static Pods
/etc/kubernetes/manifests/kube-apiserver.yaml
/etc/kubernetes/manifests/kube-controller-manager.yaml
/etc/kubernetes/manifests/kube-scheduler.yaml
# Coredns 및 기타 yaml
/etc/kubernetes/addon/test/nginx.yaml
/etc/kubernetes/addon/test/dnsutils.yaml
/etc/kubernetes/kubeadm.yaml

# Worker nodes에서 IP가 변경되는 파일 목록

# Static Pods
/etc/kubernetes/manifests/haproxy.yaml

$ sed -i 's/192.168.77.154/regi.acornsoft.io/g' /etc/apt/sources.list.d/local-repo.list

$ sudo mkdir /etc/docker/certs.d/regi.acornsoft.io

$ kubectl apply -f /etc/kubernetes/addon/calico/calico.yaml
$ kubectl apply -f /etc/kubernetes/addon/metrics-server/metrics-server.yaml

$ kubectl edit daemonset -n kube-system kube-proxy

# 예시
image: regi.acornsoft.io/registry.k8s.io/kube-proxy:v1.26.7

$ kubectl edit deployment -n kube-system coredns

# 예시
image: regi.acornsoft.io/docker.io/coredns/coredns:1.9.3

$ sudo systemctl restart kubelet

$ sudo crictl images

# 예시
ubuntu@master-1:~$ sudo crictl images
IMAGE                                                          TAG                 IMAGE ID            SIZE
regi.acornsoft.io/docker.io/calico/cni                         v3.26.1             9dee260ef7f59       93.4MB
192.168.77.154/docker.io/calico/cni                            v3.26.1             9dee260ef7f59       93.4MB
regi.acornsoft.io/docker.io/calico/node                        v3.26.1             8065b798a4d67       86.6MB
192.168.77.154/docker.io/calico/node                           v3.26.1             8065b798a4d67       86.6MB
regi.acornsoft.io/registry.k8s.io/kube-apiserver               v1.26.7             6ac727c486d08       36.1MB
192.168.77.154/registry.k8s.io/kube-apiserver                  v1.26.7             6ac727c486d08       36.1MB
regi.acornsoft.io/registry.k8s.io/kube-controller-manager      v1.26.7             17314033c0a0b       32.8MB
192.168.77.154/registry.k8s.io/kube-controller-manager         v1.26.7             17314033c0a0b       32.8MB
regi.acornsoft.io/registry.k8s.io/kube-proxy                   v1.26.7             1e7eac3bc5c0b       21.8MB
192.168.77.154/registry.k8s.io/kube-proxy                      v1.26.7             1e7eac3bc5c0b       21.8MB
regi.acornsoft.io/registry.k8s.io/kube-scheduler               v1.26.7             c1902187a39f8       17.8MB
192.168.77.154/registry.k8s.io/kube-scheduler                  v1.26.7             c1902187a39f8       17.8MB
regi.acornsoft.io/registry.k8s.io/pause                        3.9                 e6f1816883972       319kB
192.168.77.154/registry.k8s.io/pause                           3.9                 e6f1816883972       319kB
192.168.77.154/docker.io/coredns/coredns                       1.9.3               5185b96f0becf       14.8MB
192.168.77.154/registry.k8s.io/metrics-server/metrics-server   v0.6.1              e57a417f15d36       28.1MB

$ cubectl config sync -p <ssh key> -u <username>

$ cubectl update-kubeconfig -p <ssh key> -u <username>

$ vi config/cubectl.toml

[kubernetes]
version = "v1.24"

[node-pool.master]
ip = ["10.30.30.10","10.30.30.11","10.30.30.12"]

[node-pool.node]
ip = ["10.30.31.11","10.30.31.12"]

$ bin/cubectl upgrade -p ~/.ssh/id_rsa -u cocktail --kubeconfig config/acloud-client-kubeconfig

## step_1. 마스터 정보 확인
$ kubectl -n kube-public get cm cluster-info -o yaml | grep server

$ bin/cubectl config sync -p <ssh key> -u <username>

$ bin/cubectl update-kubeconfig -p <ssh key> -u <username>

$ bin/cubectl update -p <ssh key> -u <username> --kubeconfig $(pwd)/config/acloud-client-kubeconfig

$ bin/cubectl config sync -p <ssh key> -u <username>

$ bin/cubectl update-kubeconfig -p <ssh key> -u <username>

$ bin/cubectl update -p <ssh key> -u <username> --kubeconfig $(pwd)/config/acloud-client-kubeconfig

$ tree archive

archive
├── cubectl-image-v1.2.2.tgz
├── push                                                    # 새로 생성된 디렉토리
│   └── registry
└── runtime
    ├── podman-v4.4.2-linux.amd64.tgz
    ├── harbor-offline-installer-v2.7.4-linux-amd64.tgz
    └── regctl-linux-amd64.tgz

...

$ tree archive/push/registry

archive/push/registry
├── docker.io_library_golang:1.22-alpine.tar
├── docker.io_library_python:alpine3.19.tgz
├── docker.io_library_tomcat:9.0.96.tgz
└── quay.io_rockylinux_rockylinux:9.3-minimal.tgz

$ sudo bin/cubectl push registry -p ~/.ssh/id_rsa -u cocktail

$ docker login {{ registry_domain }} -u {{ id }} -p {{ password }}
$ regctl image import {{ registry_domain }}/docker.io/library/tomcat:9.0.96 {{ Image file path }}

$ cat <<EOF> config/cubectl.toml
[cubectl]
closed-network = true
local-repository-install = true

[kubernetes]
api-sans=["192.168.77.112"]

[node-pool.master]
ip = ["10.30.30.10"]

[node-pool.node]
ip = ["10.30.30.11"]

[private-registry]
install=true
registry-ip="10.30.30.220"
EOF

$ sudo bin/cubectl create -p ~/.ssh/id_rsa -u cocktail

$ export KUBECONFIG=$(pwd)/config/acloud-client-kubeconfig

# 또는

$ sudo cp -i config/acloud-client-kubeconfig $HOME/.kube/config
$ sudo chown $(id -u):$(id -g) $HOME/.kube/config

# 노드 확인
$ kubectl get nodes

$ cat /usr/lib/systemd/system/kubelet.service.d/10-kubeadm.conf
# Note: This dropin only works with kubeadm and kubelet v1.11+
[Service]
Environment="KUBELET_KUBECONFIG_ARGS=--bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf --kubeconfig=/etc/kubernetes/kubelet.conf"
Environment="KUBELET_CONFIG_ARGS=--config=/var/lib/kubelet/config.yaml"
# This is a file that "kubeadm init" and "kubeadm join" generates at runtime, populating the KUBELET_KUBEADM_ARGS variable dynamically
EnvironmentFile=-/var/lib/kubelet/kubeadm-flags.env
# This is a file that the user can use for overrides of the kubelet args as a last resort. Preferably, the user should use
# the .NodeRegistration.KubeletExtraArgs object in the configuration files instead. KUBELET_EXTRA_ARGS should be sourced from this file.
EnvironmentFile=-/etc/default/kubelet
ExecStart=
ExecStart=/usr/bin/kubelet $KUBELET_KUBECONFIG_ARGS $KUBELET_CONFIG_ARGS $KUBELET_KUBEADM_ARGS $KUBELET_EXTRA_ARGS
StandardOutput=append:[원하는 경로]/[파일 명]
StandardError=append:[원하는 경로]/[파일 명]