Loading...
Loading...
Loading...
Loading...
Loading...
Loading...
Loading...
Loading...
Loading...
Loading...
Loading...
Loading...
Loading...
Loading...
Loading...
Loading...
Loading...
Loading...
Loading...
Loading...
Loading...
Loading...
Loading...
Loading...
Loading...
Loading...
Loading...
Loading...
Loading...
Loading...
Loading...
Loading...
Loading...
Loading...
Loading...
Loading...
Loading...
Loading...
Loading...
local-repository
├── install
├── ip
├── private-ip
├── service-port
└── data-dirkubernetes
├── version
├── kube-proxy-mode
├── service-cidr
├── pod-cidr
├── node-port-range
├── audit-log-enable
├── api-sans
└── calico
└── vxlan-modenode-pool
├── data-dir
├── ssh-port
├── master
│ ├── ip
│ ├── private-ip
│ ├── lb-ip
│ └── lb-port
└── node
├── ip
└── private-ip/data/
├── backup
│ ├── cluster
│ ├── db
│ └── registry
├── containerd : containerd's persistent data location
├── etcd : etcd data location
│ └── member
├── k8s-audit : apiserver audit logs
├── kubelet : kubelet root-dir
└── log# master-1 노드를 삭제되지 않도록 보호합니다.
$ kubectl annotate node master-1 cube.acornsoft.io/protected=true --overwrite
# 해제
$ kubectl annotate node master-1 cube.acornsoft.io/protected-# master-1 노드의 SSH 연결 계정을 "ubuntu" 로 지정합니다.
$ kubectl annotate node master-1 cube.acornsoft.io/ansible-user=ubuntu --overwrite
# 해제
$ kubectl annotate node master-1 cube.acornsoft.io/ansible-user-# master-1 노드의 SSH 연결 포트를 3003 포트로 지정합니다.
$ kubectl annotate node master-1 cube.acornsoft.io/ansible-port=3003 --overwrite
# 해제
$ kubectl annotate node master-1 cube.acornsoft.io/ansible-port-## step_1. 마스터 정보 확인
$ kubectl -n kube-public get cm cluster-info -o yaml | grep server$ bin/cubectl config sync -p <ssh key> -u <username>$ bin/cubectl update-kubeconfig -p <ssh key> -u <username>$ vi config/cubectl.toml$ bin/cubectl update -p <ssh key> -u <username> --kubeconfig $(pwd)/config/acloud-client-kubeconfig$ bin/cubectl config sync -p <ssh key> -u <username>$ bin/cubectl update-kubeconfig -p <ssh key> -u <username>$ vi config/cubectl.toml$ bin/cubectl update -p <ssh key> -u <username> --kubeconfig $(pwd)/config/acloud-client-kubeconfig$ curl -sfL https://cube.k3.acornsoft.io/download-cubectl | ACCOUNT=user PASSWD=pwd VERSION=v1.0.4 sh -$ curl -sfL https://cube.k3.acornsoft.io/download-cubectl | ACCOUNT=<user> PASSWD=<password> sh -
$ curl -sfL https://cube.k3.acornsoft.io/download-cubectl | ACCOUNT=<user> PASSWD=<password> VERSION=v1.2.0 sh -
$ curl -sfL https://cube.k3.acornsoft.io/download-cubectl | ACCOUNT=<user> PASSWD=<password> VERSION=v1.2.0 OS=linux sh -
$ curl -sfL https://cube.k3.acornsoft.io/download-cubectl | ACCOUNT=<user> PASSWD=<password> VERSION=v1.2.0 OS=linux ARCH=amd64 sh -$ curl -sfL https://cube.k3.acornsoft.io/download-cubectl | ACCOUNT=<user> PASSWD=<password> AIRGAP=ubuntu2204 sh -
$ curl -sfL https://cube.k3.acornsoft.io/download-cubectl | ACCOUNT=<user> PASSWD=<password> AIRGAP=ubuntu2204 VERSION=v1.2.0 sh -
$ curl -sfL https://cube.k3.acornsoft.io/download-cubectl | ACCOUNT=<user> PASSWD=<password> AIRGAP=ubuntu2204 VERSION=v1.2.0 OS=linux sh -
$ curl -sfL https://cube.k3.acornsoft.io/download-cubectl | ACCOUNT=<user> PASSWD=<password> AIRGAP=ubuntu2204 VERSION=v1.2.0 OS=linux ARCH=amd64 sh -
$ curl -sfL https://cube.k3.acornsoft.io/download-cubectl | ACCOUNT=<user> PASSWD=<password> AIRGAP=ubuntu2204 VERSION=v1.2.0 OS=linux ARCH=amd64 K8S=v1.29 sh -
# 멀티 배포판
$ curl -sfL https://cube.k3.acornsoft.io/download-cubectl | ACCOUNT=<user> PASSWD=<password> AIRGAP=ubuntu2204,rocky9,rhel8 sh -
$ curl -sfL https://cube.k3.acornsoft.io/download-cubectl | ACCOUNT=<user> PASSWD=<password> AIRGAP=ubuntu2204,rocky9,rhel8 ARCH=amd64,arm64 sh -$ curl -sfL https://cube.k3.acornsoft.io/download-cubectl | ACCOUNT=<user> PASSWD=<password> COCKTAIL=v4.7.4 sh -
$ curl -sfL https://cube.k3.acornsoft.io/download-cubectl | ACCOUNT=<user> PASSWD=<password> COCKTAIL=v4.7.4 VERSION=v1.0.0 sh -
$ curl -sfL https://cube.k3.acornsoft.io/download-cubectl | ACCOUNT=<user> PASSWD=<password> COCKTAIL=v4.7.4 VERSION=v1.0.0 OS=linux sh -
$ curl -sfL https://cube.k3.acornsoft.io/download-cubectl | ACCOUNT=<user> PASSWD=<password> COCKTAIL=v4.7.4 VERSION=v1.0.0 OS=linux ARCH=amd64 sh -$ curl -sfL https://cube.k3.acornsoft.io/download-cubectl | ACCOUNT=<user> PASSWD=<password> AIRGAP=ubuntu2204 COCKTAIL=v4.7.4 sh -
$ curl -sfL https://cube.k3.acornsoft.io/download-cubectl | ACCOUNT=<user> PASSWD=<password> AIRGAP=ubuntu2204 COCKTAIL=v4.7.4 VERSION=v1.0.0 sh -
$ curl -sfL https://cube.k3.acornsoft.io/download-cubectl | ACCOUNT=<user> PASSWD=<password> AIRGAP=ubuntu2204 COCKTAIL=v4.7.4 VERSION=v1.0.0 OS=linux sh -
$ curl -sfL https://cube.k3.acornsoft.io/download-cubectl | ACCOUNT=<user> PASSWD=<password> AIRGAP=ubuntu2204 COCKTAIL=v4.7.4 VERSION=v1.0.0 OS=linux ARCH=amd64 sh -
$ curl -sfL https://cube.k3.acornsoft.io/download-cubectl | ACCOUNT=<user> PASSWD=<password> AIRGAP=ubuntu2204 COCKTAIL=v4.7.4 VERSION=v1.0.0 OS=linux ARCH=amd64 K8S=v1.24 sh -cubectl-v1.2.0/
├── bin/
│ └── cubectl
├── config/
│ ├── cubectl.toml
│ └── default/ : ansible-runner private-data dir. (default)/
│ ├── env/
│ ├── inventory/
│ └── certs/
├── archive/
│ ├── runtime/
│ │ ├── regctl-linux-amd64.tgz : since v1.2.0
│ │ └── harbor-offline-installer-v2.7.4-linux-amd64.tgz : since v1.2.0
│ │ └── podman-v4.4.2-linux-amd64.tgz : since v1.2.2
│ └── cubectl-image-v1.2.0.tgz : for bastion architecture since v1.2.0
├── extends/
│ └── addon/
│ ├── charts/
│ │ ├── index.yaml : runtime 생성
│ │ ├── kore-board-0.5.5.tgz
│ │ ├── csi-driver-nfs-v4.8.0.tgz
│ │ └── gpu-operator-v23.9.0.tgz
│ ├── images/
│ │ └── gpu-operator/
│ │ └── gpu-operator-v23.9.0-ubuntu2004.tgz
│ └── profile/
│ ├── kore-board/
│ │ └── default.yaml
│ ├── kore-board_20240101000000/ : 백업시 생성
│ │ └── default.yaml
│ ├── csi-driver-nfs/
│ │ └── default.yaml
│ └── gpu-operator/
│ ├── default.yaml
│ ├── ubuntu.yaml
│ └── redhat.yaml
├── logs/
└── LICENSE.txtcubectl-v1.2.0/
├── bin/
│ ├── ccp
│ └── cubectl
├── config/
├── archive/
│ ├── runtime/
│ │ ├── regctl-linux-amd64.tgz
│ │ └── harbor-offline-installer-v2.7.4-linux-amd64.tgz
│ │ └── podman-v4.4.2-linux-amd64.tgz
│ ├── cocktail-v4.7.4-R20230419.tgz
│ └── cubectl-image-v1.2.0.tgz
├── extends/
│ └── addon/
│ ├── charts/
│ │ ├── index.yaml : runtime 생성
│ │ ├── kore-board-0.5.5.tgz
│ │ ├── csi-driver-nfs-v4.8.0.tgz
│ │ └── gpu-operator-v23.9.0.tgz
│ ├── images/
│ │ └── gpu-operator/
│ │ └── gpu-operator-v23.9.0-ubuntu2004.tgz
│ └── profile/
│ ├── kore-board/
│ │ └── default.yaml
│ ├── kore-board_20240101000000/ : 백업시 생성
│ │ └── default.yaml
│ ├── csi-driver-nfs/
│ │ └── default.yaml
│ └── gpu-operator/
│ ├── default.yaml
│ ├── ubuntu.yaml
│ └── redhat.yaml
├── logs/
└── LICENSE.txtcubectl-v1.2.0/
├── bin/
│ └── cubectl
├── config/
├── archive/
│ ├── v1.29/
│ │ ├── harbor-images-v1.25.9.tgz
│ │ ├── k8s-images-v1.29.3.tgz
│ │ └── k8s-repo-ubuntu2204-amd64-v1.29.3.tgz
│ ├── runtime/
│ │ ├── calicoctl-v3.28.0-linux-amd64.tar.gz
│ │ ├── etcd-v3.5.7-linux-amd64.tar.gz
│ │ ├── harbor-offline-installer-v2.7.4-linux-amd64.tgz
│ │ ├── helm-v3.10.3-linux-amd64.tar.gz
│ │ ├── regctl-linux-amd64.tgz
│ │ └── podman-v4.4.2-linux-amd64.tgz
│ ├── local-repo-ubuntu2204-amd64-R240528.tgz
│ ├── extend-images.tgz
│ ├── util-images.tgz
│ └── cubectl-image-v1.2.0.tgz
├── extends/
│ └── addon/
│ ├── charts/
│ │ ├── index.yaml : runtime 생성
│ │ ├── kore-board-0.5.5.tgz
│ │ ├── csi-driver-nfs-v4.8.0.tgz
│ │ └── gpu-operator-v23.9.0.tgz
│ ├── images/
│ │ └── gpu-operator/
│ │ └── gpu-operator-v23.9.0-ubuntu2004.tgz
│ └── profile/
│ ├── kore-board/
│ │ └── default.yaml
│ ├── kore-board_20240101000000/ : 백업시 생성
│ │ └── default.yaml
│ ├── csi-driver-nfs/
│ │ └── default.yaml
│ └── gpu-operator/
│ ├── default.yaml
│ ├── ubuntu.yaml
│ └── redhat.yaml
├── logs/
└── LICENSE.txtcubectl-v1.2.0/
├── bin/
│ ├── cubectl
│ └── ccp
├── config/
├── archive/
│ ├── v1.29/
│ │ ├── harbor-images-v1.25.9.tgz
│ │ ├── k8s-images-v1.29.3.tgz
│ │ └── k8s-repo-ubuntu2204-amd64-v1.29.3.tgz
│ ├── runtime/
│ │ ├── calicoctl-v3.28.0-linux-amd64.tar.gz
│ │ ├── etcd-v3.5.7-linux-amd64.tar.gz
│ │ ├── harbor-offline-installer-v2.7.4-linux-amd64.tgz
│ │ ├── helm-v3.10.3-linux-amd64.tar.gz
│ │ ├── regctl-linux-amd64.tgz
│ │ └── podman-v4.4.2-linux-amd64.tgz
│ ├── cocktail-v4.8.0-R2024011603.tgz
│ ├── local-repo-ubuntu2204-amd64-R240528.tgz
│ ├── extend-images.tgz
│ ├── util-images.tgz
│ └── cubectl-image-v1.2.0.tgz
├── extends/
│ └── addon/
│ ├── charts/
│ │ ├── index.yaml : runtime 생성
│ │ ├── kore-board-0.5.5.tgz
│ │ ├── csi-driver-nfs-v4.8.0.tgz
│ │ └── gpu-operator-v23.9.0.tgz
│ ├── images/
│ │ └── gpu-operator/
│ │ └── gpu-operator-v23.9.0-ubuntu2004.tgz
│ └── profile/
│ ├── kore-board/
│ │ └── default.yaml
│ ├── kore-board_20240101000000/ : 백업시 생성
│ │ └── default.yaml
│ ├── csi-driver-nfs/
│ │ └── default.yaml
│ └── gpu-operator/
│ ├── default.yaml
│ ├── ubuntu.yaml
│ └── redhat.yaml
├── logs/
└── LICENSE.txt$ curl -sfL https://cube.k3.acornsoft.io/download-cocktail | ACCOUNT=user PASSWD=pwd sh -$ curl -sfL https://cube.k3.acornsoft.io/download-cocktail | ACCOUNT=<user> PASSWD=<password> sh -
$ curl -sfL https://cube.k3.acornsoft.io/download-cocktail | ACCOUNT=<user> PASSWD=<password> VERSION=v4.7.4 sh -
$ curl -sfL https://cube.k3.acornsoft.io/download-cocktail | ACCOUNT=<user> PASSWD=<password> VERSION=v4.7.4 CCP=v1.0.6 sh -destroy 시 이전 network-cni 설정 정보 유지되는 현상$ curl -sfL https://cube.k3.acornsoft.io/download-cubectl | ACCOUNT=user PASSWD=pwd VERSION=$(curl -s https://cube.k3.acornsoft.io/stable.txt) sh -
$ cd cubectl-v*$ cat <<EOF> config/cubectl.toml
[kubernetes]
api-sans=["192.168.77.112"]
[node-pool.master]
ip = ["10.30.30.10"]
[node-pool.node]
ip = ["10.30.30.11"]
EOF
$ sudo bin/cubectl create -p ~/.ssh/id_rsa -u cocktail$ cubectl update-kubeconfig -p ~/.ssh/id_rsa -u cocktail
$ kubectl get nodes --kubeconfig=$(pwd)/config/acloud-client-kubeconfig$ sudo bin/cubectl destroy -p ~/.ssh/id_rsa -u cocktail$ cubectl update -p ~/.ssh/id_rsa -u cocktail --kubeconfig $(pwd)/config/{{CLUSTER}}/acloud-client-kubeconfigdestroy
├── cluster
├── registry
└── storage$ cubectl destroy -p <ssh key> -u <username>$ cubectl destroy -p ~/.ssh/id_rsa -u cocktail$ cubectl destroy cluster -p ~/.ssh/id_rsa -u cocktail$ cubectl destroy registry -p ~/.ssh/id_rsa -u cocktail$ cubectl destroy localrepo -p ~/.ssh/id_rsa -u cocktail$ cubectl destroy storage -p ~/.ssh/id_rsa -u cocktail$ cubectl update-kubeconfig -p <ssh key> -u <username>$ cubectl update-kubeconfig -p ~/.ssh/id_rsa -u cocktail
$ cat config/default/acloud-client-kubeconfig
$ cubectl update-kubeconfig -p ~/.ssh/id_rsa -u cocktail -c cluster-1
$ cat config/cluster-1/acloud-client-kubeconfig$ cubectl upgrade -p <ssh key> -u <username> --kubeconfig <KUBECONFIG>$ cubectl upgrade -p ~/.ssh/id_rsa -u cocktail --kubeconfig $(pwd)/config/{{CLUSTER}}/acloud-client-kubeconfig$ cat <<EOF > ${CUBE_HOME}/config/cubectl.toml
[cubectl]
closed-network = true
local-repository-install = true
[kubernetes]
version = "v1.28.15"
[node-pool.master]
ip = ["192.168.77.132"]
private-ip = ["10.30.30.192"]
[node-pool.node]
ip = ["192.168.77.98"]
private-ip = ["10.30.30.30"]
[private-registry]
install = true
registry-ip = "192.168.77.119"
private-ip = "10.30.30.50"
EOF
$ cubectl create -p ~/.ssh/id_rsa -u cocktailcubectl
├── cluster-name
├── cert_validity_days
├── install-dir
├── closed-network
├── local-repository-install
└── local-repository-urlcubectl/
├── create
├── destroy
│ ├── cluster
│ ├── registry
│ ├── localrepo
│ └── storage
├── update
├── update-kubeconfig
├── upgrade
├── config
│ ├── view
│ ├── new
│ └── sync
│── addon
│ ├── list
│ ├── enable
│ └── disable
│── push
│ ├── registry
│ └── localrepo
├── version
└── help$ bin/cubectl config new
# cat config/cubectl.tomlcubectl.toml/
├── cubectl
│ ├── cluster-name
│ ├── cubectl.cert_validity_days
│ ├── install-dir
│ ├── closed-network
│ ├── local-repository-install
│ └── local-repository-url
├── kubernetes
│ ├── version
│ ├── kube-proxy-mode
│ ├── service-cidr
│ ├── pod-cidr
│ ├── node-port-range
│ ├── audit-log-enable
│ ├── api-sans
│ └── calico
│ └── vxlan-mode
├── node-pool
│ ├── data-dir
│ ├── ssh-port
│ ├── master
│ │ ├── ip
│ │ ├── private-ip
│ │ ├── haproxy-install
│ │ ├── lb-ip
│ │ └── lb-port
│ └── node
│ ├── ip
│ └── private-ip
├── private-registry
│ ├── install
│ ├── registry-ip
│ ├── private-ip
│ ├── registry-domain
│ ├── data-dir
│ ├── public-cert
│ └── cert-file
│ ├── ssl-certificate
│ └── ssl-certificate-key
└── shared-storage
├── install
├── storage-ip
├── private-ip
└── volume-dirconfig
├── new
├── view
└── sync$ cubectl config new$ export CONSOLE_USERNAME="admin"
$ export CONSOLE_PASSWORD="user_password"
$ bin/console start$ bin/cubectl config new
Do you really want to make a new config file?
Is this ok [y/n]: y[cubectl]
## Required
## - local-repository-install: local repository installation activate. (Required when selecting the closed network.)
## It is installed on the registry host.
## - local-repository-url: local repository service url (Required when selecting the closed network.)
## If you are installing a private repository, you can skip it. (default: registry-ip)
## Optional
## - cluster-name: use cluster name in config context (default: "kubernetes")
## - install-dir: installation scripts(harbor, shell scripts) save directory (default: "/var/lib/cubectl")
## - cert-validity-days: SSL validity days(default: 36500)
## - closed-network: Enable Air Gap (default: false)
#cluster-name = "kubernetes"
#install-dir = "/var/lib/cubectl"
#cert-validity-days = 36500
#debug-mode = false
#closed-network = false
#local-repository-install = false
#local-repository-url = "http://x.x.x.x"
[kubernetes]
## Required
## -
## Optional
## - version: Kubernetes version (default: "latest")
## If you input only the major version, the minor version automatically selects the last version.
## - container-runtime: use k8s cri (only containerd)
## - kube-proxy-mode: use k8s proxy mode [iptables | ipvs] (default: "ipvs")
## - service-cidr: k8s service network cidr (default: "10.96.0.0/20")
## - pod-cidr: k8s pod network cidr (default: "10.4.0.0/16")
## - node-port-range: k8s node port network range (default: "30000-32767")
## - audit-log-enable: k8s audit log enabled (default: true)
## - api-sans: Add k8s apiserver SAN [--apiserver-cert-extra-sans same as setting] (default: master[0] ip address)
#version = "v1.23.13"
#container-runtime = "containerd"
#kube-proxy-mode = "ipvs"
#service-cidr = "10.96.0.0/20"
#pod-cidr = "10.4.0.0/16"
#node-port-range = "30000-32767"
#audit-log-enable = true
#api-sans = ["x.x.x.x"]
[kubernetes.etcd]
## Required
## - ip: k8s control plane nodes ip address. (Required when selecting the external-etcd="true")
## - private-ip: K8s control plane nodes private ip address. (Required when selecting the external-etcd="true")
## If you use the same IP address, you can skip it.
## Optional
## - external-etcd: used external etcd than input the ip and private-ip address (default: false)
## not used than skip ip address. it is used control plane nodes as automatic.
#external-etcd = false
#ip = ["x.x.x.x"]
#private-ip = ["x.x.x.x"]
[kubernetes.calico]
## Required
## -
## Optional
## - vxlan-mode: calico VXLAN mode activate (default: false)
#vxlan-mode = false
[node-pool]
## Required
## -
## Optional
## - data-dir: data(backup, docker, log, kubelet, etcd, k8s-audit, containerd) root dir (default: "/data")
## - ssh-port: Node ssh port (default: 22)
#data-dir = "/data"
#ssh-port = 22
[node-pool.master]
## Required
## - ip: k8s control plane nodes ip address.
## - private-ip: K8s control plane nodes private ip address.
## If you use the same IP address, you can skip it.
## Optional
## - lb-ip: load balancer ip address (default: master[0] node ip address)
## - isolated: K8s control plane nodes isolated (default: true)
## - haproxy-install: used internal load-balancer (default: true)
## - lb-ip: Enter the IP address when using a load balancer (default: master[0] ip address)
## - lb-port: Enter the port when using a load balancer (default: 6443)
#ip = ["x.x.x.x","x.x.x.x","x.x.x.x"]
#private-ip = ["x.x.x.x","x.x.x.x","x.x.x.x"]
#isolated = true
#haproxy-install = true
#lb-ip = "x.x.x.x"
#lb-port = 6443
[node-pool.node]
## Required
## - ip: k8s work nodes ip address.
## - private-ip: K8s work nodes private ip address.
## If you use the same IP address, you can skip it.
## Optional
#ip = ["x.x.x.x", "x.x.x.x"]
#private-ip = ["x.x.x.x", "x.x.x.x"]
[private-registry]
## Required
## - registry-ip: Public IP address of the private registry node.
## This is a required entry used when installing a private registry.
## - private-ip: Private IP address of the private registry node.
## This is a required entry used when installing a private registry.
## If you use the same IP address, you can skip it.
## Optional
## - install: private registry install (default: false)
## - data-dir: private registry data directory (default: "/data/harbor")
## - public-cert: public cert activate (default: false)
## - cocktail-archive-file: Push cocktail addons and image files to the harbor (default:"")
#install = false
#registry-ip = "x.x.x.x"
#private-ip = "x.x.x.x"
#registry-domain = "x.x.x.x"
#data-dir = "/data/harbor"
#public-cert = false
#cocktail-archive-file = ""
[private-registry.cert-file]
## Required
## - ssl-certificate: The certificate path used when using public-cert.
## This is a required field used when using a public certificate.
## - ssl-certificate-key: The certificate-key used when using public-cert.
## This is a required field used when using a public certificate.
## Optional
#ssl-certificate = ""
#ssl-certificate-key = ""
[shared-storage]
## Required
## - storage-ip: Storage node ip address.
## This is a required field used when installing the nfs server.
## (this is using it to generate an inventory and generate an extra vars)
## - private-ip: Storage node ip address.
## This is a required field used when installing the nfs server.
## If you use the same IP address, you can skip it.
## (this is using it to generate an inventory)
## - volume-dir: Storage node data directory. (default: /data/storage)
## This is a required field used when installing the nfs server.
## (this is using it to generate an extra vars)
## Optional
## - install: NFS Server Installation (default: false)
#install = false
#storage-ip = "x.x.x.x"
#private-ip = "x.x.x.x"
#volume-dir = "/data/storage"
#nfs_version = "4.1"$ cubectl config view$ bin/cubectl config view
[cubectl]
[kubernetes]
version = "v1.23.13"
[kubernetes.etcd]
[kubernetes.calico]
[node-pool]
[node-pool.master]
ip = ["x.x.x.x","x.x.x.x","x.x.x.x"]
[node-pool.node]
[private-registry]
[private-registry.cert-file]
[shared-storage]
[prepare-airgap]
~~~해당 파일은 Control-Plane 노드의 `/etc/kubernetes/cubectl-config-cm.yaml` 에 존재합니다.~~~
### parameters
* `-p <ssh key>`
* 설치 호스트들의 Passwordless SSH 연결을 위한 Private-Key 입니다.
* `-u <username>`
* 설치 호스트들의 Passwordless SSH 연결을 위한 사용자 계정명 입니다.
* `-a, --auto-approve`
* _optional_
* y/n 질의 절차 생략.
### examples
```
$ bin/cubectl config sync -p ~/.ssh/id_rsa -u cocktail
## Inventory for Sync the configuration task.
================================================================
Node Name IP Private IP
================================================================
node-0 x.x.x.x
================================================================
Is this ok [y/n]: y
```addon
├── list
├── enable
└── disable$ cubectl addon list --kubeconfig <kubeconfig file>$ cubectl addon list
$ cubectl addon list --kubeconfig acloud-client-kubeconfig$ cubectl addon enable {{ADDON_NAME}} --kubeconfig <kubeconfig file> --profile <profile file>$ cubectl addon enable gpu-operator
$ cubectl addon enable gpu-operator --kubeconfig acloud-client-kubeconfig
$ cubectl addon enable gpu-operator --kubeconfig acloud-client-kubeconfig --profile redhat
$ cubectl addon enable gpu-operator --kubeconfig acloud-client-kubeconfig --profile redhat$ cubectl addon disable {{ADDON_NAME}} --kubeconfig <kubeconfig file>$ cubectl addon disable csi-driver-nfs
$ cubectl addon disable csi-driver-nfs --kubeconfig acloud-client-kubeconfig$ vi config/cubectl.toml
[shared-storage]
install = true
storage-ip = "192.168.11.7"
private-ip = "172.16.11.7"
#volume-dir = "/data/storage"
#nfs_version = "4.1"$ sudo bin/cubectl create -p ~/.ssh/id_rsa -u cocktailStandardOutput=append:[원하는 경로]/[파일 명], StandardError=append:[원하는 경로]/[파일 명]을 하단 부분에 추가$ vi /usr/lib/systemd/system/kubelet.service.d/10-kubeadm.conf$ cat /usr/lib/systemd/system/kubelet.service.d/10-kubeadm.conf
# Note: This dropin only works with kubeadm and kubelet v1.11+
[Service]
Environment="KUBELET_KUBECONFIG_ARGS=--bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf --kubeconfig=/etc/kubernetes/kubelet.conf"
Environment="KUBELET_CONFIG_ARGS=--config=/var/lib/kubelet/config.yaml"
# This is a file that "kubeadm init" and "kubeadm join" generates at runtime, populating the KUBELET_KUBEADM_ARGS variable dynamically
EnvironmentFile=-/var/lib/kubelet/kubeadm-flags.env
# This is a file that the user can use for overrides of the kubelet args as a last resort. Preferably, the user should use
# the .NodeRegistration.KubeletExtraArgs object in the configuration files instead. KUBELET_EXTRA_ARGS should be sourced from this file.
EnvironmentFile=-/etc/default/kubelet
ExecStart=
ExecStart=/usr/bin/kubelet $KUBELET_KUBECONFIG_ARGS $KUBELET_CONFIG_ARGS $KUBELET_KUBEADM_ARGS $KUBELET_EXTRA_ARGS
StandardOutput=append:[원하는 경로]/[파일 명]
StandardError=append:[원하는 경로]/[파일 명]$ cubectl config sync -p <ssh key> -u <username>$ cubectl update-kubeconfig -p <ssh key> -u <username>$ vi config/cubectl.toml
[kubernetes]
version = "v1.24"
[node-pool.master]
ip = ["10.30.30.10","10.30.30.11","10.30.30.12"]
[node-pool.node]
ip = ["10.30.31.11","10.30.31.12"]$ bin/cubectl upgrade -p ~/.ssh/id_rsa -u cocktail --kubeconfig config/acloud-client-kubeconfig$ sudo vi /var/lib/cubectl/cert/openssl.conf# 예시
[ alt_names_registry ]
DNS.1 = localhost
DNS.2 = registry
DNS.3 = regi.acornsoft.io
IP.1 = 127.0.0.1
IP.2 = 192.168.77.154$ sudo mkdir /etc/docker/certs.d/regi.acornsoft.io$ sudo sed -i 's/192.168.77.154/regi.acornsoft.io/g' /var/lib/cubectl/harbor/harbor.yml# 예시
hostname: regi.acornsoft.io$ cd /var/lib/cubectl/harbor/
$ sudo ./install.sh$ cd /var/lib/cubectl/harbor/common/config/core
$ vi env
# AS-IS
EXT_ENDPOINT=https://10.1.1.50
# TO-BE (접속될 접속주소, 예시:)
EXT_ENDPOINT=https://www.regi.acornsoft.io$ sudo mkdir -p /etc/containerd/certs.d/regi.acornsoft.io
$ sudo cp -r /etc/containerd/certs.d/192.168.77.154/* /etc/containerd/certs.d/regi.acornsoft.io$ sudo sed -i 's/192.168.77.154/regi.acornsoft.io/g' /etc/containerd/certs.d/regi.acornsoft.io/hosts.toml$ sudo sed -i 's/192.168.77.154/regi.acornsoft.io/g' /etc/containerd/config.toml$ sudo systemctl restart containerd$ sudo sed -i '/ca =/d' /etc/containerd/certs.d/regi.acornsoft.io/hosts.toml# 도메인주소로 변경하는 경우 ca 라인 삭제
server = "https://www.regi.acornsoft.io"
[host."https://www.regi.acornsoft.io"]$ sudo find /etc/kubernetes -type f -exec sed -i 's/192.168.77.154/regi.acornsoft.io/g' {} +# Control plane에서 IP가 변경되는 파일 목록
# Addon
/etc/kubernetes/addon/calico/calico.yaml
/etc/kubernetes/addon/metrics-server/metrics-server.yaml
# Static Pods
/etc/kubernetes/manifests/kube-apiserver.yaml
/etc/kubernetes/manifests/kube-controller-manager.yaml
/etc/kubernetes/manifests/kube-scheduler.yaml
# Coredns 및 기타 yaml
/etc/kubernetes/addon/test/nginx.yaml
/etc/kubernetes/addon/test/dnsutils.yaml
/etc/kubernetes/kubeadm.yaml# Worker nodes에서 IP가 변경되는 파일 목록
# Static Pods
/etc/kubernetes/manifests/haproxy.yaml$ sed -i 's/192.168.77.154/regi.acornsoft.io/g' /etc/apt/sources.list.d/local-repo.list$ sudo mkdir /etc/docker/certs.d/regi.acornsoft.io$ kubectl apply -f /etc/kubernetes/addon/calico/calico.yaml
$ kubectl apply -f /etc/kubernetes/addon/metrics-server/metrics-server.yaml$ kubectl edit daemonset -n kube-system kube-proxy# 예시
image: regi.acornsoft.io/registry.k8s.io/kube-proxy:v1.26.7$ kubectl edit deployment -n kube-system coredns# 예시
image: regi.acornsoft.io/docker.io/coredns/coredns:1.9.3$ sudo systemctl restart kubelet$ sudo crictl images# 예시
ubuntu@master-1:~$ sudo crictl images
IMAGE TAG IMAGE ID SIZE
regi.acornsoft.io/docker.io/calico/cni v3.26.1 9dee260ef7f59 93.4MB
192.168.77.154/docker.io/calico/cni v3.26.1 9dee260ef7f59 93.4MB
regi.acornsoft.io/docker.io/calico/node v3.26.1 8065b798a4d67 86.6MB
192.168.77.154/docker.io/calico/node v3.26.1 8065b798a4d67 86.6MB
regi.acornsoft.io/registry.k8s.io/kube-apiserver v1.26.7 6ac727c486d08 36.1MB
192.168.77.154/registry.k8s.io/kube-apiserver v1.26.7 6ac727c486d08 36.1MB
regi.acornsoft.io/registry.k8s.io/kube-controller-manager v1.26.7 17314033c0a0b 32.8MB
192.168.77.154/registry.k8s.io/kube-controller-manager v1.26.7 17314033c0a0b 32.8MB
regi.acornsoft.io/registry.k8s.io/kube-proxy v1.26.7 1e7eac3bc5c0b 21.8MB
192.168.77.154/registry.k8s.io/kube-proxy v1.26.7 1e7eac3bc5c0b 21.8MB
regi.acornsoft.io/registry.k8s.io/kube-scheduler v1.26.7 c1902187a39f8 17.8MB
192.168.77.154/registry.k8s.io/kube-scheduler v1.26.7 c1902187a39f8 17.8MB
regi.acornsoft.io/registry.k8s.io/pause 3.9 e6f1816883972 319kB
192.168.77.154/registry.k8s.io/pause 3.9 e6f1816883972 319kB
192.168.77.154/docker.io/coredns/coredns 1.9.3 5185b96f0becf 14.8MB
192.168.77.154/registry.k8s.io/metrics-server/metrics-server v0.6.1 e57a417f15d36 28.1MBprivate-registry
├── install
├── registry-ip
├── private-ip
├── registry-domain
├── admin-password
├── data-dir
├── public-cert
└── cert-file
├── ssl-certificate
└── ssl-certificate-key$ cat <<EOF> config/cubectl.toml
[cubectl]
closed-network = true
local-repository-install = true
[kubernetes]
api-sans=["192.168.77.112"]
[node-pool.master]
ip = ["10.30.30.10"]
[node-pool.node]
ip = ["10.30.30.11"]
[private-registry]
install=true
registry-ip="10.30.30.220"
EOF$ sudo bin/cubectl create -p ~/.ssh/id_rsa -u cocktail$ export KUBECONFIG=$(pwd)/config/acloud-client-kubeconfig
# 또는
$ sudo cp -i config/acloud-client-kubeconfig $HOME/.kube/config
$ sudo chown $(id -u):$(id -g) $HOME/.kube/config
# 노드 확인
$ kubectl get nodesshared-storage
├── install
├── storage-ip
├── private-ip
└── volume-dir$ vi ${CUBE_HOME}/extends/addon/profile/csi-driver-nfs/default.yaml$ bin/cubectl addon list
┌────────────────┬─────────┬──────────┬─────────┬─────────────────────────────┐
│ ADDON NAME │ VERSION │ STATUS │ PROFILE │ VALUES PATH │
├────────────────┼─────────┼──────────┼─────────┼─────────────────────────────┤
│ csi-driver-nfs │ v4.8.0 │ disabled │ │ csi-driver-nfs/default.yaml │
│ gpu-operator │ v23.9.0 │ disabled │ │ gpu-operator/default.yaml │
│ │ │ │ redhat │ gpu-operator/redhat.yaml │
│ │ │ │ ubuntu │ gpu-operator/ubuntu.yaml │
│ kore-board │ 0.5.5 │ disabled │ │ kore-board/default.yaml │
└────────────────┴─────────┴──────────┴─────────┴─────────────────────────────┘
Duration 45.097377ms time$ bin/cubectl addon enable csi-driver-nfs
addon enable start: csi-driver-nfs ...
addon enable complete: csi-driver-nfs
Duration 1m5.100330923s time$ bin/cubectl addon list
┌────────────────┬─────────┬────────────┬─────────┬─────────────────────────────┐
│ ADDON NAME │ VERSION │ STATUS │ PROFILE │ VALUES PATH │
├────────────────┼─────────┼────────────┼─────────┼─────────────────────────────┤
│ csi-driver-nfs │ v4.8.0 │ enabled ✅ │ │ csi-driver-nfs/default.yaml │
│ gpu-operator │ v23.9.0 │ disabled │ │ gpu-operator/default.yaml │
│ │ │ │ redhat │ gpu-operator/redhat.yaml │
│ │ │ │ ubuntu │ gpu-operator/ubuntu.yaml │
│ kore-board │ 0.5.5 │ disabled │ │ kore-board/default.yaml │
└────────────────┴─────────┴────────────┴─────────┴─────────────────────────────┘
Duration 188.932222ms time
$ kubectl get pods -n csi-driver-nfs
NAMESPACE NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES
csi-driver-nfs csi-nfs-controller-5bbdfbdcc9-5xn6x 4/4 Running 1 (19s ago) 75s 10.10.30.164 wk-node-2 <none> <none>
csi-driver-nfs csi-nfs-node-8xpjf 3/3 Running 0 75s 10.10.30.184 cp-node-1 <none> <none>
csi-driver-nfs csi-nfs-node-d7kz2 3/3 Running 0 75s 10.10.30.143 cp-node-2 <none> <none>
csi-driver-nfs csi-nfs-node-dx6jm 3/3 Running 0 75s 10.10.30.164 wk-node-2 <none> <none>
csi-driver-nfs csi-nfs-node-ppcwh 3/3 Running 0 75s 10.10.30.128 wk-node-1 <none> <none>
csi-driver-nfs csi-nfs-node-tvrb5 3/3 Running 0 75s 10.10.30.153 cp-node-3 <none> <none>
$ kubectl get sc
NAME PROVISIONER RECLAIMPOLICY VOLUMEBINDINGMODE ALLOWVOLUMEEXPANSION AGE
nfs-csi (default) nfs.csi.k8s.io Delete Immediate false 11m$ bin/cubectl addon disable csi-driver-nfs
addon disable start: csi-driver-nfs ...
addon disable complete: csi-driver-nfs
Duration 1.358568651s time$ curl -sfL https://cube.k3.acornsoft.io/download-package | ACCOUNT={{ ID }} PASSWD={{ PW }} PACKAGE=kubeflow VERSION=1.10.2 sh -
[INFO] starting download file kubeflow-manifests-1.10.2.tar.gz
######################################################################## 100.0%
[INFO] decompressing file kubeflow-manifests-*.tar.gz
[INFO] starting download file kubectl-*-linux-amd64
######################################################################## 100.0%
[INFO] starting download file kubeflow-images-1.10.2.tar.gz
##### 8.3%
...$ sudo vi /etc/security/limits.conf
# 추가
* soft nofile 65535
* hard nofile 65535
$ sudo vi /etc/sysctl.d/99-sysctl.conf
# 추가
fs.file-max = 2097152
fs.inotify.max_user_instances=1280
fs.inotify.max_user_watches=655360
sudo sysctl -p$ tar -zxvf ./archive/push/registry/kubeflow-images-1.10.2.tar.gz -C ./archive/push/registry
docker.io_bitnami_kubectl:1.30.4.tar
docker.io_filebrowser_filebrowser:v2.25.0.tar
docker.io_grafana_grafana:5.3.4.tar
docker.io_kong_httpbin:latest.tar
...
cubectl-v1.x.x/
├── archive
│ └── push
│ ├── localrepo
│ └── registry
│ └── kubeflow-images-1.10.2.tar.gz(해당 파일을 압축 해제 후 위치 변경)
├── bin
...
├── LICENSE.txt
├── kubeflow
│ ├── bash
│ │ └── ...
│ ├── bin
│ │ └── ...
│ └── manifest
│ └── ...
└── logs$ ./bin/cubectl-test push registry -u test -p ./config/sshkey
┌─────────────────┬───────────┬────────────┬────────────┬──────────────────────────────────────────────────────────────────────────────────────────────┐
│ KIND │ NODE │ IP │ PRIVATE IP │ PUSH IMAGES │
├─────────────────┼───────────┼────────────┼────────────┼──────────────────────────────────────────────────────────────────────────────────────────────┤
│ Push Repository │ node-regi │ 10.10.0.49 │ 10.10.0.49 │ docker.io_bitnami_kubectl:1.30.4.tar │
│ │ │ │ │ docker.io_filebrowser_filebrowser:v2.25.0.tar │
│ │ │ │ │ gcr.io_knative-releases_knative.dev_eventing_cmd_in|memory_channel|controller:v1.16.4.tar │
│ │ │ │ │ gcr.io_knative-releases_knative.dev_eventing_cmd_in|memory_channel|dispatcher:v1.16.4.tar │
...
│ │ │ │ │ quay.io_jetstack_cert-manager-controller:v1.16.1.tar │
│ │ │ │ │ quay.io_jetstack_cert-manager-webhook:v1.16.1.tar │
│ │ │ │ │ quay.io_kuberay_operator:v1.3.2.tar │
│ │ │ │ │ quay.io_oauth2-proxy_oauth2-proxy:latest.tar │
│ │ │ │ │ quay.io_oauth2-proxy_oauth2-proxy:v7.7.1.tar │
└─────────────────┴───────────┴────────────┴────────────┴──────────────────────────────────────────────────────────────────────────────────────────────┘
Is this ok [Y/n]:# 예시
test@kubeflow-cp-1:~/cubectl-v1.3.3$ export CUBE_HOME=$(pwd)
test@kubeflow-cp-1:~/cubectl-v1.3.3$ echo $CUBE_HOME
/home/test/cubectl-v1.3.3./kubeflow/bin/install base -c {cubectl -c 옵션과 동일}$ vi ${CUBE_HOME}/kubeflow/manifests/applications/pipeline/upstream/base/installs/generic/pipeline-install-config.yaml
$ vi ${CUBE_HOME}/kubeflow/manifests/applications/pipeline/upstream/base/installs/generic/postgres/pipeline-install-config.yaml
# registry.k8s.io/busybox -> docker.io/library/busybox$ vi ${CUBE_HOME}/kubeflow/manifests/applications/pipeline/upstream/base/installs/multi-user/pipelines-profile-controller/sync.py
# ghcr.io/kubeflow/kfp-frontend -> 10.10.0.49/ghcr.io/kubeflow/kfp-frontend
# ghcr.io/kubeflow/kfp-visualization-server -> 10.10.0.49/ghcr.io/kubeflow/kfp-visualization-server$ vi ${CUBE_HOME}/kubeflow/manifests/common/istio/profile.yaml
$ vi ${CUBE_HOME}/kubeflow/manifests/common/istio/istio-install/overlays/insecure/configmap-patch.yaml
# hub: gcr.io/istio-release -> hub: 10.10.0.49/gcr.io/istio-release로 변경$ vi ${CUBE_HOME}/kubeflow/manifests/common/cert-manager/base/upstream/cert-manager.yaml
# --acme-http01-solver-image=quay.io/jetstack/cert-manager-acmesolver:v1.16.1 ->
# --acme-http01-solver-image=10.10.0.49/quay.io/jetstack/cert-manager-acmesolver:v1.16.1로 변경$ vi ${CUBE_HOME}/kubeflow/manifests/applications/pipeline/upstream/third-party/argo/base/workflow-controller-deployment-patch.yaml
# quay.io/argoproj/argoexec:v3.5.14 -> 10.10.0.49/quay.io/argoproj/argoexec:v3.5.14$ vi ${CUBE_HOME}/kubeflow/manifests/applications/pipeline/upstream/base/pipeline/ml-pipeline-apiserver-deployment.yaml
# ghcr.io/kubeflow/kfp-driver:2.5.0 -> 10.10.0.49/ghcr.io/kubeflow/kfp-driver:2.5.0
# ghcr.io/kubeflow/kfp-launcher:2.5.0 -> 10.10.0.49/ghcr.io/kubeflow/kfp-launcher:2.5.0$ vi ${CUBE_HOME}/kubeflow/manifests/common/knative/knative-serving/base/upstream/serving-core.yaml
# queue-sidecar-image: ... -> queue-sidecar-image: 10.10.0.49/gcr.io/knative-releases/knative.dev/serving/cmd/queue:v1.16.2$ ./kubeflow/bin/install -r {registry ip} -c {cubectl -c 옵션과 동일} overlays/airgapped
...
validatingwebhookconfiguration.admissionregistration.k8s.io/spark-operator-webhook serverside-applied
validatingwebhookconfiguration.admissionregistration.k8s.io/trainedmodel.serving.kserve.io serverside-applied
validatingwebhookconfiguration.admissionregistration.k8s.io/validation.webhook.serving.knative.dev serverside-applied
validatingwebhookconfiguration.admissionregistration.k8s.io/validator.training-operator.kubeflow.org serverside-applied$ lspci -nnk | grep -i nvidia
00:05.0 3D controller [0302]: NVIDIA Corporation Device [10de:20b7] (rev a1)
Subsystem: NVIDIA Corporation Device [10de:1532]
Kernel modules: nvidiafb$ bin/cubectl addon list
┌────────────────┬─────────┬──────────┬─────────┬─────────────────────────────┐
│ ADDON NAME │ VERSION │ STATUS │ PROFILE │ VALUES PATH │
├────────────────┼─────────┼──────────┼─────────┼─────────────────────────────┤
│ csi-driver-nfs │ v4.8.0 │ disabled │ │ csi-driver-nfs/default.yaml │
│ gpu-operator │ v23.9.0 │ disabled │ │ gpu-operator/default.yaml │
│ │ │ │ redhat │ gpu-operator/redhat.yaml │
│ │ │ │ ubuntu │ gpu-operator/ubuntu.yaml │
│ kore-board │ 0.5.5 │ disabled │ │ kore-board/default.yaml │
└────────────────┴─────────┴──────────┴─────────┴─────────────────────────────┘
Duration 73.639078ms time$ vi ${CUBE_HOME}/extends/addon/profile/gpu-operator/default.yaml$ bin/cubectl addon enable gpu-operator
addon enable start: gpu-operator ...
addon enable complete: gpu-operator
Duration 52.093538621s time$ bin/cubectl addon list
┌────────────────┬─────────┬────────────┬─────────┬─────────────────────────────┐
│ ADDON NAME │ VERSION │ STATUS │ PROFILE │ VALUES PATH │
├────────────────┼─────────┼────────────┼─────────┼─────────────────────────────┤
│ csi-driver-nfs │ v4.8.0 │ disabled │ │ csi-driver-nfs/default.yaml │
│ gpu-operator │ v23.9.0 │ enabled ✅ │ │ gpu-operator/default.yaml │
│ │ │ │ redhat │ gpu-operator/redhat.yaml │
│ │ │ │ ubuntu │ gpu-operator/ubuntu.yaml │
│ kore-board │ 0.5.5 │ disabled │ │ kore-board/default.yaml │
└────────────────┴─────────┴────────────┴─────────┴─────────────────────────────┘
Duration 75.061448ms time
$ kubectl get pods -n gpu-operator
NAMESPACE NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES
gpu-operator gpu-operator-5564789746-rlpzk 1/1 Running 0 65s 10.4.185.65 cp-node-1 <none> <none>
gpu-operator gpu-operator-node-feature-discovery-gc-78b479ccc6-ngfnd 1/1 Running 0 65s 10.4.211.67 wk-node-1 <none> <none>
gpu-operator gpu-operator-node-feature-discovery-master-569bfcd8bc-5xb8h 1/1 Running 0 65s 10.4.111.193 cp-node-3 <none> <none>
gpu-operator gpu-operator-node-feature-discovery-worker-dlxxh 1/1 Running 0 65s 10.4.111.194 cp-node-3 <none> <none>
gpu-operator gpu-operator-node-feature-discovery-worker-fmlmb 1/1 Running 0 65s 10.4.185.66 cp-node-1 <none> <none>
gpu-operator gpu-operator-node-feature-discovery-worker-gqn8z 1/1 Running 0 65s 10.4.238.68 cp-node-2 <none> <none>
gpu-operator gpu-operator-node-feature-discovery-worker-pksh4 1/1 Running 0 65s 10.4.109.2 wk-node-2 <none> <none>
gpu-operator gpu-operator-node-feature-discovery-worker-xx6gb 1/1 Running 0 65s 10.4.211.66 wk-node-1 <none> <none>$ bin/cubectl addon disable gpu-operator# 1g.5gb 인 경우
$ kubectl label nodes $NODE nvidia.com/mig.config=all-1g.5gb --overwrite$ kubectl -n gpu-operator exec -it nvidia-dcgm-exporter-gc6bm bash
root@nvidia-dcgm-exporter-gc6bm:/# nvidia-smi
Thu Dec 7 06:02:55 2023
+---------------------------------------------------------------------------------------+
| NVIDIA-SMI 535.104.12 Driver Version: 535.104.12 CUDA Version: 12.2 |
|-----------------------------------------+----------------------+----------------------+
| GPU Name Persistence-M | Bus-Id Disp.A | Volatile Uncorr. ECC |
| Fan Temp Perf Pwr:Usage/Cap | Memory-Usage | GPU-Util Compute M. |
| | | MIG M. |
|=========================================+======================+======================|
| 0 NVIDIA A30 On | 00000000:00:05.0 Off | On |
| N/A 63C P0 72W / 165W | N/A | N/A Default |
| | | Enabled |
+-----------------------------------------+----------------------+----------------------+
+---------------------------------------------------------------------------------------+
| MIG devices: |
+------------------+--------------------------------+-----------+-----------------------+
| GPU GI CI MIG | Memory-Usage | Vol| Shared |
| ID ID Dev | BAR1-Usage | SM Unc| CE ENC DEC OFA JPG |
| | | ECC| |
|==================+================================+===========+=======================|
| 0 3 0 0 | 12MiB / 5952MiB | 14 0 | 1 0 1 0 0 |
| | 0MiB / 8191MiB | | |
+------------------+--------------------------------+-----------+-----------------------+
| 0 4 0 1 | 12MiB / 5952MiB | 14 0 | 1 0 1 0 0 |
| | 0MiB / 8191MiB | | |
+------------------+--------------------------------+-----------+-----------------------+
| 0 5 0 2 | 12MiB / 5952MiB | 14 0 | 1 0 1 0 0 |
| | 0MiB / 8191MiB | | |
+------------------+--------------------------------+-----------+-----------------------+
| 0 6 0 3 | 12MiB / 5952MiB | 14 0 | 1 0 1 0 0 |
| | 0MiB / 8191MiB | | |
+------------------+--------------------------------+-----------+-----------------------+
+---------------------------------------------------------------------------------------+
| Processes: |
| GPU GI CI PID Type Process name GPU Memory |
| ID ID Usage |
|=======================================================================================|
| No running processes found |
+---------------------------------------------------------------------------------------+
root@nvidia-dcgm-exporter-gc6bm:/# nvidia-smi -L
GPU 0: NVIDIA A30 (UUID: GPU-79e36614-3f62-d3dd-cdd0-48b00aa446e0)
MIG 1g.6gb Device 0: (UUID: MIG-39f52290-ccf4-5e32-b8b8-cc1877a32051)
MIG 1g.6gb Device 1: (UUID: MIG-dbf3834e-128b-5965-88b7-2e3d2fe5a0aa)
MIG 1g.6gb Device 2: (UUID: MIG-c735f798-c9d5-5c0e-972c-e0bc6cdb05e7)
MIG 1g.6gb Device 3: (UUID: MIG-233d355f-f84e-530d-8526-797b5a867669)# 4개로 분할하는 경우
$ cat <<EOF > time-slicing-config-all.yaml
apiVersion: v1
kind: ConfigMap
metadata:
name: time-slicing-config-all
data:
any: |-
version: v1
flags:
migStrategy: none
sharing:
timeSlicing:
resources:
- name: nvidia.com/gpu
replicas: 4
EOF
$ kubectl apply -n gpu-operator -f time-slicing-config-all.yaml$ kubectl patch clusterpolicy/cluster-policy -n gpu-operator --type merge -p '{"spec": {"devicePlugin": {"config": {"name": "time-slicing-config-all", "default": "any"}}}}'$ kubectl describe no $GPU-NODE$ curl -sfL https://cube.k3.acornsoft.io/download-addon | ACCOUNT=<user> PASSWD=<password> NAME=gpu sh -
$ curl -sfL https://cube.k3.acornsoft.io/download-addon | ACCOUNT=<user> PASSWD=<password> NAME=gpu VERSION=v23.9.0 sh -
$ curl -sfL https://cube.k3.acornsoft.io/download-addon | ACCOUNT=<user> PASSWD=<password> NAME=gpu VERSION=v23.9.0 OSEXT=ubuntu2004 sh -$ tree archive
archive
├── cubectl-image-v1.2.2.tgz
├── push # 새로 생성된 디렉토리
│ └── registry
└── runtime
├── podman-v4.4.2-linux.amd64.tgz
├── harbor-offline-installer-v2.7.4-linux-amd64.tgz
└── regctl-linux-amd64.tgz
...
$ tree archive/push/registry
archive/push/registry
├── docker.io_library_golang:1.22-alpine.tar
├── docker.io_library_python:alpine3.19.tgz
├── docker.io_library_tomcat:9.0.96.tgz
└── quay.io_rockylinux_rockylinux:9.3-minimal.tgz$ sudo bin/cubectl push registry -p ~/.ssh/id_rsa -u cocktail$ docker login {{ registry_domain }} -u {{ id }} -p {{ password }}
$ regctl image import {{ registry_domain }}/docker.io/library/tomcat:9.0.96 {{ Image file path }}$CUBE_HOME/
└── cubectl-v1.x.x/
├── archive
├── bin
├── config
├── extends/
│ └── addon/
│ ├── chart/
│ │ ├── csi-driver-nfs-v4.8.0.tgz
│ │ ├── gpu-operator-v23.9.0.tgz
│ │ └── kore-board-0.5.5.tgz
│ ├── profile/
│ │ ├── csi-driver-nfs/
│ │ │ └── default.yaml
│ │ ├── gpu-operator/
│ │ │ ├── default.yaml
│ │ │ ├── redhat.yaml
│ │ │ └── ubuntu.yaml
│ │ └── kore-board/
│ │ └── default.yaml
│ └── images/
│ └── gpu-operator/
│ └── gpu-operator-v23.9.0-ubuntu2204.tgz
│
├── logs
└── LICENSE.txt$ curl -sfL https://cube.k3.acornsoft.io/download-addon | ACCOUNT=<user> PASSWD=<password> NAME=cilium sh -
$ curl -sfL https://cube.k3.acornsoft.io/download-addon | ACCOUNT=<user> PASSWD=<password> NAME=cilium VERSION=v1.17.3 sh -
...$ tar -zxvf ./extends/addon/images/cilium/cilium-v1.17.3.tgz -C ./archive/push/registry
x docker.io_library_busybox:1.37.0.tar
...
x quay.io_cilium_operator:v1.17.3.tar
x quay.io_cilium_startup-script:c54c7edeab7fde4da68e59acd319ab24af242c3f.tarcubectl-v1.3.1
├── archive
│ ├── cubectl-image-v1.3.1.tgz
│ ├── extend-images.tgz
│ ├── local-repo-ubuntu2204-amd64-R250314.tgz
│ ├── push
│ │ ├── localrepo
│ │ └── registry
│ │ ├── docker.io_library_busybox:1.37.0.tar
│ │ ├── ghcr.io_spiffe_spire-agent:1.9.6.tar
│ │ ├── ghcr.io_spiffe_spire-server:1.9.6.tar
│ │ ├── quay.io_cilium_certgen:v0.2.1.tar
│ │ ├── quay.io_cilium_cilium-envoy:v1.32.5-1744305768-f9ddca7dcd91f7ca25a505560e655c47d3dec2cf.tar
│ │ ├── quay.io_cilium_cilium:v1.17.3.tar
│ │ ├── quay.io_cilium_clustermesh-apiserver:v1.17.3.tar
│ │ ├── quay.io_cilium_hubble-relay:v1.17.3.tar
│ │ ├── quay.io_cilium_hubble-ui-backend:v0.13.2.tar
│ │ ├── quay.io_cilium_hubble-ui:v0.13.2.tar
│ │ ├── quay.io_cilium_operator-alibabacloud:v1.17.3.tar
│ │ ├── quay.io_cilium_operator-aws:v1.17.3.tar
│ │ ├── quay.io_cilium_operator-azure:v1.17.3.tar
│ │ ├── quay.io_cilium_operator-generic:v1.17.3.tar
│ │ ├── quay.io_cilium_operator:v1.17.3.tar
│ │ └── quay.io_cilium_startup-script:c54c7edeab7fde4da68e59acd319ab24af242c3f.tar
...
├── extends
│ └── addon
│ ├── charts
│ │ ├── cilium-v1.17.3.tgz
│ │ ├── csi-driver-nfs-v4.6.0.tgz
│ │ ├── csi-driver-nfs-v4.8.0.tgz
│ │ ├── csi-driver-nfs-v4.9.0.tgz
│ │ ├── gpu-operator-v23.9.0.tgz
│ │ ├── gpu-operator-v24.9.0.tgz
│ │ └── kore-board-0.5.5.tgz
│ ├── images
│ │ └── cilium
│ │ └── cilium-v1.17.3.tgz
│ └── profile
│ ├── cilium
│ │ ├── default-airgap-clustermesh.yaml
│ │ ├── default-airgap.yaml
│ │ ├── default-clustermesh.yaml
│ │ └── default.yaml
│ ├── csi-driver-nfs
│ │ └── default.yaml
│ ├── gpu-operator
│ │ └── default.yaml
│ └── kore-board
│ └── default.yaml
└── LICENSE.txt$ bin/cubectl push registry -u test -p ~/Desktop/key-list/node-ssh
┌─────────────────┬───────────┬────────────────┬───────────────┬─────────────────────────────────────────────────────────────────────────────────────────────┐
│ KIND │ NODE │ IP │ PRIVATE IP │ PUSH IMAGES │
├─────────────────┼───────────┼────────────────┼───────────────┼─────────────────────────────────────────────────────────────────────────────────────────────┤
│ Push Repository │ node-regi │ 192.168.88.188 │ 10.50.240.253 │ docker.io_library_busybox:1.37.0.tar │
│ │ │ │ │ ghcr.io_spiffe_spire-agent:1.9.6.tar │
│ │ │ │ │ ghcr.io_spiffe_spire-server:1.9.6.tar │
│ │ │ │ │ quay.io_cilium_certgen:v0.2.1.tar │
│ │ │ │ │ quay.io_cilium_cilium-envoy:v1.32.5-1744305768-f9ddca7dcd91f7ca25a505560e655c47d3dec2cf.tar │
│ │ │ │ │ quay.io_cilium_cilium:v1.17.3.tar │
│ │ │ │ │ quay.io_cilium_clustermesh-apiserver:v1.17.3.tar │
│ │ │ │ │ quay.io_cilium_hubble-relay:v1.17.3.tar │
│ │ │ │ │ quay.io_cilium_hubble-ui-backend:v0.13.2.tar │
│ │ │ │ │ quay.io_cilium_hubble-ui:v0.13.2.tar │
│ │ │ │ │ quay.io_cilium_operator-alibabacloud:v1.17.3.tar │
│ │ │ │ │ quay.io_cilium_operator-aws:v1.17.3.tar │
│ │ │ │ │ quay.io_cilium_operator-azure:v1.17.3.tar │
│ │ │ │ │ quay.io_cilium_operator-generic:v1.17.3.tar │
│ │ │ │ │ quay.io_cilium_operator:v1.17.3.tar │
│ │ │ │ │ quay.io_cilium_startup-script:c54c7edeab7fde4da68e59acd319ab24af242c3f.tar │
└─────────────────┴───────────┴────────────────┴───────────────┴─────────────────────────────────────────────────────────────────────────────────────────────┘$ sudo kubectl create secret generic -n kube-system cilium-etcd-secrets --from-file=etcd-client-ca.crt=/etc/kubernetes/pki/etcd/ca.crt --from-file=etcd-client.key=/etc/kubernetes/pki/etcd/healthcheck-client.key --from-file=etcd-client.crt=/etc/kubernetes/pki/etcd/healthcheck-client.crt --kubeconfig [ kubeconfig path ]
secret/cilium-etcd-secrets created$ bin/cubectl addon list
┌────────────────┬─────────┬─────────┬────────────────────────────┬────────────────────────────────────────┐
│ ADDON NAME │ VERSION │ STATUS │ PROFILE │ VALUES PATH │
├────────────────┼─────────┼─────────┼────────────────────────────┼────────────────────────────────────────┤
│ gpu-operator │ v24.9.0 │ unknown │ │ gpu-operator/default.yaml │
│ cilium │ 1.17.3 │ unknown │ default-airgap-clustermesh │ cilium/default-airgap-clustermesh.yaml │
│ │ │ │ default-airgap │ cilium/default-airgap.yaml │
│ │ │ │ default-clustermesh │ cilium/default-clustermesh.yaml │
│ │ │ │ │ cilium/default.yaml │
│ csi-driver-nfs │ v4.10.0 │ unknown │ │ csi-driver-nfs/default.yaml │
└────────────────┴─────────┴─────────┴────────────────────────────┴────────────────────────────────────────┘
Duration 30.379208ms time$ vi ${CUBE_HOME}/extends/addon/profile/cilium/default-airgap.yaml
...$ bin/cubectl addon enable cilium -n kube-system --profile default-airgap.yaml
addon enable start: cilium ...
addon enable complete: cilium
Duration 1m16.91672575s time$ bin/cubectl addon list
┌────────────────┬─────────┬────────────┬────────────────────────────┬────────────────────────────────────────┐
│ ADDON NAME │ VERSION │ STATUS │ PROFILE │ VALUES PATH │
├────────────────┼─────────┼────────────┼────────────────────────────┼────────────────────────────────────────┤
│ cilium │ 1.17.3 │ enabled ✅ │ default-airgap-clustermesh │ cilium/default-airgap-clustermesh.yaml │
│ │ │ │ default-airgap │ cilium/default-airgap.yaml │
│ │ │ │ default-clustermesh │ cilium/default-clustermesh.yaml │
│ │ │ │ │ cilium/default.yaml │
│ csi-driver-nfs │ v4.10.0 │ disabled │ │ csi-driver-nfs/default.yaml │
│ gpu-operator │ v24.9.0 │ disabled │ │ gpu-operator/default.yaml │
│ kore-board │ 0.5.5 │ disabled │ │ kore-board/default.yaml │
└────────────────┴─────────┴────────────┴────────────────────────────┴────────────────────────────────────────┘
Duration 241.054958ms time
$ kubectl get pod -A
NAMESPACE NAME READY STATUS RESTARTS AGE
kube-system cilium-cz8j8 1/1 Running 0 3m3s
kube-system cilium-envoy-lqrrk 1/1 Running 0 3m3s
kube-system cilium-envoy-z87nh 1/1 Running 0 3m3s
kube-system cilium-hfrmw 1/1 Running 0 3m3s
kube-system cilium-operator-86d86869fd-444fv 1/1 Running 0 3m3s
kube-system cilium-operator-86d86869fd-bn848 1/1 Running 0 3m3s
kube-system coredns-7b9dbb4b67-2255m 1/1 Running 0 6m38s
kube-system coredns-7b9dbb4b67-vdlkv 1/1 Running 0 6m38s
kube-system haproxy-temp-wk-node-1 1/1 Running 0 4m56s
kube-system kube-apiserver-temp-cp-node-1 1/1 Running 0 6m39s
kube-system kube-controller-manager-temp-cp-node-1 1/1 Running 0 6m39s
kube-system kube-proxy-24xsp 1/1 Running 0 4m56s
kube-system kube-proxy-bl97s 1/1 Running 0 6m23s
kube-system kube-scheduler-temp-cp-node-1 1/1 Running 0 6m39s
kube-system metrics-server-5ccf9c5678-rbrc5 1/1 Running 0 6m18s$ bin/cubectl addon disable cilium --kubeconfig [ kubeconfig path ]
addon disable start: cilium ...
Error: addon disable fail: uninstallation completed with 1 error(s): context deadline exceeded
exit status 1$ kubectl get ns -A
$ kubectl get ns
NAME STATUS AGE
cilium-secrets Terminating 66s
default Active 9m37s
kube-node-lease Active 9m37s
kube-public Active 9m37s
kube-system Active 9m37s
$ kubectl get namespace cilium-secrets -o json > tmp-ns.json
$ vi tmp-ns.json
# 해당 파일 수정 시 spec 부분을 빈칸으로 만들어준다.
# ex) spec: {},
$ kubectl replace --raw "/api/v1/namespaces/cilium-secrets/finalize" -f ./tmp-ns.json
{"kind":"Namespace","apiVersion":"v1","metadata":{"name":"cilium-secrets","uid":"e47bd9f0-983f-414a-a351-62e2e7a9753a","resourceVersion":"1744",
...
"reason":"ContentHasNoFinalizers","message":"All content-preserving finalizers finished"}]}}$ curl -sfL https://cube.k3.acornsoft.io/download-package | ACCOUNT=<user> PASSWD=<password> PACKAGE=<package name> sh -
$ curl -sfL https://cube.k3.acornsoft.io/download-package | ACCOUNT=<user> PASSWD=<password> PACKAGE=<package name> VERSION=<version> sh -
$ curl -sfL https://cube.k3.acornsoft.io/download-package | ACCOUNT=<user> PASSWD=<password> PACKAGE=<package name> VERSION=<version> OS=<linux | darwin> ARCH=<CPU architecuture> sh -
[INFO] starting download file kubeflow-manifests-1.10.2.tar.gz
######################################################################## 100.0%
[INFO] decompressing file kubeflow-manifests-*.tar.gz
[INFO] starting download file kubectl-*-linux-amd64
...$CUBE_HOME/
└── cubectl-v1.x.x/
├── archive
│ └── push
│ ├── localrepo
│ └── registry
│ └── kubeflow-images-1.10.2.tar.gz
├── bin
...
├── LICENSE.txt
├── kubeflow
└── logs