sudo apt-get update
# apt-transport-https may be a dummy package; if so, you can skip that package
sudo apt-get install -y apt-transport-https ca-certificates curl gpg

# If the folder `/etc/apt/keyrings` does not exist, it should be created before the curl command, read the note below.
# sudo mkdir -p -m 755 /etc/apt/keyrings
curl -fsSL https://pkgs.k8s.io/core:/stable:/v1.29/deb/Release.key | sudo gpg --dearmor -o /etc/apt/keyrings/kubernetes-apt-keyring.gpg

# This overwrites any existing configuration in /etc/apt/sources.list.d/kubernetes.list
echo 'deb [signed-by=/etc/apt/keyrings/kubernetes-apt-keyring.gpg] https://pkgs.k8s.io/core:/stable:/v1.29/deb/ /' | sudo tee /etc/apt/sources.list.d/kubernetes.list

sudo apt-get update
sudo apt-get install -y kubelet kubeadm kubectl
sudo apt-mark hold kubelet kubeadm kubectl

sudo kubeadm init

[init] Using Kubernetes version: v1.29.2
[preflight] Running pre-flight checks
error execution phase preflight: [preflight] Some fatal errors occurred:
	[ERROR IsPrivilegedUser]: user is not running as root
[preflight] If you know what you are doing, you can make a check non-fatal with `--ignore-preflight-errors=...`
To see the stack trace of this error execute with --v=5 or higher

kubectl create deployment nginx --image nginx

error: failed to create deployment: Post "http://localhost:8080/apis/apps/v1/namespaces/default/deployments?fieldManager=kubectl-create&fieldValidation=Strict": dial tcp 127.0.0.1:8080: connect: connection refused

sudo kubeadm init --v=5

[preflight] Some fatal errors occurred:
	[ERROR Mem]: the system RAM (917 MB) is less than the minimum 1700 MB
[preflight] If you know what you are doing, you can make a check non-fatal with `--ignore-preflight-errors=...`

cat <<EOF | sudo tee /etc/modules-load.d/k8s.conf
overlay
br_netfilter
EOF

sudo modprobe overlay
sudo modprobe br_netfilter

# sysctl params required by setup, params persist across reboots
cat <<EOF | sudo tee /etc/sysctl.d/k8s.conf
net.bridge.bridge-nf-call-iptables  = 1
net.bridge.bridge-nf-call-ip6tables = 1
net.ipv4.ip_forward                 = 1
EOF

# Apply sysctl params without reboot
sudo sysctl --system

lsmod | grep br_netfilter
lsmod | grep overlay

sysctl net.bridge.bridge-nf-call-iptables net.bridge.bridge-nf-call-ip6tables net.ipv4.ip_forward

# Add Docker's official GPG key:
sudo apt-get update
sudo apt-get install ca-certificates curl
sudo install -m 0755 -d /etc/apt/keyrings
sudo curl -fsSL https://download.docker.com/linux/ubuntu/gpg -o /etc/apt/keyrings/docker.asc
sudo chmod a+r /etc/apt/keyrings/docker.asc

# Add the repository to Apt sources:
echo \
  "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/ubuntu \
  $(. /etc/os-release && echo "$VERSION_CODENAME") stable" | \
  sudo tee /etc/apt/sources.list.d/docker.list > /dev/null
sudo apt-get update

# sudo apt-get install docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin
# docker에서 깔라고 알려주는거라 다 깔라는데.... 난.... compose는 깔지 않을래....
sudo apt-get install docker-ce docker-ce-cli containerd.io docker-buildx-plugin -y

sudo mkdir -p /etc/containerd
containerd config default | sudo tee /etc/containerd/config.toml

Note:
If you installed containerd from a package (for example, RPM or .deb), you may find that the CRI integration plugin is disabled by default.

You need CRI support enabled to use containerd with Kubernetes. Make sure that cri is not included in thedisabled_plugins list within /etc/containerd/config.toml; if you made changes to that file, also restart containerd.

If you experience container crash loops after the initial cluster installation or after installing a CNI, the containerd configuration provided with the package might contain incompatible configuration parameters. Consider resetting the containerd configuration with containerd config default > /etc/containerd/config.toml as specified in getting-started.md and then set the configuration parameters specified above accordingly.

sudo systemctl daemon-reload
# sudo systemctl start containerd
# start로 했다가 변경 사항이 반영되지 않아서, 안되더라 꼭 restart 를 하라
sudo systemctl stop containerd
sudo systemctl restart containerd
sudo systemctl enable --now containerd

/usr/bin/kubelet --container-runtime=remote --container-runtime-endpoint=unix:///run/containerd/containerd.sock --pod-manifest-path=/etc/kubernetes/manifests

--container-runtime-endpoint string     Default: "unix:///run/containerd/containerd.sock"

sudo kubeadm init

[init] Using Kubernetes version: v1.29.2
[preflight] Running pre-flight checks
error execution phase preflight: [preflight] Some fatal errors occurred:
	[ERROR CRI]: container runtime is not running: output: time="2024-02-26T00:18:17Z" level=fatal msg="validate service connection: validate CRI v1 runtime API for endpoint \"unix:///var/run/containerd/containerd.sock\": rpc error: code = Unavailable desc = connection error: desc = \"transport: Error while dialing: dial unix /var/run/containerd/containerd.sock: connect: no such file or directory\""
, error: exit status 1
	[ERROR FileContent--proc-sys-net-bridge-bridge-nf-call-iptables]: /proc/sys/net/bridge/bridge-nf-call-iptables does not exist
	[ERROR FileContent--proc-sys-net-ipv4-ip_forward]: /proc/sys/net/ipv4/ip_forward contents are not set to 1
[preflight] If you know what you are doing, you can make a check non-fatal with `--ignore-preflight-errors=...`
To see the stack trace of this error execute with --v=5 or higher

[init] Using Kubernetes version: v1.29.2
[preflight] Running pre-flight checks
error execution phase preflight: [preflight] Some fatal errors occurred:
	[ERROR CRI]: container runtime is not running: output: time="2024-02-26T00:45:43Z" level=fatal msg="validate service connection: validate CRI v1 runtime API for endpoint \"unix:///var/run/containerd/containerd.sock\": rpc error: code = Unimplemented desc = unknown service runtime.v1.RuntimeService"
, error: exit status 1
[preflight] If you know what you are doing, you can make a check non-fatal with `--ignore-preflight-errors=...`
To see the stack trace of this error execute with --v=5 or higher

enabled_plugins = ["cri"]

sudo systemctl daemon-reload
# sudo systemctl start containerd
# start로 했다가 변경 사항이 반영되지 않아서, 안되더라 꼭 restart 를 하라
sudo systemctl stop containerd
sudo systemctl restart containerd
sudo systemctl enable --now containerd

VERSION="v1.29.0" # check latest version in /releases page
wget https://github.com/kubernetes-sigs/cri-tools/releases/download/$VERSION/crictl-$VERSION-linux-arm64.tar.gz
sudo tar zxvf crictl-$VERSION-linux-arm64.tar.gz -C /usr/local/bin
rm -f crictl-$VERSION-linux-amd64.tar.gz

ubuntu@ip-10-0-145-244:~$ crictl
NAME:
   crictl - client for CRI

USAGE:
   crictl [global options] command [command options] [arguments...]

VERSION:
   v1.29.0

cat << "EOF" | sudo tee /etc/crictl.yaml
runtime-endpoint: unix:///run/containerd/containerd.sock
image-endpoint: unix:///run/containerd/containerd.sock
timeout: 10
debug: false
EOF

cat << EOF | sudo tee /etc/kubernetes/manifests/web.yaml
apiVersion: v1
kind: Pod
metadata:
  name: static-web
spec:
  containers:
  - name: web
    image: nginx
    ports:
    - name: web
      containerPort: 80
      hostPort: 8080
      protocol: TCP
EOF

sudo /usr/bin/kubelet --container-runtime-endpoint=unix:///run/containerd/containerd.sock --pod-manifest-path=/etc/kubernetes/manifests

E0226 07:03:44.971905    6463 pod_workers.go:1298] "Error syncing pod, skipping" err="network is not ready: container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:Network plugin returns error: cni plugin not initialized" pod="default/static-web-ip-10-0-136-250" podUID="103f5f5baab872f8bf4f0d41fb78b05a"

wget https://github.com/containernetworking/plugins/releases/download/v1.4.0/cni-plugins-linux-arm64-v1.4.0.tgz
mkdir -p /opt/cni/bin
tar Cxzvf /opt/cni/bin cni-plugins-linux-arm64-v1.4.0.tgz

ubuntu@ip-10-0-136-250:~$ /opt/cni/bin/b
bandwidth  bridge
ubuntu@ip-10-0-136-250:~$ /opt/cni/bin/b
bandwidth  bridge
ubuntu@ip-10-0-136-250:~$ /opt/cni/bin/bandwidth
CNI bandwidth plugin v1.3.0
CNI protocol versions supported: 0.3.0, 0.3.1, 0.4.0, 1.0.0
ubuntu@ip-10-0-136-250:~$

sudo cat << EOF | sudo tee /etc/cni/net.d/10-containerd-net.conflist
{
  "cniVersion": "1.0.0",
  "name": "containerd-net",
  "plugins": [
    {
      "type": "bridge",
      "bridge": "cni0",
      "isGateway": true,
      "ipMasq": true,
      "promiscMode": true,
      "ipam": {
        "type": "host-local",
        "ranges": [
          [{
            "subnet": "10.88.0.0/16"
          }],
          [{
            "subnet": "2001:4860:4860::/64"
          }]
        ],
        "routes": [
          { "dst": "0.0.0.0/0" },
          { "dst": "::/0" }
        ]
      }
    },
    {
      "type": "portmap",
      "capabilities": {"portMappings": true}
    }
  ]
}
EOF

sudo systemctl daemon-reload
# sudo systemctl start containerd
# start로 했다가 변경 사항이 반영되지 않아서, 안되더라 꼭 restart 를 하라
sudo systemctl stop containerd
sudo systemctl restart containerd
sudo systemctl enable --now containerd

sudo crictl pods

ubuntu@ip-10-0-136-250:~$ sudo crictl pods
POD ID              CREATED              STATE               NAME                         NAMESPACE           ATTEMPT             RUNTIME
9e0988cda0127       About a minute ago   Ready               static-web-ip-10-0-136-250   default             0                   (default)

PODID=$(sudo crictl pods | awk /static-web/{'print $1'})
PODIP=$(sudo crictl inspectp -o go-template --template '{{.status.network.ip}}' ${PODID})

curl $PODIP:80

<!DOCTYPE html>
<html>
<head>
<title>Welcome to nginx!</title>
<style>
html { color-scheme: light dark; }
body { width: 35em; margin: 0 auto;
font-family: Tahoma, Verdana, Arial, sans-serif; }
</style>
</head>
<body>
<h1>Welcome to nginx!</h1>
<p>If you see this page, the nginx web server is successfully installed and
working. Further configuration is required.</p>

<p>For online documentation and support please refer to
<a href="http://nginx.org/">nginx.org</a>.<br/>
Commercial support is available at
<a href="http://nginx.com/">nginx.com</a>.</p>

<p><em>Thank you for using nginx.</em></p>
</body>
</html>

ROUTABLE_NODE_IP=$(ip -o r g 1 | awk '{print $7}')

curl $ROUTABLE_NODE_IP:8080

# /etc/containerd/config.toml
disabled_plugins = [] -> enabled_plugins = ["cri"]
SystemdCgroup = false