Setting Up Multi-Node Storage

sudo microk8s enable core/rook-ceph

sudo microk8s.kubectl wait deployment -n rook-ceph rook-ceph-operator --for condition=Available=True --timeout=180s

BLOCK_DEVICE="/dev/sdb"  # Replace with your actual device

# Check if it's a block device
if [ ! -b "$BLOCK_DEVICE" ]; then
    echo "Error: $BLOCK_DEVICE is not a block device"
    exit 1
fi

# Check if it's not mounted
if [ -n "$(lsblk -o MOUNTPOINT $BLOCK_DEVICE | tail -n +2)" ]; then
    echo "Error: $BLOCK_DEVICE is mounted"
    exit 1
fi

# Check if it has no filesystem
if [ -n "$(lsblk -o FSTYPE $BLOCK_DEVICE | tail -n +2)" ]; then
    echo "Error: $BLOCK_DEVICE has a filesystem"
    exit 1
fi
 

MULTI_NODE=3  # Number of nodes in your cluster
BLOCK_DEVICE="/dev/sdb"  # Your block device
HOSTNAME=$(hostname)

sudo microk8s.kubectl apply -f - <<EOF
apiVersion: ceph.rook.io/v1
kind: CephCluster
metadata:
  name: trifecta-rook-ceph
  namespace: rook-ceph
spec:
  cephVersion:
    image: quay.io/ceph/ceph:v18
  dataDirHostPath: /var/lib/rook
  mon:
    count: ${MULTI_NODE}
    allowMultiplePerNode: false
  dashboard:
    enabled: false
  storage:
    useAllNodes: false
    useAllDevices: false
    nodes:
    - name: "${HOSTNAME}"
      devices:
      - name: "${BLOCK_DEVICE}"
EOF
 

# Verify block device
BLOCK_DEVICE="/dev/sdb"  # Replace with your device

# Add this node to the Ceph cluster
HOSTNAME=$(hostname)
sudo microk8s.kubectl patch cephclusters.ceph.rook.io --type json -n rook-ceph trifecta-rook-ceph -p '[
  {
    "op": "add",
    "path": "/spec/storage/nodes/-",
    "value": {
      "name": "'$HOSTNAME'",
      "devices": [
        {
          "name": "'$BLOCK_DEVICE'"
        }
      ]
    }
  }
]'

MULTI_NODE=3  # Adjust to your actual node count

# Create unreplicated storage class
sudo microk8s.kubectl apply -f - <<EOF
apiVersion: ceph.rook.io/v1
kind: CephBlockPool
metadata:
  name: replicapool-1
  namespace: rook-ceph
spec:
  failureDomain: host
  replicated:
    size: 1
---
apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
   name: unreplicated
provisioner: rook-ceph.rbd.csi.ceph.com
parameters:
    clusterID: rook-ceph
    pool: replicapool-1
    imageFormat: "2"
    imageFeatures: layering
    csi.storage.k8s.io/provisioner-secret-name: rook-csi-rbd-provisioner
    csi.storage.k8s.io/provisioner-secret-namespace: rook-ceph
    csi.storage.k8s.io/controller-expand-secret-name: rook-csi-rbd-provisioner
    csi.storage.k8s.io/controller-expand-secret-namespace: rook-ceph
    csi.storage.k8s.io/node-stage-secret-name: rook-csi-rbd-node
    csi.storage.k8s.io/node-stage-secret-namespace: rook-ceph
    csi.storage.k8s.io/fstype: ext4
reclaimPolicy: Delete
allowVolumeExpansion: true
EOF

# Create replicated storage class (if more than 1 node)
if [ "$MULTI_NODE" -gt 1 ]; then
  TOP_END=3
  if [ "$MULTI_NODE" -lt 3 ]; then
    TOP_END="$MULTI_NODE"
  fi

  sudo microk8s.kubectl apply -f - <<EOF
apiVersion: ceph.rook.io/v1
kind: CephBlockPool
metadata:
  name: replicapool-${TOP_END}
  namespace: rook-ceph
spec:
  failureDomain: host
  replicated:
    size: ${TOP_END}
---
apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
   name: replicated
provisioner: rook-ceph.rbd.csi.ceph.com
parameters:
    clusterID: rook-ceph
    pool: replicapool-${TOP_END}
    imageFormat: "2"
    imageFeatures: layering
    csi.storage.k8s.io/provisioner-secret-name: rook-csi-rbd-provisioner
    csi.storage.k8s.io/provisioner-secret-namespace: rook-ceph
    csi.storage.k8s.io/controller-expand-secret-name: rook-csi-rbd-provisioner
    csi.storage.k8s.io/controller-expand-secret-namespace: rook-ceph
    csi.storage.k8s.io/node-stage-secret-name: rook-csi-rbd-node
    csi.storage.k8s.io/node-stage-secret-namespace: rook-ceph
    csi.storage.k8s.io/fstype: ext4
reclaimPolicy: Delete
allowVolumeExpansion: true
EOF
fi

sudo microk8s.kubectl patch storageclass unreplicated -p '{"metadata": {"annotations":{"storageclass.kubernetes.io/is-default-class":"true"}}}'