From Ever changing code
Jump to navigation Jump to search

Ceph RBD is a remote block device just like iSCSI. A volume is presented to your pod as a raw disk drive and can be formatted with any filesystem you desire. For obvious reasons an RBD volume cannot be shared. Ceph stores everything in RADOS as objects, in the case of RBD images.

An RBD image is a virtual hard drive image formed of RADOS objects. It is literally just a contiguous block of bytes just like a raw Linux block device.

ceph us distributed storage it's not raw device block storage. RBD is Ceph's RADOS Block Device. cephfs are two ways how ceph serves the data but via gateways can provide S3, NFS, SMB, iSCSI and it's object storage. It's using something called placement groups to distribute the data and it's using special algorithm that pre-defines where the chunk of data is so if client has info it does not need to ask server where data block/object is but goes directly to the specific node which holds it.


Configure access to ceph cluster

# Ubuntu 20.04
sudo apt install ceph-common

# Config file
cat > ~/.ceph/ceph.conf <<EOF
mon_host = XXXXXXX
keyring = /home/myuser/.ceph/ceph.client.admin.keyring # requires full absolute path
auth_cluster_required = cephx
auth_service_required = cephx
auth_client_required = cephx

cat > ~/.ceph/ceph.client.admin.keyring <<EOF
    key = XXXXXXXXXXXX==

# Test
ceph -c ~/.ceph/ceph.conf status 
    id:     aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeeee
    health: HEALTH_OK
    mon: 3 daemons, quorum dc1ceph11-2222,dc2ceph21-3333,dc3ceph31-4444 (age 4h)
    mgr: dc1ceph11-2222.dddddd(active, since 51m), standbys: dc2ceph21-3333.eeeeee
    mds: devcephfs:1 {0=devcephfs.dc3ceph31-4444.nmngty=up:active} 2 up:standby
    osd: 20 osds: 19 up (since 4d), 19 in (since 4d)
    rgw: 1 daemon active (admin)
  task status:
    scrub status:
        mds.devcephfs.dc3ceph31-4444.nmngty: idle
    pools:   22 pools, 449 pgs
    objects: 10.77M objects, 25 TiB
    usage:   54 TiB used, 85 TiB / 139 TiB avail
    pgs:     447 active+clean
             2   active+clean+scrubbing+deep
    client:   27 MiB/s rd, 5.6 MiB/s wr, 3.88k op/s rd, 191 op/s wr

# Aliases
alias ceph="ceph -c ~/.ceph/ceph.conf"
alias rbd="rbd -c ~/.ceph/ceph.conf"

Raw Block Device Operations

# List block device images
rbd ls {poolname} -c ~/.ceph/ceph.conf 

# Create a block device image
rbd create {image-name} --size {megabytes} --pool {pool-name}
rbd create dev-prometheus-data-0-vol --size 3072 --pool DEV_block # 3Gb

rbd info --image dev-prometheus-data-0-vol --pool DEV_block -c ~/.ceph/ceph.conf 
rbd image 'dev-prometheus-data-0-vol':
	size 3 GiB in 768 objects
	order 22 (4 MiB objects)
	snapshot_count: 0
	id: 0f582d6be65aaa
	block_name_prefix: rbd_data.0f582d6be65aaa
	format: 2
	features: layering, exclusive-lock, object-map, fast-diff, deep-flatten
	create_timestamp: Tue Nov 11 13:30:53 2021
	access_timestamp: Tue Nov 11 13:30:53 2021
	modify_timestamp: Tue Nov 11 13:30:53 2021

# Resize the rbd image
rbd resize --image dev-prometheus-data-0-vol --size 4096 --pool DEV_block -c ~/.ceph/ceph.conf
Resizing image: 100% complete...done.

# Expand the filesystem
## Map block device to local machine
sudo rbd device map {pool-name}/{image-name} --id {user-name}
sudo rbd -c ~/.ceph/ceph.conf device map DEV_block/dev-prometheus-data-0-vol
/dev/rbd0 # <- output is showing new device mapping

## List mapped devices
rbd device list -c ~/.ceph/ceph.conf
id  pool       namespace  image                      snap  device   
0   DEV_block             dev-prometheus-data-0-vol  -     /dev/rbd0

## Show the raw block device(rbd) filesystem
sudo blkid | grep rbd
/dev/rbd0: UUID="aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa" TYPE="ext4"

## Expand the filesystem
### Check filesystem integrity. Required if resize2fs requests it: eg. resize2fs 1.45.5 (07-Jan-2020), Please run 'e2fsck -f /dev/rbd0' first.
sudo e2fsck -f /dev/rbd0

## Expand resize2fs
sudo resize2fs /dev/rbd0

# Mount
mkdir -p ~/mnt/rdb0
sudo mount /dev/rbd0 ~/mnt/rdb0

df -h /dev/rbd0
Filesystem      Size  Used Avail Use% Mounted on
/dev/rbd0         3G  0.1G  2.9G 0.1% /home/vagrant/mnt/rdb0

# Umount and umap
sudo umount /dev/rbd0 ~/mnt/rdb0
sudo rbd -c ~/.ceph/ceph.conf unmap /dev/rbd0 # release the image from being mapped to local machine

Delete images and snapshots

# Delete a cloned image
rbd -c ~/.ceph/ceph.conf remove ${POOL}/${IMAGE}

# Delete a snapshot
rbd -c ~/.ceph/ceph.conf info --pool    ${POOL} $IMAGE
rbd -c ~/.ceph/ceph.conf snap ls        ${POOL}/$IMAGE             # Check if the snapshot is protected
rbd -c ~/.ceph/ceph.conf snap unprotect ${POOL}/$IMAGE@${SNAPSHOT} # Unprotect the parent snapshot
rbd -c ~/.ceph/ceph.conf snap rm        ${POOL}/$IMAGE@${SNAPSHOT}