On 6/7/23 14:22, Frank Schilder wrote:
Hi Stefan,
yes, ceph-volume OSDs.
Requirements:
Kernel version requirement and higher: 5.9
cryptsetup: 2.3.4 and higher. Preferably 2.4.x (automatic alignment of
sector size based on physical disk properties).
RAW device:
cryptsetup luksFormat /dev/device
LVM device:
cryptsetup luksFormat /dev/LVM/LV-here (this is what ceph-volume uses)
Afterwards unlock the device with "cryptsetup open": cryptsetup open
/dev/sdz name_that_makes_sense_for_you
To use it with ceph-volume, the following changes can be made, here a
diff against 16.2.11:
--- before: /usr/lib/python3/dist-packages/ceph_volume/util/disk.py
+++ after:
/home/stefan/git/bit-ceph/configs/osd/usr/lib/python3/dist-packages/ceph_volume/util/disk.py
@@ -194,6 +194,26 @@
devices.append(_lsblk_parser(line))
return devices
+
+def is_rotational(device):
+ """
+ Returns true if a device is rotational or not.
+ """
+ labels = ['ROTA']
+ command = ['lsblk', '-P', '-p', '-o', ','.join(labels), device]
+ out, err, rc = process.call(command)
+ device = []
+ for line in out:
+ device.append(_lsblk_parser(line))
+
+ for i in device:
+ if type(i) is dict and "ROTA" in i.keys():
+ if i['ROTA'] == '1':
+ rotational = True
+ else:
+ rotational = False
+
+ return rotational
def udevadm_property(device, properties=[]):
@@ -900,6 +920,10 @@
metadata['sectorsize'] = get_file_contents(sysdir +
"/queue/logical_block_size",
fallback_sectorsize)
+ fallback_rotational = '1'
+ metadata['rotational'] = get_file_contents(sysdir +
+ "/queue/rotational",
+ fallback_rotational)
metadata['size'] = float(size) * 512
metadata['human_readable_size'] =
human_readable_size(metadata['size'])
metadata['path'] = diskname
--- before: /usr/lib/python3/dist-packages/ceph_volume/util/encryption.py
+++ after:
/home/stefan/git/bit-ceph/configs/osd/usr/lib/python3/dist-packages/ceph_volume/util/encryption.py
@@ -1,11 +1,12 @@
import base64
import os
import logging
+import subprocess
from ceph_volume import process, conf, terminal
from ceph_volume.util import constants, system
from ceph_volume.util.device import Device
from .prepare import write_keyring
-from .disk import lsblk, device_family, get_part_entry_type
+from .disk import lsblk, device_family, get_part_entry_type, is_rotational
logger = logging.getLogger(__name__)
mlogger = terminal.MultiLogger(__name__)
@@ -23,10 +24,52 @@
if key_size not in ['256', '512']:
logger.warning(("Invalid value set for osd_dmcrypt_key_size
({}). "
- "Falling back to {}bits".format(key_size,
default_key_size)))
+ "Falling back to {} bits".format(key_size,
default_key_size)))
return default_key_size
return key_size
+
+def execCmd(cmdline):
+ cmd = subprocess.Popen(cmdline, shell=True, stdout=subprocess.PIPE)
+ stdout, _ = cmd.communicate()
+ return stdout.decode("utf-8")
+
+def bypass_workqueues(device):
+ """
+ 1) Check if cryptsetup has support for no_read_workqueue.
+ If so, it also supports no_write_workqueue.
+ cryptsetup has support for this since 2.3.4
+
+ 2) Also check if we are using a flash (non-rotational) device
+ or not. I.e. Spinning media might still benefit from queuing.
+
+ Only if both 1) and 2) are true we do _not_ want to use work queues
+ Note:
+ crypt version 1.22 and higher have support for this. Example:
+ modprobe dm-crypt
+ dmsetup targets
+
+ integrity v1.10.0
+ crypt v1.23.0
+ striped v1.6.0
+ linear v1.4.0
+ error v1.5.0
+
+ If dm-crypt does not have support for no_read_workqueue /
+ no_write_workqueue the options will be discarded and
+ logged by the linux kernel:
+
+ device-mapper: table: major:minor: crypt: Invalid feature arguments
+ device-mapper: ioctl: error adding target to table
+
+ The encrypted device will come online without the options active.
+ """
+ cryptsetup_help = execCmd("cryptsetup --help 2>/dev/null")
+
+ if '--perf-no_read_workqueue' not in cryptsetup_help or
is_rotational(device):
+ return []
+
+ return ['--perf-no_read_workqueue', '--perf-no_write_workqueue']
def create_dmcrypt_key():
"""
@@ -78,7 +121,8 @@
'--type', 'plain',
'--key-size', '256',
]
-
+ for extra_opts in bypass_workqueues(device):
+ command.insert(1, extra_opts)
process.call(command, stdin=key, terminal_verbose=True,
show_command=True)
@@ -103,6 +147,8 @@
device,
mapping,
]
+ for extra_opts in bypass_workqueues(device):
+ command.insert(1, extra_opts)
process.call(command, stdin=key, terminal_verbose=True,
show_command=True)
But in your case you might want to change the logic a bit so regardless
of rotational media or not the queues are bypassed (make the ROTA
function always return true for example).
To change Ceph container images ... I used this quick and dirty procedure:
docker run -it ceph-image bash
Make the above changes to ceph-volume
exit from container
docker ps -a
Look for the id of the most recently used container (that you just exited)
docker commit container-hash alternative_name_of_container_image
if you want to upload to registry: docker image tag
alternative_name_of_container_image
docker_registry_url:5000/alternative_name_of_container_image
docker image push
docker_registry_url:5000/alternative_name_of_container_image
for cephadm users:
docker image ls --digests <- look for the sha256 hash of the image
ceph config-key set config/global/container_image
docker_registry_url:5000/alternative_name_of_container_image:latest@sha256:the-hash-here
redeploy containers with this new image
Let me know if you need other info to make this work for you. Looking
forward to the results.
Gr. Stefan
_______________________________________________
ceph-users mailing list -- ceph-users@xxxxxxx
To unsubscribe send an email to ceph-users-leave@xxxxxxx
