1// SPDX-License-Identifier: GPL-2.0 2/* 3 * Copyright 2019 Google LLC 4 */ 5 6/** 7 * DOC: The Keyslot Manager 8 * 9 * Many devices with inline encryption support have a limited number of "slots" 10 * into which encryption contexts may be programmed, and requests can be tagged 11 * with a slot number to specify the key to use for en/decryption. 12 * 13 * As the number of slots is limited, and programming keys is expensive on 14 * many inline encryption hardware, we don't want to program the same key into 15 * multiple slots - if multiple requests are using the same key, we want to 16 * program just one slot with that key and use that slot for all requests. 17 * 18 * The keyslot manager manages these keyslots appropriately, and also acts as 19 * an abstraction between the inline encryption hardware and the upper layers. 20 * 21 * Lower layer devices will set up a keyslot manager in their request queue 22 * and tell it how to perform device specific operations like programming/ 23 * evicting keys from keyslots. 24 * 25 * Upper layers will call blk_ksm_get_slot_for_key() to program a 26 * key into some slot in the inline encryption hardware. 27 */ 28 29#define pr_fmt(fmt) "blk-crypto: " fmt 30 31#include <linux/keyslot-manager.h> 32#include <linux/atomic.h> 33#include <linux/mutex.h> 34#include <linux/pm_runtime.h> 35#include <linux/wait.h> 36#include <linux/blkdev.h> 37 38struct blk_ksm_keyslot { 39 atomic_t slot_refs; 40 struct list_head idle_slot_node; 41 struct hlist_node hash_node; 42 const struct blk_crypto_key *key; 43 struct blk_keyslot_manager *ksm; 44}; 45 46static inline void blk_ksm_hw_enter(struct blk_keyslot_manager *ksm) 47{ 48 /* 49 * Calling into the driver requires ksm->lock held and the device 50 * resumed. But we must resume the device first, since that can acquire 51 * and release ksm->lock via blk_ksm_reprogram_all_keys(). 52 */ 53 if (ksm->dev) 54 pm_runtime_get_sync(ksm->dev); 55 down_write(&ksm->lock); 56} 57 58static inline void blk_ksm_hw_exit(struct blk_keyslot_manager *ksm) 59{ 60 up_write(&ksm->lock); 61 if (ksm->dev) 62 pm_runtime_put_sync(ksm->dev); 63} 64 65/** 66 * blk_ksm_init() - Initialize a keyslot manager 67 * @ksm: The keyslot_manager to initialize. 68 * @num_slots: The number of key slots to manage. 69 * 70 * Allocate memory for keyslots and initialize a keyslot manager. Called by 71 * e.g. storage drivers to set up a keyslot manager in their request_queue. 72 * 73 * Return: 0 on success, or else a negative error code. 74 */ 75int blk_ksm_init(struct blk_keyslot_manager *ksm, unsigned int num_slots) 76{ 77 unsigned int slot; 78 unsigned int i; 79 unsigned int slot_hashtable_size; 80 81 memset(ksm, 0, sizeof(*ksm)); 82 83 if (num_slots == 0) 84 return -EINVAL; 85 86 ksm->slots = kvcalloc(num_slots, sizeof(ksm->slots[0]), GFP_KERNEL); 87 if (!ksm->slots) 88 return -ENOMEM; 89 90 ksm->num_slots = num_slots; 91 92 init_rwsem(&ksm->lock); 93 94 init_waitqueue_head(&ksm->idle_slots_wait_queue); 95 INIT_LIST_HEAD(&ksm->idle_slots); 96 97 for (slot = 0; slot < num_slots; slot++) { 98 ksm->slots[slot].ksm = ksm; 99 list_add_tail(&ksm->slots[slot].idle_slot_node, 100 &ksm->idle_slots); 101 } 102 103 spin_lock_init(&ksm->idle_slots_lock); 104 105 slot_hashtable_size = roundup_pow_of_two(num_slots); 106 /* 107 * hash_ptr() assumes bits != 0, so ensure the hash table has at least 2 108 * buckets. This only makes a difference when there is only 1 keyslot. 109 */ 110 if (slot_hashtable_size < 2) 111 slot_hashtable_size = 2; 112 113 ksm->log_slot_ht_size = ilog2(slot_hashtable_size); 114 ksm->slot_hashtable = kvmalloc_array(slot_hashtable_size, 115 sizeof(ksm->slot_hashtable[0]), 116 GFP_KERNEL); 117 if (!ksm->slot_hashtable) 118 goto err_destroy_ksm; 119 for (i = 0; i < slot_hashtable_size; i++) 120 INIT_HLIST_HEAD(&ksm->slot_hashtable[i]); 121 122 return 0; 123 124err_destroy_ksm: 125 blk_ksm_destroy(ksm); 126 return -ENOMEM; 127} 128EXPORT_SYMBOL_GPL(blk_ksm_init); 129 130static inline struct hlist_head * 131blk_ksm_hash_bucket_for_key(struct blk_keyslot_manager *ksm, 132 const struct blk_crypto_key *key) 133{ 134 return &ksm->slot_hashtable[hash_ptr(key, ksm->log_slot_ht_size)]; 135} 136 137static void blk_ksm_remove_slot_from_lru_list(struct blk_ksm_keyslot *slot) 138{ 139 struct blk_keyslot_manager *ksm = slot->ksm; 140 unsigned long flags; 141 142 spin_lock_irqsave(&ksm->idle_slots_lock, flags); 143 list_del(&slot->idle_slot_node); 144 spin_unlock_irqrestore(&ksm->idle_slots_lock, flags); 145} 146 147static struct blk_ksm_keyslot *blk_ksm_find_keyslot( 148 struct blk_keyslot_manager *ksm, 149 const struct blk_crypto_key *key) 150{ 151 const struct hlist_head *head = blk_ksm_hash_bucket_for_key(ksm, key); 152 struct blk_ksm_keyslot *slotp; 153 154 hlist_for_each_entry(slotp, head, hash_node) { 155 if (slotp->key == key) 156 return slotp; 157 } 158 return NULL; 159} 160 161static struct blk_ksm_keyslot *blk_ksm_find_and_grab_keyslot( 162 struct blk_keyslot_manager *ksm, 163 const struct blk_crypto_key *key) 164{ 165 struct blk_ksm_keyslot *slot; 166 167 slot = blk_ksm_find_keyslot(ksm, key); 168 if (!slot) 169 return NULL; 170 if (atomic_inc_return(&slot->slot_refs) == 1) { 171 /* Took first reference to this slot; remove it from LRU list */ 172 blk_ksm_remove_slot_from_lru_list(slot); 173 } 174 return slot; 175} 176 177unsigned int blk_ksm_get_slot_idx(struct blk_ksm_keyslot *slot) 178{ 179 return slot - slot->ksm->slots; 180} 181EXPORT_SYMBOL_GPL(blk_ksm_get_slot_idx); 182 183/** 184 * blk_ksm_get_slot_for_key() - Program a key into a keyslot. 185 * @ksm: The keyslot manager to program the key into. 186 * @key: Pointer to the key object to program, including the raw key, crypto 187 * mode, and data unit size. 188 * @slot_ptr: A pointer to return the pointer of the allocated keyslot. 189 * 190 * Get a keyslot that's been programmed with the specified key. If one already 191 * exists, return it with incremented refcount. Otherwise, wait for a keyslot 192 * to become idle and program it. 193 * 194 * Context: Process context. Takes and releases ksm->lock. 195 * Return: BLK_STS_OK on success (and keyslot is set to the pointer of the 196 * allocated keyslot), or some other blk_status_t otherwise (and 197 * keyslot is set to NULL). 198 */ 199blk_status_t blk_ksm_get_slot_for_key(struct blk_keyslot_manager *ksm, 200 const struct blk_crypto_key *key, 201 struct blk_ksm_keyslot **slot_ptr) 202{ 203 struct blk_ksm_keyslot *slot; 204 int slot_idx; 205 int err; 206 207 *slot_ptr = NULL; 208 down_read(&ksm->lock); 209 slot = blk_ksm_find_and_grab_keyslot(ksm, key); 210 up_read(&ksm->lock); 211 if (slot) 212 goto success; 213 214 for (;;) { 215 blk_ksm_hw_enter(ksm); 216 slot = blk_ksm_find_and_grab_keyslot(ksm, key); 217 if (slot) { 218 blk_ksm_hw_exit(ksm); 219 goto success; 220 } 221 222 /* 223 * If we're here, that means there wasn't a slot that was 224 * already programmed with the key. So try to program it. 225 */ 226 if (!list_empty(&ksm->idle_slots)) 227 break; 228 229 blk_ksm_hw_exit(ksm); 230 wait_event(ksm->idle_slots_wait_queue, 231 !list_empty(&ksm->idle_slots)); 232 } 233 234 slot = list_first_entry(&ksm->idle_slots, struct blk_ksm_keyslot, 235 idle_slot_node); 236 slot_idx = blk_ksm_get_slot_idx(slot); 237 238 err = ksm->ksm_ll_ops.keyslot_program(ksm, key, slot_idx); 239 if (err) { 240 wake_up(&ksm->idle_slots_wait_queue); 241 blk_ksm_hw_exit(ksm); 242 return errno_to_blk_status(err); 243 } 244 245 /* Move this slot to the hash list for the new key. */ 246 if (slot->key) 247 hlist_del(&slot->hash_node); 248 slot->key = key; 249 hlist_add_head(&slot->hash_node, blk_ksm_hash_bucket_for_key(ksm, key)); 250 251 atomic_set(&slot->slot_refs, 1); 252 253 blk_ksm_remove_slot_from_lru_list(slot); 254 255 blk_ksm_hw_exit(ksm); 256success: 257 *slot_ptr = slot; 258 return BLK_STS_OK; 259} 260 261/** 262 * blk_ksm_put_slot() - Release a reference to a slot 263 * @slot: The keyslot to release the reference of. 264 * 265 * Context: Any context. 266 */ 267void blk_ksm_put_slot(struct blk_ksm_keyslot *slot) 268{ 269 struct blk_keyslot_manager *ksm; 270 unsigned long flags; 271 272 if (!slot) 273 return; 274 275 ksm = slot->ksm; 276 277 if (atomic_dec_and_lock_irqsave(&slot->slot_refs, 278 &ksm->idle_slots_lock, flags)) { 279 list_add_tail(&slot->idle_slot_node, &ksm->idle_slots); 280 spin_unlock_irqrestore(&ksm->idle_slots_lock, flags); 281 wake_up(&ksm->idle_slots_wait_queue); 282 } 283} 284 285/** 286 * blk_ksm_crypto_cfg_supported() - Find out if a crypto configuration is 287 * supported by a ksm. 288 * @ksm: The keyslot manager to check 289 * @cfg: The crypto configuration to check for. 290 * 291 * Checks for crypto_mode/data unit size/dun bytes support. 292 * 293 * Return: Whether or not this ksm supports the specified crypto config. 294 */ 295bool blk_ksm_crypto_cfg_supported(struct blk_keyslot_manager *ksm, 296 const struct blk_crypto_config *cfg) 297{ 298 if (!ksm) 299 return false; 300 if (!(ksm->crypto_modes_supported[cfg->crypto_mode] & 301 cfg->data_unit_size)) 302 return false; 303 if (ksm->max_dun_bytes_supported < cfg->dun_bytes) 304 return false; 305 return true; 306} 307 308/* 309 * This is an internal function that evicts a key from an inline encryption 310 * device that can be either a real device or the blk-crypto-fallback "device". 311 * It is used only by blk_crypto_evict_key(); see that function for details. 312 */ 313int blk_ksm_evict_key(struct blk_keyslot_manager *ksm, 314 const struct blk_crypto_key *key) 315{ 316 struct blk_ksm_keyslot *slot; 317 int err; 318 319 blk_ksm_hw_enter(ksm); 320 slot = blk_ksm_find_keyslot(ksm, key); 321 if (!slot) { 322 /* 323 * Not an error, since a key not in use by I/O is not guaranteed 324 * to be in a keyslot. There can be more keys than keyslots. 325 */ 326 err = 0; 327 goto out; 328 } 329 330 if (WARN_ON_ONCE(atomic_read(&slot->slot_refs) != 0)) { 331 /* BUG: key is still in use by I/O */ 332 err = -EBUSY; 333 goto out_remove; 334 } 335 err = ksm->ksm_ll_ops.keyslot_evict(ksm, key, 336 blk_ksm_get_slot_idx(slot)); 337out_remove: 338 /* 339 * Callers free the key even on error, so unlink the key from the hash 340 * table and clear slot->key even on error. 341 */ 342 hlist_del(&slot->hash_node); 343 slot->key = NULL; 344out: 345 blk_ksm_hw_exit(ksm); 346 return err; 347} 348 349/** 350 * blk_ksm_reprogram_all_keys() - Re-program all keyslots. 351 * @ksm: The keyslot manager 352 * 353 * Re-program all keyslots that are supposed to have a key programmed. This is 354 * intended only for use by drivers for hardware that loses its keys on reset. 355 * 356 * Context: Process context. Takes and releases ksm->lock. 357 */ 358void blk_ksm_reprogram_all_keys(struct blk_keyslot_manager *ksm) 359{ 360 unsigned int slot; 361 362 /* This is for device initialization, so don't resume the device */ 363 down_write(&ksm->lock); 364 for (slot = 0; slot < ksm->num_slots; slot++) { 365 const struct blk_crypto_key *key = ksm->slots[slot].key; 366 int err; 367 368 if (!key) 369 continue; 370 371 err = ksm->ksm_ll_ops.keyslot_program(ksm, key, slot); 372 WARN_ON(err); 373 } 374 up_write(&ksm->lock); 375} 376EXPORT_SYMBOL_GPL(blk_ksm_reprogram_all_keys); 377 378void blk_ksm_destroy(struct blk_keyslot_manager *ksm) 379{ 380 if (!ksm) 381 return; 382 kvfree(ksm->slot_hashtable); 383 kvfree_sensitive(ksm->slots, sizeof(ksm->slots[0]) * ksm->num_slots); 384 memzero_explicit(ksm, sizeof(*ksm)); 385} 386EXPORT_SYMBOL_GPL(blk_ksm_destroy); 387 388bool blk_ksm_register(struct blk_keyslot_manager *ksm, struct request_queue *q) 389{ 390 if (blk_integrity_queue_supports_integrity(q)) { 391 pr_warn("Integrity and hardware inline encryption are not supported together. Disabling hardware inline encryption.\n"); 392 return false; 393 } 394 q->ksm = ksm; 395 return true; 396} 397EXPORT_SYMBOL_GPL(blk_ksm_register); 398 399void blk_ksm_unregister(struct request_queue *q) 400{ 401 q->ksm = NULL; 402} 403