This is a note to let you know that I've just added the patch titled random: remove batched entropy locking to the 4.19-stable tree which can be found at: http://www.kernel.org/git/?p=linux/kernel/git/stable/stable-queue.git;a=summary The filename of the patch is: random-remove-batched-entropy-locking.patch and it can be found in the queue-4.19 subdirectory. If you, or anyone else, feels it should not be added to the stable tree, please let <stable@xxxxxxxxxxxxxxx> know about it. >From foo@baz Fri Jun 17 08:58:56 AM CEST 2022 From: "Jason A. Donenfeld" <Jason@xxxxxxxxx> Date: Fri, 28 Jan 2022 23:29:45 +0100 Subject: random: remove batched entropy locking From: "Jason A. Donenfeld" <Jason@xxxxxxxxx> commit 77760fd7f7ae3dfd03668204e708d1568d75447d upstream. Rather than use spinlocks to protect batched entropy, we can instead disable interrupts locally, since we're dealing with per-cpu data, and manage resets with a basic generation counter. At the same time, we can't quite do this on PREEMPT_RT, where we still want spinlocks-as- mutexes semantics. So we use a local_lock_t, which provides the right behavior for each. Because this is a per-cpu lock, that generation counter is still doing the necessary CPU-to-CPU communication. This should improve performance a bit. It will also fix the linked splat that Jonathan received with a PROVE_RAW_LOCK_NESTING=y. Reviewed-by: Sebastian Andrzej Siewior <bigeasy@xxxxxxxxxxxxx> Reviewed-by: Dominik Brodowski <linux@xxxxxxxxxxxxxxxxxxxx> Reviewed-by: Eric Biggers <ebiggers@xxxxxxxxxx> Suggested-by: Andy Lutomirski <luto@xxxxxxxxxx> Reported-by: Jonathan Neuschäfer <j.neuschaefer@xxxxxxx> Tested-by: Jonathan Neuschäfer <j.neuschaefer@xxxxxxx> Link: https://lore.kernel.org/lkml/YfMa0QgsjCVdRAvJ@latitude/ Signed-off-by: Jason A. Donenfeld <Jason@xxxxxxxxx> Signed-off-by: Greg Kroah-Hartman <gregkh@xxxxxxxxxxxxxxxxxxx> --- drivers/char/random.c | 58 +++++++++++++++++++++++--------------------------- 1 file changed, 27 insertions(+), 31 deletions(-) --- a/drivers/char/random.c +++ b/drivers/char/random.c @@ -1719,13 +1719,15 @@ struct ctl_table random_table[] = { }; #endif /* CONFIG_SYSCTL */ +static atomic_t batch_generation = ATOMIC_INIT(0); + struct batched_entropy { union { u64 entropy_u64[CHACHA20_BLOCK_SIZE / sizeof(u64)]; u32 entropy_u32[CHACHA20_BLOCK_SIZE / sizeof(u32)]; }; unsigned int position; - spinlock_t batch_lock; + int generation; }; /* @@ -1736,9 +1738,7 @@ struct batched_entropy { * wait_for_random_bytes() should be called and return 0 at least once at any * point prior. */ -static DEFINE_PER_CPU(struct batched_entropy, batched_entropy_u64) = { - .batch_lock = __SPIN_LOCK_UNLOCKED(batched_entropy_u64.lock), -}; +static DEFINE_PER_CPU(struct batched_entropy, batched_entropy_u64); u64 get_random_u64(void) { @@ -1746,67 +1746,63 @@ u64 get_random_u64(void) unsigned long flags; struct batched_entropy *batch; static void *previous; + int next_gen; warn_unseeded_randomness(&previous); + local_irq_save(flags); batch = raw_cpu_ptr(&batched_entropy_u64); - spin_lock_irqsave(&batch->batch_lock, flags); - if (batch->position % ARRAY_SIZE(batch->entropy_u64) == 0) { + + next_gen = atomic_read(&batch_generation); + if (batch->position % ARRAY_SIZE(batch->entropy_u64) == 0 || + next_gen != batch->generation) { extract_crng((u8 *)batch->entropy_u64); batch->position = 0; + batch->generation = next_gen; } + ret = batch->entropy_u64[batch->position++]; - spin_unlock_irqrestore(&batch->batch_lock, flags); + local_irq_restore(flags); return ret; } EXPORT_SYMBOL(get_random_u64); -static DEFINE_PER_CPU(struct batched_entropy, batched_entropy_u32) = { - .batch_lock = __SPIN_LOCK_UNLOCKED(batched_entropy_u32.lock), -}; +static DEFINE_PER_CPU(struct batched_entropy, batched_entropy_u32); + u32 get_random_u32(void) { u32 ret; unsigned long flags; struct batched_entropy *batch; static void *previous; + int next_gen; warn_unseeded_randomness(&previous); + local_irq_save(flags); batch = raw_cpu_ptr(&batched_entropy_u32); - spin_lock_irqsave(&batch->batch_lock, flags); - if (batch->position % ARRAY_SIZE(batch->entropy_u32) == 0) { + + next_gen = atomic_read(&batch_generation); + if (batch->position % ARRAY_SIZE(batch->entropy_u32) == 0 || + next_gen != batch->generation) { extract_crng((u8 *)batch->entropy_u32); batch->position = 0; + batch->generation = next_gen; } + ret = batch->entropy_u32[batch->position++]; - spin_unlock_irqrestore(&batch->batch_lock, flags); + local_irq_restore(flags); return ret; } EXPORT_SYMBOL(get_random_u32); /* It's important to invalidate all potential batched entropy that might * be stored before the crng is initialized, which we can do lazily by - * simply resetting the counter to zero so that it's re-extracted on the - * next usage. */ + * bumping the generation counter. + */ static void invalidate_batched_entropy(void) { - int cpu; - unsigned long flags; - - for_each_possible_cpu(cpu) { - struct batched_entropy *batched_entropy; - - batched_entropy = per_cpu_ptr(&batched_entropy_u32, cpu); - spin_lock_irqsave(&batched_entropy->batch_lock, flags); - batched_entropy->position = 0; - spin_unlock(&batched_entropy->batch_lock); - - batched_entropy = per_cpu_ptr(&batched_entropy_u64, cpu); - spin_lock(&batched_entropy->batch_lock); - batched_entropy->position = 0; - spin_unlock_irqrestore(&batched_entropy->batch_lock, flags); - } + atomic_inc(&batch_generation); } /** Patches currently in stable-queue which might be from Jason@xxxxxxxxx are queue-4.19/random-do-not-take-pool-spinlock-at-boot.patch queue-4.19/random-remove-kernel.random.read_wakeup_threshold.patch queue-4.19/random-simplify-arithmetic-function-flow-in-account.patch queue-4.19/random-order-timer-entropy-functions-below-interrupt-functions.patch queue-4.19/random-introduce-drain_entropy-helper-to-declutter-crng_reseed.patch queue-4.19/random-fix-locking-in-crng_fast_load.patch queue-4.19/random-cleanup-uuid-handling.patch queue-4.19/random-group-userspace-read-write-functions.patch queue-4.19/random-make-credit_entropy_bits-always-safe.patch queue-4.19/latent_entropy-avoid-build-error-when-plugin-cflags-are-not-set.patch queue-4.19/revert-hwrng-core-freeze-khwrng-thread-during-suspend.patch queue-4.19/drivers-char-random.c-make-primary_crng-static.patch queue-4.19/random-do-not-re-init-if-crng_reseed-completes-before-primary-init.patch queue-4.19/random-use-proper-jiffies-comparison-macro.patch queue-4.19/init-call-time_init-before-rand_initialize.patch queue-4.19/random-split-primary-secondary-crng-init-paths.patch queue-4.19/random-pull-add_hwgenerator_randomness-declaration-into-random.h.patch queue-4.19/random-unify-early-init-crng-load-accounting.patch queue-4.19/drivers-char-random.c-remove-unused-stuct-poolinfo-poolbits.patch queue-4.19/random-use-blake2s-instead-of-sha1-in-extraction.patch queue-4.19/random-convert-to-using-fops-write_iter.patch queue-4.19/random-initialize-chacha20-constants-with-correct-endianness.patch queue-4.19/random-remove-incomplete-last_data-logic.patch queue-4.19/random-group-entropy-extraction-functions.patch queue-4.19/random-add-proper-spdx-header.patch queue-4.19/linux-random.h-remove-arch_has_random-arch_has_random_seed.patch queue-4.19/random-rewrite-header-introductory-comment.patch queue-4.19/random-make-dev-random-be-almost-like-dev-urandom.patch queue-4.19/random-remove-ifdef-d-out-interrupt-bench.patch queue-4.19/um-use-fallback-for-random_get_entropy-instead-of-zero.patch queue-4.19/lib-crypto-sha1-re-roll-loops-to-reduce-code-size.patch queue-4.19/random-tie-batched-entropy-generation-to-base_crng-generation.patch queue-4.19/sparc-use-fallback-for-random_get_entropy-instead-of-zero.patch queue-4.19/random-use-linear-min-entropy-accumulation-crediting.patch queue-4.19/random-remove-batched-entropy-locking.patch queue-4.19/xtensa-use-fallback-for-random_get_entropy-instead-of-zero.patch queue-4.19/powerpc-remove-arch_has_random-arch_has_random_seed.patch queue-4.19/fdt-add-support-for-rng-seed.patch queue-4.19/random-continually-use-hwgenerator-randomness.patch queue-4.19/random-access-input_pool_data-directly-rather-than-through-pointer.patch queue-4.19/random-add-arch_get_random_-long_early.patch queue-4.19/random-inline-leaves-of-rand_initialize.patch queue-4.19/random-cleanup-poolinfo-abstraction.patch queue-4.19/random-wire-up-fops-splice_-read-write-_iter.patch queue-4.19/random-handle-latent-entropy-and-command-line-from-random_init.patch queue-4.19/random-remove-use_input_pool-parameter-from-crng_reseed.patch queue-4.19/random-credit-architectural-init-the-exact-amount.patch queue-4.19/ia64-define-get_cycles-macro-for-arch-override.patch queue-4.19/random-replace-custom-notifier-chain-with-standard-one.patch queue-4.19/random-support-freezable-kthreads-in-add_hwgenerator_randomness.patch queue-4.19/random-document-get_random_int-family.patch queue-4.19/random-remove-the-blocking-pool.patch queue-4.19/random-avoid-initializing-twice-in-credit-race.patch queue-4.19/random-avoid-warnings-for-config_numa-builds.patch queue-4.19/crypto-drbg-add-fips-140-2-ctrng-for-noise-source.patch queue-4.19/random-mark-bootloader-randomness-code-as-__init.patch queue-4.19/random-zero-buffer-after-reading-entropy-from-userspace.patch queue-4.19/random-remove-whitespace-and-reorder-includes.patch queue-4.19/random-ignore-grnd_random-in-getentropy-2.patch queue-4.19/random-clear-fast-pool-crng-and-batches-in-cpuhp-bring-up.patch queue-4.19/random-document-add_hwgenerator_randomness-with-other-input-functions.patch queue-4.19/random-fix-typo-in-add_timer_randomness.patch queue-4.19/random-do-crng-pre-init-loading-in-worker-rather-than-irq.patch queue-4.19/powerpc-use-bool-in-archrandom.h.patch queue-4.19/random-do-not-split-fast-init-input-in-add_hwgenerator_randomness.patch queue-4.19/timekeeping-add-raw-clock-fallback-for-random_get_entropy.patch queue-4.19/random-early-initialization-of-chacha-constants.patch queue-4.19/crypto-drbg-prepare-for-more-fine-grained-tracking-of-seeding-state.patch queue-4.19/random-delete-code-to-pull-data-into-pools.patch queue-4.19/crypto-drbg-always-try-to-free-jitter-rng-instance.patch queue-4.19/random-simplify-entropy-debiting.patch queue-4.19/random-don-t-reset-crng_init_cnt-on-urandom_read.patch queue-4.19/random-skip-fast_init-if-hwrng-provides-large-chunk-of-entropy.patch queue-4.19/random-use-siphash-as-interrupt-entropy-accumulator.patch queue-4.19/random-avoid-checking-crng_ready-twice-in-random_init.patch queue-4.19/random-fix-soft-lockup-when-trying-to-read-from-an-uninitialized-blocking-pool.patch queue-4.19/random-group-sysctl-functions.patch queue-4.19/random-don-t-let-644-read-only-sysctls-be-written-to.patch queue-4.19/random-document-crng_fast_key_erasure-destination-possibility.patch queue-4.19/random-only-wake-up-writers-after-zap-if-threshold-was-passed.patch queue-4.19/random-use-wait_event_freezable-in-add_hwgenerator_randomness.patch queue-4.19/random-check-for-signal-and-try-earlier-when-generating-entropy.patch queue-4.19/random-check-for-signals-every-page_size-chunk-of-dev-random.patch queue-4.19/arm-use-fallback-for-random_get_entropy-instead-of-zero.patch queue-4.19/random-absorb-fast-pool-into-input-pool-after-fast-load.patch queue-4.19/random-give-sysctl_random_min_urandom_seed-a-more-sensible-value.patch queue-4.19/crypto-blake2s-generic-c-library-implementation-and-selftest.patch queue-4.19/random-cleanup-fractional-entropy-shift-constants.patch queue-4.19/random-use-rdseed-instead-of-rdrand-in-entropy-extraction.patch queue-4.19/random-move-rand_initialize-earlier.patch queue-4.19/random-don-t-wake-crng_init_wait-when-crng_init-1.patch queue-4.19/random-add-a-urandom_read_nowait-for-random-apis-that-don-t-warn.patch queue-4.19/random-do-not-sign-extend-bytes-for-rotation-when-mixing.patch queue-4.19/random-move-initialization-functions-out-of-hot-pages.patch queue-4.19/random-remove-dead-code-left-over-from-blocking-pool.patch queue-4.19/drivers-char-random.c-constify-poolinfo_table.patch queue-4.19/crypto-drbg-track-whether-drbg-was-seeded-with-rng_is_initialized.patch queue-4.19/random-use-computational-hash-for-entropy-extraction.patch queue-4.19/random-add-and-use-pr_fmt.patch queue-4.19/random-round-robin-registers-as-ulong-not-u32.patch queue-4.19/random-always-wake-up-entropy-writers-after-extraction.patch queue-4.19/s390-remove-arch_has_random-arch_has_random_seed.patch queue-4.19/random-do-not-xor-rdrand-when-writing-into-dev-random.patch queue-4.19/crypto-drbg-make-reseeding-from-get_random_bytes-synchronous.patch queue-4.19/random-convert-to-entropy_bits-for-better-code-readability.patch queue-4.19/char-random-add-a-newline-at-the-end-of-the-file.patch queue-4.19/random-move-randomize_page-into-mm-where-it-belongs.patch queue-4.19/random-only-call-crng_finalize_init-for-primary_crng.patch queue-4.19/random-cleanup-integer-types.patch queue-4.19/random-re-add-removed-comment-about-get_random_-u32-u64-reseeding.patch queue-4.19/random-unify-cycles_t-and-jiffies-usage-and-types.patch queue-4.19/random-insist-on-random_get_entropy-existing-in-order-to-simplify.patch queue-4.19/random-group-initialization-wait-functions.patch queue-4.19/linux-random.h-mark-config_arch_random-functions-__must_check.patch queue-4.19/random-remove-unused-extract_entropy-reserved-argument.patch queue-4.19/random-check-for-signal_pending-outside-of-need_resched-check.patch queue-4.19/random-access-primary_pool-directly-rather-than-through-pointer.patch queue-4.19/random-fix-sysctl-documentation-nits.patch queue-4.19/random-remove-unused-tracepoints.patch queue-4.19/random-only-read-from-dev-random-after-its-pool-has-received-128-bits.patch queue-4.19/nios2-use-fallback-for-random_get_entropy-instead-of-zero.patch queue-4.19/random-treat-bootloader-trust-toggle-the-same-way-as-cpu-trust-toggle.patch queue-4.19/random-make-consistent-usage-of-crng_ready.patch queue-4.19/lib-crypto-blake2s-move-hmac-construction-into-wireguard.patch queue-4.19/parisc-define-get_cycles-macro-for-arch-override.patch queue-4.19/x86-tsc-use-fallback-for-random_get_entropy-instead-of-zero.patch queue-4.19/random-add-grnd_insecure-to-return-best-effort-non-cryptographic-bytes.patch queue-4.19/crypto-drbg-move-dynamic-reseed_threshold-adjustments-to-__drbg_seed.patch queue-4.19/random-check-for-signals-after-page-of-pool-writes.patch queue-4.19/random-make-random_get_entropy-return-an-unsigned-long.patch queue-4.19/random-check-for-crng_init-0-in-add_device_randomness.patch queue-4.19/random-remove-unnecessary-unlikely.patch queue-4.19/random-defer-fast-pool-mixing-to-worker.patch queue-4.19/random-harmonize-crng-init-done-messages.patch queue-4.19/crypto-blake2s-include-linux-bug.h-instead-of-asm-bug.h.patch queue-4.19/random-use-static-branch-for-crng_ready.patch queue-4.19/random-rather-than-entropy_store-abstraction-use-global.patch queue-4.19/random-remove-extern-from-functions-in-header.patch queue-4.19/siphash-use-one-source-of-truth-for-siphash-permutations.patch queue-4.19/random-group-entropy-collection-functions.patch queue-4.19/random-de-duplicate-input_pool-constants.patch queue-4.19/random-mix-build-time-latent-entropy-into-pool-at-init.patch queue-4.19/random-remove-useless-header-comment.patch queue-4.19/linux-random.h-use-false-with-bool.patch queue-4.19/maintainers-co-maintain-random.c.patch queue-4.19/random-remove-outdated-int_max-6-check-in-urandom_read.patch queue-4.19/m68k-use-fallback-for-random_get_entropy-instead-of-zero.patch queue-4.19/alpha-define-get_cycles-macro-for-arch-override.patch queue-4.19/random-mix-bootloader-randomness-into-pool.patch queue-4.19/random-remove-some-dead-code-of-poolinfo.patch queue-4.19/random-do-not-use-batches-when-crng_ready.patch queue-4.19/crypto-drbg-always-seeded-with-sp800-90b-compliant-noise-source.patch queue-4.19/s390-define-get_cycles-macro-for-arch-override.patch queue-4.19/random-do-not-pretend-to-handle-premature-next-security-model.patch queue-4.19/random-avoid-arch_get_random_seed_long-when-collecting-irq-randomness.patch queue-4.19/random-use-is_enabled-config_numa-instead-of-ifdefs.patch queue-4.19/random-avoid-superfluous-call-to-rdrand-in-crng-extraction.patch queue-4.19/random-use-symbolic-constants-for-crng_init-states.patch queue-4.19/random-reseed-more-often-immediately-after-booting.patch queue-4.19/random-ensure-early-rdseed-goes-through-mixer-on-init.patch queue-4.19/random-deobfuscate-irq-u32-u64-contributions.patch queue-4.19/random-do-not-use-input-pool-from-hard-irqs.patch queue-4.19/random-help-compiler-out-with-fast_mix-by-using-simpler-arguments.patch queue-4.19/revert-random-use-static-branch-for-crng_ready.patch queue-4.19/random-fix-crash-on-multiple-early-calls-to-add_bootloader_randomness.patch queue-4.19/random-account-for-arch-randomness-in-bits.patch queue-4.19/crypto-blake2s-adjust-include-guard-naming.patch queue-4.19/random-do-not-allow-user-to-keep-crng-key-around-on-stack.patch queue-4.19/x86-remove-arch_has_random-arch_has_random_seed.patch queue-4.19/random-remove-ratelimiting-for-in-kernel-unseeded-randomness.patch queue-4.19/random-remove-unused-irq_flags-argument-from-add_interrupt_randomness.patch queue-4.19/random-prepend-remaining-pool-constants-with-pool_.patch queue-4.19/powerpc-define-get_cycles-macro-for-arch-override.patch queue-4.19/random-remove-unused-output_pool-constants.patch queue-4.19/mips-use-fallback-for-random_get_entropy-instead-of-just-c0-random.patch queue-4.19/random-use-hash-function-for-crng_slow_load.patch queue-4.19/random-fix-typo-in-comments.patch queue-4.19/random-use-proper-return-types-on-get_random_-int-long-_wait.patch