Gentoo Archives: gentoo-commits

From: Mike Pagano <mpagano@g.o>
To: gentoo-commits@l.g.o
Subject: [gentoo-commits] proj/linux-patches:4.14 commit in: /
Date: Sat, 25 Jun 2022 10:23:31
Message-Id: 1656152593.83d84df380be14fa5ccdddab856e74ccd428913e.mpagano@gentoo
1 commit: 83d84df380be14fa5ccdddab856e74ccd428913e
2 Author: Mike Pagano <mpagano <AT> gentoo <DOT> org>
3 AuthorDate: Sat Jun 25 10:23:13 2022 +0000
4 Commit: Mike Pagano <mpagano <AT> gentoo <DOT> org>
5 CommitDate: Sat Jun 25 10:23:13 2022 +0000
6 URL: https://gitweb.gentoo.org/proj/linux-patches.git/commit/?id=83d84df3
7
8 Linux patch 4.14.285
9
10 Signed-off-by: Mike Pagano <mpagano <AT> gentoo.org>
11
12 0000_README | 4 +
13 1284_linux-4.14.285.patch | 7496 +++++++++++++++++++++++++++++++++++++++++++++
14 2 files changed, 7500 insertions(+)
15
16 diff --git a/0000_README b/0000_README
17 index d5dcb6b7..4620290c 100644
18 --- a/0000_README
19 +++ b/0000_README
20 @@ -1183,6 +1183,10 @@ Patch: 1283_linux-4.14.284.patch
21 From: https://www.kernel.org
22 Desc: Linux 4.14.284
23
24 +Patch: 1284_linux-4.14.285.patch
25 +From: https://www.kernel.org
26 +Desc: Linux 4.14.285
27 +
28 Patch: 1500_XATTR_USER_PREFIX.patch
29 From: https://bugs.gentoo.org/show_bug.cgi?id=470644
30 Desc: Support for namespace user.pax.* on tmpfs.
31
32 diff --git a/1284_linux-4.14.285.patch b/1284_linux-4.14.285.patch
33 new file mode 100644
34 index 00000000..34aefc0e
35 --- /dev/null
36 +++ b/1284_linux-4.14.285.patch
37 @@ -0,0 +1,7496 @@
38 +diff --git a/Documentation/admin-guide/kernel-parameters.txt b/Documentation/admin-guide/kernel-parameters.txt
39 +index 0c3c72a0ba97b..681d429c64269 100644
40 +--- a/Documentation/admin-guide/kernel-parameters.txt
41 ++++ b/Documentation/admin-guide/kernel-parameters.txt
42 +@@ -3526,6 +3526,18 @@
43 + ramdisk_size= [RAM] Sizes of RAM disks in kilobytes
44 + See Documentation/blockdev/ramdisk.txt.
45 +
46 ++ random.trust_cpu={on,off}
47 ++ [KNL] Enable or disable trusting the use of the
48 ++ CPU's random number generator (if available) to
49 ++ fully seed the kernel's CRNG. Default is controlled
50 ++ by CONFIG_RANDOM_TRUST_CPU.
51 ++
52 ++ random.trust_bootloader={on,off}
53 ++ [KNL] Enable or disable trusting the use of a
54 ++ seed passed by the bootloader (if available) to
55 ++ fully seed the kernel's CRNG. Default is controlled
56 ++ by CONFIG_RANDOM_TRUST_BOOTLOADER.
57 ++
58 + ras=option[,option,...] [KNL] RAS-specific options
59 +
60 + cec_disable [X86]
61 +diff --git a/Documentation/sysctl/kernel.txt b/Documentation/sysctl/kernel.txt
62 +index 3c8f5bfdf6dae..37b612a17c461 100644
63 +--- a/Documentation/sysctl/kernel.txt
64 ++++ b/Documentation/sysctl/kernel.txt
65 +@@ -781,9 +781,40 @@ The kernel command line parameter printk.devkmsg= overrides this and is
66 + a one-time setting until next reboot: once set, it cannot be changed by
67 + this sysctl interface anymore.
68 +
69 +-==============================================================
70 ++pty
71 ++===
72 ++
73 ++See Documentation/filesystems/devpts.rst.
74 ++
75 ++
76 ++random
77 ++======
78 ++
79 ++This is a directory, with the following entries:
80 ++
81 ++* ``boot_id``: a UUID generated the first time this is retrieved, and
82 ++ unvarying after that;
83 ++
84 ++* ``uuid``: a UUID generated every time this is retrieved (this can
85 ++ thus be used to generate UUIDs at will);
86 ++
87 ++* ``entropy_avail``: the pool's entropy count, in bits;
88 ++
89 ++* ``poolsize``: the entropy pool size, in bits;
90 ++
91 ++* ``urandom_min_reseed_secs``: obsolete (used to determine the minimum
92 ++ number of seconds between urandom pool reseeding). This file is
93 ++ writable for compatibility purposes, but writing to it has no effect
94 ++ on any RNG behavior;
95 ++
96 ++* ``write_wakeup_threshold``: when the entropy count drops below this
97 ++ (as a number of bits), processes waiting to write to ``/dev/random``
98 ++ are woken up. This file is writable for compatibility purposes, but
99 ++ writing to it has no effect on any RNG behavior.
100 ++
101 +
102 +-randomize_va_space:
103 ++randomize_va_space
104 ++==================
105 +
106 + This option can be used to select the type of process address
107 + space randomization that is used in the system, for architectures
108 +diff --git a/MAINTAINERS b/MAINTAINERS
109 +index e2dd302345c27..b18207d0c744a 100644
110 +--- a/MAINTAINERS
111 ++++ b/MAINTAINERS
112 +@@ -11307,6 +11307,7 @@ F: drivers/block/brd.c
113 +
114 + RANDOM NUMBER DRIVER
115 + M: "Theodore Ts'o" <tytso@×××.edu>
116 ++M: Jason A. Donenfeld <Jason@×××××.com>
117 + S: Maintained
118 + F: drivers/char/random.c
119 +
120 +diff --git a/Makefile b/Makefile
121 +index c6bdc627db2c7..abdee02ff6735 100644
122 +--- a/Makefile
123 ++++ b/Makefile
124 +@@ -1,7 +1,7 @@
125 + # SPDX-License-Identifier: GPL-2.0
126 + VERSION = 4
127 + PATCHLEVEL = 14
128 +-SUBLEVEL = 284
129 ++SUBLEVEL = 285
130 + EXTRAVERSION =
131 + NAME = Petit Gorille
132 +
133 +diff --git a/arch/alpha/include/asm/timex.h b/arch/alpha/include/asm/timex.h
134 +index b565cc6f408e9..f89798da8a147 100644
135 +--- a/arch/alpha/include/asm/timex.h
136 ++++ b/arch/alpha/include/asm/timex.h
137 +@@ -28,5 +28,6 @@ static inline cycles_t get_cycles (void)
138 + __asm__ __volatile__ ("rpcc %0" : "=r"(ret));
139 + return ret;
140 + }
141 ++#define get_cycles get_cycles
142 +
143 + #endif
144 +diff --git a/arch/arm/include/asm/timex.h b/arch/arm/include/asm/timex.h
145 +index f6fcc67ef06ef..c06d38f0df8ec 100644
146 +--- a/arch/arm/include/asm/timex.h
147 ++++ b/arch/arm/include/asm/timex.h
148 +@@ -14,5 +14,6 @@
149 +
150 + typedef unsigned long cycles_t;
151 + #define get_cycles() ({ cycles_t c; read_current_timer(&c) ? 0 : c; })
152 ++#define random_get_entropy() (((unsigned long)get_cycles()) ?: random_get_entropy_fallback())
153 +
154 + #endif
155 +diff --git a/arch/arm64/kernel/ftrace.c b/arch/arm64/kernel/ftrace.c
156 +index 6eefd5873aef4..cd0b2fc94d3b6 100644
157 +--- a/arch/arm64/kernel/ftrace.c
158 ++++ b/arch/arm64/kernel/ftrace.c
159 +@@ -72,7 +72,7 @@ int ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr)
160 + {
161 + unsigned long pc = rec->ip;
162 + u32 old, new;
163 +- long offset = (long)pc - (long)addr;
164 ++ long offset = (long)addr - (long)pc;
165 +
166 + if (offset < -SZ_128M || offset >= SZ_128M) {
167 + #ifdef CONFIG_ARM64_MODULE_PLTS
168 +@@ -151,7 +151,7 @@ int ftrace_make_nop(struct module *mod, struct dyn_ftrace *rec,
169 + unsigned long pc = rec->ip;
170 + bool validate = true;
171 + u32 old = 0, new;
172 +- long offset = (long)pc - (long)addr;
173 ++ long offset = (long)addr - (long)pc;
174 +
175 + if (offset < -SZ_128M || offset >= SZ_128M) {
176 + #ifdef CONFIG_ARM64_MODULE_PLTS
177 +diff --git a/arch/ia64/include/asm/timex.h b/arch/ia64/include/asm/timex.h
178 +index 869a3ac6bf23a..7ccc077a60bed 100644
179 +--- a/arch/ia64/include/asm/timex.h
180 ++++ b/arch/ia64/include/asm/timex.h
181 +@@ -39,6 +39,7 @@ get_cycles (void)
182 + ret = ia64_getreg(_IA64_REG_AR_ITC);
183 + return ret;
184 + }
185 ++#define get_cycles get_cycles
186 +
187 + extern void ia64_cpu_local_tick (void);
188 + extern unsigned long long ia64_native_sched_clock (void);
189 +diff --git a/arch/m68k/include/asm/timex.h b/arch/m68k/include/asm/timex.h
190 +index 6a21d93582805..f4a7a340f4cae 100644
191 +--- a/arch/m68k/include/asm/timex.h
192 ++++ b/arch/m68k/include/asm/timex.h
193 +@@ -35,7 +35,7 @@ static inline unsigned long random_get_entropy(void)
194 + {
195 + if (mach_random_get_entropy)
196 + return mach_random_get_entropy();
197 +- return 0;
198 ++ return random_get_entropy_fallback();
199 + }
200 + #define random_get_entropy random_get_entropy
201 +
202 +diff --git a/arch/mips/include/asm/timex.h b/arch/mips/include/asm/timex.h
203 +index 8026baf46e729..2e107886f97ac 100644
204 +--- a/arch/mips/include/asm/timex.h
205 ++++ b/arch/mips/include/asm/timex.h
206 +@@ -76,25 +76,24 @@ static inline cycles_t get_cycles(void)
207 + else
208 + return 0; /* no usable counter */
209 + }
210 ++#define get_cycles get_cycles
211 +
212 + /*
213 + * Like get_cycles - but where c0_count is not available we desperately
214 + * use c0_random in an attempt to get at least a little bit of entropy.
215 +- *
216 +- * R6000 and R6000A neither have a count register nor a random register.
217 +- * That leaves no entropy source in the CPU itself.
218 + */
219 + static inline unsigned long random_get_entropy(void)
220 + {
221 +- unsigned int prid = read_c0_prid();
222 +- unsigned int imp = prid & PRID_IMP_MASK;
223 ++ unsigned int c0_random;
224 +
225 +- if (can_use_mips_counter(prid))
226 ++ if (can_use_mips_counter(read_c0_prid()))
227 + return read_c0_count();
228 +- else if (likely(imp != PRID_IMP_R6000 && imp != PRID_IMP_R6000A))
229 +- return read_c0_random();
230 ++
231 ++ if (cpu_has_3kex)
232 ++ c0_random = (read_c0_random() >> 8) & 0x3f;
233 + else
234 +- return 0; /* no usable register */
235 ++ c0_random = read_c0_random() & 0x3f;
236 ++ return (random_get_entropy_fallback() << 6) | (0x3f - c0_random);
237 + }
238 + #define random_get_entropy random_get_entropy
239 +
240 +diff --git a/arch/nios2/include/asm/timex.h b/arch/nios2/include/asm/timex.h
241 +index 2f2abb28ec2fd..9c9b50599ea30 100644
242 +--- a/arch/nios2/include/asm/timex.h
243 ++++ b/arch/nios2/include/asm/timex.h
244 +@@ -20,5 +20,8 @@
245 + typedef unsigned long cycles_t;
246 +
247 + extern cycles_t get_cycles(void);
248 ++#define get_cycles get_cycles
249 ++
250 ++#define random_get_entropy() (((unsigned long)get_cycles()) ?: random_get_entropy_fallback())
251 +
252 + #endif
253 +diff --git a/arch/parisc/include/asm/timex.h b/arch/parisc/include/asm/timex.h
254 +index 45537cd4d1d39..1cd2bd3eef33b 100644
255 +--- a/arch/parisc/include/asm/timex.h
256 ++++ b/arch/parisc/include/asm/timex.h
257 +@@ -12,9 +12,10 @@
258 +
259 + typedef unsigned long cycles_t;
260 +
261 +-static inline cycles_t get_cycles (void)
262 ++static inline cycles_t get_cycles(void)
263 + {
264 + return mfctl(16);
265 + }
266 ++#define get_cycles get_cycles
267 +
268 + #endif
269 +diff --git a/arch/powerpc/include/asm/archrandom.h b/arch/powerpc/include/asm/archrandom.h
270 +index a09595f00cabe..f0f16b4fc5ea1 100644
271 +--- a/arch/powerpc/include/asm/archrandom.h
272 ++++ b/arch/powerpc/include/asm/archrandom.h
273 +@@ -6,27 +6,28 @@
274 +
275 + #include <asm/machdep.h>
276 +
277 +-static inline int arch_get_random_long(unsigned long *v)
278 ++static inline bool arch_get_random_long(unsigned long *v)
279 + {
280 +- return 0;
281 ++ return false;
282 + }
283 +
284 +-static inline int arch_get_random_int(unsigned int *v)
285 ++static inline bool arch_get_random_int(unsigned int *v)
286 + {
287 +- return 0;
288 ++ return false;
289 + }
290 +
291 +-static inline int arch_get_random_seed_long(unsigned long *v)
292 ++static inline bool arch_get_random_seed_long(unsigned long *v)
293 + {
294 + if (ppc_md.get_random_seed)
295 + return ppc_md.get_random_seed(v);
296 +
297 +- return 0;
298 ++ return false;
299 + }
300 +-static inline int arch_get_random_seed_int(unsigned int *v)
301 ++
302 ++static inline bool arch_get_random_seed_int(unsigned int *v)
303 + {
304 + unsigned long val;
305 +- int rc;
306 ++ bool rc;
307 +
308 + rc = arch_get_random_seed_long(&val);
309 + if (rc)
310 +@@ -34,16 +35,6 @@ static inline int arch_get_random_seed_int(unsigned int *v)
311 +
312 + return rc;
313 + }
314 +-
315 +-static inline int arch_has_random(void)
316 +-{
317 +- return 0;
318 +-}
319 +-
320 +-static inline int arch_has_random_seed(void)
321 +-{
322 +- return !!ppc_md.get_random_seed;
323 +-}
324 + #endif /* CONFIG_ARCH_RANDOM */
325 +
326 + #ifdef CONFIG_PPC_POWERNV
327 +diff --git a/arch/powerpc/include/asm/timex.h b/arch/powerpc/include/asm/timex.h
328 +index 926b9f91a3ef2..7401d6a684c55 100644
329 +--- a/arch/powerpc/include/asm/timex.h
330 ++++ b/arch/powerpc/include/asm/timex.h
331 +@@ -50,6 +50,7 @@ static inline cycles_t get_cycles(void)
332 + return ret;
333 + #endif
334 + }
335 ++#define get_cycles get_cycles
336 +
337 + #endif /* __KERNEL__ */
338 + #endif /* _ASM_POWERPC_TIMEX_H */
339 +diff --git a/arch/s390/include/asm/archrandom.h b/arch/s390/include/asm/archrandom.h
340 +index e9f7d7a57f999..ddf97715ee53b 100644
341 +--- a/arch/s390/include/asm/archrandom.h
342 ++++ b/arch/s390/include/asm/archrandom.h
343 +@@ -26,18 +26,6 @@ static void s390_arch_random_generate(u8 *buf, unsigned int nbytes)
344 + atomic64_add(nbytes, &s390_arch_random_counter);
345 + }
346 +
347 +-static inline bool arch_has_random(void)
348 +-{
349 +- if (static_branch_likely(&s390_arch_random_available))
350 +- return true;
351 +- return false;
352 +-}
353 +-
354 +-static inline bool arch_has_random_seed(void)
355 +-{
356 +- return arch_has_random();
357 +-}
358 +-
359 + static inline bool arch_get_random_long(unsigned long *v)
360 + {
361 + if (static_branch_likely(&s390_arch_random_available)) {
362 +diff --git a/arch/s390/include/asm/timex.h b/arch/s390/include/asm/timex.h
363 +index b6a4ce9dafafb..99a7e028232d8 100644
364 +--- a/arch/s390/include/asm/timex.h
365 ++++ b/arch/s390/include/asm/timex.h
366 +@@ -177,6 +177,7 @@ static inline cycles_t get_cycles(void)
367 + {
368 + return (cycles_t) get_tod_clock() >> 2;
369 + }
370 ++#define get_cycles get_cycles
371 +
372 + int get_phys_clock(unsigned long *clock);
373 + void init_cpu_timer(void);
374 +diff --git a/arch/s390/mm/pgtable.c b/arch/s390/mm/pgtable.c
375 +index aa6b9487c8bbc..783e07934a3d9 100644
376 +--- a/arch/s390/mm/pgtable.c
377 ++++ b/arch/s390/mm/pgtable.c
378 +@@ -684,7 +684,7 @@ void ptep_zap_key(struct mm_struct *mm, unsigned long addr, pte_t *ptep)
379 + pgste_val(pgste) |= PGSTE_GR_BIT | PGSTE_GC_BIT;
380 + ptev = pte_val(*ptep);
381 + if (!(ptev & _PAGE_INVALID) && (ptev & _PAGE_WRITE))
382 +- page_set_storage_key(ptev & PAGE_MASK, PAGE_DEFAULT_KEY, 1);
383 ++ page_set_storage_key(ptev & PAGE_MASK, PAGE_DEFAULT_KEY, 0);
384 + pgste_set_unlock(ptep, pgste);
385 + preempt_enable();
386 + }
387 +diff --git a/arch/sparc/include/asm/timex_32.h b/arch/sparc/include/asm/timex_32.h
388 +index 542915b462097..f86326a6f89e0 100644
389 +--- a/arch/sparc/include/asm/timex_32.h
390 ++++ b/arch/sparc/include/asm/timex_32.h
391 +@@ -9,8 +9,6 @@
392 +
393 + #define CLOCK_TICK_RATE 1193180 /* Underlying HZ */
394 +
395 +-/* XXX Maybe do something better at some point... -DaveM */
396 +-typedef unsigned long cycles_t;
397 +-#define get_cycles() (0)
398 ++#include <asm-generic/timex.h>
399 +
400 + #endif
401 +diff --git a/arch/um/include/asm/timex.h b/arch/um/include/asm/timex.h
402 +index e392a9a5bc9bd..9f27176adb26d 100644
403 +--- a/arch/um/include/asm/timex.h
404 ++++ b/arch/um/include/asm/timex.h
405 +@@ -2,13 +2,8 @@
406 + #ifndef __UM_TIMEX_H
407 + #define __UM_TIMEX_H
408 +
409 +-typedef unsigned long cycles_t;
410 +-
411 +-static inline cycles_t get_cycles (void)
412 +-{
413 +- return 0;
414 +-}
415 +-
416 + #define CLOCK_TICK_RATE (HZ)
417 +
418 ++#include <asm-generic/timex.h>
419 ++
420 + #endif
421 +diff --git a/arch/x86/include/asm/archrandom.h b/arch/x86/include/asm/archrandom.h
422 +index 3ac991d81e74d..4d3cac3c9b250 100644
423 +--- a/arch/x86/include/asm/archrandom.h
424 ++++ b/arch/x86/include/asm/archrandom.h
425 +@@ -86,10 +86,6 @@ static inline bool rdseed_int(unsigned int *v)
426 + return ok;
427 + }
428 +
429 +-/* Conditional execution based on CPU type */
430 +-#define arch_has_random() static_cpu_has(X86_FEATURE_RDRAND)
431 +-#define arch_has_random_seed() static_cpu_has(X86_FEATURE_RDSEED)
432 +-
433 + /*
434 + * These are the generic interfaces; they must not be declared if the
435 + * stubs in <linux/random.h> are to be invoked,
436 +@@ -99,22 +95,22 @@ static inline bool rdseed_int(unsigned int *v)
437 +
438 + static inline bool arch_get_random_long(unsigned long *v)
439 + {
440 +- return arch_has_random() ? rdrand_long(v) : false;
441 ++ return static_cpu_has(X86_FEATURE_RDRAND) ? rdrand_long(v) : false;
442 + }
443 +
444 + static inline bool arch_get_random_int(unsigned int *v)
445 + {
446 +- return arch_has_random() ? rdrand_int(v) : false;
447 ++ return static_cpu_has(X86_FEATURE_RDRAND) ? rdrand_int(v) : false;
448 + }
449 +
450 + static inline bool arch_get_random_seed_long(unsigned long *v)
451 + {
452 +- return arch_has_random_seed() ? rdseed_long(v) : false;
453 ++ return static_cpu_has(X86_FEATURE_RDSEED) ? rdseed_long(v) : false;
454 + }
455 +
456 + static inline bool arch_get_random_seed_int(unsigned int *v)
457 + {
458 +- return arch_has_random_seed() ? rdseed_int(v) : false;
459 ++ return static_cpu_has(X86_FEATURE_RDSEED) ? rdseed_int(v) : false;
460 + }
461 +
462 + extern void x86_init_rdrand(struct cpuinfo_x86 *c);
463 +diff --git a/arch/x86/include/asm/timex.h b/arch/x86/include/asm/timex.h
464 +index a4a8b1b16c0c1..956e4145311b1 100644
465 +--- a/arch/x86/include/asm/timex.h
466 ++++ b/arch/x86/include/asm/timex.h
467 +@@ -5,6 +5,15 @@
468 + #include <asm/processor.h>
469 + #include <asm/tsc.h>
470 +
471 ++static inline unsigned long random_get_entropy(void)
472 ++{
473 ++ if (!IS_ENABLED(CONFIG_X86_TSC) &&
474 ++ !cpu_feature_enabled(X86_FEATURE_TSC))
475 ++ return random_get_entropy_fallback();
476 ++ return rdtsc();
477 ++}
478 ++#define random_get_entropy random_get_entropy
479 ++
480 + /* Assume we use the PIT time source for the clock tick */
481 + #define CLOCK_TICK_RATE PIT_TICK_RATE
482 +
483 +diff --git a/arch/x86/include/asm/tsc.h b/arch/x86/include/asm/tsc.h
484 +index 8da0efb13544d..036feaff46ed2 100644
485 +--- a/arch/x86/include/asm/tsc.h
486 ++++ b/arch/x86/include/asm/tsc.h
487 +@@ -22,13 +22,12 @@ extern void disable_TSC(void);
488 +
489 + static inline cycles_t get_cycles(void)
490 + {
491 +-#ifndef CONFIG_X86_TSC
492 +- if (!boot_cpu_has(X86_FEATURE_TSC))
493 ++ if (!IS_ENABLED(CONFIG_X86_TSC) &&
494 ++ !cpu_feature_enabled(X86_FEATURE_TSC))
495 + return 0;
496 +-#endif
497 +-
498 + return rdtsc();
499 + }
500 ++#define get_cycles get_cycles
501 +
502 + extern struct system_counterval_t convert_art_to_tsc(u64 art);
503 +
504 +diff --git a/arch/xtensa/include/asm/timex.h b/arch/xtensa/include/asm/timex.h
505 +index f9b389d4e9739..d866bc847d8dc 100644
506 +--- a/arch/xtensa/include/asm/timex.h
507 ++++ b/arch/xtensa/include/asm/timex.h
508 +@@ -30,10 +30,6 @@
509 +
510 + extern unsigned long ccount_freq;
511 +
512 +-typedef unsigned long long cycles_t;
513 +-
514 +-#define get_cycles() (0)
515 +-
516 + void local_timer_setup(unsigned cpu);
517 +
518 + /*
519 +@@ -69,4 +65,6 @@ static inline void set_linux_timer (unsigned long ccompare)
520 + WSR_CCOMPARE(LINUX_TIMER, ccompare);
521 + }
522 +
523 ++#include <asm-generic/timex.h>
524 ++
525 + #endif /* _XTENSA_TIMEX_H */
526 +diff --git a/certs/blacklist_hashes.c b/certs/blacklist_hashes.c
527 +index 344892337be07..d5961aa3d3380 100644
528 +--- a/certs/blacklist_hashes.c
529 ++++ b/certs/blacklist_hashes.c
530 +@@ -1,7 +1,7 @@
531 + // SPDX-License-Identifier: GPL-2.0
532 + #include "blacklist.h"
533 +
534 +-const char __initdata *const blacklist_hashes[] = {
535 ++const char __initconst *const blacklist_hashes[] = {
536 + #include CONFIG_SYSTEM_BLACKLIST_HASH_LIST
537 + , NULL
538 + };
539 +diff --git a/crypto/chacha20_generic.c b/crypto/chacha20_generic.c
540 +index 4a45fa4890c0e..24827d00fb679 100644
541 +--- a/crypto/chacha20_generic.c
542 ++++ b/crypto/chacha20_generic.c
543 +@@ -22,7 +22,8 @@ static inline u32 le32_to_cpuvp(const void *p)
544 + static void chacha20_docrypt(u32 *state, u8 *dst, const u8 *src,
545 + unsigned int bytes)
546 + {
547 +- u8 stream[CHACHA20_BLOCK_SIZE];
548 ++ /* aligned to potentially speed up crypto_xor() */
549 ++ u8 stream[CHACHA20_BLOCK_SIZE] __aligned(sizeof(long));
550 +
551 + if (dst != src)
552 + memcpy(dst, src, bytes);
553 +diff --git a/crypto/drbg.c b/crypto/drbg.c
554 +index 6c3221313753a..1a555db4eef0c 100644
555 +--- a/crypto/drbg.c
556 ++++ b/crypto/drbg.c
557 +@@ -219,6 +219,57 @@ static inline unsigned short drbg_sec_strength(drbg_flag_t flags)
558 + }
559 + }
560 +
561 ++/*
562 ++ * FIPS 140-2 continuous self test for the noise source
563 ++ * The test is performed on the noise source input data. Thus, the function
564 ++ * implicitly knows the size of the buffer to be equal to the security
565 ++ * strength.
566 ++ *
567 ++ * Note, this function disregards the nonce trailing the entropy data during
568 ++ * initial seeding.
569 ++ *
570 ++ * drbg->drbg_mutex must have been taken.
571 ++ *
572 ++ * @drbg DRBG handle
573 ++ * @entropy buffer of seed data to be checked
574 ++ *
575 ++ * return:
576 ++ * 0 on success
577 ++ * -EAGAIN on when the CTRNG is not yet primed
578 ++ * < 0 on error
579 ++ */
580 ++static int drbg_fips_continuous_test(struct drbg_state *drbg,
581 ++ const unsigned char *entropy)
582 ++{
583 ++ unsigned short entropylen = drbg_sec_strength(drbg->core->flags);
584 ++ int ret = 0;
585 ++
586 ++ if (!IS_ENABLED(CONFIG_CRYPTO_FIPS))
587 ++ return 0;
588 ++
589 ++ /* skip test if we test the overall system */
590 ++ if (list_empty(&drbg->test_data.list))
591 ++ return 0;
592 ++ /* only perform test in FIPS mode */
593 ++ if (!fips_enabled)
594 ++ return 0;
595 ++
596 ++ if (!drbg->fips_primed) {
597 ++ /* Priming of FIPS test */
598 ++ memcpy(drbg->prev, entropy, entropylen);
599 ++ drbg->fips_primed = true;
600 ++ /* priming: another round is needed */
601 ++ return -EAGAIN;
602 ++ }
603 ++ ret = memcmp(drbg->prev, entropy, entropylen);
604 ++ if (!ret)
605 ++ panic("DRBG continuous self test failed\n");
606 ++ memcpy(drbg->prev, entropy, entropylen);
607 ++
608 ++ /* the test shall pass when the two values are not equal */
609 ++ return 0;
610 ++}
611 ++
612 + /*
613 + * Convert an integer into a byte representation of this integer.
614 + * The byte representation is big-endian
615 +@@ -986,55 +1037,79 @@ static const struct drbg_state_ops drbg_hash_ops = {
616 + ******************************************************************/
617 +
618 + static inline int __drbg_seed(struct drbg_state *drbg, struct list_head *seed,
619 +- int reseed)
620 ++ int reseed, enum drbg_seed_state new_seed_state)
621 + {
622 + int ret = drbg->d_ops->update(drbg, seed, reseed);
623 +
624 + if (ret)
625 + return ret;
626 +
627 +- drbg->seeded = true;
628 ++ drbg->seeded = new_seed_state;
629 + /* 10.1.1.2 / 10.1.1.3 step 5 */
630 + drbg->reseed_ctr = 1;
631 +
632 ++ switch (drbg->seeded) {
633 ++ case DRBG_SEED_STATE_UNSEEDED:
634 ++ /* Impossible, but handle it to silence compiler warnings. */
635 ++ case DRBG_SEED_STATE_PARTIAL:
636 ++ /*
637 ++ * Require frequent reseeds until the seed source is
638 ++ * fully initialized.
639 ++ */
640 ++ drbg->reseed_threshold = 50;
641 ++ break;
642 ++
643 ++ case DRBG_SEED_STATE_FULL:
644 ++ /*
645 ++ * Seed source has become fully initialized, frequent
646 ++ * reseeds no longer required.
647 ++ */
648 ++ drbg->reseed_threshold = drbg_max_requests(drbg);
649 ++ break;
650 ++ }
651 ++
652 + return ret;
653 + }
654 +
655 +-static void drbg_async_seed(struct work_struct *work)
656 ++static inline int drbg_get_random_bytes(struct drbg_state *drbg,
657 ++ unsigned char *entropy,
658 ++ unsigned int entropylen)
659 ++{
660 ++ int ret;
661 ++
662 ++ do {
663 ++ get_random_bytes(entropy, entropylen);
664 ++ ret = drbg_fips_continuous_test(drbg, entropy);
665 ++ if (ret && ret != -EAGAIN)
666 ++ return ret;
667 ++ } while (ret);
668 ++
669 ++ return 0;
670 ++}
671 ++
672 ++static int drbg_seed_from_random(struct drbg_state *drbg)
673 + {
674 + struct drbg_string data;
675 + LIST_HEAD(seedlist);
676 +- struct drbg_state *drbg = container_of(work, struct drbg_state,
677 +- seed_work);
678 + unsigned int entropylen = drbg_sec_strength(drbg->core->flags);
679 + unsigned char entropy[32];
680 ++ int ret;
681 +
682 + BUG_ON(!entropylen);
683 + BUG_ON(entropylen > sizeof(entropy));
684 +- get_random_bytes(entropy, entropylen);
685 +
686 + drbg_string_fill(&data, entropy, entropylen);
687 + list_add_tail(&data.list, &seedlist);
688 +
689 +- mutex_lock(&drbg->drbg_mutex);
690 +-
691 +- /* If nonblocking pool is initialized, deactivate Jitter RNG */
692 +- crypto_free_rng(drbg->jent);
693 +- drbg->jent = NULL;
694 +-
695 +- /* Set seeded to false so that if __drbg_seed fails the
696 +- * next generate call will trigger a reseed.
697 +- */
698 +- drbg->seeded = false;
699 +-
700 +- __drbg_seed(drbg, &seedlist, true);
701 +-
702 +- if (drbg->seeded)
703 +- drbg->reseed_threshold = drbg_max_requests(drbg);
704 ++ ret = drbg_get_random_bytes(drbg, entropy, entropylen);
705 ++ if (ret)
706 ++ goto out;
707 +
708 +- mutex_unlock(&drbg->drbg_mutex);
709 ++ ret = __drbg_seed(drbg, &seedlist, true, DRBG_SEED_STATE_FULL);
710 +
711 ++out:
712 + memzero_explicit(entropy, entropylen);
713 ++ return ret;
714 + }
715 +
716 + /*
717 +@@ -1056,6 +1131,7 @@ static int drbg_seed(struct drbg_state *drbg, struct drbg_string *pers,
718 + unsigned int entropylen = drbg_sec_strength(drbg->core->flags);
719 + struct drbg_string data1;
720 + LIST_HEAD(seedlist);
721 ++ enum drbg_seed_state new_seed_state = DRBG_SEED_STATE_FULL;
722 +
723 + /* 9.1 / 9.2 / 9.3.1 step 3 */
724 + if (pers && pers->len > (drbg_max_addtl(drbg))) {
725 +@@ -1083,7 +1159,12 @@ static int drbg_seed(struct drbg_state *drbg, struct drbg_string *pers,
726 + BUG_ON((entropylen * 2) > sizeof(entropy));
727 +
728 + /* Get seed from in-kernel /dev/urandom */
729 +- get_random_bytes(entropy, entropylen);
730 ++ if (!rng_is_initialized())
731 ++ new_seed_state = DRBG_SEED_STATE_PARTIAL;
732 ++
733 ++ ret = drbg_get_random_bytes(drbg, entropy, entropylen);
734 ++ if (ret)
735 ++ goto out;
736 +
737 + if (!drbg->jent) {
738 + drbg_string_fill(&data1, entropy, entropylen);
739 +@@ -1096,7 +1177,23 @@ static int drbg_seed(struct drbg_state *drbg, struct drbg_string *pers,
740 + entropylen);
741 + if (ret) {
742 + pr_devel("DRBG: jent failed with %d\n", ret);
743 +- return ret;
744 ++
745 ++ /*
746 ++ * Do not treat the transient failure of the
747 ++ * Jitter RNG as an error that needs to be
748 ++ * reported. The combined number of the
749 ++ * maximum reseed threshold times the maximum
750 ++ * number of Jitter RNG transient errors is
751 ++ * less than the reseed threshold required by
752 ++ * SP800-90A allowing us to treat the
753 ++ * transient errors as such.
754 ++ *
755 ++ * However, we mandate that at least the first
756 ++ * seeding operation must succeed with the
757 ++ * Jitter RNG.
758 ++ */
759 ++ if (!reseed || ret != -EAGAIN)
760 ++ goto out;
761 + }
762 +
763 + drbg_string_fill(&data1, entropy, entropylen * 2);
764 +@@ -1121,8 +1218,9 @@ static int drbg_seed(struct drbg_state *drbg, struct drbg_string *pers,
765 + memset(drbg->C, 0, drbg_statelen(drbg));
766 + }
767 +
768 +- ret = __drbg_seed(drbg, &seedlist, reseed);
769 ++ ret = __drbg_seed(drbg, &seedlist, reseed, new_seed_state);
770 +
771 ++out:
772 + memzero_explicit(entropy, entropylen * 2);
773 +
774 + return ret;
775 +@@ -1144,6 +1242,11 @@ static inline void drbg_dealloc_state(struct drbg_state *drbg)
776 + drbg->reseed_ctr = 0;
777 + drbg->d_ops = NULL;
778 + drbg->core = NULL;
779 ++ if (IS_ENABLED(CONFIG_CRYPTO_FIPS)) {
780 ++ kzfree(drbg->prev);
781 ++ drbg->prev = NULL;
782 ++ drbg->fips_primed = false;
783 ++ }
784 + }
785 +
786 + /*
787 +@@ -1213,6 +1316,14 @@ static inline int drbg_alloc_state(struct drbg_state *drbg)
788 + drbg->scratchpad = PTR_ALIGN(drbg->scratchpadbuf, ret + 1);
789 + }
790 +
791 ++ if (IS_ENABLED(CONFIG_CRYPTO_FIPS)) {
792 ++ drbg->prev = kzalloc(drbg_sec_strength(drbg->core->flags),
793 ++ GFP_KERNEL);
794 ++ if (!drbg->prev)
795 ++ goto fini;
796 ++ drbg->fips_primed = false;
797 ++ }
798 ++
799 + return 0;
800 +
801 + fini:
802 +@@ -1285,19 +1396,25 @@ static int drbg_generate(struct drbg_state *drbg,
803 + * here. The spec is a bit convoluted here, we make it simpler.
804 + */
805 + if (drbg->reseed_threshold < drbg->reseed_ctr)
806 +- drbg->seeded = false;
807 ++ drbg->seeded = DRBG_SEED_STATE_UNSEEDED;
808 +
809 +- if (drbg->pr || !drbg->seeded) {
810 ++ if (drbg->pr || drbg->seeded == DRBG_SEED_STATE_UNSEEDED) {
811 + pr_devel("DRBG: reseeding before generation (prediction "
812 + "resistance: %s, state %s)\n",
813 + drbg->pr ? "true" : "false",
814 +- drbg->seeded ? "seeded" : "unseeded");
815 ++ (drbg->seeded == DRBG_SEED_STATE_FULL ?
816 ++ "seeded" : "unseeded"));
817 + /* 9.3.1 steps 7.1 through 7.3 */
818 + len = drbg_seed(drbg, addtl, true);
819 + if (len)
820 + goto err;
821 + /* 9.3.1 step 7.4 */
822 + addtl = NULL;
823 ++ } else if (rng_is_initialized() &&
824 ++ drbg->seeded == DRBG_SEED_STATE_PARTIAL) {
825 ++ len = drbg_seed_from_random(drbg);
826 ++ if (len)
827 ++ goto err;
828 + }
829 +
830 + if (addtl && 0 < addtl->len)
831 +@@ -1390,51 +1507,15 @@ static int drbg_generate_long(struct drbg_state *drbg,
832 + return 0;
833 + }
834 +
835 +-static void drbg_schedule_async_seed(struct random_ready_callback *rdy)
836 +-{
837 +- struct drbg_state *drbg = container_of(rdy, struct drbg_state,
838 +- random_ready);
839 +-
840 +- schedule_work(&drbg->seed_work);
841 +-}
842 +-
843 + static int drbg_prepare_hrng(struct drbg_state *drbg)
844 + {
845 +- int err;
846 +-
847 + /* We do not need an HRNG in test mode. */
848 + if (list_empty(&drbg->test_data.list))
849 + return 0;
850 +
851 +- INIT_WORK(&drbg->seed_work, drbg_async_seed);
852 +-
853 +- drbg->random_ready.owner = THIS_MODULE;
854 +- drbg->random_ready.func = drbg_schedule_async_seed;
855 +-
856 +- err = add_random_ready_callback(&drbg->random_ready);
857 +-
858 +- switch (err) {
859 +- case 0:
860 +- break;
861 +-
862 +- case -EALREADY:
863 +- err = 0;
864 +- /* fall through */
865 +-
866 +- default:
867 +- drbg->random_ready.func = NULL;
868 +- return err;
869 +- }
870 +-
871 + drbg->jent = crypto_alloc_rng("jitterentropy_rng", 0, 0);
872 +
873 +- /*
874 +- * Require frequent reseeds until the seed source is fully
875 +- * initialized.
876 +- */
877 +- drbg->reseed_threshold = 50;
878 +-
879 +- return err;
880 ++ return 0;
881 + }
882 +
883 + /*
884 +@@ -1477,7 +1558,7 @@ static int drbg_instantiate(struct drbg_state *drbg, struct drbg_string *pers,
885 + if (!drbg->core) {
886 + drbg->core = &drbg_cores[coreref];
887 + drbg->pr = pr;
888 +- drbg->seeded = false;
889 ++ drbg->seeded = DRBG_SEED_STATE_UNSEEDED;
890 + drbg->reseed_threshold = drbg_max_requests(drbg);
891 +
892 + ret = drbg_alloc_state(drbg);
893 +@@ -1528,12 +1609,9 @@ free_everything:
894 + */
895 + static int drbg_uninstantiate(struct drbg_state *drbg)
896 + {
897 +- if (drbg->random_ready.func) {
898 +- del_random_ready_callback(&drbg->random_ready);
899 +- cancel_work_sync(&drbg->seed_work);
900 ++ if (!IS_ERR_OR_NULL(drbg->jent))
901 + crypto_free_rng(drbg->jent);
902 +- drbg->jent = NULL;
903 +- }
904 ++ drbg->jent = NULL;
905 +
906 + if (drbg->d_ops)
907 + drbg->d_ops->crypto_fini(drbg);
908 +diff --git a/crypto/md4.c b/crypto/md4.c
909 +index 3515af425cc91..810fefb0a007a 100644
910 +--- a/crypto/md4.c
911 ++++ b/crypto/md4.c
912 +@@ -64,23 +64,6 @@ static inline u32 H(u32 x, u32 y, u32 z)
913 + #define ROUND2(a,b,c,d,k,s) (a = lshift(a + G(b,c,d) + k + (u32)0x5A827999,s))
914 + #define ROUND3(a,b,c,d,k,s) (a = lshift(a + H(b,c,d) + k + (u32)0x6ED9EBA1,s))
915 +
916 +-/* XXX: this stuff can be optimized */
917 +-static inline void le32_to_cpu_array(u32 *buf, unsigned int words)
918 +-{
919 +- while (words--) {
920 +- __le32_to_cpus(buf);
921 +- buf++;
922 +- }
923 +-}
924 +-
925 +-static inline void cpu_to_le32_array(u32 *buf, unsigned int words)
926 +-{
927 +- while (words--) {
928 +- __cpu_to_le32s(buf);
929 +- buf++;
930 +- }
931 +-}
932 +-
933 + static void md4_transform(u32 *hash, u32 const *in)
934 + {
935 + u32 a, b, c, d;
936 +diff --git a/crypto/md5.c b/crypto/md5.c
937 +index f7ae1a48225ba..f776ef43d6217 100644
938 +--- a/crypto/md5.c
939 ++++ b/crypto/md5.c
940 +@@ -32,23 +32,6 @@ const u8 md5_zero_message_hash[MD5_DIGEST_SIZE] = {
941 + };
942 + EXPORT_SYMBOL_GPL(md5_zero_message_hash);
943 +
944 +-/* XXX: this stuff can be optimized */
945 +-static inline void le32_to_cpu_array(u32 *buf, unsigned int words)
946 +-{
947 +- while (words--) {
948 +- __le32_to_cpus(buf);
949 +- buf++;
950 +- }
951 +-}
952 +-
953 +-static inline void cpu_to_le32_array(u32 *buf, unsigned int words)
954 +-{
955 +- while (words--) {
956 +- __cpu_to_le32s(buf);
957 +- buf++;
958 +- }
959 +-}
960 +-
961 + #define F1(x, y, z) (z ^ (x & (y ^ z)))
962 + #define F2(x, y, z) F1(z, x, y)
963 + #define F3(x, y, z) (x ^ y ^ z)
964 +diff --git a/drivers/ata/libata-core.c b/drivers/ata/libata-core.c
965 +index d3a7b3bb5043c..08dc37a62f5a9 100644
966 +--- a/drivers/ata/libata-core.c
967 ++++ b/drivers/ata/libata-core.c
968 +@@ -6180,7 +6180,7 @@ struct ata_host *ata_host_alloc_pinfo(struct device *dev,
969 + const struct ata_port_info * const * ppi,
970 + int n_ports)
971 + {
972 +- const struct ata_port_info *pi;
973 ++ const struct ata_port_info *pi = &ata_dummy_port_info;
974 + struct ata_host *host;
975 + int i, j;
976 +
977 +@@ -6188,7 +6188,7 @@ struct ata_host *ata_host_alloc_pinfo(struct device *dev,
978 + if (!host)
979 + return NULL;
980 +
981 +- for (i = 0, j = 0, pi = NULL; i < host->n_ports; i++) {
982 ++ for (i = 0, j = 0; i < host->n_ports; i++) {
983 + struct ata_port *ap = host->ports[i];
984 +
985 + if (ppi[j])
986 +diff --git a/drivers/char/Kconfig b/drivers/char/Kconfig
987 +index 88316f86cc952..e329d1cc019ae 100644
988 +--- a/drivers/char/Kconfig
989 ++++ b/drivers/char/Kconfig
990 +@@ -588,5 +588,41 @@ config TILE_SROM
991 +
992 + source "drivers/char/xillybus/Kconfig"
993 +
994 +-endmenu
995 ++config RANDOM_TRUST_CPU
996 ++ bool "Initialize RNG using CPU RNG instructions"
997 ++ default y
998 ++ depends on ARCH_RANDOM
999 ++ help
1000 ++ Initialize the RNG using random numbers supplied by the CPU's
1001 ++ RNG instructions (e.g. RDRAND), if supported and available. These
1002 ++ random numbers are never used directly, but are rather hashed into
1003 ++ the main input pool, and this happens regardless of whether or not
1004 ++ this option is enabled. Instead, this option controls whether the
1005 ++ they are credited and hence can initialize the RNG. Additionally,
1006 ++ other sources of randomness are always used, regardless of this
1007 ++ setting. Enabling this implies trusting that the CPU can supply high
1008 ++ quality and non-backdoored random numbers.
1009 ++
1010 ++ Say Y here unless you have reason to mistrust your CPU or believe
1011 ++ its RNG facilities may be faulty. This may also be configured at
1012 ++ boot time with "random.trust_cpu=on/off".
1013 ++
1014 ++config RANDOM_TRUST_BOOTLOADER
1015 ++ bool "Initialize RNG using bootloader-supplied seed"
1016 ++ default y
1017 ++ help
1018 ++ Initialize the RNG using a seed supplied by the bootloader or boot
1019 ++ environment (e.g. EFI or a bootloader-generated device tree). This
1020 ++ seed is not used directly, but is rather hashed into the main input
1021 ++ pool, and this happens regardless of whether or not this option is
1022 ++ enabled. Instead, this option controls whether the seed is credited
1023 ++ and hence can initialize the RNG. Additionally, other sources of
1024 ++ randomness are always used, regardless of this setting. Enabling
1025 ++ this implies trusting that the bootloader can supply high quality and
1026 ++ non-backdoored seeds.
1027 ++
1028 ++ Say Y here unless you have reason to mistrust your bootloader or
1029 ++ believe its RNG facilities may be faulty. This may also be configured
1030 ++ at boot time with "random.trust_bootloader=on/off".
1031 +
1032 ++endmenu
1033 +diff --git a/drivers/char/hw_random/core.c b/drivers/char/hw_random/core.c
1034 +index 21b98771312fb..303fd55f9be05 100644
1035 +--- a/drivers/char/hw_random/core.c
1036 ++++ b/drivers/char/hw_random/core.c
1037 +@@ -15,6 +15,7 @@
1038 + #include <linux/err.h>
1039 + #include <linux/fs.h>
1040 + #include <linux/hw_random.h>
1041 ++#include <linux/random.h>
1042 + #include <linux/kernel.h>
1043 + #include <linux/kthread.h>
1044 + #include <linux/sched/signal.h>
1045 +diff --git a/drivers/char/random.c b/drivers/char/random.c
1046 +index 1eb0cdbe87866..1331571e2b12d 100644
1047 +--- a/drivers/char/random.c
1048 ++++ b/drivers/char/random.c
1049 +@@ -1,242 +1,29 @@
1050 ++// SPDX-License-Identifier: (GPL-2.0 OR BSD-3-Clause)
1051 + /*
1052 +- * random.c -- A strong random number generator
1053 +- *
1054 +- * Copyright (C) 2017 Jason A. Donenfeld <Jason@×××××.com>. All
1055 +- * Rights Reserved.
1056 +- *
1057 ++ * Copyright (C) 2017-2022 Jason A. Donenfeld <Jason@×××××.com>. All Rights Reserved.
1058 + * Copyright Matt Mackall <mpm@×××××××.com>, 2003, 2004, 2005
1059 +- *
1060 +- * Copyright Theodore Ts'o, 1994, 1995, 1996, 1997, 1998, 1999. All
1061 +- * rights reserved.
1062 +- *
1063 +- * Redistribution and use in source and binary forms, with or without
1064 +- * modification, are permitted provided that the following conditions
1065 +- * are met:
1066 +- * 1. Redistributions of source code must retain the above copyright
1067 +- * notice, and the entire permission notice in its entirety,
1068 +- * including the disclaimer of warranties.
1069 +- * 2. Redistributions in binary form must reproduce the above copyright
1070 +- * notice, this list of conditions and the following disclaimer in the
1071 +- * documentation and/or other materials provided with the distribution.
1072 +- * 3. The name of the author may not be used to endorse or promote
1073 +- * products derived from this software without specific prior
1074 +- * written permission.
1075 +- *
1076 +- * ALTERNATIVELY, this product may be distributed under the terms of
1077 +- * the GNU General Public License, in which case the provisions of the GPL are
1078 +- * required INSTEAD OF the above restrictions. (This clause is
1079 +- * necessary due to a potential bad interaction between the GPL and
1080 +- * the restrictions contained in a BSD-style copyright.)
1081 +- *
1082 +- * THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESS OR IMPLIED
1083 +- * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
1084 +- * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ALL OF
1085 +- * WHICH ARE HEREBY DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE
1086 +- * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
1087 +- * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT
1088 +- * OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
1089 +- * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
1090 +- * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
1091 +- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
1092 +- * USE OF THIS SOFTWARE, EVEN IF NOT ADVISED OF THE POSSIBILITY OF SUCH
1093 +- * DAMAGE.
1094 ++ * Copyright Theodore Ts'o, 1994, 1995, 1996, 1997, 1998, 1999. All rights reserved.
1095 ++ *
1096 ++ * This driver produces cryptographically secure pseudorandom data. It is divided
1097 ++ * into roughly six sections, each with a section header:
1098 ++ *
1099 ++ * - Initialization and readiness waiting.
1100 ++ * - Fast key erasure RNG, the "crng".
1101 ++ * - Entropy accumulation and extraction routines.
1102 ++ * - Entropy collection routines.
1103 ++ * - Userspace reader/writer interfaces.
1104 ++ * - Sysctl interface.
1105 ++ *
1106 ++ * The high level overview is that there is one input pool, into which
1107 ++ * various pieces of data are hashed. Prior to initialization, some of that
1108 ++ * data is then "credited" as having a certain number of bits of entropy.
1109 ++ * When enough bits of entropy are available, the hash is finalized and
1110 ++ * handed as a key to a stream cipher that expands it indefinitely for
1111 ++ * various consumers. This key is periodically refreshed as the various
1112 ++ * entropy collectors, described below, add data to the input pool.
1113 + */
1114 +
1115 +-/*
1116 +- * (now, with legal B.S. out of the way.....)
1117 +- *
1118 +- * This routine gathers environmental noise from device drivers, etc.,
1119 +- * and returns good random numbers, suitable for cryptographic use.
1120 +- * Besides the obvious cryptographic uses, these numbers are also good
1121 +- * for seeding TCP sequence numbers, and other places where it is
1122 +- * desirable to have numbers which are not only random, but hard to
1123 +- * predict by an attacker.
1124 +- *
1125 +- * Theory of operation
1126 +- * ===================
1127 +- *
1128 +- * Computers are very predictable devices. Hence it is extremely hard
1129 +- * to produce truly random numbers on a computer --- as opposed to
1130 +- * pseudo-random numbers, which can easily generated by using a
1131 +- * algorithm. Unfortunately, it is very easy for attackers to guess
1132 +- * the sequence of pseudo-random number generators, and for some
1133 +- * applications this is not acceptable. So instead, we must try to
1134 +- * gather "environmental noise" from the computer's environment, which
1135 +- * must be hard for outside attackers to observe, and use that to
1136 +- * generate random numbers. In a Unix environment, this is best done
1137 +- * from inside the kernel.
1138 +- *
1139 +- * Sources of randomness from the environment include inter-keyboard
1140 +- * timings, inter-interrupt timings from some interrupts, and other
1141 +- * events which are both (a) non-deterministic and (b) hard for an
1142 +- * outside observer to measure. Randomness from these sources are
1143 +- * added to an "entropy pool", which is mixed using a CRC-like function.
1144 +- * This is not cryptographically strong, but it is adequate assuming
1145 +- * the randomness is not chosen maliciously, and it is fast enough that
1146 +- * the overhead of doing it on every interrupt is very reasonable.
1147 +- * As random bytes are mixed into the entropy pool, the routines keep
1148 +- * an *estimate* of how many bits of randomness have been stored into
1149 +- * the random number generator's internal state.
1150 +- *
1151 +- * When random bytes are desired, they are obtained by taking the SHA
1152 +- * hash of the contents of the "entropy pool". The SHA hash avoids
1153 +- * exposing the internal state of the entropy pool. It is believed to
1154 +- * be computationally infeasible to derive any useful information
1155 +- * about the input of SHA from its output. Even if it is possible to
1156 +- * analyze SHA in some clever way, as long as the amount of data
1157 +- * returned from the generator is less than the inherent entropy in
1158 +- * the pool, the output data is totally unpredictable. For this
1159 +- * reason, the routine decreases its internal estimate of how many
1160 +- * bits of "true randomness" are contained in the entropy pool as it
1161 +- * outputs random numbers.
1162 +- *
1163 +- * If this estimate goes to zero, the routine can still generate
1164 +- * random numbers; however, an attacker may (at least in theory) be
1165 +- * able to infer the future output of the generator from prior
1166 +- * outputs. This requires successful cryptanalysis of SHA, which is
1167 +- * not believed to be feasible, but there is a remote possibility.
1168 +- * Nonetheless, these numbers should be useful for the vast majority
1169 +- * of purposes.
1170 +- *
1171 +- * Exported interfaces ---- output
1172 +- * ===============================
1173 +- *
1174 +- * There are three exported interfaces; the first is one designed to
1175 +- * be used from within the kernel:
1176 +- *
1177 +- * void get_random_bytes(void *buf, int nbytes);
1178 +- *
1179 +- * This interface will return the requested number of random bytes,
1180 +- * and place it in the requested buffer.
1181 +- *
1182 +- * The two other interfaces are two character devices /dev/random and
1183 +- * /dev/urandom. /dev/random is suitable for use when very high
1184 +- * quality randomness is desired (for example, for key generation or
1185 +- * one-time pads), as it will only return a maximum of the number of
1186 +- * bits of randomness (as estimated by the random number generator)
1187 +- * contained in the entropy pool.
1188 +- *
1189 +- * The /dev/urandom device does not have this limit, and will return
1190 +- * as many bytes as are requested. As more and more random bytes are
1191 +- * requested without giving time for the entropy pool to recharge,
1192 +- * this will result in random numbers that are merely cryptographically
1193 +- * strong. For many applications, however, this is acceptable.
1194 +- *
1195 +- * Exported interfaces ---- input
1196 +- * ==============================
1197 +- *
1198 +- * The current exported interfaces for gathering environmental noise
1199 +- * from the devices are:
1200 +- *
1201 +- * void add_device_randomness(const void *buf, unsigned int size);
1202 +- * void add_input_randomness(unsigned int type, unsigned int code,
1203 +- * unsigned int value);
1204 +- * void add_interrupt_randomness(int irq, int irq_flags);
1205 +- * void add_disk_randomness(struct gendisk *disk);
1206 +- *
1207 +- * add_device_randomness() is for adding data to the random pool that
1208 +- * is likely to differ between two devices (or possibly even per boot).
1209 +- * This would be things like MAC addresses or serial numbers, or the
1210 +- * read-out of the RTC. This does *not* add any actual entropy to the
1211 +- * pool, but it initializes the pool to different values for devices
1212 +- * that might otherwise be identical and have very little entropy
1213 +- * available to them (particularly common in the embedded world).
1214 +- *
1215 +- * add_input_randomness() uses the input layer interrupt timing, as well as
1216 +- * the event type information from the hardware.
1217 +- *
1218 +- * add_interrupt_randomness() uses the interrupt timing as random
1219 +- * inputs to the entropy pool. Using the cycle counters and the irq source
1220 +- * as inputs, it feeds the randomness roughly once a second.
1221 +- *
1222 +- * add_disk_randomness() uses what amounts to the seek time of block
1223 +- * layer request events, on a per-disk_devt basis, as input to the
1224 +- * entropy pool. Note that high-speed solid state drives with very low
1225 +- * seek times do not make for good sources of entropy, as their seek
1226 +- * times are usually fairly consistent.
1227 +- *
1228 +- * All of these routines try to estimate how many bits of randomness a
1229 +- * particular randomness source. They do this by keeping track of the
1230 +- * first and second order deltas of the event timings.
1231 +- *
1232 +- * Ensuring unpredictability at system startup
1233 +- * ============================================
1234 +- *
1235 +- * When any operating system starts up, it will go through a sequence
1236 +- * of actions that are fairly predictable by an adversary, especially
1237 +- * if the start-up does not involve interaction with a human operator.
1238 +- * This reduces the actual number of bits of unpredictability in the
1239 +- * entropy pool below the value in entropy_count. In order to
1240 +- * counteract this effect, it helps to carry information in the
1241 +- * entropy pool across shut-downs and start-ups. To do this, put the
1242 +- * following lines an appropriate script which is run during the boot
1243 +- * sequence:
1244 +- *
1245 +- * echo "Initializing random number generator..."
1246 +- * random_seed=/var/run/random-seed
1247 +- * # Carry a random seed from start-up to start-up
1248 +- * # Load and then save the whole entropy pool
1249 +- * if [ -f $random_seed ]; then
1250 +- * cat $random_seed >/dev/urandom
1251 +- * else
1252 +- * touch $random_seed
1253 +- * fi
1254 +- * chmod 600 $random_seed
1255 +- * dd if=/dev/urandom of=$random_seed count=1 bs=512
1256 +- *
1257 +- * and the following lines in an appropriate script which is run as
1258 +- * the system is shutdown:
1259 +- *
1260 +- * # Carry a random seed from shut-down to start-up
1261 +- * # Save the whole entropy pool
1262 +- * echo "Saving random seed..."
1263 +- * random_seed=/var/run/random-seed
1264 +- * touch $random_seed
1265 +- * chmod 600 $random_seed
1266 +- * dd if=/dev/urandom of=$random_seed count=1 bs=512
1267 +- *
1268 +- * For example, on most modern systems using the System V init
1269 +- * scripts, such code fragments would be found in
1270 +- * /etc/rc.d/init.d/random. On older Linux systems, the correct script
1271 +- * location might be in /etc/rcb.d/rc.local or /etc/rc.d/rc.0.
1272 +- *
1273 +- * Effectively, these commands cause the contents of the entropy pool
1274 +- * to be saved at shut-down time and reloaded into the entropy pool at
1275 +- * start-up. (The 'dd' in the addition to the bootup script is to
1276 +- * make sure that /etc/random-seed is different for every start-up,
1277 +- * even if the system crashes without executing rc.0.) Even with
1278 +- * complete knowledge of the start-up activities, predicting the state
1279 +- * of the entropy pool requires knowledge of the previous history of
1280 +- * the system.
1281 +- *
1282 +- * Configuring the /dev/random driver under Linux
1283 +- * ==============================================
1284 +- *
1285 +- * The /dev/random driver under Linux uses minor numbers 8 and 9 of
1286 +- * the /dev/mem major number (#1). So if your system does not have
1287 +- * /dev/random and /dev/urandom created already, they can be created
1288 +- * by using the commands:
1289 +- *
1290 +- * mknod /dev/random c 1 8
1291 +- * mknod /dev/urandom c 1 9
1292 +- *
1293 +- * Acknowledgements:
1294 +- * =================
1295 +- *
1296 +- * Ideas for constructing this random number generator were derived
1297 +- * from Pretty Good Privacy's random number generator, and from private
1298 +- * discussions with Phil Karn. Colin Plumb provided a faster random
1299 +- * number generator, which speed up the mixing function of the entropy
1300 +- * pool, taken from PGPfone. Dale Worley has also contributed many
1301 +- * useful ideas and suggestions to improve this driver.
1302 +- *
1303 +- * Any flaws in the design are solely my responsibility, and should
1304 +- * not be attributed to the Phil, Colin, or any of authors of PGP.
1305 +- *
1306 +- * Further background information on this topic may be obtained from
1307 +- * RFC 1750, "Randomness Recommendations for Security", by Donald
1308 +- * Eastlake, Steve Crocker, and Jeff Schiller.
1309 +- */
1310 ++#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
1311 +
1312 + #include <linux/utsname.h>
1313 + #include <linux/module.h>
1314 +@@ -256,8 +43,6 @@
1315 + #include <linux/spinlock.h>
1316 + #include <linux/kthread.h>
1317 + #include <linux/percpu.h>
1318 +-#include <linux/cryptohash.h>
1319 +-#include <linux/fips.h>
1320 + #include <linux/ptrace.h>
1321 + #include <linux/workqueue.h>
1322 + #include <linux/irq.h>
1323 +@@ -265,1549 +50,1067 @@
1324 + #include <linux/syscalls.h>
1325 + #include <linux/completion.h>
1326 + #include <linux/uuid.h>
1327 ++#include <linux/uaccess.h>
1328 ++#include <linux/siphash.h>
1329 ++#include <linux/uio.h>
1330 + #include <crypto/chacha20.h>
1331 +-
1332 ++#include <crypto/blake2s.h>
1333 + #include <asm/processor.h>
1334 +-#include <linux/uaccess.h>
1335 + #include <asm/irq.h>
1336 + #include <asm/irq_regs.h>
1337 + #include <asm/io.h>
1338 +
1339 +-#define CREATE_TRACE_POINTS
1340 +-#include <trace/events/random.h>
1341 +-
1342 +-/* #define ADD_INTERRUPT_BENCH */
1343 ++/*********************************************************************
1344 ++ *
1345 ++ * Initialization and readiness waiting.
1346 ++ *
1347 ++ * Much of the RNG infrastructure is devoted to various dependencies
1348 ++ * being able to wait until the RNG has collected enough entropy and
1349 ++ * is ready for safe consumption.
1350 ++ *
1351 ++ *********************************************************************/
1352 +
1353 + /*
1354 +- * Configuration information
1355 ++ * crng_init is protected by base_crng->lock, and only increases
1356 ++ * its value (from empty->early->ready).
1357 + */
1358 +-#define INPUT_POOL_SHIFT 12
1359 +-#define INPUT_POOL_WORDS (1 << (INPUT_POOL_SHIFT-5))
1360 +-#define OUTPUT_POOL_SHIFT 10
1361 +-#define OUTPUT_POOL_WORDS (1 << (OUTPUT_POOL_SHIFT-5))
1362 +-#define SEC_XFER_SIZE 512
1363 +-#define EXTRACT_SIZE 10
1364 +-
1365 ++static enum {
1366 ++ CRNG_EMPTY = 0, /* Little to no entropy collected */
1367 ++ CRNG_EARLY = 1, /* At least POOL_EARLY_BITS collected */
1368 ++ CRNG_READY = 2 /* Fully initialized with POOL_READY_BITS collected */
1369 ++} crng_init __read_mostly = CRNG_EMPTY;
1370 ++#define crng_ready() (likely(crng_init >= CRNG_READY))
1371 ++/* Various types of waiters for crng_init->CRNG_READY transition. */
1372 ++static DECLARE_WAIT_QUEUE_HEAD(crng_init_wait);
1373 ++static struct fasync_struct *fasync;
1374 ++static DEFINE_SPINLOCK(random_ready_chain_lock);
1375 ++static RAW_NOTIFIER_HEAD(random_ready_chain);
1376 +
1377 +-#define LONGS(x) (((x) + sizeof(unsigned long) - 1)/sizeof(unsigned long))
1378 ++/* Control how we warn userspace. */
1379 ++static struct ratelimit_state urandom_warning =
1380 ++ RATELIMIT_STATE_INIT("warn_urandom_randomness", HZ, 3);
1381 ++static int ratelimit_disable __read_mostly =
1382 ++ IS_ENABLED(CONFIG_WARN_ALL_UNSEEDED_RANDOM);
1383 ++module_param_named(ratelimit_disable, ratelimit_disable, int, 0644);
1384 ++MODULE_PARM_DESC(ratelimit_disable, "Disable random ratelimit suppression");
1385 +
1386 + /*
1387 +- * To allow fractional bits to be tracked, the entropy_count field is
1388 +- * denominated in units of 1/8th bits.
1389 ++ * Returns whether or not the input pool has been seeded and thus guaranteed
1390 ++ * to supply cryptographically secure random numbers. This applies to: the
1391 ++ * /dev/urandom device, the get_random_bytes function, and the get_random_{u32,
1392 ++ * ,u64,int,long} family of functions.
1393 + *
1394 +- * 2*(ENTROPY_SHIFT + log2(poolbits)) must <= 31, or the multiply in
1395 +- * credit_entropy_bits() needs to be 64 bits wide.
1396 ++ * Returns: true if the input pool has been seeded.
1397 ++ * false if the input pool has not been seeded.
1398 + */
1399 +-#define ENTROPY_SHIFT 3
1400 +-#define ENTROPY_BITS(r) ((r)->entropy_count >> ENTROPY_SHIFT)
1401 ++bool rng_is_initialized(void)
1402 ++{
1403 ++ return crng_ready();
1404 ++}
1405 ++EXPORT_SYMBOL(rng_is_initialized);
1406 +
1407 +-/*
1408 +- * The minimum number of bits of entropy before we wake up a read on
1409 +- * /dev/random. Should be enough to do a significant reseed.
1410 +- */
1411 +-static int random_read_wakeup_bits = 64;
1412 ++/* Used by wait_for_random_bytes(), and considered an entropy collector, below. */
1413 ++static void try_to_generate_entropy(void);
1414 +
1415 + /*
1416 +- * If the entropy count falls under this number of bits, then we
1417 +- * should wake up processes which are selecting or polling on write
1418 +- * access to /dev/random.
1419 ++ * Wait for the input pool to be seeded and thus guaranteed to supply
1420 ++ * cryptographically secure random numbers. This applies to: the /dev/urandom
1421 ++ * device, the get_random_bytes function, and the get_random_{u32,u64,int,long}
1422 ++ * family of functions. Using any of these functions without first calling
1423 ++ * this function forfeits the guarantee of security.
1424 ++ *
1425 ++ * Returns: 0 if the input pool has been seeded.
1426 ++ * -ERESTARTSYS if the function was interrupted by a signal.
1427 + */
1428 +-static int random_write_wakeup_bits = 28 * OUTPUT_POOL_WORDS;
1429 ++int wait_for_random_bytes(void)
1430 ++{
1431 ++ while (!crng_ready()) {
1432 ++ int ret;
1433 ++
1434 ++ try_to_generate_entropy();
1435 ++ ret = wait_event_interruptible_timeout(crng_init_wait, crng_ready(), HZ);
1436 ++ if (ret)
1437 ++ return ret > 0 ? 0 : ret;
1438 ++ }
1439 ++ return 0;
1440 ++}
1441 ++EXPORT_SYMBOL(wait_for_random_bytes);
1442 +
1443 + /*
1444 +- * Originally, we used a primitive polynomial of degree .poolwords
1445 +- * over GF(2). The taps for various sizes are defined below. They
1446 +- * were chosen to be evenly spaced except for the last tap, which is 1
1447 +- * to get the twisting happening as fast as possible.
1448 +- *
1449 +- * For the purposes of better mixing, we use the CRC-32 polynomial as
1450 +- * well to make a (modified) twisted Generalized Feedback Shift
1451 +- * Register. (See M. Matsumoto & Y. Kurita, 1992. Twisted GFSR
1452 +- * generators. ACM Transactions on Modeling and Computer Simulation
1453 +- * 2(3):179-194. Also see M. Matsumoto & Y. Kurita, 1994. Twisted
1454 +- * GFSR generators II. ACM Transactions on Modeling and Computer
1455 +- * Simulation 4:254-266)
1456 +- *
1457 +- * Thanks to Colin Plumb for suggesting this.
1458 +- *
1459 +- * The mixing operation is much less sensitive than the output hash,
1460 +- * where we use SHA-1. All that we want of mixing operation is that
1461 +- * it be a good non-cryptographic hash; i.e. it not produce collisions
1462 +- * when fed "random" data of the sort we expect to see. As long as
1463 +- * the pool state differs for different inputs, we have preserved the
1464 +- * input entropy and done a good job. The fact that an intelligent
1465 +- * attacker can construct inputs that will produce controlled
1466 +- * alterations to the pool's state is not important because we don't
1467 +- * consider such inputs to contribute any randomness. The only
1468 +- * property we need with respect to them is that the attacker can't
1469 +- * increase his/her knowledge of the pool's state. Since all
1470 +- * additions are reversible (knowing the final state and the input,
1471 +- * you can reconstruct the initial state), if an attacker has any
1472 +- * uncertainty about the initial state, he/she can only shuffle that
1473 +- * uncertainty about, but never cause any collisions (which would
1474 +- * decrease the uncertainty).
1475 ++ * Add a callback function that will be invoked when the input
1476 ++ * pool is initialised.
1477 + *
1478 +- * Our mixing functions were analyzed by Lacharme, Roeck, Strubel, and
1479 +- * Videau in their paper, "The Linux Pseudorandom Number Generator
1480 +- * Revisited" (see: http://eprint.iacr.org/2012/251.pdf). In their
1481 +- * paper, they point out that we are not using a true Twisted GFSR,
1482 +- * since Matsumoto & Kurita used a trinomial feedback polynomial (that
1483 +- * is, with only three taps, instead of the six that we are using).
1484 +- * As a result, the resulting polynomial is neither primitive nor
1485 +- * irreducible, and hence does not have a maximal period over
1486 +- * GF(2**32). They suggest a slight change to the generator
1487 +- * polynomial which improves the resulting TGFSR polynomial to be
1488 +- * irreducible, which we have made here.
1489 ++ * returns: 0 if callback is successfully added
1490 ++ * -EALREADY if pool is already initialised (callback not called)
1491 + */
1492 +-static struct poolinfo {
1493 +- int poolbitshift, poolwords, poolbytes, poolbits, poolfracbits;
1494 +-#define S(x) ilog2(x)+5, (x), (x)*4, (x)*32, (x) << (ENTROPY_SHIFT+5)
1495 +- int tap1, tap2, tap3, tap4, tap5;
1496 +-} poolinfo_table[] = {
1497 +- /* was: x^128 + x^103 + x^76 + x^51 +x^25 + x + 1 */
1498 +- /* x^128 + x^104 + x^76 + x^51 +x^25 + x + 1 */
1499 +- { S(128), 104, 76, 51, 25, 1 },
1500 +- /* was: x^32 + x^26 + x^20 + x^14 + x^7 + x + 1 */
1501 +- /* x^32 + x^26 + x^19 + x^14 + x^7 + x + 1 */
1502 +- { S(32), 26, 19, 14, 7, 1 },
1503 +-#if 0
1504 +- /* x^2048 + x^1638 + x^1231 + x^819 + x^411 + x + 1 -- 115 */
1505 +- { S(2048), 1638, 1231, 819, 411, 1 },
1506 +-
1507 +- /* x^1024 + x^817 + x^615 + x^412 + x^204 + x + 1 -- 290 */
1508 +- { S(1024), 817, 615, 412, 204, 1 },
1509 +-
1510 +- /* x^1024 + x^819 + x^616 + x^410 + x^207 + x^2 + 1 -- 115 */
1511 +- { S(1024), 819, 616, 410, 207, 2 },
1512 +-
1513 +- /* x^512 + x^411 + x^308 + x^208 + x^104 + x + 1 -- 225 */
1514 +- { S(512), 411, 308, 208, 104, 1 },
1515 +-
1516 +- /* x^512 + x^409 + x^307 + x^206 + x^102 + x^2 + 1 -- 95 */
1517 +- { S(512), 409, 307, 206, 102, 2 },
1518 +- /* x^512 + x^409 + x^309 + x^205 + x^103 + x^2 + 1 -- 95 */
1519 +- { S(512), 409, 309, 205, 103, 2 },
1520 +-
1521 +- /* x^256 + x^205 + x^155 + x^101 + x^52 + x + 1 -- 125 */
1522 +- { S(256), 205, 155, 101, 52, 1 },
1523 +-
1524 +- /* x^128 + x^103 + x^78 + x^51 + x^27 + x^2 + 1 -- 70 */
1525 +- { S(128), 103, 78, 51, 27, 2 },
1526 +-
1527 +- /* x^64 + x^52 + x^39 + x^26 + x^14 + x + 1 -- 15 */
1528 +- { S(64), 52, 39, 26, 14, 1 },
1529 +-#endif
1530 +-};
1531 ++int __cold register_random_ready_notifier(struct notifier_block *nb)
1532 ++{
1533 ++ unsigned long flags;
1534 ++ int ret = -EALREADY;
1535 ++
1536 ++ if (crng_ready())
1537 ++ return ret;
1538 ++
1539 ++ spin_lock_irqsave(&random_ready_chain_lock, flags);
1540 ++ if (!crng_ready())
1541 ++ ret = raw_notifier_chain_register(&random_ready_chain, nb);
1542 ++ spin_unlock_irqrestore(&random_ready_chain_lock, flags);
1543 ++ return ret;
1544 ++}
1545 +
1546 + /*
1547 +- * Static global variables
1548 ++ * Delete a previously registered readiness callback function.
1549 + */
1550 +-static DECLARE_WAIT_QUEUE_HEAD(random_read_wait);
1551 +-static DECLARE_WAIT_QUEUE_HEAD(random_write_wait);
1552 +-static struct fasync_struct *fasync;
1553 +-
1554 +-static DEFINE_SPINLOCK(random_ready_list_lock);
1555 +-static LIST_HEAD(random_ready_list);
1556 ++int __cold unregister_random_ready_notifier(struct notifier_block *nb)
1557 ++{
1558 ++ unsigned long flags;
1559 ++ int ret;
1560 +
1561 +-struct crng_state {
1562 +- __u32 state[16];
1563 +- unsigned long init_time;
1564 +- spinlock_t lock;
1565 +-};
1566 ++ spin_lock_irqsave(&random_ready_chain_lock, flags);
1567 ++ ret = raw_notifier_chain_unregister(&random_ready_chain, nb);
1568 ++ spin_unlock_irqrestore(&random_ready_chain_lock, flags);
1569 ++ return ret;
1570 ++}
1571 +
1572 +-struct crng_state primary_crng = {
1573 +- .lock = __SPIN_LOCK_UNLOCKED(primary_crng.lock),
1574 +-};
1575 ++static void __cold process_random_ready_list(void)
1576 ++{
1577 ++ unsigned long flags;
1578 +
1579 +-/*
1580 +- * crng_init = 0 --> Uninitialized
1581 +- * 1 --> Initialized
1582 +- * 2 --> Initialized from input_pool
1583 +- *
1584 +- * crng_init is protected by primary_crng->lock, and only increases
1585 +- * its value (from 0->1->2).
1586 +- */
1587 +-static int crng_init = 0;
1588 +-#define crng_ready() (likely(crng_init > 1))
1589 +-static int crng_init_cnt = 0;
1590 +-static unsigned long crng_global_init_time = 0;
1591 +-#define CRNG_INIT_CNT_THRESH (2*CHACHA20_KEY_SIZE)
1592 +-static void _extract_crng(struct crng_state *crng,
1593 +- __u8 out[CHACHA20_BLOCK_SIZE]);
1594 +-static void _crng_backtrack_protect(struct crng_state *crng,
1595 +- __u8 tmp[CHACHA20_BLOCK_SIZE], int used);
1596 +-static void process_random_ready_list(void);
1597 +-static void _get_random_bytes(void *buf, int nbytes);
1598 +-
1599 +-static struct ratelimit_state unseeded_warning =
1600 +- RATELIMIT_STATE_INIT("warn_unseeded_randomness", HZ, 3);
1601 +-static struct ratelimit_state urandom_warning =
1602 +- RATELIMIT_STATE_INIT("warn_urandom_randomness", HZ, 3);
1603 ++ spin_lock_irqsave(&random_ready_chain_lock, flags);
1604 ++ raw_notifier_call_chain(&random_ready_chain, 0, NULL);
1605 ++ spin_unlock_irqrestore(&random_ready_chain_lock, flags);
1606 ++}
1607 +
1608 +-static int ratelimit_disable __read_mostly;
1609 ++#define warn_unseeded_randomness() \
1610 ++ if (IS_ENABLED(CONFIG_WARN_ALL_UNSEEDED_RANDOM) && !crng_ready()) \
1611 ++ pr_notice("%s called from %pS with crng_init=%d\n", \
1612 ++ __func__, (void *)_RET_IP_, crng_init)
1613 +
1614 +-module_param_named(ratelimit_disable, ratelimit_disable, int, 0644);
1615 +-MODULE_PARM_DESC(ratelimit_disable, "Disable random ratelimit suppression");
1616 +
1617 +-/**********************************************************************
1618 ++/*********************************************************************
1619 + *
1620 +- * OS independent entropy store. Here are the functions which handle
1621 +- * storing entropy in an entropy pool.
1622 ++ * Fast key erasure RNG, the "crng".
1623 + *
1624 +- **********************************************************************/
1625 ++ * These functions expand entropy from the entropy extractor into
1626 ++ * long streams for external consumption using the "fast key erasure"
1627 ++ * RNG described at <https://blog.cr.yp.to/20170723-random.html>.
1628 ++ *
1629 ++ * There are a few exported interfaces for use by other drivers:
1630 ++ *
1631 ++ * void get_random_bytes(void *buf, size_t len)
1632 ++ * u32 get_random_u32()
1633 ++ * u64 get_random_u64()
1634 ++ * unsigned int get_random_int()
1635 ++ * unsigned long get_random_long()
1636 ++ *
1637 ++ * These interfaces will return the requested number of random bytes
1638 ++ * into the given buffer or as a return value. This is equivalent to
1639 ++ * a read from /dev/urandom. The u32, u64, int, and long family of
1640 ++ * functions may be higher performance for one-off random integers,
1641 ++ * because they do a bit of buffering and do not invoke reseeding
1642 ++ * until the buffer is emptied.
1643 ++ *
1644 ++ *********************************************************************/
1645 +
1646 +-struct entropy_store;
1647 +-struct entropy_store {
1648 +- /* read-only data: */
1649 +- const struct poolinfo *poolinfo;
1650 +- __u32 *pool;
1651 +- const char *name;
1652 +- struct entropy_store *pull;
1653 +- struct work_struct push_work;
1654 +-
1655 +- /* read-write data: */
1656 +- unsigned long last_pulled;
1657 +- spinlock_t lock;
1658 +- unsigned short add_ptr;
1659 +- unsigned short input_rotate;
1660 +- int entropy_count;
1661 +- int entropy_total;
1662 +- unsigned int initialized:1;
1663 +- unsigned int last_data_init:1;
1664 +- __u8 last_data[EXTRACT_SIZE];
1665 ++enum {
1666 ++ CRNG_RESEED_START_INTERVAL = HZ,
1667 ++ CRNG_RESEED_INTERVAL = 60 * HZ
1668 + };
1669 +
1670 +-static ssize_t extract_entropy(struct entropy_store *r, void *buf,
1671 +- size_t nbytes, int min, int rsvd);
1672 +-static ssize_t _extract_entropy(struct entropy_store *r, void *buf,
1673 +- size_t nbytes, int fips);
1674 +-
1675 +-static void crng_reseed(struct crng_state *crng, struct entropy_store *r);
1676 +-static void push_to_pool(struct work_struct *work);
1677 +-static __u32 input_pool_data[INPUT_POOL_WORDS] __latent_entropy;
1678 +-static __u32 blocking_pool_data[OUTPUT_POOL_WORDS] __latent_entropy;
1679 +-
1680 +-static struct entropy_store input_pool = {
1681 +- .poolinfo = &poolinfo_table[0],
1682 +- .name = "input",
1683 +- .lock = __SPIN_LOCK_UNLOCKED(input_pool.lock),
1684 +- .pool = input_pool_data
1685 ++static struct {
1686 ++ u8 key[CHACHA20_KEY_SIZE] __aligned(__alignof__(long));
1687 ++ unsigned long birth;
1688 ++ unsigned long generation;
1689 ++ spinlock_t lock;
1690 ++} base_crng = {
1691 ++ .lock = __SPIN_LOCK_UNLOCKED(base_crng.lock)
1692 + };
1693 +
1694 +-static struct entropy_store blocking_pool = {
1695 +- .poolinfo = &poolinfo_table[1],
1696 +- .name = "blocking",
1697 +- .pull = &input_pool,
1698 +- .lock = __SPIN_LOCK_UNLOCKED(blocking_pool.lock),
1699 +- .pool = blocking_pool_data,
1700 +- .push_work = __WORK_INITIALIZER(blocking_pool.push_work,
1701 +- push_to_pool),
1702 ++struct crng {
1703 ++ u8 key[CHACHA20_KEY_SIZE];
1704 ++ unsigned long generation;
1705 + };
1706 +
1707 +-static __u32 const twist_table[8] = {
1708 +- 0x00000000, 0x3b6e20c8, 0x76dc4190, 0x4db26158,
1709 +- 0xedb88320, 0xd6d6a3e8, 0x9b64c2b0, 0xa00ae278 };
1710 +-
1711 +-/*
1712 +- * This function adds bytes into the entropy "pool". It does not
1713 +- * update the entropy estimate. The caller should call
1714 +- * credit_entropy_bits if this is appropriate.
1715 +- *
1716 +- * The pool is stirred with a primitive polynomial of the appropriate
1717 +- * degree, and then twisted. We twist by three bits at a time because
1718 +- * it's cheap to do so and helps slightly in the expected case where
1719 +- * the entropy is concentrated in the low-order bits.
1720 +- */
1721 +-static void _mix_pool_bytes(struct entropy_store *r, const void *in,
1722 +- int nbytes)
1723 +-{
1724 +- unsigned long i, tap1, tap2, tap3, tap4, tap5;
1725 +- int input_rotate;
1726 +- int wordmask = r->poolinfo->poolwords - 1;
1727 +- const char *bytes = in;
1728 +- __u32 w;
1729 +-
1730 +- tap1 = r->poolinfo->tap1;
1731 +- tap2 = r->poolinfo->tap2;
1732 +- tap3 = r->poolinfo->tap3;
1733 +- tap4 = r->poolinfo->tap4;
1734 +- tap5 = r->poolinfo->tap5;
1735 +-
1736 +- input_rotate = r->input_rotate;
1737 +- i = r->add_ptr;
1738 +-
1739 +- /* mix one byte at a time to simplify size handling and churn faster */
1740 +- while (nbytes--) {
1741 +- w = rol32(*bytes++, input_rotate);
1742 +- i = (i - 1) & wordmask;
1743 +-
1744 +- /* XOR in the various taps */
1745 +- w ^= r->pool[i];
1746 +- w ^= r->pool[(i + tap1) & wordmask];
1747 +- w ^= r->pool[(i + tap2) & wordmask];
1748 +- w ^= r->pool[(i + tap3) & wordmask];
1749 +- w ^= r->pool[(i + tap4) & wordmask];
1750 +- w ^= r->pool[(i + tap5) & wordmask];
1751 +-
1752 +- /* Mix the result back in with a twist */
1753 +- r->pool[i] = (w >> 3) ^ twist_table[w & 7];
1754 +-
1755 +- /*
1756 +- * Normally, we add 7 bits of rotation to the pool.
1757 +- * At the beginning of the pool, add an extra 7 bits
1758 +- * rotation, so that successive passes spread the
1759 +- * input bits across the pool evenly.
1760 +- */
1761 +- input_rotate = (input_rotate + (i ? 7 : 14)) & 31;
1762 +- }
1763 +-
1764 +- r->input_rotate = input_rotate;
1765 +- r->add_ptr = i;
1766 +-}
1767 ++static DEFINE_PER_CPU(struct crng, crngs) = {
1768 ++ .generation = ULONG_MAX
1769 ++};
1770 +
1771 +-static void __mix_pool_bytes(struct entropy_store *r, const void *in,
1772 +- int nbytes)
1773 +-{
1774 +- trace_mix_pool_bytes_nolock(r->name, nbytes, _RET_IP_);
1775 +- _mix_pool_bytes(r, in, nbytes);
1776 +-}
1777 ++/* Used by crng_reseed() and crng_make_state() to extract a new seed from the input pool. */
1778 ++static void extract_entropy(void *buf, size_t len);
1779 +
1780 +-static void mix_pool_bytes(struct entropy_store *r, const void *in,
1781 +- int nbytes)
1782 ++/* This extracts a new crng key from the input pool. */
1783 ++static void crng_reseed(void)
1784 + {
1785 + unsigned long flags;
1786 ++ unsigned long next_gen;
1787 ++ u8 key[CHACHA20_KEY_SIZE];
1788 +
1789 +- trace_mix_pool_bytes(r->name, nbytes, _RET_IP_);
1790 +- spin_lock_irqsave(&r->lock, flags);
1791 +- _mix_pool_bytes(r, in, nbytes);
1792 +- spin_unlock_irqrestore(&r->lock, flags);
1793 +-}
1794 ++ extract_entropy(key, sizeof(key));
1795 +
1796 +-struct fast_pool {
1797 +- __u32 pool[4];
1798 +- unsigned long last;
1799 +- unsigned short reg_idx;
1800 +- unsigned char count;
1801 +-};
1802 ++ /*
1803 ++ * We copy the new key into the base_crng, overwriting the old one,
1804 ++ * and update the generation counter. We avoid hitting ULONG_MAX,
1805 ++ * because the per-cpu crngs are initialized to ULONG_MAX, so this
1806 ++ * forces new CPUs that come online to always initialize.
1807 ++ */
1808 ++ spin_lock_irqsave(&base_crng.lock, flags);
1809 ++ memcpy(base_crng.key, key, sizeof(base_crng.key));
1810 ++ next_gen = base_crng.generation + 1;
1811 ++ if (next_gen == ULONG_MAX)
1812 ++ ++next_gen;
1813 ++ WRITE_ONCE(base_crng.generation, next_gen);
1814 ++ WRITE_ONCE(base_crng.birth, jiffies);
1815 ++ if (!crng_ready())
1816 ++ crng_init = CRNG_READY;
1817 ++ spin_unlock_irqrestore(&base_crng.lock, flags);
1818 ++ memzero_explicit(key, sizeof(key));
1819 ++}
1820 +
1821 + /*
1822 +- * This is a fast mixing routine used by the interrupt randomness
1823 +- * collector. It's hardcoded for an 128 bit pool and assumes that any
1824 +- * locks that might be needed are taken by the caller.
1825 ++ * This generates a ChaCha block using the provided key, and then
1826 ++ * immediately overwites that key with half the block. It returns
1827 ++ * the resultant ChaCha state to the user, along with the second
1828 ++ * half of the block containing 32 bytes of random data that may
1829 ++ * be used; random_data_len may not be greater than 32.
1830 ++ *
1831 ++ * The returned ChaCha state contains within it a copy of the old
1832 ++ * key value, at index 4, so the state should always be zeroed out
1833 ++ * immediately after using in order to maintain forward secrecy.
1834 ++ * If the state cannot be erased in a timely manner, then it is
1835 ++ * safer to set the random_data parameter to &chacha_state[4] so
1836 ++ * that this function overwrites it before returning.
1837 + */
1838 +-static void fast_mix(struct fast_pool *f)
1839 ++static void crng_fast_key_erasure(u8 key[CHACHA20_KEY_SIZE],
1840 ++ u32 chacha_state[CHACHA20_BLOCK_SIZE / sizeof(u32)],
1841 ++ u8 *random_data, size_t random_data_len)
1842 + {
1843 +- __u32 a = f->pool[0], b = f->pool[1];
1844 +- __u32 c = f->pool[2], d = f->pool[3];
1845 ++ u8 first_block[CHACHA20_BLOCK_SIZE];
1846 +
1847 +- a += b; c += d;
1848 +- b = rol32(b, 6); d = rol32(d, 27);
1849 +- d ^= a; b ^= c;
1850 ++ BUG_ON(random_data_len > 32);
1851 +
1852 +- a += b; c += d;
1853 +- b = rol32(b, 16); d = rol32(d, 14);
1854 +- d ^= a; b ^= c;
1855 ++ chacha_init_consts(chacha_state);
1856 ++ memcpy(&chacha_state[4], key, CHACHA20_KEY_SIZE);
1857 ++ memset(&chacha_state[12], 0, sizeof(u32) * 4);
1858 ++ chacha20_block(chacha_state, first_block);
1859 +
1860 +- a += b; c += d;
1861 +- b = rol32(b, 6); d = rol32(d, 27);
1862 +- d ^= a; b ^= c;
1863 +-
1864 +- a += b; c += d;
1865 +- b = rol32(b, 16); d = rol32(d, 14);
1866 +- d ^= a; b ^= c;
1867 +-
1868 +- f->pool[0] = a; f->pool[1] = b;
1869 +- f->pool[2] = c; f->pool[3] = d;
1870 +- f->count++;
1871 ++ memcpy(key, first_block, CHACHA20_KEY_SIZE);
1872 ++ memcpy(random_data, first_block + CHACHA20_KEY_SIZE, random_data_len);
1873 ++ memzero_explicit(first_block, sizeof(first_block));
1874 + }
1875 +
1876 +-static void process_random_ready_list(void)
1877 ++/*
1878 ++ * Return whether the crng seed is considered to be sufficiently old
1879 ++ * that a reseeding is needed. This happens if the last reseeding
1880 ++ * was CRNG_RESEED_INTERVAL ago, or during early boot, at an interval
1881 ++ * proportional to the uptime.
1882 ++ */
1883 ++static bool crng_has_old_seed(void)
1884 + {
1885 +- unsigned long flags;
1886 +- struct random_ready_callback *rdy, *tmp;
1887 +-
1888 +- spin_lock_irqsave(&random_ready_list_lock, flags);
1889 +- list_for_each_entry_safe(rdy, tmp, &random_ready_list, list) {
1890 +- struct module *owner = rdy->owner;
1891 +-
1892 +- list_del_init(&rdy->list);
1893 +- rdy->func(rdy);
1894 +- module_put(owner);
1895 ++ static bool early_boot = true;
1896 ++ unsigned long interval = CRNG_RESEED_INTERVAL;
1897 ++
1898 ++ if (unlikely(READ_ONCE(early_boot))) {
1899 ++ time64_t uptime = ktime_get_seconds();
1900 ++ if (uptime >= CRNG_RESEED_INTERVAL / HZ * 2)
1901 ++ WRITE_ONCE(early_boot, false);
1902 ++ else
1903 ++ interval = max_t(unsigned int, CRNG_RESEED_START_INTERVAL,
1904 ++ (unsigned int)uptime / 2 * HZ);
1905 + }
1906 +- spin_unlock_irqrestore(&random_ready_list_lock, flags);
1907 ++ return time_is_before_jiffies(READ_ONCE(base_crng.birth) + interval);
1908 + }
1909 +
1910 + /*
1911 +- * Credit (or debit) the entropy store with n bits of entropy.
1912 +- * Use credit_entropy_bits_safe() if the value comes from userspace
1913 +- * or otherwise should be checked for extreme values.
1914 ++ * This function returns a ChaCha state that you may use for generating
1915 ++ * random data. It also returns up to 32 bytes on its own of random data
1916 ++ * that may be used; random_data_len may not be greater than 32.
1917 + */
1918 +-static void credit_entropy_bits(struct entropy_store *r, int nbits)
1919 ++static void crng_make_state(u32 chacha_state[CHACHA20_BLOCK_SIZE / sizeof(u32)],
1920 ++ u8 *random_data, size_t random_data_len)
1921 + {
1922 +- int entropy_count, orig;
1923 +- const int pool_size = r->poolinfo->poolfracbits;
1924 +- int nfrac = nbits << ENTROPY_SHIFT;
1925 +-
1926 +- if (!nbits)
1927 +- return;
1928 ++ unsigned long flags;
1929 ++ struct crng *crng;
1930 +
1931 +-retry:
1932 +- entropy_count = orig = ACCESS_ONCE(r->entropy_count);
1933 +- if (nfrac < 0) {
1934 +- /* Debit */
1935 +- entropy_count += nfrac;
1936 +- } else {
1937 +- /*
1938 +- * Credit: we have to account for the possibility of
1939 +- * overwriting already present entropy. Even in the
1940 +- * ideal case of pure Shannon entropy, new contributions
1941 +- * approach the full value asymptotically:
1942 +- *
1943 +- * entropy <- entropy + (pool_size - entropy) *
1944 +- * (1 - exp(-add_entropy/pool_size))
1945 +- *
1946 +- * For add_entropy <= pool_size/2 then
1947 +- * (1 - exp(-add_entropy/pool_size)) >=
1948 +- * (add_entropy/pool_size)*0.7869...
1949 +- * so we can approximate the exponential with
1950 +- * 3/4*add_entropy/pool_size and still be on the
1951 +- * safe side by adding at most pool_size/2 at a time.
1952 +- *
1953 +- * The use of pool_size-2 in the while statement is to
1954 +- * prevent rounding artifacts from making the loop
1955 +- * arbitrarily long; this limits the loop to log2(pool_size)*2
1956 +- * turns no matter how large nbits is.
1957 +- */
1958 +- int pnfrac = nfrac;
1959 +- const int s = r->poolinfo->poolbitshift + ENTROPY_SHIFT + 2;
1960 +- /* The +2 corresponds to the /4 in the denominator */
1961 +-
1962 +- do {
1963 +- unsigned int anfrac = min(pnfrac, pool_size/2);
1964 +- unsigned int add =
1965 +- ((pool_size - entropy_count)*anfrac*3) >> s;
1966 +-
1967 +- entropy_count += add;
1968 +- pnfrac -= anfrac;
1969 +- } while (unlikely(entropy_count < pool_size-2 && pnfrac));
1970 +- }
1971 ++ BUG_ON(random_data_len > 32);
1972 +
1973 +- if (unlikely(entropy_count < 0)) {
1974 +- pr_warn("random: negative entropy/overflow: pool %s count %d\n",
1975 +- r->name, entropy_count);
1976 +- WARN_ON(1);
1977 +- entropy_count = 0;
1978 +- } else if (entropy_count > pool_size)
1979 +- entropy_count = pool_size;
1980 +- if (cmpxchg(&r->entropy_count, orig, entropy_count) != orig)
1981 +- goto retry;
1982 +-
1983 +- r->entropy_total += nbits;
1984 +- if (!r->initialized && r->entropy_total > 128) {
1985 +- r->initialized = 1;
1986 +- r->entropy_total = 0;
1987 ++ /*
1988 ++ * For the fast path, we check whether we're ready, unlocked first, and
1989 ++ * then re-check once locked later. In the case where we're really not
1990 ++ * ready, we do fast key erasure with the base_crng directly, extracting
1991 ++ * when crng_init is CRNG_EMPTY.
1992 ++ */
1993 ++ if (!crng_ready()) {
1994 ++ bool ready;
1995 ++
1996 ++ spin_lock_irqsave(&base_crng.lock, flags);
1997 ++ ready = crng_ready();
1998 ++ if (!ready) {
1999 ++ if (crng_init == CRNG_EMPTY)
2000 ++ extract_entropy(base_crng.key, sizeof(base_crng.key));
2001 ++ crng_fast_key_erasure(base_crng.key, chacha_state,
2002 ++ random_data, random_data_len);
2003 ++ }
2004 ++ spin_unlock_irqrestore(&base_crng.lock, flags);
2005 ++ if (!ready)
2006 ++ return;
2007 + }
2008 +
2009 +- trace_credit_entropy_bits(r->name, nbits,
2010 +- entropy_count >> ENTROPY_SHIFT,
2011 +- r->entropy_total, _RET_IP_);
2012 +-
2013 +- if (r == &input_pool) {
2014 +- int entropy_bits = entropy_count >> ENTROPY_SHIFT;
2015 ++ /*
2016 ++ * If the base_crng is old enough, we reseed, which in turn bumps the
2017 ++ * generation counter that we check below.
2018 ++ */
2019 ++ if (unlikely(crng_has_old_seed()))
2020 ++ crng_reseed();
2021 +
2022 +- if (crng_init < 2 && entropy_bits >= 128) {
2023 +- crng_reseed(&primary_crng, r);
2024 +- entropy_bits = r->entropy_count >> ENTROPY_SHIFT;
2025 +- }
2026 ++ local_irq_save(flags);
2027 ++ crng = raw_cpu_ptr(&crngs);
2028 +
2029 +- /* should we wake readers? */
2030 +- if (entropy_bits >= random_read_wakeup_bits) {
2031 +- wake_up_interruptible(&random_read_wait);
2032 +- kill_fasync(&fasync, SIGIO, POLL_IN);
2033 +- }
2034 +- /* If the input pool is getting full, send some
2035 +- * entropy to the blocking pool until it is 75% full.
2036 +- */
2037 +- if (entropy_bits > random_write_wakeup_bits &&
2038 +- r->initialized &&
2039 +- r->entropy_total >= 2*random_read_wakeup_bits) {
2040 +- struct entropy_store *other = &blocking_pool;
2041 +-
2042 +- if (other->entropy_count <=
2043 +- 3 * other->poolinfo->poolfracbits / 4) {
2044 +- schedule_work(&other->push_work);
2045 +- r->entropy_total = 0;
2046 +- }
2047 +- }
2048 ++ /*
2049 ++ * If our per-cpu crng is older than the base_crng, then it means
2050 ++ * somebody reseeded the base_crng. In that case, we do fast key
2051 ++ * erasure on the base_crng, and use its output as the new key
2052 ++ * for our per-cpu crng. This brings us up to date with base_crng.
2053 ++ */
2054 ++ if (unlikely(crng->generation != READ_ONCE(base_crng.generation))) {
2055 ++ spin_lock(&base_crng.lock);
2056 ++ crng_fast_key_erasure(base_crng.key, chacha_state,
2057 ++ crng->key, sizeof(crng->key));
2058 ++ crng->generation = base_crng.generation;
2059 ++ spin_unlock(&base_crng.lock);
2060 + }
2061 ++
2062 ++ /*
2063 ++ * Finally, when we've made it this far, our per-cpu crng has an up
2064 ++ * to date key, and we can do fast key erasure with it to produce
2065 ++ * some random data and a ChaCha state for the caller. All other
2066 ++ * branches of this function are "unlikely", so most of the time we
2067 ++ * should wind up here immediately.
2068 ++ */
2069 ++ crng_fast_key_erasure(crng->key, chacha_state, random_data, random_data_len);
2070 ++ local_irq_restore(flags);
2071 + }
2072 +
2073 +-static int credit_entropy_bits_safe(struct entropy_store *r, int nbits)
2074 ++static void _get_random_bytes(void *buf, size_t len)
2075 + {
2076 +- const int nbits_max = r->poolinfo->poolwords * 32;
2077 ++ u32 chacha_state[CHACHA20_BLOCK_SIZE / sizeof(u32)];
2078 ++ u8 tmp[CHACHA20_BLOCK_SIZE];
2079 ++ size_t first_block_len;
2080 +
2081 +- if (nbits < 0)
2082 +- return -EINVAL;
2083 +-
2084 +- /* Cap the value to avoid overflows */
2085 +- nbits = min(nbits, nbits_max);
2086 ++ if (!len)
2087 ++ return;
2088 +
2089 +- credit_entropy_bits(r, nbits);
2090 +- return 0;
2091 +-}
2092 ++ first_block_len = min_t(size_t, 32, len);
2093 ++ crng_make_state(chacha_state, buf, first_block_len);
2094 ++ len -= first_block_len;
2095 ++ buf += first_block_len;
2096 +
2097 +-/*********************************************************************
2098 +- *
2099 +- * CRNG using CHACHA20
2100 +- *
2101 +- *********************************************************************/
2102 ++ while (len) {
2103 ++ if (len < CHACHA20_BLOCK_SIZE) {
2104 ++ chacha20_block(chacha_state, tmp);
2105 ++ memcpy(buf, tmp, len);
2106 ++ memzero_explicit(tmp, sizeof(tmp));
2107 ++ break;
2108 ++ }
2109 +
2110 +-#define CRNG_RESEED_INTERVAL (300*HZ)
2111 ++ chacha20_block(chacha_state, buf);
2112 ++ if (unlikely(chacha_state[12] == 0))
2113 ++ ++chacha_state[13];
2114 ++ len -= CHACHA20_BLOCK_SIZE;
2115 ++ buf += CHACHA20_BLOCK_SIZE;
2116 ++ }
2117 +
2118 +-static DECLARE_WAIT_QUEUE_HEAD(crng_init_wait);
2119 ++ memzero_explicit(chacha_state, sizeof(chacha_state));
2120 ++}
2121 +
2122 +-#ifdef CONFIG_NUMA
2123 + /*
2124 +- * Hack to deal with crazy userspace progams when they are all trying
2125 +- * to access /dev/urandom in parallel. The programs are almost
2126 +- * certainly doing something terribly wrong, but we'll work around
2127 +- * their brain damage.
2128 ++ * This function is the exported kernel interface. It returns some
2129 ++ * number of good random numbers, suitable for key generation, seeding
2130 ++ * TCP sequence numbers, etc. It does not rely on the hardware random
2131 ++ * number generator. For random bytes direct from the hardware RNG
2132 ++ * (when available), use get_random_bytes_arch(). In order to ensure
2133 ++ * that the randomness provided by this function is okay, the function
2134 ++ * wait_for_random_bytes() should be called and return 0 at least once
2135 ++ * at any point prior.
2136 + */
2137 +-static struct crng_state **crng_node_pool __read_mostly;
2138 +-#endif
2139 +-
2140 +-static void invalidate_batched_entropy(void);
2141 +-
2142 +-static void crng_initialize(struct crng_state *crng)
2143 ++void get_random_bytes(void *buf, size_t len)
2144 + {
2145 +- int i;
2146 +- unsigned long rv;
2147 +-
2148 +- memcpy(&crng->state[0], "expand 32-byte k", 16);
2149 +- if (crng == &primary_crng)
2150 +- _extract_entropy(&input_pool, &crng->state[4],
2151 +- sizeof(__u32) * 12, 0);
2152 +- else
2153 +- _get_random_bytes(&crng->state[4], sizeof(__u32) * 12);
2154 +- for (i = 4; i < 16; i++) {
2155 +- if (!arch_get_random_seed_long(&rv) &&
2156 +- !arch_get_random_long(&rv))
2157 +- rv = random_get_entropy();
2158 +- crng->state[i] ^= rv;
2159 +- }
2160 +- crng->init_time = jiffies - CRNG_RESEED_INTERVAL - 1;
2161 ++ warn_unseeded_randomness();
2162 ++ _get_random_bytes(buf, len);
2163 + }
2164 ++EXPORT_SYMBOL(get_random_bytes);
2165 +
2166 +-#ifdef CONFIG_NUMA
2167 +-static void do_numa_crng_init(struct work_struct *work)
2168 ++static ssize_t get_random_bytes_user(struct iov_iter *iter)
2169 + {
2170 +- int i;
2171 +- struct crng_state *crng;
2172 +- struct crng_state **pool;
2173 +-
2174 +- pool = kcalloc(nr_node_ids, sizeof(*pool), GFP_KERNEL|__GFP_NOFAIL);
2175 +- for_each_online_node(i) {
2176 +- crng = kmalloc_node(sizeof(struct crng_state),
2177 +- GFP_KERNEL | __GFP_NOFAIL, i);
2178 +- spin_lock_init(&crng->lock);
2179 +- crng_initialize(crng);
2180 +- pool[i] = crng;
2181 +- }
2182 +- /* pairs with READ_ONCE() in select_crng() */
2183 +- if (cmpxchg_release(&crng_node_pool, NULL, pool) != NULL) {
2184 +- for_each_node(i)
2185 +- kfree(pool[i]);
2186 +- kfree(pool);
2187 +- }
2188 +-}
2189 ++ u32 chacha_state[CHACHA20_BLOCK_SIZE / sizeof(u32)];
2190 ++ u8 block[CHACHA20_BLOCK_SIZE];
2191 ++ size_t ret = 0, copied;
2192 +
2193 +-static DECLARE_WORK(numa_crng_init_work, do_numa_crng_init);
2194 ++ if (unlikely(!iov_iter_count(iter)))
2195 ++ return 0;
2196 +
2197 +-static void numa_crng_init(void)
2198 +-{
2199 +- schedule_work(&numa_crng_init_work);
2200 +-}
2201 ++ /*
2202 ++ * Immediately overwrite the ChaCha key at index 4 with random
2203 ++ * bytes, in case userspace causes copy_to_user() below to sleep
2204 ++ * forever, so that we still retain forward secrecy in that case.
2205 ++ */
2206 ++ crng_make_state(chacha_state, (u8 *)&chacha_state[4], CHACHA20_KEY_SIZE);
2207 ++ /*
2208 ++ * However, if we're doing a read of len <= 32, we don't need to
2209 ++ * use chacha_state after, so we can simply return those bytes to
2210 ++ * the user directly.
2211 ++ */
2212 ++ if (iov_iter_count(iter) <= CHACHA20_KEY_SIZE) {
2213 ++ ret = copy_to_iter(&chacha_state[4], CHACHA20_KEY_SIZE, iter);
2214 ++ goto out_zero_chacha;
2215 ++ }
2216 +
2217 +-static struct crng_state *select_crng(void)
2218 +-{
2219 +- struct crng_state **pool;
2220 +- int nid = numa_node_id();
2221 ++ for (;;) {
2222 ++ chacha20_block(chacha_state, block);
2223 ++ if (unlikely(chacha_state[12] == 0))
2224 ++ ++chacha_state[13];
2225 ++
2226 ++ copied = copy_to_iter(block, sizeof(block), iter);
2227 ++ ret += copied;
2228 ++ if (!iov_iter_count(iter) || copied != sizeof(block))
2229 ++ break;
2230 +
2231 +- /* pairs with cmpxchg_release() in do_numa_crng_init() */
2232 +- pool = READ_ONCE(crng_node_pool);
2233 +- if (pool && pool[nid])
2234 +- return pool[nid];
2235 ++ BUILD_BUG_ON(PAGE_SIZE % sizeof(block) != 0);
2236 ++ if (ret % PAGE_SIZE == 0) {
2237 ++ if (signal_pending(current))
2238 ++ break;
2239 ++ cond_resched();
2240 ++ }
2241 ++ }
2242 +
2243 +- return &primary_crng;
2244 ++ memzero_explicit(block, sizeof(block));
2245 ++out_zero_chacha:
2246 ++ memzero_explicit(chacha_state, sizeof(chacha_state));
2247 ++ return ret ? ret : -EFAULT;
2248 + }
2249 +-#else
2250 +-static void numa_crng_init(void) {}
2251 +
2252 +-static struct crng_state *select_crng(void)
2253 ++/*
2254 ++ * Batched entropy returns random integers. The quality of the random
2255 ++ * number is good as /dev/urandom. In order to ensure that the randomness
2256 ++ * provided by this function is okay, the function wait_for_random_bytes()
2257 ++ * should be called and return 0 at least once at any point prior.
2258 ++ */
2259 ++
2260 ++#define DEFINE_BATCHED_ENTROPY(type) \
2261 ++struct batch_ ##type { \
2262 ++ /* \
2263 ++ * We make this 1.5x a ChaCha block, so that we get the \
2264 ++ * remaining 32 bytes from fast key erasure, plus one full \
2265 ++ * block from the detached ChaCha state. We can increase \
2266 ++ * the size of this later if needed so long as we keep the \
2267 ++ * formula of (integer_blocks + 0.5) * CHACHA20_BLOCK_SIZE. \
2268 ++ */ \
2269 ++ type entropy[CHACHA20_BLOCK_SIZE * 3 / (2 * sizeof(type))]; \
2270 ++ unsigned long generation; \
2271 ++ unsigned int position; \
2272 ++}; \
2273 ++ \
2274 ++static DEFINE_PER_CPU(struct batch_ ##type, batched_entropy_ ##type) = { \
2275 ++ .position = UINT_MAX \
2276 ++}; \
2277 ++ \
2278 ++type get_random_ ##type(void) \
2279 ++{ \
2280 ++ type ret; \
2281 ++ unsigned long flags; \
2282 ++ struct batch_ ##type *batch; \
2283 ++ unsigned long next_gen; \
2284 ++ \
2285 ++ warn_unseeded_randomness(); \
2286 ++ \
2287 ++ if (!crng_ready()) { \
2288 ++ _get_random_bytes(&ret, sizeof(ret)); \
2289 ++ return ret; \
2290 ++ } \
2291 ++ \
2292 ++ local_irq_save(flags); \
2293 ++ batch = raw_cpu_ptr(&batched_entropy_##type); \
2294 ++ \
2295 ++ next_gen = READ_ONCE(base_crng.generation); \
2296 ++ if (batch->position >= ARRAY_SIZE(batch->entropy) || \
2297 ++ next_gen != batch->generation) { \
2298 ++ _get_random_bytes(batch->entropy, sizeof(batch->entropy)); \
2299 ++ batch->position = 0; \
2300 ++ batch->generation = next_gen; \
2301 ++ } \
2302 ++ \
2303 ++ ret = batch->entropy[batch->position]; \
2304 ++ batch->entropy[batch->position] = 0; \
2305 ++ ++batch->position; \
2306 ++ local_irq_restore(flags); \
2307 ++ return ret; \
2308 ++} \
2309 ++EXPORT_SYMBOL(get_random_ ##type);
2310 ++
2311 ++DEFINE_BATCHED_ENTROPY(u64)
2312 ++DEFINE_BATCHED_ENTROPY(u32)
2313 ++
2314 ++#ifdef CONFIG_SMP
2315 ++/*
2316 ++ * This function is called when the CPU is coming up, with entry
2317 ++ * CPUHP_RANDOM_PREPARE, which comes before CPUHP_WORKQUEUE_PREP.
2318 ++ */
2319 ++int __cold random_prepare_cpu(unsigned int cpu)
2320 + {
2321 +- return &primary_crng;
2322 ++ /*
2323 ++ * When the cpu comes back online, immediately invalidate both
2324 ++ * the per-cpu crng and all batches, so that we serve fresh
2325 ++ * randomness.
2326 ++ */
2327 ++ per_cpu_ptr(&crngs, cpu)->generation = ULONG_MAX;
2328 ++ per_cpu_ptr(&batched_entropy_u32, cpu)->position = UINT_MAX;
2329 ++ per_cpu_ptr(&batched_entropy_u64, cpu)->position = UINT_MAX;
2330 ++ return 0;
2331 + }
2332 + #endif
2333 +
2334 + /*
2335 +- * crng_fast_load() can be called by code in the interrupt service
2336 +- * path. So we can't afford to dilly-dally.
2337 ++ * This function will use the architecture-specific hardware random
2338 ++ * number generator if it is available. It is not recommended for
2339 ++ * use. Use get_random_bytes() instead. It returns the number of
2340 ++ * bytes filled in.
2341 + */
2342 +-static int crng_fast_load(const char *cp, size_t len)
2343 ++size_t __must_check get_random_bytes_arch(void *buf, size_t len)
2344 + {
2345 +- unsigned long flags;
2346 +- char *p;
2347 ++ size_t left = len;
2348 ++ u8 *p = buf;
2349 +
2350 +- if (!spin_trylock_irqsave(&primary_crng.lock, flags))
2351 +- return 0;
2352 +- if (crng_init != 0) {
2353 +- spin_unlock_irqrestore(&primary_crng.lock, flags);
2354 +- return 0;
2355 +- }
2356 +- p = (unsigned char *) &primary_crng.state[4];
2357 +- while (len > 0 && crng_init_cnt < CRNG_INIT_CNT_THRESH) {
2358 +- p[crng_init_cnt % CHACHA20_KEY_SIZE] ^= *cp;
2359 +- cp++; crng_init_cnt++; len--;
2360 +- }
2361 +- spin_unlock_irqrestore(&primary_crng.lock, flags);
2362 +- if (crng_init_cnt >= CRNG_INIT_CNT_THRESH) {
2363 +- invalidate_batched_entropy();
2364 +- crng_init = 1;
2365 +- wake_up_interruptible(&crng_init_wait);
2366 +- pr_notice("random: fast init done\n");
2367 ++ while (left) {
2368 ++ unsigned long v;
2369 ++ size_t block_len = min_t(size_t, left, sizeof(unsigned long));
2370 ++
2371 ++ if (!arch_get_random_long(&v))
2372 ++ break;
2373 ++
2374 ++ memcpy(p, &v, block_len);
2375 ++ p += block_len;
2376 ++ left -= block_len;
2377 + }
2378 +- return 1;
2379 ++
2380 ++ return len - left;
2381 + }
2382 ++EXPORT_SYMBOL(get_random_bytes_arch);
2383 +
2384 +-/*
2385 +- * crng_slow_load() is called by add_device_randomness, which has two
2386 +- * attributes. (1) We can't trust the buffer passed to it is
2387 +- * guaranteed to be unpredictable (so it might not have any entropy at
2388 +- * all), and (2) it doesn't have the performance constraints of
2389 +- * crng_fast_load().
2390 ++
2391 ++/**********************************************************************
2392 + *
2393 +- * So we do something more comprehensive which is guaranteed to touch
2394 +- * all of the primary_crng's state, and which uses a LFSR with a
2395 +- * period of 255 as part of the mixing algorithm. Finally, we do
2396 +- * *not* advance crng_init_cnt since buffer we may get may be something
2397 +- * like a fixed DMI table (for example), which might very well be
2398 +- * unique to the machine, but is otherwise unvarying.
2399 +- */
2400 +-static int crng_slow_load(const char *cp, size_t len)
2401 +-{
2402 +- unsigned long flags;
2403 +- static unsigned char lfsr = 1;
2404 +- unsigned char tmp;
2405 +- unsigned i, max = CHACHA20_KEY_SIZE;
2406 +- const char * src_buf = cp;
2407 +- char * dest_buf = (char *) &primary_crng.state[4];
2408 +-
2409 +- if (!spin_trylock_irqsave(&primary_crng.lock, flags))
2410 +- return 0;
2411 +- if (crng_init != 0) {
2412 +- spin_unlock_irqrestore(&primary_crng.lock, flags);
2413 +- return 0;
2414 +- }
2415 +- if (len > max)
2416 +- max = len;
2417 +-
2418 +- for (i = 0; i < max ; i++) {
2419 +- tmp = lfsr;
2420 +- lfsr >>= 1;
2421 +- if (tmp & 1)
2422 +- lfsr ^= 0xE1;
2423 +- tmp = dest_buf[i % CHACHA20_KEY_SIZE];
2424 +- dest_buf[i % CHACHA20_KEY_SIZE] ^= src_buf[i % len] ^ lfsr;
2425 +- lfsr += (tmp << 3) | (tmp >> 5);
2426 +- }
2427 +- spin_unlock_irqrestore(&primary_crng.lock, flags);
2428 +- return 1;
2429 +-}
2430 +-
2431 +-static void crng_reseed(struct crng_state *crng, struct entropy_store *r)
2432 +-{
2433 +- unsigned long flags;
2434 +- int i, num;
2435 +- union {
2436 +- __u8 block[CHACHA20_BLOCK_SIZE];
2437 +- __u32 key[8];
2438 +- } buf;
2439 +-
2440 +- if (r) {
2441 +- num = extract_entropy(r, &buf, 32, 16, 0);
2442 +- if (num == 0)
2443 +- return;
2444 +- } else {
2445 +- _extract_crng(&primary_crng, buf.block);
2446 +- _crng_backtrack_protect(&primary_crng, buf.block,
2447 +- CHACHA20_KEY_SIZE);
2448 +- }
2449 +- spin_lock_irqsave(&crng->lock, flags);
2450 +- for (i = 0; i < 8; i++) {
2451 +- unsigned long rv;
2452 +- if (!arch_get_random_seed_long(&rv) &&
2453 +- !arch_get_random_long(&rv))
2454 +- rv = random_get_entropy();
2455 +- crng->state[i+4] ^= buf.key[i] ^ rv;
2456 +- }
2457 +- memzero_explicit(&buf, sizeof(buf));
2458 +- WRITE_ONCE(crng->init_time, jiffies);
2459 +- spin_unlock_irqrestore(&crng->lock, flags);
2460 +- if (crng == &primary_crng && crng_init < 2) {
2461 +- invalidate_batched_entropy();
2462 +- numa_crng_init();
2463 +- crng_init = 2;
2464 +- process_random_ready_list();
2465 +- wake_up_interruptible(&crng_init_wait);
2466 +- pr_notice("random: crng init done\n");
2467 +- if (unseeded_warning.missed) {
2468 +- pr_notice("random: %d get_random_xx warning(s) missed "
2469 +- "due to ratelimiting\n",
2470 +- unseeded_warning.missed);
2471 +- unseeded_warning.missed = 0;
2472 +- }
2473 +- if (urandom_warning.missed) {
2474 +- pr_notice("random: %d urandom warning(s) missed "
2475 +- "due to ratelimiting\n",
2476 +- urandom_warning.missed);
2477 +- urandom_warning.missed = 0;
2478 +- }
2479 +- }
2480 +-}
2481 +-
2482 +-static void _extract_crng(struct crng_state *crng,
2483 +- __u8 out[CHACHA20_BLOCK_SIZE])
2484 +-{
2485 +- unsigned long v, flags, init_time;
2486 +-
2487 +- if (crng_ready()) {
2488 +- init_time = READ_ONCE(crng->init_time);
2489 +- if (time_after(READ_ONCE(crng_global_init_time), init_time) ||
2490 +- time_after(jiffies, init_time + CRNG_RESEED_INTERVAL))
2491 +- crng_reseed(crng, crng == &primary_crng ?
2492 +- &input_pool : NULL);
2493 +- }
2494 +- spin_lock_irqsave(&crng->lock, flags);
2495 +- if (arch_get_random_long(&v))
2496 +- crng->state[14] ^= v;
2497 +- chacha20_block(&crng->state[0], out);
2498 +- if (crng->state[12] == 0)
2499 +- crng->state[13]++;
2500 +- spin_unlock_irqrestore(&crng->lock, flags);
2501 +-}
2502 +-
2503 +-static void extract_crng(__u8 out[CHACHA20_BLOCK_SIZE])
2504 +-{
2505 +- _extract_crng(select_crng(), out);
2506 +-}
2507 +-
2508 +-/*
2509 +- * Use the leftover bytes from the CRNG block output (if there is
2510 +- * enough) to mutate the CRNG key to provide backtracking protection.
2511 +- */
2512 +-static void _crng_backtrack_protect(struct crng_state *crng,
2513 +- __u8 tmp[CHACHA20_BLOCK_SIZE], int used)
2514 +-{
2515 +- unsigned long flags;
2516 +- __u32 *s, *d;
2517 +- int i;
2518 +-
2519 +- used = round_up(used, sizeof(__u32));
2520 +- if (used + CHACHA20_KEY_SIZE > CHACHA20_BLOCK_SIZE) {
2521 +- extract_crng(tmp);
2522 +- used = 0;
2523 +- }
2524 +- spin_lock_irqsave(&crng->lock, flags);
2525 +- s = (__u32 *) &tmp[used];
2526 +- d = &crng->state[4];
2527 +- for (i=0; i < 8; i++)
2528 +- *d++ ^= *s++;
2529 +- spin_unlock_irqrestore(&crng->lock, flags);
2530 +-}
2531 +-
2532 +-static void crng_backtrack_protect(__u8 tmp[CHACHA20_BLOCK_SIZE], int used)
2533 +-{
2534 +- _crng_backtrack_protect(select_crng(), tmp, used);
2535 +-}
2536 +-
2537 +-static ssize_t extract_crng_user(void __user *buf, size_t nbytes)
2538 +-{
2539 +- ssize_t ret = 0, i = CHACHA20_BLOCK_SIZE;
2540 +- __u8 tmp[CHACHA20_BLOCK_SIZE];
2541 +- int large_request = (nbytes > 256);
2542 +-
2543 +- while (nbytes) {
2544 +- if (large_request && need_resched()) {
2545 +- if (signal_pending(current)) {
2546 +- if (ret == 0)
2547 +- ret = -ERESTARTSYS;
2548 +- break;
2549 +- }
2550 +- schedule();
2551 +- }
2552 +-
2553 +- extract_crng(tmp);
2554 +- i = min_t(int, nbytes, CHACHA20_BLOCK_SIZE);
2555 +- if (copy_to_user(buf, tmp, i)) {
2556 +- ret = -EFAULT;
2557 +- break;
2558 +- }
2559 +-
2560 +- nbytes -= i;
2561 +- buf += i;
2562 +- ret += i;
2563 +- }
2564 +- crng_backtrack_protect(tmp, i);
2565 +-
2566 +- /* Wipe data just written to memory */
2567 +- memzero_explicit(tmp, sizeof(tmp));
2568 +-
2569 +- return ret;
2570 +-}
2571 +-
2572 +-
2573 +-/*********************************************************************
2574 ++ * Entropy accumulation and extraction routines.
2575 + *
2576 +- * Entropy input management
2577 ++ * Callers may add entropy via:
2578 + *
2579 +- *********************************************************************/
2580 ++ * static void mix_pool_bytes(const void *buf, size_t len)
2581 ++ *
2582 ++ * After which, if added entropy should be credited:
2583 ++ *
2584 ++ * static void credit_init_bits(size_t bits)
2585 ++ *
2586 ++ * Finally, extract entropy via:
2587 ++ *
2588 ++ * static void extract_entropy(void *buf, size_t len)
2589 ++ *
2590 ++ **********************************************************************/
2591 +
2592 +-/* There is one of these per entropy source */
2593 +-struct timer_rand_state {
2594 +- cycles_t last_time;
2595 +- long last_delta, last_delta2;
2596 +- unsigned dont_count_entropy:1;
2597 ++enum {
2598 ++ POOL_BITS = BLAKE2S_HASH_SIZE * 8,
2599 ++ POOL_READY_BITS = POOL_BITS, /* When crng_init->CRNG_READY */
2600 ++ POOL_EARLY_BITS = POOL_READY_BITS / 2 /* When crng_init->CRNG_EARLY */
2601 + };
2602 +
2603 +-#define INIT_TIMER_RAND_STATE { INITIAL_JIFFIES, };
2604 ++static struct {
2605 ++ struct blake2s_state hash;
2606 ++ spinlock_t lock;
2607 ++ unsigned int init_bits;
2608 ++} input_pool = {
2609 ++ .hash.h = { BLAKE2S_IV0 ^ (0x01010000 | BLAKE2S_HASH_SIZE),
2610 ++ BLAKE2S_IV1, BLAKE2S_IV2, BLAKE2S_IV3, BLAKE2S_IV4,
2611 ++ BLAKE2S_IV5, BLAKE2S_IV6, BLAKE2S_IV7 },
2612 ++ .hash.outlen = BLAKE2S_HASH_SIZE,
2613 ++ .lock = __SPIN_LOCK_UNLOCKED(input_pool.lock),
2614 ++};
2615 ++
2616 ++static void _mix_pool_bytes(const void *buf, size_t len)
2617 ++{
2618 ++ blake2s_update(&input_pool.hash, buf, len);
2619 ++}
2620 +
2621 + /*
2622 +- * Add device- or boot-specific data to the input pool to help
2623 +- * initialize it.
2624 +- *
2625 +- * None of this adds any entropy; it is meant to avoid the problem of
2626 +- * the entropy pool having similar initial state across largely
2627 +- * identical devices.
2628 ++ * This function adds bytes into the input pool. It does not
2629 ++ * update the initialization bit counter; the caller should call
2630 ++ * credit_init_bits if this is appropriate.
2631 + */
2632 +-void add_device_randomness(const void *buf, unsigned int size)
2633 ++static void mix_pool_bytes(const void *buf, size_t len)
2634 + {
2635 +- unsigned long time = random_get_entropy() ^ jiffies;
2636 + unsigned long flags;
2637 +
2638 +- if (!crng_ready() && size)
2639 +- crng_slow_load(buf, size);
2640 +-
2641 +- trace_add_device_randomness(size, _RET_IP_);
2642 + spin_lock_irqsave(&input_pool.lock, flags);
2643 +- _mix_pool_bytes(&input_pool, buf, size);
2644 +- _mix_pool_bytes(&input_pool, &time, sizeof(time));
2645 ++ _mix_pool_bytes(buf, len);
2646 + spin_unlock_irqrestore(&input_pool.lock, flags);
2647 + }
2648 +-EXPORT_SYMBOL(add_device_randomness);
2649 +-
2650 +-static struct timer_rand_state input_timer_state = INIT_TIMER_RAND_STATE;
2651 +
2652 + /*
2653 +- * This function adds entropy to the entropy "pool" by using timing
2654 +- * delays. It uses the timer_rand_state structure to make an estimate
2655 +- * of how many bits of entropy this call has added to the pool.
2656 +- *
2657 +- * The number "num" is also added to the pool - it should somehow describe
2658 +- * the type of event which just happened. This is currently 0-255 for
2659 +- * keyboard scan codes, and 256 upwards for interrupts.
2660 +- *
2661 ++ * This is an HKDF-like construction for using the hashed collected entropy
2662 ++ * as a PRF key, that's then expanded block-by-block.
2663 + */
2664 +-static void add_timer_randomness(struct timer_rand_state *state, unsigned num)
2665 ++static void extract_entropy(void *buf, size_t len)
2666 + {
2667 +- struct entropy_store *r;
2668 ++ unsigned long flags;
2669 ++ u8 seed[BLAKE2S_HASH_SIZE], next_key[BLAKE2S_HASH_SIZE];
2670 + struct {
2671 +- long jiffies;
2672 +- unsigned cycles;
2673 +- unsigned num;
2674 +- } sample;
2675 +- long delta, delta2, delta3;
2676 +-
2677 +- preempt_disable();
2678 +-
2679 +- sample.jiffies = jiffies;
2680 +- sample.cycles = random_get_entropy();
2681 +- sample.num = num;
2682 +- r = &input_pool;
2683 +- mix_pool_bytes(r, &sample, sizeof(sample));
2684 +-
2685 +- /*
2686 +- * Calculate number of bits of randomness we probably added.
2687 +- * We take into account the first, second and third-order deltas
2688 +- * in order to make our estimate.
2689 +- */
2690 +-
2691 +- if (!state->dont_count_entropy) {
2692 +- delta = sample.jiffies - state->last_time;
2693 +- state->last_time = sample.jiffies;
2694 +-
2695 +- delta2 = delta - state->last_delta;
2696 +- state->last_delta = delta;
2697 +-
2698 +- delta3 = delta2 - state->last_delta2;
2699 +- state->last_delta2 = delta2;
2700 +-
2701 +- if (delta < 0)
2702 +- delta = -delta;
2703 +- if (delta2 < 0)
2704 +- delta2 = -delta2;
2705 +- if (delta3 < 0)
2706 +- delta3 = -delta3;
2707 +- if (delta > delta2)
2708 +- delta = delta2;
2709 +- if (delta > delta3)
2710 +- delta = delta3;
2711 +-
2712 +- /*
2713 +- * delta is now minimum absolute delta.
2714 +- * Round down by 1 bit on general principles,
2715 +- * and limit entropy entimate to 12 bits.
2716 +- */
2717 +- credit_entropy_bits(r, min_t(int, fls(delta>>1), 11));
2718 ++ unsigned long rdseed[32 / sizeof(long)];
2719 ++ size_t counter;
2720 ++ } block;
2721 ++ size_t i;
2722 ++
2723 ++ for (i = 0; i < ARRAY_SIZE(block.rdseed); ++i) {
2724 ++ if (!arch_get_random_seed_long(&block.rdseed[i]) &&
2725 ++ !arch_get_random_long(&block.rdseed[i]))
2726 ++ block.rdseed[i] = random_get_entropy();
2727 + }
2728 +- preempt_enable();
2729 +-}
2730 +
2731 +-void add_input_randomness(unsigned int type, unsigned int code,
2732 +- unsigned int value)
2733 +-{
2734 +- static unsigned char last_value;
2735 +-
2736 +- /* ignore autorepeat and the like */
2737 +- if (value == last_value)
2738 +- return;
2739 +-
2740 +- last_value = value;
2741 +- add_timer_randomness(&input_timer_state,
2742 +- (type << 4) ^ code ^ (code >> 4) ^ value);
2743 +- trace_add_input_randomness(ENTROPY_BITS(&input_pool));
2744 +-}
2745 +-EXPORT_SYMBOL_GPL(add_input_randomness);
2746 ++ spin_lock_irqsave(&input_pool.lock, flags);
2747 +
2748 +-static DEFINE_PER_CPU(struct fast_pool, irq_randomness);
2749 ++ /* seed = HASHPRF(last_key, entropy_input) */
2750 ++ blake2s_final(&input_pool.hash, seed);
2751 +
2752 +-#ifdef ADD_INTERRUPT_BENCH
2753 +-static unsigned long avg_cycles, avg_deviation;
2754 ++ /* next_key = HASHPRF(seed, RDSEED || 0) */
2755 ++ block.counter = 0;
2756 ++ blake2s(next_key, (u8 *)&block, seed, sizeof(next_key), sizeof(block), sizeof(seed));
2757 ++ blake2s_init_key(&input_pool.hash, BLAKE2S_HASH_SIZE, next_key, sizeof(next_key));
2758 +
2759 +-#define AVG_SHIFT 8 /* Exponential average factor k=1/256 */
2760 +-#define FIXED_1_2 (1 << (AVG_SHIFT-1))
2761 ++ spin_unlock_irqrestore(&input_pool.lock, flags);
2762 ++ memzero_explicit(next_key, sizeof(next_key));
2763 ++
2764 ++ while (len) {
2765 ++ i = min_t(size_t, len, BLAKE2S_HASH_SIZE);
2766 ++ /* output = HASHPRF(seed, RDSEED || ++counter) */
2767 ++ ++block.counter;
2768 ++ blake2s(buf, (u8 *)&block, seed, i, sizeof(block), sizeof(seed));
2769 ++ len -= i;
2770 ++ buf += i;
2771 ++ }
2772 +
2773 +-static void add_interrupt_bench(cycles_t start)
2774 +-{
2775 +- long delta = random_get_entropy() - start;
2776 +-
2777 +- /* Use a weighted moving average */
2778 +- delta = delta - ((avg_cycles + FIXED_1_2) >> AVG_SHIFT);
2779 +- avg_cycles += delta;
2780 +- /* And average deviation */
2781 +- delta = abs(delta) - ((avg_deviation + FIXED_1_2) >> AVG_SHIFT);
2782 +- avg_deviation += delta;
2783 ++ memzero_explicit(seed, sizeof(seed));
2784 ++ memzero_explicit(&block, sizeof(block));
2785 + }
2786 +-#else
2787 +-#define add_interrupt_bench(x)
2788 +-#endif
2789 +
2790 +-static __u32 get_reg(struct fast_pool *f, struct pt_regs *regs)
2791 +-{
2792 +- __u32 *ptr = (__u32 *) regs;
2793 +- unsigned int idx;
2794 +-
2795 +- if (regs == NULL)
2796 +- return 0;
2797 +- idx = READ_ONCE(f->reg_idx);
2798 +- if (idx >= sizeof(struct pt_regs) / sizeof(__u32))
2799 +- idx = 0;
2800 +- ptr += idx++;
2801 +- WRITE_ONCE(f->reg_idx, idx);
2802 +- return *ptr;
2803 +-}
2804 ++#define credit_init_bits(bits) if (!crng_ready()) _credit_init_bits(bits)
2805 +
2806 +-void add_interrupt_randomness(int irq, int irq_flags)
2807 ++static void __cold _credit_init_bits(size_t bits)
2808 + {
2809 +- struct entropy_store *r;
2810 +- struct fast_pool *fast_pool = this_cpu_ptr(&irq_randomness);
2811 +- struct pt_regs *regs = get_irq_regs();
2812 +- unsigned long now = jiffies;
2813 +- cycles_t cycles = random_get_entropy();
2814 +- __u32 c_high, j_high;
2815 +- __u64 ip;
2816 +- unsigned long seed;
2817 +- int credit = 0;
2818 +-
2819 +- if (cycles == 0)
2820 +- cycles = get_reg(fast_pool, regs);
2821 +- c_high = (sizeof(cycles) > 4) ? cycles >> 32 : 0;
2822 +- j_high = (sizeof(now) > 4) ? now >> 32 : 0;
2823 +- fast_pool->pool[0] ^= cycles ^ j_high ^ irq;
2824 +- fast_pool->pool[1] ^= now ^ c_high;
2825 +- ip = regs ? instruction_pointer(regs) : _RET_IP_;
2826 +- fast_pool->pool[2] ^= ip;
2827 +- fast_pool->pool[3] ^= (sizeof(ip) > 4) ? ip >> 32 :
2828 +- get_reg(fast_pool, regs);
2829 +-
2830 +- fast_mix(fast_pool);
2831 +- add_interrupt_bench(cycles);
2832 +-
2833 +- if (unlikely(crng_init == 0)) {
2834 +- if ((fast_pool->count >= 64) &&
2835 +- crng_fast_load((char *) fast_pool->pool,
2836 +- sizeof(fast_pool->pool))) {
2837 +- fast_pool->count = 0;
2838 +- fast_pool->last = now;
2839 +- }
2840 +- return;
2841 +- }
2842 ++ unsigned int new, orig, add;
2843 ++ unsigned long flags;
2844 +
2845 +- if ((fast_pool->count < 64) &&
2846 +- !time_after(now, fast_pool->last + HZ))
2847 ++ if (!bits)
2848 + return;
2849 +
2850 +- r = &input_pool;
2851 +- if (!spin_trylock(&r->lock))
2852 +- return;
2853 ++ add = min_t(size_t, bits, POOL_BITS);
2854 +
2855 +- fast_pool->last = now;
2856 +- __mix_pool_bytes(r, &fast_pool->pool, sizeof(fast_pool->pool));
2857 ++ do {
2858 ++ orig = READ_ONCE(input_pool.init_bits);
2859 ++ new = min_t(unsigned int, POOL_BITS, orig + add);
2860 ++ } while (cmpxchg(&input_pool.init_bits, orig, new) != orig);
2861 +
2862 +- /*
2863 +- * If we have architectural seed generator, produce a seed and
2864 +- * add it to the pool. For the sake of paranoia don't let the
2865 +- * architectural seed generator dominate the input from the
2866 +- * interrupt noise.
2867 +- */
2868 +- if (arch_get_random_seed_long(&seed)) {
2869 +- __mix_pool_bytes(r, &seed, sizeof(seed));
2870 +- credit = 1;
2871 ++ if (orig < POOL_READY_BITS && new >= POOL_READY_BITS) {
2872 ++ crng_reseed(); /* Sets crng_init to CRNG_READY under base_crng.lock. */
2873 ++ process_random_ready_list();
2874 ++ wake_up_interruptible(&crng_init_wait);
2875 ++ kill_fasync(&fasync, SIGIO, POLL_IN);
2876 ++ pr_notice("crng init done\n");
2877 ++ if (urandom_warning.missed)
2878 ++ pr_notice("%d urandom warning(s) missed due to ratelimiting\n",
2879 ++ urandom_warning.missed);
2880 ++ } else if (orig < POOL_EARLY_BITS && new >= POOL_EARLY_BITS) {
2881 ++ spin_lock_irqsave(&base_crng.lock, flags);
2882 ++ /* Check if crng_init is CRNG_EMPTY, to avoid race with crng_reseed(). */
2883 ++ if (crng_init == CRNG_EMPTY) {
2884 ++ extract_entropy(base_crng.key, sizeof(base_crng.key));
2885 ++ crng_init = CRNG_EARLY;
2886 ++ }
2887 ++ spin_unlock_irqrestore(&base_crng.lock, flags);
2888 + }
2889 +- spin_unlock(&r->lock);
2890 +-
2891 +- fast_pool->count = 0;
2892 +-
2893 +- /* award one bit for the contents of the fast pool */
2894 +- credit_entropy_bits(r, credit + 1);
2895 + }
2896 +-EXPORT_SYMBOL_GPL(add_interrupt_randomness);
2897 +
2898 +-#ifdef CONFIG_BLOCK
2899 +-void add_disk_randomness(struct gendisk *disk)
2900 +-{
2901 +- if (!disk || !disk->random)
2902 +- return;
2903 +- /* first major is 1, so we get >= 0x200 here */
2904 +- add_timer_randomness(disk->random, 0x100 + disk_devt(disk));
2905 +- trace_add_disk_randomness(disk_devt(disk), ENTROPY_BITS(&input_pool));
2906 +-}
2907 +-EXPORT_SYMBOL_GPL(add_disk_randomness);
2908 +-#endif
2909 +
2910 +-/*********************************************************************
2911 ++/**********************************************************************
2912 + *
2913 +- * Entropy extraction routines
2914 ++ * Entropy collection routines.
2915 + *
2916 +- *********************************************************************/
2917 +-
2918 +-/*
2919 +- * This utility inline function is responsible for transferring entropy
2920 +- * from the primary pool to the secondary extraction pool. We make
2921 +- * sure we pull enough for a 'catastrophic reseed'.
2922 +- */
2923 +-static void _xfer_secondary_pool(struct entropy_store *r, size_t nbytes);
2924 +-static void xfer_secondary_pool(struct entropy_store *r, size_t nbytes)
2925 +-{
2926 +- if (!r->pull ||
2927 +- r->entropy_count >= (nbytes << (ENTROPY_SHIFT + 3)) ||
2928 +- r->entropy_count > r->poolinfo->poolfracbits)
2929 +- return;
2930 +-
2931 +- _xfer_secondary_pool(r, nbytes);
2932 +-}
2933 +-
2934 +-static void _xfer_secondary_pool(struct entropy_store *r, size_t nbytes)
2935 +-{
2936 +- __u32 tmp[OUTPUT_POOL_WORDS];
2937 +-
2938 +- int bytes = nbytes;
2939 +-
2940 +- /* pull at least as much as a wakeup */
2941 +- bytes = max_t(int, bytes, random_read_wakeup_bits / 8);
2942 +- /* but never more than the buffer size */
2943 +- bytes = min_t(int, bytes, sizeof(tmp));
2944 +-
2945 +- trace_xfer_secondary_pool(r->name, bytes * 8, nbytes * 8,
2946 +- ENTROPY_BITS(r), ENTROPY_BITS(r->pull));
2947 +- bytes = extract_entropy(r->pull, tmp, bytes,
2948 +- random_read_wakeup_bits / 8, 0);
2949 +- mix_pool_bytes(r, tmp, bytes);
2950 +- credit_entropy_bits(r, bytes*8);
2951 +-}
2952 ++ * The following exported functions are used for pushing entropy into
2953 ++ * the above entropy accumulation routines:
2954 ++ *
2955 ++ * void add_device_randomness(const void *buf, size_t len);
2956 ++ * void add_hwgenerator_randomness(const void *buf, size_t len, size_t entropy);
2957 ++ * void add_bootloader_randomness(const void *buf, size_t len);
2958 ++ * void add_interrupt_randomness(int irq);
2959 ++ * void add_input_randomness(unsigned int type, unsigned int code, unsigned int value);
2960 ++ * void add_disk_randomness(struct gendisk *disk);
2961 ++ *
2962 ++ * add_device_randomness() adds data to the input pool that
2963 ++ * is likely to differ between two devices (or possibly even per boot).
2964 ++ * This would be things like MAC addresses or serial numbers, or the
2965 ++ * read-out of the RTC. This does *not* credit any actual entropy to
2966 ++ * the pool, but it initializes the pool to different values for devices
2967 ++ * that might otherwise be identical and have very little entropy
2968 ++ * available to them (particularly common in the embedded world).
2969 ++ *
2970 ++ * add_hwgenerator_randomness() is for true hardware RNGs, and will credit
2971 ++ * entropy as specified by the caller. If the entropy pool is full it will
2972 ++ * block until more entropy is needed.
2973 ++ *
2974 ++ * add_bootloader_randomness() is called by bootloader drivers, such as EFI
2975 ++ * and device tree, and credits its input depending on whether or not the
2976 ++ * configuration option CONFIG_RANDOM_TRUST_BOOTLOADER is set.
2977 ++ *
2978 ++ * add_interrupt_randomness() uses the interrupt timing as random
2979 ++ * inputs to the entropy pool. Using the cycle counters and the irq source
2980 ++ * as inputs, it feeds the input pool roughly once a second or after 64
2981 ++ * interrupts, crediting 1 bit of entropy for whichever comes first.
2982 ++ *
2983 ++ * add_input_randomness() uses the input layer interrupt timing, as well
2984 ++ * as the event type information from the hardware.
2985 ++ *
2986 ++ * add_disk_randomness() uses what amounts to the seek time of block
2987 ++ * layer request events, on a per-disk_devt basis, as input to the
2988 ++ * entropy pool. Note that high-speed solid state drives with very low
2989 ++ * seek times do not make for good sources of entropy, as their seek
2990 ++ * times are usually fairly consistent.
2991 ++ *
2992 ++ * The last two routines try to estimate how many bits of entropy
2993 ++ * to credit. They do this by keeping track of the first and second
2994 ++ * order deltas of the event timings.
2995 ++ *
2996 ++ **********************************************************************/
2997 +
2998 +-/*
2999 +- * Used as a workqueue function so that when the input pool is getting
3000 +- * full, we can "spill over" some entropy to the output pools. That
3001 +- * way the output pools can store some of the excess entropy instead
3002 +- * of letting it go to waste.
3003 +- */
3004 +-static void push_to_pool(struct work_struct *work)
3005 ++static bool trust_cpu __initdata = IS_ENABLED(CONFIG_RANDOM_TRUST_CPU);
3006 ++static bool trust_bootloader __initdata = IS_ENABLED(CONFIG_RANDOM_TRUST_BOOTLOADER);
3007 ++static int __init parse_trust_cpu(char *arg)
3008 + {
3009 +- struct entropy_store *r = container_of(work, struct entropy_store,
3010 +- push_work);
3011 +- BUG_ON(!r);
3012 +- _xfer_secondary_pool(r, random_read_wakeup_bits/8);
3013 +- trace_push_to_pool(r->name, r->entropy_count >> ENTROPY_SHIFT,
3014 +- r->pull->entropy_count >> ENTROPY_SHIFT);
3015 ++ return kstrtobool(arg, &trust_cpu);
3016 + }
3017 +-
3018 +-/*
3019 +- * This function decides how many bytes to actually take from the
3020 +- * given pool, and also debits the entropy count accordingly.
3021 +- */
3022 +-static size_t account(struct entropy_store *r, size_t nbytes, int min,
3023 +- int reserved)
3024 ++static int __init parse_trust_bootloader(char *arg)
3025 + {
3026 +- int entropy_count, orig, have_bytes;
3027 +- size_t ibytes, nfrac;
3028 +-
3029 +- BUG_ON(r->entropy_count > r->poolinfo->poolfracbits);
3030 +-
3031 +- /* Can we pull enough? */
3032 +-retry:
3033 +- entropy_count = orig = ACCESS_ONCE(r->entropy_count);
3034 +- ibytes = nbytes;
3035 +- /* never pull more than available */
3036 +- have_bytes = entropy_count >> (ENTROPY_SHIFT + 3);
3037 +-
3038 +- if ((have_bytes -= reserved) < 0)
3039 +- have_bytes = 0;
3040 +- ibytes = min_t(size_t, ibytes, have_bytes);
3041 +- if (ibytes < min)
3042 +- ibytes = 0;
3043 +-
3044 +- if (unlikely(entropy_count < 0)) {
3045 +- pr_warn("random: negative entropy count: pool %s count %d\n",
3046 +- r->name, entropy_count);
3047 +- WARN_ON(1);
3048 +- entropy_count = 0;
3049 +- }
3050 +- nfrac = ibytes << (ENTROPY_SHIFT + 3);
3051 +- if ((size_t) entropy_count > nfrac)
3052 +- entropy_count -= nfrac;
3053 +- else
3054 +- entropy_count = 0;
3055 +-
3056 +- if (cmpxchg(&r->entropy_count, orig, entropy_count) != orig)
3057 +- goto retry;
3058 +-
3059 +- trace_debit_entropy(r->name, 8 * ibytes);
3060 +- if (ibytes &&
3061 +- (r->entropy_count >> ENTROPY_SHIFT) < random_write_wakeup_bits) {
3062 +- wake_up_interruptible(&random_write_wait);
3063 +- kill_fasync(&fasync, SIGIO, POLL_OUT);
3064 +- }
3065 +-
3066 +- return ibytes;
3067 ++ return kstrtobool(arg, &trust_bootloader);
3068 + }
3069 ++early_param("random.trust_cpu", parse_trust_cpu);
3070 ++early_param("random.trust_bootloader", parse_trust_bootloader);
3071 +
3072 + /*
3073 +- * This function does the actual extraction for extract_entropy and
3074 +- * extract_entropy_user.
3075 +- *
3076 +- * Note: we assume that .poolwords is a multiple of 16 words.
3077 ++ * The first collection of entropy occurs at system boot while interrupts
3078 ++ * are still turned off. Here we push in latent entropy, RDSEED, a timestamp,
3079 ++ * utsname(), and the command line. Depending on the above configuration knob,
3080 ++ * RDSEED may be considered sufficient for initialization. Note that much
3081 ++ * earlier setup may already have pushed entropy into the input pool by the
3082 ++ * time we get here.
3083 + */
3084 +-static void extract_buf(struct entropy_store *r, __u8 *out)
3085 ++int __init random_init(const char *command_line)
3086 + {
3087 +- int i;
3088 +- union {
3089 +- __u32 w[5];
3090 +- unsigned long l[LONGS(20)];
3091 +- } hash;
3092 +- __u32 workspace[SHA_WORKSPACE_WORDS];
3093 +- unsigned long flags;
3094 +-
3095 +- /*
3096 +- * If we have an architectural hardware random number
3097 +- * generator, use it for SHA's initial vector
3098 +- */
3099 +- sha_init(hash.w);
3100 +- for (i = 0; i < LONGS(20); i++) {
3101 +- unsigned long v;
3102 +- if (!arch_get_random_long(&v))
3103 +- break;
3104 +- hash.l[i] = v;
3105 +- }
3106 +-
3107 +- /* Generate a hash across the pool, 16 words (512 bits) at a time */
3108 +- spin_lock_irqsave(&r->lock, flags);
3109 +- for (i = 0; i < r->poolinfo->poolwords; i += 16)
3110 +- sha_transform(hash.w, (__u8 *)(r->pool + i), workspace);
3111 +-
3112 +- /*
3113 +- * We mix the hash back into the pool to prevent backtracking
3114 +- * attacks (where the attacker knows the state of the pool
3115 +- * plus the current outputs, and attempts to find previous
3116 +- * ouputs), unless the hash function can be inverted. By
3117 +- * mixing at least a SHA1 worth of hash data back, we make
3118 +- * brute-forcing the feedback as hard as brute-forcing the
3119 +- * hash.
3120 +- */
3121 +- __mix_pool_bytes(r, hash.w, sizeof(hash.w));
3122 +- spin_unlock_irqrestore(&r->lock, flags);
3123 +-
3124 +- memzero_explicit(workspace, sizeof(workspace));
3125 +-
3126 +- /*
3127 +- * In case the hash function has some recognizable output
3128 +- * pattern, we fold it in half. Thus, we always feed back
3129 +- * twice as much data as we output.
3130 +- */
3131 +- hash.w[0] ^= hash.w[3];
3132 +- hash.w[1] ^= hash.w[4];
3133 +- hash.w[2] ^= rol32(hash.w[2], 16);
3134 +-
3135 +- memcpy(out, &hash, EXTRACT_SIZE);
3136 +- memzero_explicit(&hash, sizeof(hash));
3137 +-}
3138 +-
3139 +-static ssize_t _extract_entropy(struct entropy_store *r, void *buf,
3140 +- size_t nbytes, int fips)
3141 +-{
3142 +- ssize_t ret = 0, i;
3143 +- __u8 tmp[EXTRACT_SIZE];
3144 +- unsigned long flags;
3145 ++ ktime_t now = ktime_get_real();
3146 ++ unsigned int i, arch_bits;
3147 ++ unsigned long entropy;
3148 +
3149 +- while (nbytes) {
3150 +- extract_buf(r, tmp);
3151 ++#if defined(LATENT_ENTROPY_PLUGIN)
3152 ++ static const u8 compiletime_seed[BLAKE2S_BLOCK_SIZE] __initconst __latent_entropy;
3153 ++ _mix_pool_bytes(compiletime_seed, sizeof(compiletime_seed));
3154 ++#endif
3155 +
3156 +- if (fips) {
3157 +- spin_lock_irqsave(&r->lock, flags);
3158 +- if (!memcmp(tmp, r->last_data, EXTRACT_SIZE))
3159 +- panic("Hardware RNG duplicated output!\n");
3160 +- memcpy(r->last_data, tmp, EXTRACT_SIZE);
3161 +- spin_unlock_irqrestore(&r->lock, flags);
3162 ++ for (i = 0, arch_bits = BLAKE2S_BLOCK_SIZE * 8;
3163 ++ i < BLAKE2S_BLOCK_SIZE; i += sizeof(entropy)) {
3164 ++ if (!arch_get_random_seed_long_early(&entropy) &&
3165 ++ !arch_get_random_long_early(&entropy)) {
3166 ++ entropy = random_get_entropy();
3167 ++ arch_bits -= sizeof(entropy) * 8;
3168 + }
3169 +- i = min_t(int, nbytes, EXTRACT_SIZE);
3170 +- memcpy(buf, tmp, i);
3171 +- nbytes -= i;
3172 +- buf += i;
3173 +- ret += i;
3174 ++ _mix_pool_bytes(&entropy, sizeof(entropy));
3175 + }
3176 ++ _mix_pool_bytes(&now, sizeof(now));
3177 ++ _mix_pool_bytes(utsname(), sizeof(*(utsname())));
3178 ++ _mix_pool_bytes(command_line, strlen(command_line));
3179 ++ add_latent_entropy();
3180 +
3181 +- /* Wipe data just returned from memory */
3182 +- memzero_explicit(tmp, sizeof(tmp));
3183 ++ if (crng_ready())
3184 ++ crng_reseed();
3185 ++ else if (trust_cpu)
3186 ++ _credit_init_bits(arch_bits);
3187 +
3188 +- return ret;
3189 ++ return 0;
3190 + }
3191 +
3192 + /*
3193 +- * This function extracts randomness from the "entropy pool", and
3194 +- * returns it in a buffer.
3195 ++ * Add device- or boot-specific data to the input pool to help
3196 ++ * initialize it.
3197 + *
3198 +- * The min parameter specifies the minimum amount we can pull before
3199 +- * failing to avoid races that defeat catastrophic reseeding while the
3200 +- * reserved parameter indicates how much entropy we must leave in the
3201 +- * pool after each pull to avoid starving other readers.
3202 ++ * None of this adds any entropy; it is meant to avoid the problem of
3203 ++ * the entropy pool having similar initial state across largely
3204 ++ * identical devices.
3205 + */
3206 +-static ssize_t extract_entropy(struct entropy_store *r, void *buf,
3207 +- size_t nbytes, int min, int reserved)
3208 ++void add_device_randomness(const void *buf, size_t len)
3209 + {
3210 +- __u8 tmp[EXTRACT_SIZE];
3211 ++ unsigned long entropy = random_get_entropy();
3212 + unsigned long flags;
3213 +
3214 +- /* if last_data isn't primed, we need EXTRACT_SIZE extra bytes */
3215 +- if (fips_enabled) {
3216 +- spin_lock_irqsave(&r->lock, flags);
3217 +- if (!r->last_data_init) {
3218 +- r->last_data_init = 1;
3219 +- spin_unlock_irqrestore(&r->lock, flags);
3220 +- trace_extract_entropy(r->name, EXTRACT_SIZE,
3221 +- ENTROPY_BITS(r), _RET_IP_);
3222 +- xfer_secondary_pool(r, EXTRACT_SIZE);
3223 +- extract_buf(r, tmp);
3224 +- spin_lock_irqsave(&r->lock, flags);
3225 +- memcpy(r->last_data, tmp, EXTRACT_SIZE);
3226 +- }
3227 +- spin_unlock_irqrestore(&r->lock, flags);
3228 +- }
3229 +-
3230 +- trace_extract_entropy(r->name, nbytes, ENTROPY_BITS(r), _RET_IP_);
3231 +- xfer_secondary_pool(r, nbytes);
3232 +- nbytes = account(r, nbytes, min, reserved);
3233 +-
3234 +- return _extract_entropy(r, buf, nbytes, fips_enabled);
3235 +-}
3236 +-
3237 +-/*
3238 +- * This function extracts randomness from the "entropy pool", and
3239 +- * returns it in a userspace buffer.
3240 +- */
3241 +-static ssize_t extract_entropy_user(struct entropy_store *r, void __user *buf,
3242 +- size_t nbytes)
3243 +-{
3244 +- ssize_t ret = 0, i;
3245 +- __u8 tmp[EXTRACT_SIZE];
3246 +- int large_request = (nbytes > 256);
3247 +-
3248 +- trace_extract_entropy_user(r->name, nbytes, ENTROPY_BITS(r), _RET_IP_);
3249 +- xfer_secondary_pool(r, nbytes);
3250 +- nbytes = account(r, nbytes, 0, 0);
3251 +-
3252 +- while (nbytes) {
3253 +- if (large_request && need_resched()) {
3254 +- if (signal_pending(current)) {
3255 +- if (ret == 0)
3256 +- ret = -ERESTARTSYS;
3257 +- break;
3258 +- }
3259 +- schedule();
3260 +- }
3261 +-
3262 +- extract_buf(r, tmp);
3263 +- i = min_t(int, nbytes, EXTRACT_SIZE);
3264 +- if (copy_to_user(buf, tmp, i)) {
3265 +- ret = -EFAULT;
3266 +- break;
3267 +- }
3268 +-
3269 +- nbytes -= i;
3270 +- buf += i;
3271 +- ret += i;
3272 +- }
3273 +-
3274 +- /* Wipe data just returned from memory */
3275 +- memzero_explicit(tmp, sizeof(tmp));
3276 +-
3277 +- return ret;
3278 +-}
3279 +-
3280 +-#define warn_unseeded_randomness(previous) \
3281 +- _warn_unseeded_randomness(__func__, (void *) _RET_IP_, (previous))
3282 +-
3283 +-static void _warn_unseeded_randomness(const char *func_name, void *caller,
3284 +- void **previous)
3285 +-{
3286 +-#ifdef CONFIG_WARN_ALL_UNSEEDED_RANDOM
3287 +- const bool print_once = false;
3288 +-#else
3289 +- static bool print_once __read_mostly;
3290 +-#endif
3291 +-
3292 +- if (print_once ||
3293 +- crng_ready() ||
3294 +- (previous && (caller == READ_ONCE(*previous))))
3295 +- return;
3296 +- WRITE_ONCE(*previous, caller);
3297 +-#ifndef CONFIG_WARN_ALL_UNSEEDED_RANDOM
3298 +- print_once = true;
3299 +-#endif
3300 +- if (__ratelimit(&unseeded_warning))
3301 +- pr_notice("random: %s called from %pS with crng_init=%d\n",
3302 +- func_name, caller, crng_init);
3303 ++ spin_lock_irqsave(&input_pool.lock, flags);
3304 ++ _mix_pool_bytes(&entropy, sizeof(entropy));
3305 ++ _mix_pool_bytes(buf, len);
3306 ++ spin_unlock_irqrestore(&input_pool.lock, flags);
3307 + }
3308 ++EXPORT_SYMBOL(add_device_randomness);
3309 +
3310 + /*
3311 +- * This function is the exported kernel interface. It returns some
3312 +- * number of good random numbers, suitable for key generation, seeding
3313 +- * TCP sequence numbers, etc. It does not rely on the hardware random
3314 +- * number generator. For random bytes direct from the hardware RNG
3315 +- * (when available), use get_random_bytes_arch(). In order to ensure
3316 +- * that the randomness provided by this function is okay, the function
3317 +- * wait_for_random_bytes() should be called and return 0 at least once
3318 +- * at any point prior.
3319 ++ * Interface for in-kernel drivers of true hardware RNGs.
3320 ++ * Those devices may produce endless random bits and will be throttled
3321 ++ * when our pool is full.
3322 + */
3323 +-static void _get_random_bytes(void *buf, int nbytes)
3324 +-{
3325 +- __u8 tmp[CHACHA20_BLOCK_SIZE];
3326 +-
3327 +- trace_get_random_bytes(nbytes, _RET_IP_);
3328 +-
3329 +- while (nbytes >= CHACHA20_BLOCK_SIZE) {
3330 +- extract_crng(buf);
3331 +- buf += CHACHA20_BLOCK_SIZE;
3332 +- nbytes -= CHACHA20_BLOCK_SIZE;
3333 +- }
3334 +-
3335 +- if (nbytes > 0) {
3336 +- extract_crng(tmp);
3337 +- memcpy(buf, tmp, nbytes);
3338 +- crng_backtrack_protect(tmp, nbytes);
3339 +- } else
3340 +- crng_backtrack_protect(tmp, CHACHA20_BLOCK_SIZE);
3341 +- memzero_explicit(tmp, sizeof(tmp));
3342 +-}
3343 +-
3344 +-void get_random_bytes(void *buf, int nbytes)
3345 ++void add_hwgenerator_randomness(const void *buf, size_t len, size_t entropy)
3346 + {
3347 +- static void *previous;
3348 ++ mix_pool_bytes(buf, len);
3349 ++ credit_init_bits(entropy);
3350 +
3351 +- warn_unseeded_randomness(&previous);
3352 +- _get_random_bytes(buf, nbytes);
3353 ++ /*
3354 ++ * Throttle writing to once every CRNG_RESEED_INTERVAL, unless
3355 ++ * we're not yet initialized.
3356 ++ */
3357 ++ if (!kthread_should_stop() && crng_ready())
3358 ++ schedule_timeout_interruptible(CRNG_RESEED_INTERVAL);
3359 + }
3360 +-EXPORT_SYMBOL(get_random_bytes);
3361 ++EXPORT_SYMBOL_GPL(add_hwgenerator_randomness);
3362 +
3363 + /*
3364 +- * Wait for the urandom pool to be seeded and thus guaranteed to supply
3365 +- * cryptographically secure random numbers. This applies to: the /dev/urandom
3366 +- * device, the get_random_bytes function, and the get_random_{u32,u64,int,long}
3367 +- * family of functions. Using any of these functions without first calling
3368 +- * this function forfeits the guarantee of security.
3369 +- *
3370 +- * Returns: 0 if the urandom pool has been seeded.
3371 +- * -ERESTARTSYS if the function was interrupted by a signal.
3372 ++ * Handle random seed passed by bootloader, and credit it if
3373 ++ * CONFIG_RANDOM_TRUST_BOOTLOADER is set.
3374 + */
3375 +-int wait_for_random_bytes(void)
3376 ++void __init add_bootloader_randomness(const void *buf, size_t len)
3377 + {
3378 +- if (likely(crng_ready()))
3379 +- return 0;
3380 +- return wait_event_interruptible(crng_init_wait, crng_ready());
3381 ++ mix_pool_bytes(buf, len);
3382 ++ if (trust_bootloader)
3383 ++ credit_init_bits(len * 8);
3384 + }
3385 +-EXPORT_SYMBOL(wait_for_random_bytes);
3386 +
3387 +-/*
3388 +- * Add a callback function that will be invoked when the nonblocking
3389 +- * pool is initialised.
3390 +- *
3391 +- * returns: 0 if callback is successfully added
3392 +- * -EALREADY if pool is already initialised (callback not called)
3393 +- * -ENOENT if module for callback is not alive
3394 +- */
3395 +-int add_random_ready_callback(struct random_ready_callback *rdy)
3396 +-{
3397 +- struct module *owner;
3398 +- unsigned long flags;
3399 +- int err = -EALREADY;
3400 +-
3401 +- if (crng_ready())
3402 +- return err;
3403 +-
3404 +- owner = rdy->owner;
3405 +- if (!try_module_get(owner))
3406 +- return -ENOENT;
3407 +-
3408 +- spin_lock_irqsave(&random_ready_list_lock, flags);
3409 +- if (crng_ready())
3410 +- goto out;
3411 +-
3412 +- owner = NULL;
3413 +-
3414 +- list_add(&rdy->list, &random_ready_list);
3415 +- err = 0;
3416 +-
3417 +-out:
3418 +- spin_unlock_irqrestore(&random_ready_list_lock, flags);
3419 ++struct fast_pool {
3420 ++ struct work_struct mix;
3421 ++ unsigned long pool[4];
3422 ++ unsigned long last;
3423 ++ unsigned int count;
3424 ++};
3425 +
3426 +- module_put(owner);
3427 ++static DEFINE_PER_CPU(struct fast_pool, irq_randomness) = {
3428 ++#ifdef CONFIG_64BIT
3429 ++#define FASTMIX_PERM SIPHASH_PERMUTATION
3430 ++ .pool = { SIPHASH_CONST_0, SIPHASH_CONST_1, SIPHASH_CONST_2, SIPHASH_CONST_3 }
3431 ++#else
3432 ++#define FASTMIX_PERM HSIPHASH_PERMUTATION
3433 ++ .pool = { HSIPHASH_CONST_0, HSIPHASH_CONST_1, HSIPHASH_CONST_2, HSIPHASH_CONST_3 }
3434 ++#endif
3435 ++};
3436 +
3437 +- return err;
3438 ++/*
3439 ++ * This is [Half]SipHash-1-x, starting from an empty key. Because
3440 ++ * the key is fixed, it assumes that its inputs are non-malicious,
3441 ++ * and therefore this has no security on its own. s represents the
3442 ++ * four-word SipHash state, while v represents a two-word input.
3443 ++ */
3444 ++static void fast_mix(unsigned long s[4], unsigned long v1, unsigned long v2)
3445 ++{
3446 ++ s[3] ^= v1;
3447 ++ FASTMIX_PERM(s[0], s[1], s[2], s[3]);
3448 ++ s[0] ^= v1;
3449 ++ s[3] ^= v2;
3450 ++ FASTMIX_PERM(s[0], s[1], s[2], s[3]);
3451 ++ s[0] ^= v2;
3452 + }
3453 +-EXPORT_SYMBOL(add_random_ready_callback);
3454 +
3455 ++#ifdef CONFIG_SMP
3456 + /*
3457 +- * Delete a previously registered readiness callback function.
3458 ++ * This function is called when the CPU has just come online, with
3459 ++ * entry CPUHP_AP_RANDOM_ONLINE, just after CPUHP_AP_WORKQUEUE_ONLINE.
3460 + */
3461 +-void del_random_ready_callback(struct random_ready_callback *rdy)
3462 ++int __cold random_online_cpu(unsigned int cpu)
3463 + {
3464 +- unsigned long flags;
3465 +- struct module *owner = NULL;
3466 ++ /*
3467 ++ * During CPU shutdown and before CPU onlining, add_interrupt_
3468 ++ * randomness() may schedule mix_interrupt_randomness(), and
3469 ++ * set the MIX_INFLIGHT flag. However, because the worker can
3470 ++ * be scheduled on a different CPU during this period, that
3471 ++ * flag will never be cleared. For that reason, we zero out
3472 ++ * the flag here, which runs just after workqueues are onlined
3473 ++ * for the CPU again. This also has the effect of setting the
3474 ++ * irq randomness count to zero so that new accumulated irqs
3475 ++ * are fresh.
3476 ++ */
3477 ++ per_cpu_ptr(&irq_randomness, cpu)->count = 0;
3478 ++ return 0;
3479 ++}
3480 ++#endif
3481 ++
3482 ++static void mix_interrupt_randomness(struct work_struct *work)
3483 ++{
3484 ++ struct fast_pool *fast_pool = container_of(work, struct fast_pool, mix);
3485 ++ /*
3486 ++ * The size of the copied stack pool is explicitly 2 longs so that we
3487 ++ * only ever ingest half of the siphash output each time, retaining
3488 ++ * the other half as the next "key" that carries over. The entropy is
3489 ++ * supposed to be sufficiently dispersed between bits so on average
3490 ++ * we don't wind up "losing" some.
3491 ++ */
3492 ++ unsigned long pool[2];
3493 ++ unsigned int count;
3494 +
3495 +- spin_lock_irqsave(&random_ready_list_lock, flags);
3496 +- if (!list_empty(&rdy->list)) {
3497 +- list_del_init(&rdy->list);
3498 +- owner = rdy->owner;
3499 ++ /* Check to see if we're running on the wrong CPU due to hotplug. */
3500 ++ local_irq_disable();
3501 ++ if (fast_pool != this_cpu_ptr(&irq_randomness)) {
3502 ++ local_irq_enable();
3503 ++ return;
3504 + }
3505 +- spin_unlock_irqrestore(&random_ready_list_lock, flags);
3506 +
3507 +- module_put(owner);
3508 ++ /*
3509 ++ * Copy the pool to the stack so that the mixer always has a
3510 ++ * consistent view, before we reenable irqs again.
3511 ++ */
3512 ++ memcpy(pool, fast_pool->pool, sizeof(pool));
3513 ++ count = fast_pool->count;
3514 ++ fast_pool->count = 0;
3515 ++ fast_pool->last = jiffies;
3516 ++ local_irq_enable();
3517 ++
3518 ++ mix_pool_bytes(pool, sizeof(pool));
3519 ++ credit_init_bits(max(1u, (count & U16_MAX) / 64));
3520 ++
3521 ++ memzero_explicit(pool, sizeof(pool));
3522 + }
3523 +-EXPORT_SYMBOL(del_random_ready_callback);
3524 +
3525 +-/*
3526 +- * This function will use the architecture-specific hardware random
3527 +- * number generator if it is available. The arch-specific hw RNG will
3528 +- * almost certainly be faster than what we can do in software, but it
3529 +- * is impossible to verify that it is implemented securely (as
3530 +- * opposed, to, say, the AES encryption of a sequence number using a
3531 +- * key known by the NSA). So it's useful if we need the speed, but
3532 +- * only if we're willing to trust the hardware manufacturer not to
3533 +- * have put in a back door.
3534 +- */
3535 +-void get_random_bytes_arch(void *buf, int nbytes)
3536 ++void add_interrupt_randomness(int irq)
3537 + {
3538 +- char *p = buf;
3539 ++ enum { MIX_INFLIGHT = 1U << 31 };
3540 ++ unsigned long entropy = random_get_entropy();
3541 ++ struct fast_pool *fast_pool = this_cpu_ptr(&irq_randomness);
3542 ++ struct pt_regs *regs = get_irq_regs();
3543 ++ unsigned int new_count;
3544 +
3545 +- trace_get_random_bytes_arch(nbytes, _RET_IP_);
3546 +- while (nbytes) {
3547 +- unsigned long v;
3548 +- int chunk = min(nbytes, (int)sizeof(unsigned long));
3549 ++ fast_mix(fast_pool->pool, entropy,
3550 ++ (regs ? instruction_pointer(regs) : _RET_IP_) ^ swab(irq));
3551 ++ new_count = ++fast_pool->count;
3552 +
3553 +- if (!arch_get_random_long(&v))
3554 +- break;
3555 +-
3556 +- memcpy(p, &v, chunk);
3557 +- p += chunk;
3558 +- nbytes -= chunk;
3559 +- }
3560 ++ if (new_count & MIX_INFLIGHT)
3561 ++ return;
3562 ++
3563 ++ if (new_count < 64 && !time_is_before_jiffies(fast_pool->last + HZ))
3564 ++ return;
3565 +
3566 +- if (nbytes)
3567 +- get_random_bytes(p, nbytes);
3568 ++ if (unlikely(!fast_pool->mix.func))
3569 ++ INIT_WORK(&fast_pool->mix, mix_interrupt_randomness);
3570 ++ fast_pool->count |= MIX_INFLIGHT;
3571 ++ queue_work_on(raw_smp_processor_id(), system_highpri_wq, &fast_pool->mix);
3572 + }
3573 +-EXPORT_SYMBOL(get_random_bytes_arch);
3574 ++EXPORT_SYMBOL_GPL(add_interrupt_randomness);
3575 +
3576 ++/* There is one of these per entropy source */
3577 ++struct timer_rand_state {
3578 ++ unsigned long last_time;
3579 ++ long last_delta, last_delta2;
3580 ++};
3581 +
3582 + /*
3583 +- * init_std_data - initialize pool with system data
3584 +- *
3585 +- * @r: pool to initialize
3586 +- *
3587 +- * This function clears the pool's entropy count and mixes some system
3588 +- * data into the pool to prepare it for use. The pool is not cleared
3589 +- * as that can only decrease the entropy in the pool.
3590 ++ * This function adds entropy to the entropy "pool" by using timing
3591 ++ * delays. It uses the timer_rand_state structure to make an estimate
3592 ++ * of how many bits of entropy this call has added to the pool. The
3593 ++ * value "num" is also added to the pool; it should somehow describe
3594 ++ * the type of event that just happened.
3595 + */
3596 +-static void init_std_data(struct entropy_store *r)
3597 ++static void add_timer_randomness(struct timer_rand_state *state, unsigned int num)
3598 + {
3599 +- int i;
3600 +- ktime_t now = ktime_get_real();
3601 +- unsigned long rv;
3602 +-
3603 +- r->last_pulled = jiffies;
3604 +- mix_pool_bytes(r, &now, sizeof(now));
3605 +- for (i = r->poolinfo->poolbytes; i > 0; i -= sizeof(rv)) {
3606 +- if (!arch_get_random_seed_long(&rv) &&
3607 +- !arch_get_random_long(&rv))
3608 +- rv = random_get_entropy();
3609 +- mix_pool_bytes(r, &rv, sizeof(rv));
3610 ++ unsigned long entropy = random_get_entropy(), now = jiffies, flags;
3611 ++ long delta, delta2, delta3;
3612 ++ unsigned int bits;
3613 ++
3614 ++ /*
3615 ++ * If we're in a hard IRQ, add_interrupt_randomness() will be called
3616 ++ * sometime after, so mix into the fast pool.
3617 ++ */
3618 ++ if (in_irq()) {
3619 ++ fast_mix(this_cpu_ptr(&irq_randomness)->pool, entropy, num);
3620 ++ } else {
3621 ++ spin_lock_irqsave(&input_pool.lock, flags);
3622 ++ _mix_pool_bytes(&entropy, sizeof(entropy));
3623 ++ _mix_pool_bytes(&num, sizeof(num));
3624 ++ spin_unlock_irqrestore(&input_pool.lock, flags);
3625 + }
3626 +- mix_pool_bytes(r, utsname(), sizeof(*(utsname())));
3627 ++
3628 ++ if (crng_ready())
3629 ++ return;
3630 ++
3631 ++ /*
3632 ++ * Calculate number of bits of randomness we probably added.
3633 ++ * We take into account the first, second and third-order deltas
3634 ++ * in order to make our estimate.
3635 ++ */
3636 ++ delta = now - READ_ONCE(state->last_time);
3637 ++ WRITE_ONCE(state->last_time, now);
3638 ++
3639 ++ delta2 = delta - READ_ONCE(state->last_delta);
3640 ++ WRITE_ONCE(state->last_delta, delta);
3641 ++
3642 ++ delta3 = delta2 - READ_ONCE(state->last_delta2);
3643 ++ WRITE_ONCE(state->last_delta2, delta2);
3644 ++
3645 ++ if (delta < 0)
3646 ++ delta = -delta;
3647 ++ if (delta2 < 0)
3648 ++ delta2 = -delta2;
3649 ++ if (delta3 < 0)
3650 ++ delta3 = -delta3;
3651 ++ if (delta > delta2)
3652 ++ delta = delta2;
3653 ++ if (delta > delta3)
3654 ++ delta = delta3;
3655 ++
3656 ++ /*
3657 ++ * delta is now minimum absolute delta. Round down by 1 bit
3658 ++ * on general principles, and limit entropy estimate to 11 bits.
3659 ++ */
3660 ++ bits = min(fls(delta >> 1), 11);
3661 ++
3662 ++ /*
3663 ++ * As mentioned above, if we're in a hard IRQ, add_interrupt_randomness()
3664 ++ * will run after this, which uses a different crediting scheme of 1 bit
3665 ++ * per every 64 interrupts. In order to let that function do accounting
3666 ++ * close to the one in this function, we credit a full 64/64 bit per bit,
3667 ++ * and then subtract one to account for the extra one added.
3668 ++ */
3669 ++ if (in_irq())
3670 ++ this_cpu_ptr(&irq_randomness)->count += max(1u, bits * 64) - 1;
3671 ++ else
3672 ++ _credit_init_bits(bits);
3673 + }
3674 +
3675 +-/*
3676 +- * Note that setup_arch() may call add_device_randomness()
3677 +- * long before we get here. This allows seeding of the pools
3678 +- * with some platform dependent data very early in the boot
3679 +- * process. But it limits our options here. We must use
3680 +- * statically allocated structures that already have all
3681 +- * initializations complete at compile time. We should also
3682 +- * take care not to overwrite the precious per platform data
3683 +- * we were given.
3684 +- */
3685 +-static int rand_initialize(void)
3686 ++void add_input_randomness(unsigned int type, unsigned int code, unsigned int value)
3687 + {
3688 +- init_std_data(&input_pool);
3689 +- init_std_data(&blocking_pool);
3690 +- crng_initialize(&primary_crng);
3691 +- crng_global_init_time = jiffies;
3692 +- if (ratelimit_disable) {
3693 +- urandom_warning.interval = 0;
3694 +- unseeded_warning.interval = 0;
3695 +- }
3696 +- return 0;
3697 ++ static unsigned char last_value;
3698 ++ static struct timer_rand_state input_timer_state = { INITIAL_JIFFIES };
3699 ++
3700 ++ /* Ignore autorepeat and the like. */
3701 ++ if (value == last_value)
3702 ++ return;
3703 ++
3704 ++ last_value = value;
3705 ++ add_timer_randomness(&input_timer_state,
3706 ++ (type << 4) ^ code ^ (code >> 4) ^ value);
3707 + }
3708 +-early_initcall(rand_initialize);
3709 ++EXPORT_SYMBOL_GPL(add_input_randomness);
3710 +
3711 + #ifdef CONFIG_BLOCK
3712 +-void rand_initialize_disk(struct gendisk *disk)
3713 ++void add_disk_randomness(struct gendisk *disk)
3714 ++{
3715 ++ if (!disk || !disk->random)
3716 ++ return;
3717 ++ /* First major is 1, so we get >= 0x200 here. */
3718 ++ add_timer_randomness(disk->random, 0x100 + disk_devt(disk));
3719 ++}
3720 ++EXPORT_SYMBOL_GPL(add_disk_randomness);
3721 ++
3722 ++void __cold rand_initialize_disk(struct gendisk *disk)
3723 + {
3724 + struct timer_rand_state *state;
3725 +
3726 +@@ -1823,134 +1126,189 @@ void rand_initialize_disk(struct gendisk *disk)
3727 + }
3728 + #endif
3729 +
3730 +-static ssize_t
3731 +-_random_read(int nonblock, char __user *buf, size_t nbytes)
3732 ++/*
3733 ++ * Each time the timer fires, we expect that we got an unpredictable
3734 ++ * jump in the cycle counter. Even if the timer is running on another
3735 ++ * CPU, the timer activity will be touching the stack of the CPU that is
3736 ++ * generating entropy..
3737 ++ *
3738 ++ * Note that we don't re-arm the timer in the timer itself - we are
3739 ++ * happy to be scheduled away, since that just makes the load more
3740 ++ * complex, but we do not want the timer to keep ticking unless the
3741 ++ * entropy loop is running.
3742 ++ *
3743 ++ * So the re-arming always happens in the entropy loop itself.
3744 ++ */
3745 ++static void __cold entropy_timer(unsigned long data)
3746 + {
3747 +- ssize_t n;
3748 ++ credit_init_bits(1);
3749 ++}
3750 +
3751 +- if (nbytes == 0)
3752 +- return 0;
3753 ++/*
3754 ++ * If we have an actual cycle counter, see if we can
3755 ++ * generate enough entropy with timing noise
3756 ++ */
3757 ++static void __cold try_to_generate_entropy(void)
3758 ++{
3759 ++ struct {
3760 ++ unsigned long entropy;
3761 ++ struct timer_list timer;
3762 ++ } stack;
3763 +
3764 +- nbytes = min_t(size_t, nbytes, SEC_XFER_SIZE);
3765 +- while (1) {
3766 +- n = extract_entropy_user(&blocking_pool, buf, nbytes);
3767 +- if (n < 0)
3768 +- return n;
3769 +- trace_random_read(n*8, (nbytes-n)*8,
3770 +- ENTROPY_BITS(&blocking_pool),
3771 +- ENTROPY_BITS(&input_pool));
3772 +- if (n > 0)
3773 +- return n;
3774 +-
3775 +- /* Pool is (near) empty. Maybe wait and retry. */
3776 +- if (nonblock)
3777 +- return -EAGAIN;
3778 ++ stack.entropy = random_get_entropy();
3779 +
3780 +- wait_event_interruptible(random_read_wait,
3781 +- ENTROPY_BITS(&input_pool) >=
3782 +- random_read_wakeup_bits);
3783 +- if (signal_pending(current))
3784 +- return -ERESTARTSYS;
3785 ++ /* Slow counter - or none. Don't even bother */
3786 ++ if (stack.entropy == random_get_entropy())
3787 ++ return;
3788 ++
3789 ++ __setup_timer_on_stack(&stack.timer, entropy_timer, 0, 0);
3790 ++ while (!crng_ready() && !signal_pending(current)) {
3791 ++ if (!timer_pending(&stack.timer))
3792 ++ mod_timer(&stack.timer, jiffies + 1);
3793 ++ mix_pool_bytes(&stack.entropy, sizeof(stack.entropy));
3794 ++ schedule();
3795 ++ stack.entropy = random_get_entropy();
3796 + }
3797 +-}
3798 +
3799 +-static ssize_t
3800 +-random_read(struct file *file, char __user *buf, size_t nbytes, loff_t *ppos)
3801 +-{
3802 +- return _random_read(file->f_flags & O_NONBLOCK, buf, nbytes);
3803 ++ del_timer_sync(&stack.timer);
3804 ++ destroy_timer_on_stack(&stack.timer);
3805 ++ mix_pool_bytes(&stack.entropy, sizeof(stack.entropy));
3806 + }
3807 +
3808 +-static ssize_t
3809 +-urandom_read(struct file *file, char __user *buf, size_t nbytes, loff_t *ppos)
3810 ++
3811 ++/**********************************************************************
3812 ++ *
3813 ++ * Userspace reader/writer interfaces.
3814 ++ *
3815 ++ * getrandom(2) is the primary modern interface into the RNG and should
3816 ++ * be used in preference to anything else.
3817 ++ *
3818 ++ * Reading from /dev/random has the same functionality as calling
3819 ++ * getrandom(2) with flags=0. In earlier versions, however, it had
3820 ++ * vastly different semantics and should therefore be avoided, to
3821 ++ * prevent backwards compatibility issues.
3822 ++ *
3823 ++ * Reading from /dev/urandom has the same functionality as calling
3824 ++ * getrandom(2) with flags=GRND_INSECURE. Because it does not block
3825 ++ * waiting for the RNG to be ready, it should not be used.
3826 ++ *
3827 ++ * Writing to either /dev/random or /dev/urandom adds entropy to
3828 ++ * the input pool but does not credit it.
3829 ++ *
3830 ++ * Polling on /dev/random indicates when the RNG is initialized, on
3831 ++ * the read side, and when it wants new entropy, on the write side.
3832 ++ *
3833 ++ * Both /dev/random and /dev/urandom have the same set of ioctls for
3834 ++ * adding entropy, getting the entropy count, zeroing the count, and
3835 ++ * reseeding the crng.
3836 ++ *
3837 ++ **********************************************************************/
3838 ++
3839 ++SYSCALL_DEFINE3(getrandom, char __user *, ubuf, size_t, len, unsigned int, flags)
3840 + {
3841 +- unsigned long flags;
3842 +- static int maxwarn = 10;
3843 ++ struct iov_iter iter;
3844 ++ struct iovec iov;
3845 + int ret;
3846 +
3847 +- if (!crng_ready() && maxwarn > 0) {
3848 +- maxwarn--;
3849 +- if (__ratelimit(&urandom_warning))
3850 +- printk(KERN_NOTICE "random: %s: uninitialized "
3851 +- "urandom read (%zd bytes read)\n",
3852 +- current->comm, nbytes);
3853 +- spin_lock_irqsave(&primary_crng.lock, flags);
3854 +- crng_init_cnt = 0;
3855 +- spin_unlock_irqrestore(&primary_crng.lock, flags);
3856 ++ if (flags & ~(GRND_NONBLOCK | GRND_RANDOM | GRND_INSECURE))
3857 ++ return -EINVAL;
3858 ++
3859 ++ /*
3860 ++ * Requesting insecure and blocking randomness at the same time makes
3861 ++ * no sense.
3862 ++ */
3863 ++ if ((flags & (GRND_INSECURE | GRND_RANDOM)) == (GRND_INSECURE | GRND_RANDOM))
3864 ++ return -EINVAL;
3865 ++
3866 ++ if (!crng_ready() && !(flags & GRND_INSECURE)) {
3867 ++ if (flags & GRND_NONBLOCK)
3868 ++ return -EAGAIN;
3869 ++ ret = wait_for_random_bytes();
3870 ++ if (unlikely(ret))
3871 ++ return ret;
3872 + }
3873 +- nbytes = min_t(size_t, nbytes, INT_MAX >> (ENTROPY_SHIFT + 3));
3874 +- ret = extract_crng_user(buf, nbytes);
3875 +- trace_urandom_read(8 * nbytes, 0, ENTROPY_BITS(&input_pool));
3876 +- return ret;
3877 ++
3878 ++ ret = import_single_range(READ, ubuf, len, &iov, &iter);
3879 ++ if (unlikely(ret))
3880 ++ return ret;
3881 ++ return get_random_bytes_user(&iter);
3882 + }
3883 +
3884 +-static unsigned int
3885 +-random_poll(struct file *file, poll_table * wait)
3886 ++static unsigned int random_poll(struct file *file, poll_table *wait)
3887 + {
3888 +- unsigned int mask;
3889 +-
3890 +- poll_wait(file, &random_read_wait, wait);
3891 +- poll_wait(file, &random_write_wait, wait);
3892 +- mask = 0;
3893 +- if (ENTROPY_BITS(&input_pool) >= random_read_wakeup_bits)
3894 +- mask |= POLLIN | POLLRDNORM;
3895 +- if (ENTROPY_BITS(&input_pool) < random_write_wakeup_bits)
3896 +- mask |= POLLOUT | POLLWRNORM;
3897 +- return mask;
3898 ++ poll_wait(file, &crng_init_wait, wait);
3899 ++ return crng_ready() ? POLLIN | POLLRDNORM : POLLOUT | POLLWRNORM;
3900 + }
3901 +
3902 +-static int
3903 +-write_pool(struct entropy_store *r, const char __user *buffer, size_t count)
3904 ++static ssize_t write_pool_user(struct iov_iter *iter)
3905 + {
3906 +- size_t bytes;
3907 +- __u32 t, buf[16];
3908 +- const char __user *p = buffer;
3909 ++ u8 block[BLAKE2S_BLOCK_SIZE];
3910 ++ ssize_t ret = 0;
3911 ++ size_t copied;
3912 +
3913 +- while (count > 0) {
3914 +- int b, i = 0;
3915 ++ if (unlikely(!iov_iter_count(iter)))
3916 ++ return 0;
3917 +
3918 +- bytes = min(count, sizeof(buf));
3919 +- if (copy_from_user(&buf, p, bytes))
3920 +- return -EFAULT;
3921 ++ for (;;) {
3922 ++ copied = copy_from_iter(block, sizeof(block), iter);
3923 ++ ret += copied;
3924 ++ mix_pool_bytes(block, copied);
3925 ++ if (!iov_iter_count(iter) || copied != sizeof(block))
3926 ++ break;
3927 +
3928 +- for (b = bytes ; b > 0 ; b -= sizeof(__u32), i++) {
3929 +- if (!arch_get_random_int(&t))
3930 ++ BUILD_BUG_ON(PAGE_SIZE % sizeof(block) != 0);
3931 ++ if (ret % PAGE_SIZE == 0) {
3932 ++ if (signal_pending(current))
3933 + break;
3934 +- buf[i] ^= t;
3935 ++ cond_resched();
3936 + }
3937 ++ }
3938 ++
3939 ++ memzero_explicit(block, sizeof(block));
3940 ++ return ret ? ret : -EFAULT;
3941 ++}
3942 ++
3943 ++static ssize_t random_write_iter(struct kiocb *kiocb, struct iov_iter *iter)
3944 ++{
3945 ++ return write_pool_user(iter);
3946 ++}
3947 +
3948 +- count -= bytes;
3949 +- p += bytes;
3950 ++static ssize_t urandom_read_iter(struct kiocb *kiocb, struct iov_iter *iter)
3951 ++{
3952 ++ static int maxwarn = 10;
3953 +
3954 +- mix_pool_bytes(r, buf, bytes);
3955 +- cond_resched();
3956 ++ if (!crng_ready()) {
3957 ++ if (!ratelimit_disable && maxwarn <= 0)
3958 ++ ++urandom_warning.missed;
3959 ++ else if (ratelimit_disable || __ratelimit(&urandom_warning)) {
3960 ++ --maxwarn;
3961 ++ pr_notice("%s: uninitialized urandom read (%zu bytes read)\n",
3962 ++ current->comm, iov_iter_count(iter));
3963 ++ }
3964 + }
3965 +
3966 +- return 0;
3967 ++ return get_random_bytes_user(iter);
3968 + }
3969 +
3970 +-static ssize_t random_write(struct file *file, const char __user *buffer,
3971 +- size_t count, loff_t *ppos)
3972 ++static ssize_t random_read_iter(struct kiocb *kiocb, struct iov_iter *iter)
3973 + {
3974 +- size_t ret;
3975 ++ int ret;
3976 +
3977 +- ret = write_pool(&input_pool, buffer, count);
3978 +- if (ret)
3979 ++ ret = wait_for_random_bytes();
3980 ++ if (ret != 0)
3981 + return ret;
3982 +-
3983 +- return (ssize_t)count;
3984 ++ return get_random_bytes_user(iter);
3985 + }
3986 +
3987 + static long random_ioctl(struct file *f, unsigned int cmd, unsigned long arg)
3988 + {
3989 +- int size, ent_count;
3990 + int __user *p = (int __user *)arg;
3991 +- int retval;
3992 ++ int ent_count;
3993 +
3994 + switch (cmd) {
3995 + case RNDGETENTCNT:
3996 +- /* inherently racy, no point locking */
3997 +- ent_count = ENTROPY_BITS(&input_pool);
3998 +- if (put_user(ent_count, p))
3999 ++ /* Inherently racy, no point locking. */
4000 ++ if (put_user(input_pool.init_bits, p))
4001 + return -EFAULT;
4002 + return 0;
4003 + case RNDADDTOENTCNT:
4004 +@@ -1958,39 +1316,48 @@ static long random_ioctl(struct file *f, unsigned int cmd, unsigned long arg)
4005 + return -EPERM;
4006 + if (get_user(ent_count, p))
4007 + return -EFAULT;
4008 +- return credit_entropy_bits_safe(&input_pool, ent_count);
4009 +- case RNDADDENTROPY:
4010 ++ if (ent_count < 0)
4011 ++ return -EINVAL;
4012 ++ credit_init_bits(ent_count);
4013 ++ return 0;
4014 ++ case RNDADDENTROPY: {
4015 ++ struct iov_iter iter;
4016 ++ struct iovec iov;
4017 ++ ssize_t ret;
4018 ++ int len;
4019 ++
4020 + if (!capable(CAP_SYS_ADMIN))
4021 + return -EPERM;
4022 + if (get_user(ent_count, p++))
4023 + return -EFAULT;
4024 + if (ent_count < 0)
4025 + return -EINVAL;
4026 +- if (get_user(size, p++))
4027 ++ if (get_user(len, p++))
4028 ++ return -EFAULT;
4029 ++ ret = import_single_range(WRITE, p, len, &iov, &iter);
4030 ++ if (unlikely(ret))
4031 ++ return ret;
4032 ++ ret = write_pool_user(&iter);
4033 ++ if (unlikely(ret < 0))
4034 ++ return ret;
4035 ++ /* Since we're crediting, enforce that it was all written into the pool. */
4036 ++ if (unlikely(ret != len))
4037 + return -EFAULT;
4038 +- retval = write_pool(&input_pool, (const char __user *)p,
4039 +- size);
4040 +- if (retval < 0)
4041 +- return retval;
4042 +- return credit_entropy_bits_safe(&input_pool, ent_count);
4043 ++ credit_init_bits(ent_count);
4044 ++ return 0;
4045 ++ }
4046 + case RNDZAPENTCNT:
4047 + case RNDCLEARPOOL:
4048 +- /*
4049 +- * Clear the entropy pool counters. We no longer clear
4050 +- * the entropy pool, as that's silly.
4051 +- */
4052 ++ /* No longer has any effect. */
4053 + if (!capable(CAP_SYS_ADMIN))
4054 + return -EPERM;
4055 +- input_pool.entropy_count = 0;
4056 +- blocking_pool.entropy_count = 0;
4057 + return 0;
4058 + case RNDRESEEDCRNG:
4059 + if (!capable(CAP_SYS_ADMIN))
4060 + return -EPERM;
4061 +- if (crng_init < 2)
4062 ++ if (!crng_ready())
4063 + return -ENODATA;
4064 +- crng_reseed(&primary_crng, &input_pool);
4065 +- WRITE_ONCE(crng_global_init_time, jiffies - 1);
4066 ++ crng_reseed();
4067 + return 0;
4068 + default:
4069 + return -EINVAL;
4070 +@@ -2003,49 +1370,54 @@ static int random_fasync(int fd, struct file *filp, int on)
4071 + }
4072 +
4073 + const struct file_operations random_fops = {
4074 +- .read = random_read,
4075 +- .write = random_write,
4076 +- .poll = random_poll,
4077 ++ .read_iter = random_read_iter,
4078 ++ .write_iter = random_write_iter,
4079 ++ .poll = random_poll,
4080 + .unlocked_ioctl = random_ioctl,
4081 + .fasync = random_fasync,
4082 + .llseek = noop_llseek,
4083 ++ .splice_read = generic_file_splice_read,
4084 ++ .splice_write = iter_file_splice_write,
4085 + };
4086 +
4087 + const struct file_operations urandom_fops = {
4088 +- .read = urandom_read,
4089 +- .write = random_write,
4090 ++ .read_iter = urandom_read_iter,
4091 ++ .write_iter = random_write_iter,
4092 + .unlocked_ioctl = random_ioctl,
4093 + .fasync = random_fasync,
4094 + .llseek = noop_llseek,
4095 ++ .splice_read = generic_file_splice_read,
4096 ++ .splice_write = iter_file_splice_write,
4097 + };
4098 +
4099 +-SYSCALL_DEFINE3(getrandom, char __user *, buf, size_t, count,
4100 +- unsigned int, flags)
4101 +-{
4102 +- int ret;
4103 +-
4104 +- if (flags & ~(GRND_NONBLOCK|GRND_RANDOM))
4105 +- return -EINVAL;
4106 +-
4107 +- if (count > INT_MAX)
4108 +- count = INT_MAX;
4109 +-
4110 +- if (flags & GRND_RANDOM)
4111 +- return _random_read(flags & GRND_NONBLOCK, buf, count);
4112 +-
4113 +- if (!crng_ready()) {
4114 +- if (flags & GRND_NONBLOCK)
4115 +- return -EAGAIN;
4116 +- ret = wait_for_random_bytes();
4117 +- if (unlikely(ret))
4118 +- return ret;
4119 +- }
4120 +- return urandom_read(NULL, buf, count, NULL);
4121 +-}
4122 +
4123 + /********************************************************************
4124 + *
4125 +- * Sysctl interface
4126 ++ * Sysctl interface.
4127 ++ *
4128 ++ * These are partly unused legacy knobs with dummy values to not break
4129 ++ * userspace and partly still useful things. They are usually accessible
4130 ++ * in /proc/sys/kernel/random/ and are as follows:
4131 ++ *
4132 ++ * - boot_id - a UUID representing the current boot.
4133 ++ *
4134 ++ * - uuid - a random UUID, different each time the file is read.
4135 ++ *
4136 ++ * - poolsize - the number of bits of entropy that the input pool can
4137 ++ * hold, tied to the POOL_BITS constant.
4138 ++ *
4139 ++ * - entropy_avail - the number of bits of entropy currently in the
4140 ++ * input pool. Always <= poolsize.
4141 ++ *
4142 ++ * - write_wakeup_threshold - the amount of entropy in the input pool
4143 ++ * below which write polls to /dev/random will unblock, requesting
4144 ++ * more entropy, tied to the POOL_READY_BITS constant. It is writable
4145 ++ * to avoid breaking old userspaces, but writing to it does not
4146 ++ * change any behavior of the RNG.
4147 ++ *
4148 ++ * - urandom_min_reseed_secs - fixed to the value CRNG_RESEED_INTERVAL.
4149 ++ * It is writable to avoid breaking old userspaces, but writing
4150 ++ * to it does not change any behavior of the RNG.
4151 + *
4152 + ********************************************************************/
4153 +
4154 +@@ -2053,26 +1425,28 @@ SYSCALL_DEFINE3(getrandom, char __user *, buf, size_t, count,
4155 +
4156 + #include <linux/sysctl.h>
4157 +
4158 +-static int min_read_thresh = 8, min_write_thresh;
4159 +-static int max_read_thresh = OUTPUT_POOL_WORDS * 32;
4160 +-static int max_write_thresh = INPUT_POOL_WORDS * 32;
4161 +-static int random_min_urandom_seed = 60;
4162 +-static char sysctl_bootid[16];
4163 ++static int sysctl_random_min_urandom_seed = CRNG_RESEED_INTERVAL / HZ;
4164 ++static int sysctl_random_write_wakeup_bits = POOL_READY_BITS;
4165 ++static int sysctl_poolsize = POOL_BITS;
4166 ++static u8 sysctl_bootid[UUID_SIZE];
4167 +
4168 + /*
4169 + * This function is used to return both the bootid UUID, and random
4170 +- * UUID. The difference is in whether table->data is NULL; if it is,
4171 ++ * UUID. The difference is in whether table->data is NULL; if it is,
4172 + * then a new UUID is generated and returned to the user.
4173 +- *
4174 +- * If the user accesses this via the proc interface, the UUID will be
4175 +- * returned as an ASCII string in the standard UUID format; if via the
4176 +- * sysctl system call, as 16 bytes of binary data.
4177 + */
4178 +-static int proc_do_uuid(struct ctl_table *table, int write,
4179 +- void __user *buffer, size_t *lenp, loff_t *ppos)
4180 ++static int proc_do_uuid(struct ctl_table *table, int write, void __user *buf,
4181 ++ size_t *lenp, loff_t *ppos)
4182 + {
4183 +- struct ctl_table fake_table;
4184 +- unsigned char buf[64], tmp_uuid[16], *uuid;
4185 ++ u8 tmp_uuid[UUID_SIZE], *uuid;
4186 ++ char uuid_string[UUID_STRING_LEN + 1];
4187 ++ struct ctl_table fake_table = {
4188 ++ .data = uuid_string,
4189 ++ .maxlen = UUID_STRING_LEN
4190 ++ };
4191 ++
4192 ++ if (write)
4193 ++ return -EPERM;
4194 +
4195 + uuid = table->data;
4196 + if (!uuid) {
4197 +@@ -2087,32 +1461,17 @@ static int proc_do_uuid(struct ctl_table *table, int write,
4198 + spin_unlock(&bootid_spinlock);
4199 + }
4200 +
4201 +- sprintf(buf, "%pU", uuid);
4202 +-
4203 +- fake_table.data = buf;
4204 +- fake_table.maxlen = sizeof(buf);
4205 +-
4206 +- return proc_dostring(&fake_table, write, buffer, lenp, ppos);
4207 ++ snprintf(uuid_string, sizeof(uuid_string), "%pU", uuid);
4208 ++ return proc_dostring(&fake_table, 0, buf, lenp, ppos);
4209 + }
4210 +
4211 +-/*
4212 +- * Return entropy available scaled to integral bits
4213 +- */
4214 +-static int proc_do_entropy(struct ctl_table *table, int write,
4215 +- void __user *buffer, size_t *lenp, loff_t *ppos)
4216 ++/* The same as proc_dointvec, but writes don't change anything. */
4217 ++static int proc_do_rointvec(struct ctl_table *table, int write, void __user *buf,
4218 ++ size_t *lenp, loff_t *ppos)
4219 + {
4220 +- struct ctl_table fake_table;
4221 +- int entropy_count;
4222 +-
4223 +- entropy_count = *(int *)table->data >> ENTROPY_SHIFT;
4224 +-
4225 +- fake_table.data = &entropy_count;
4226 +- fake_table.maxlen = sizeof(entropy_count);
4227 +-
4228 +- return proc_dointvec(&fake_table, write, buffer, lenp, ppos);
4229 ++ return write ? 0 : proc_dointvec(table, 0, buf, lenp, ppos);
4230 + }
4231 +
4232 +-static int sysctl_poolsize = INPUT_POOL_WORDS * 32;
4233 + extern struct ctl_table random_table[];
4234 + struct ctl_table random_table[] = {
4235 + {
4236 +@@ -2124,213 +1483,36 @@ struct ctl_table random_table[] = {
4237 + },
4238 + {
4239 + .procname = "entropy_avail",
4240 ++ .data = &input_pool.init_bits,
4241 + .maxlen = sizeof(int),
4242 + .mode = 0444,
4243 +- .proc_handler = proc_do_entropy,
4244 +- .data = &input_pool.entropy_count,
4245 +- },
4246 +- {
4247 +- .procname = "read_wakeup_threshold",
4248 +- .data = &random_read_wakeup_bits,
4249 +- .maxlen = sizeof(int),
4250 +- .mode = 0644,
4251 +- .proc_handler = proc_dointvec_minmax,
4252 +- .extra1 = &min_read_thresh,
4253 +- .extra2 = &max_read_thresh,
4254 ++ .proc_handler = proc_dointvec,
4255 + },
4256 + {
4257 + .procname = "write_wakeup_threshold",
4258 +- .data = &random_write_wakeup_bits,
4259 ++ .data = &sysctl_random_write_wakeup_bits,
4260 + .maxlen = sizeof(int),
4261 + .mode = 0644,
4262 +- .proc_handler = proc_dointvec_minmax,
4263 +- .extra1 = &min_write_thresh,
4264 +- .extra2 = &max_write_thresh,
4265 ++ .proc_handler = proc_do_rointvec,
4266 + },
4267 + {
4268 + .procname = "urandom_min_reseed_secs",
4269 +- .data = &random_min_urandom_seed,
4270 ++ .data = &sysctl_random_min_urandom_seed,
4271 + .maxlen = sizeof(int),
4272 + .mode = 0644,
4273 +- .proc_handler = proc_dointvec,
4274 ++ .proc_handler = proc_do_rointvec,
4275 + },
4276 + {
4277 + .procname = "boot_id",
4278 + .data = &sysctl_bootid,
4279 +- .maxlen = 16,
4280 + .mode = 0444,
4281 + .proc_handler = proc_do_uuid,
4282 + },
4283 + {
4284 + .procname = "uuid",
4285 +- .maxlen = 16,
4286 + .mode = 0444,
4287 + .proc_handler = proc_do_uuid,
4288 + },
4289 +-#ifdef ADD_INTERRUPT_BENCH
4290 +- {
4291 +- .procname = "add_interrupt_avg_cycles",
4292 +- .data = &avg_cycles,
4293 +- .maxlen = sizeof(avg_cycles),
4294 +- .mode = 0444,
4295 +- .proc_handler = proc_doulongvec_minmax,
4296 +- },
4297 +- {
4298 +- .procname = "add_interrupt_avg_deviation",
4299 +- .data = &avg_deviation,
4300 +- .maxlen = sizeof(avg_deviation),
4301 +- .mode = 0444,
4302 +- .proc_handler = proc_doulongvec_minmax,
4303 +- },
4304 +-#endif
4305 + { }
4306 + };
4307 +-#endif /* CONFIG_SYSCTL */
4308 +-
4309 +-struct batched_entropy {
4310 +- union {
4311 +- u64 entropy_u64[CHACHA20_BLOCK_SIZE / sizeof(u64)];
4312 +- u32 entropy_u32[CHACHA20_BLOCK_SIZE / sizeof(u32)];
4313 +- };
4314 +- unsigned int position;
4315 +- spinlock_t batch_lock;
4316 +-};
4317 +-
4318 +-/*
4319 +- * Get a random word for internal kernel use only. The quality of the random
4320 +- * number is good as /dev/urandom, but there is no backtrack protection, with
4321 +- * the goal of being quite fast and not depleting entropy. In order to ensure
4322 +- * that the randomness provided by this function is okay, the function
4323 +- * wait_for_random_bytes() should be called and return 0 at least once at any
4324 +- * point prior.
4325 +- */
4326 +-static DEFINE_PER_CPU(struct batched_entropy, batched_entropy_u64) = {
4327 +- .batch_lock = __SPIN_LOCK_UNLOCKED(batched_entropy_u64.lock),
4328 +-};
4329 +-
4330 +-u64 get_random_u64(void)
4331 +-{
4332 +- u64 ret;
4333 +- unsigned long flags;
4334 +- struct batched_entropy *batch;
4335 +- static void *previous;
4336 +-
4337 +- warn_unseeded_randomness(&previous);
4338 +-
4339 +- batch = raw_cpu_ptr(&batched_entropy_u64);
4340 +- spin_lock_irqsave(&batch->batch_lock, flags);
4341 +- if (batch->position % ARRAY_SIZE(batch->entropy_u64) == 0) {
4342 +- extract_crng((u8 *)batch->entropy_u64);
4343 +- batch->position = 0;
4344 +- }
4345 +- ret = batch->entropy_u64[batch->position++];
4346 +- spin_unlock_irqrestore(&batch->batch_lock, flags);
4347 +- return ret;
4348 +-}
4349 +-EXPORT_SYMBOL(get_random_u64);
4350 +-
4351 +-static DEFINE_PER_CPU(struct batched_entropy, batched_entropy_u32) = {
4352 +- .batch_lock = __SPIN_LOCK_UNLOCKED(batched_entropy_u32.lock),
4353 +-};
4354 +-u32 get_random_u32(void)
4355 +-{
4356 +- u32 ret;
4357 +- unsigned long flags;
4358 +- struct batched_entropy *batch;
4359 +- static void *previous;
4360 +-
4361 +- warn_unseeded_randomness(&previous);
4362 +-
4363 +- batch = raw_cpu_ptr(&batched_entropy_u32);
4364 +- spin_lock_irqsave(&batch->batch_lock, flags);
4365 +- if (batch->position % ARRAY_SIZE(batch->entropy_u32) == 0) {
4366 +- extract_crng((u8 *)batch->entropy_u32);
4367 +- batch->position = 0;
4368 +- }
4369 +- ret = batch->entropy_u32[batch->position++];
4370 +- spin_unlock_irqrestore(&batch->batch_lock, flags);
4371 +- return ret;
4372 +-}
4373 +-EXPORT_SYMBOL(get_random_u32);
4374 +-
4375 +-/* It's important to invalidate all potential batched entropy that might
4376 +- * be stored before the crng is initialized, which we can do lazily by
4377 +- * simply resetting the counter to zero so that it's re-extracted on the
4378 +- * next usage. */
4379 +-static void invalidate_batched_entropy(void)
4380 +-{
4381 +- int cpu;
4382 +- unsigned long flags;
4383 +-
4384 +- for_each_possible_cpu (cpu) {
4385 +- struct batched_entropy *batched_entropy;
4386 +-
4387 +- batched_entropy = per_cpu_ptr(&batched_entropy_u32, cpu);
4388 +- spin_lock_irqsave(&batched_entropy->batch_lock, flags);
4389 +- batched_entropy->position = 0;
4390 +- spin_unlock(&batched_entropy->batch_lock);
4391 +-
4392 +- batched_entropy = per_cpu_ptr(&batched_entropy_u64, cpu);
4393 +- spin_lock(&batched_entropy->batch_lock);
4394 +- batched_entropy->position = 0;
4395 +- spin_unlock_irqrestore(&batched_entropy->batch_lock, flags);
4396 +- }
4397 +-}
4398 +-
4399 +-/**
4400 +- * randomize_page - Generate a random, page aligned address
4401 +- * @start: The smallest acceptable address the caller will take.
4402 +- * @range: The size of the area, starting at @start, within which the
4403 +- * random address must fall.
4404 +- *
4405 +- * If @start + @range would overflow, @range is capped.
4406 +- *
4407 +- * NOTE: Historical use of randomize_range, which this replaces, presumed that
4408 +- * @start was already page aligned. We now align it regardless.
4409 +- *
4410 +- * Return: A page aligned address within [start, start + range). On error,
4411 +- * @start is returned.
4412 +- */
4413 +-unsigned long
4414 +-randomize_page(unsigned long start, unsigned long range)
4415 +-{
4416 +- if (!PAGE_ALIGNED(start)) {
4417 +- range -= PAGE_ALIGN(start) - start;
4418 +- start = PAGE_ALIGN(start);
4419 +- }
4420 +-
4421 +- if (start > ULONG_MAX - range)
4422 +- range = ULONG_MAX - start;
4423 +-
4424 +- range >>= PAGE_SHIFT;
4425 +-
4426 +- if (range == 0)
4427 +- return start;
4428 +-
4429 +- return start + (get_random_long() % range << PAGE_SHIFT);
4430 +-}
4431 +-
4432 +-/* Interface for in-kernel drivers of true hardware RNGs.
4433 +- * Those devices may produce endless random bits and will be throttled
4434 +- * when our pool is full.
4435 +- */
4436 +-void add_hwgenerator_randomness(const char *buffer, size_t count,
4437 +- size_t entropy)
4438 +-{
4439 +- struct entropy_store *poolp = &input_pool;
4440 +-
4441 +- if (unlikely(crng_init == 0)) {
4442 +- crng_fast_load(buffer, count);
4443 +- return;
4444 +- }
4445 +-
4446 +- /* Suspend writing if we're above the trickle threshold.
4447 +- * We'll be woken up again once below random_write_wakeup_thresh,
4448 +- * or when the calling thread is about to terminate.
4449 +- */
4450 +- wait_event_interruptible(random_write_wait, kthread_should_stop() ||
4451 +- ENTROPY_BITS(&input_pool) <= random_write_wakeup_bits);
4452 +- mix_pool_bytes(poolp, buffer, count);
4453 +- credit_entropy_bits(poolp, entropy);
4454 +-}
4455 +-EXPORT_SYMBOL_GPL(add_hwgenerator_randomness);
4456 ++#endif /* CONFIG_SYSCTL */
4457 +diff --git a/drivers/hv/vmbus_drv.c b/drivers/hv/vmbus_drv.c
4458 +index 45b8ccdfb0852..cdf7d39362fd0 100644
4459 +--- a/drivers/hv/vmbus_drv.c
4460 ++++ b/drivers/hv/vmbus_drv.c
4461 +@@ -1064,7 +1064,7 @@ static void vmbus_isr(void)
4462 + tasklet_schedule(&hv_cpu->msg_dpc);
4463 + }
4464 +
4465 +- add_interrupt_randomness(HYPERVISOR_CALLBACK_VECTOR, 0);
4466 ++ add_interrupt_randomness(HYPERVISOR_CALLBACK_VECTOR);
4467 + }
4468 +
4469 +
4470 +diff --git a/drivers/irqchip/irq-gic-realview.c b/drivers/irqchip/irq-gic-realview.c
4471 +index b4c1924f02554..38fab02ffe9d0 100644
4472 +--- a/drivers/irqchip/irq-gic-realview.c
4473 ++++ b/drivers/irqchip/irq-gic-realview.c
4474 +@@ -57,6 +57,7 @@ realview_gic_of_init(struct device_node *node, struct device_node *parent)
4475 +
4476 + /* The PB11MPCore GIC needs to be configured in the syscon */
4477 + map = syscon_node_to_regmap(np);
4478 ++ of_node_put(np);
4479 + if (!IS_ERR(map)) {
4480 + /* new irq mode with no DCC */
4481 + regmap_write(map, REALVIEW_SYS_LOCK_OFFSET,
4482 +diff --git a/drivers/misc/atmel-ssc.c b/drivers/misc/atmel-ssc.c
4483 +index f9caf233e2cc0..48521861beb52 100644
4484 +--- a/drivers/misc/atmel-ssc.c
4485 ++++ b/drivers/misc/atmel-ssc.c
4486 +@@ -235,9 +235,9 @@ static int ssc_probe(struct platform_device *pdev)
4487 + clk_disable_unprepare(ssc->clk);
4488 +
4489 + ssc->irq = platform_get_irq(pdev, 0);
4490 +- if (!ssc->irq) {
4491 ++ if (ssc->irq < 0) {
4492 + dev_dbg(&pdev->dev, "could not get irq\n");
4493 +- return -ENXIO;
4494 ++ return ssc->irq;
4495 + }
4496 +
4497 + mutex_lock(&user_lock);
4498 +diff --git a/drivers/net/ethernet/broadcom/bgmac-bcma.c b/drivers/net/ethernet/broadcom/bgmac-bcma.c
4499 +index 6322594ab2600..98f1057650daa 100644
4500 +--- a/drivers/net/ethernet/broadcom/bgmac-bcma.c
4501 ++++ b/drivers/net/ethernet/broadcom/bgmac-bcma.c
4502 +@@ -317,7 +317,6 @@ static void bgmac_remove(struct bcma_device *core)
4503 + bcma_mdio_mii_unregister(bgmac->mii_bus);
4504 + bgmac_enet_remove(bgmac);
4505 + bcma_set_drvdata(core, NULL);
4506 +- kfree(bgmac);
4507 + }
4508 +
4509 + static struct bcma_driver bgmac_bcma_driver = {
4510 +diff --git a/drivers/net/ethernet/intel/i40e/i40e_ethtool.c b/drivers/net/ethernet/intel/i40e/i40e_ethtool.c
4511 +index 751ac56168843..21648dab13e07 100644
4512 +--- a/drivers/net/ethernet/intel/i40e/i40e_ethtool.c
4513 ++++ b/drivers/net/ethernet/intel/i40e/i40e_ethtool.c
4514 +@@ -1874,15 +1874,16 @@ static void i40e_diag_test(struct net_device *netdev,
4515 +
4516 + set_bit(__I40E_TESTING, pf->state);
4517 +
4518 ++ if (test_bit(__I40E_RESET_RECOVERY_PENDING, pf->state) ||
4519 ++ test_bit(__I40E_RESET_INTR_RECEIVED, pf->state)) {
4520 ++ dev_warn(&pf->pdev->dev,
4521 ++ "Cannot start offline testing when PF is in reset state.\n");
4522 ++ goto skip_ol_tests;
4523 ++ }
4524 ++
4525 + if (i40e_active_vfs(pf) || i40e_active_vmdqs(pf)) {
4526 + dev_warn(&pf->pdev->dev,
4527 + "Please take active VFs and Netqueues offline and restart the adapter before running NIC diagnostics\n");
4528 +- data[I40E_ETH_TEST_REG] = 1;
4529 +- data[I40E_ETH_TEST_EEPROM] = 1;
4530 +- data[I40E_ETH_TEST_INTR] = 1;
4531 +- data[I40E_ETH_TEST_LINK] = 1;
4532 +- eth_test->flags |= ETH_TEST_FL_FAILED;
4533 +- clear_bit(__I40E_TESTING, pf->state);
4534 + goto skip_ol_tests;
4535 + }
4536 +
4537 +@@ -1929,9 +1930,17 @@ static void i40e_diag_test(struct net_device *netdev,
4538 + data[I40E_ETH_TEST_INTR] = 0;
4539 + }
4540 +
4541 +-skip_ol_tests:
4542 +-
4543 + netif_info(pf, drv, netdev, "testing finished\n");
4544 ++ return;
4545 ++
4546 ++skip_ol_tests:
4547 ++ data[I40E_ETH_TEST_REG] = 1;
4548 ++ data[I40E_ETH_TEST_EEPROM] = 1;
4549 ++ data[I40E_ETH_TEST_INTR] = 1;
4550 ++ data[I40E_ETH_TEST_LINK] = 1;
4551 ++ eth_test->flags |= ETH_TEST_FL_FAILED;
4552 ++ clear_bit(__I40E_TESTING, pf->state);
4553 ++ netif_info(pf, drv, netdev, "testing failed\n");
4554 + }
4555 +
4556 + static void i40e_get_wol(struct net_device *netdev,
4557 +diff --git a/drivers/net/ethernet/mediatek/mtk_eth_soc.c b/drivers/net/ethernet/mediatek/mtk_eth_soc.c
4558 +index ade72b46e93cd..fb48efa707979 100644
4559 +--- a/drivers/net/ethernet/mediatek/mtk_eth_soc.c
4560 ++++ b/drivers/net/ethernet/mediatek/mtk_eth_soc.c
4561 +@@ -596,6 +596,17 @@ static inline void mtk_rx_get_desc(struct mtk_rx_dma *rxd,
4562 + rxd->rxd4 = READ_ONCE(dma_rxd->rxd4);
4563 + }
4564 +
4565 ++static void *mtk_max_lro_buf_alloc(gfp_t gfp_mask)
4566 ++{
4567 ++ unsigned int size = mtk_max_frag_size(MTK_MAX_LRO_RX_LENGTH);
4568 ++ unsigned long data;
4569 ++
4570 ++ data = __get_free_pages(gfp_mask | __GFP_COMP | __GFP_NOWARN,
4571 ++ get_order(size));
4572 ++
4573 ++ return (void *)data;
4574 ++}
4575 ++
4576 + /* the qdma core needs scratch memory to be setup */
4577 + static int mtk_init_fq_dma(struct mtk_eth *eth)
4578 + {
4579 +@@ -1005,7 +1016,10 @@ static int mtk_poll_rx(struct napi_struct *napi, int budget,
4580 + goto release_desc;
4581 +
4582 + /* alloc new buffer */
4583 +- new_data = napi_alloc_frag(ring->frag_size);
4584 ++ if (ring->frag_size <= PAGE_SIZE)
4585 ++ new_data = napi_alloc_frag(ring->frag_size);
4586 ++ else
4587 ++ new_data = mtk_max_lro_buf_alloc(GFP_ATOMIC);
4588 + if (unlikely(!new_data)) {
4589 + netdev->stats.rx_dropped++;
4590 + goto release_desc;
4591 +@@ -1315,7 +1329,10 @@ static int mtk_rx_alloc(struct mtk_eth *eth, int ring_no, int rx_flag)
4592 + return -ENOMEM;
4593 +
4594 + for (i = 0; i < rx_dma_size; i++) {
4595 +- ring->data[i] = netdev_alloc_frag(ring->frag_size);
4596 ++ if (ring->frag_size <= PAGE_SIZE)
4597 ++ ring->data[i] = netdev_alloc_frag(ring->frag_size);
4598 ++ else
4599 ++ ring->data[i] = mtk_max_lro_buf_alloc(GFP_KERNEL);
4600 + if (!ring->data[i])
4601 + return -ENOMEM;
4602 + }
4603 +diff --git a/drivers/nfc/nfcmrvl/usb.c b/drivers/nfc/nfcmrvl/usb.c
4604 +index deb953290f8f5..e88147c47c178 100644
4605 +--- a/drivers/nfc/nfcmrvl/usb.c
4606 ++++ b/drivers/nfc/nfcmrvl/usb.c
4607 +@@ -400,13 +400,25 @@ static void nfcmrvl_play_deferred(struct nfcmrvl_usb_drv_data *drv_data)
4608 + int err;
4609 +
4610 + while ((urb = usb_get_from_anchor(&drv_data->deferred))) {
4611 ++ usb_anchor_urb(urb, &drv_data->tx_anchor);
4612 ++
4613 + err = usb_submit_urb(urb, GFP_ATOMIC);
4614 +- if (err)
4615 ++ if (err) {
4616 ++ kfree(urb->setup_packet);
4617 ++ usb_unanchor_urb(urb);
4618 ++ usb_free_urb(urb);
4619 + break;
4620 ++ }
4621 +
4622 + drv_data->tx_in_flight++;
4623 ++ usb_free_urb(urb);
4624 ++ }
4625 ++
4626 ++ /* Cleanup the rest deferred urbs. */
4627 ++ while ((urb = usb_get_from_anchor(&drv_data->deferred))) {
4628 ++ kfree(urb->setup_packet);
4629 ++ usb_free_urb(urb);
4630 + }
4631 +- usb_scuttle_anchored_urbs(&drv_data->deferred);
4632 + }
4633 +
4634 + static int nfcmrvl_resume(struct usb_interface *intf)
4635 +diff --git a/drivers/of/fdt.c b/drivers/of/fdt.c
4636 +index b0bf2cb4f5484..1d4090d2b91ed 100644
4637 +--- a/drivers/of/fdt.c
4638 ++++ b/drivers/of/fdt.c
4639 +@@ -27,6 +27,7 @@
4640 + #include <linux/debugfs.h>
4641 + #include <linux/serial_core.h>
4642 + #include <linux/sysfs.h>
4643 ++#include <linux/random.h>
4644 +
4645 + #include <asm/setup.h> /* for COMMAND_LINE_SIZE */
4646 + #include <asm/page.h>
4647 +@@ -1117,6 +1118,7 @@ int __init early_init_dt_scan_chosen(unsigned long node, const char *uname,
4648 + {
4649 + int l;
4650 + const char *p;
4651 ++ const void *rng_seed;
4652 +
4653 + pr_debug("search \"chosen\", depth: %d, uname: %s\n", depth, uname);
4654 +
4655 +@@ -1151,6 +1153,14 @@ int __init early_init_dt_scan_chosen(unsigned long node, const char *uname,
4656 +
4657 + pr_debug("Command line is: %s\n", (char*)data);
4658 +
4659 ++ rng_seed = of_get_flat_dt_prop(node, "rng-seed", &l);
4660 ++ if (rng_seed && l > 0) {
4661 ++ add_bootloader_randomness(rng_seed, l);
4662 ++
4663 ++ /* try to clear seed so it won't be found. */
4664 ++ fdt_nop_property(initial_boot_params, node, "rng-seed");
4665 ++ }
4666 ++
4667 + /* break now */
4668 + return 1;
4669 + }
4670 +@@ -1262,8 +1272,6 @@ bool __init early_init_dt_verify(void *params)
4671 +
4672 + /* Setup flat device-tree pointer */
4673 + initial_boot_params = params;
4674 +- of_fdt_crc32 = crc32_be(~0, initial_boot_params,
4675 +- fdt_totalsize(initial_boot_params));
4676 + return true;
4677 + }
4678 +
4679 +@@ -1289,6 +1297,8 @@ bool __init early_init_dt_scan(void *params)
4680 + return false;
4681 +
4682 + early_init_dt_scan_nodes();
4683 ++ of_fdt_crc32 = crc32_be(~0, initial_boot_params,
4684 ++ fdt_totalsize(initial_boot_params));
4685 + return true;
4686 + }
4687 +
4688 +diff --git a/drivers/scsi/ipr.c b/drivers/scsi/ipr.c
4689 +index b172f0a020834..99b37e19fecab 100644
4690 +--- a/drivers/scsi/ipr.c
4691 ++++ b/drivers/scsi/ipr.c
4692 +@@ -9788,7 +9788,7 @@ static int ipr_alloc_mem(struct ipr_ioa_cfg *ioa_cfg)
4693 + GFP_KERNEL);
4694 +
4695 + if (!ioa_cfg->hrrq[i].host_rrq) {
4696 +- while (--i > 0)
4697 ++ while (--i >= 0)
4698 + dma_free_coherent(&pdev->dev,
4699 + sizeof(u32) * ioa_cfg->hrrq[i].size,
4700 + ioa_cfg->hrrq[i].host_rrq,
4701 +@@ -10060,7 +10060,7 @@ static int ipr_request_other_msi_irqs(struct ipr_ioa_cfg *ioa_cfg,
4702 + ioa_cfg->vectors_info[i].desc,
4703 + &ioa_cfg->hrrq[i]);
4704 + if (rc) {
4705 +- while (--i >= 0)
4706 ++ while (--i > 0)
4707 + free_irq(pci_irq_vector(pdev, i),
4708 + &ioa_cfg->hrrq[i]);
4709 + return rc;
4710 +diff --git a/drivers/scsi/lpfc/lpfc_nportdisc.c b/drivers/scsi/lpfc/lpfc_nportdisc.c
4711 +index da6685700b04c..70733d751bb8e 100644
4712 +--- a/drivers/scsi/lpfc/lpfc_nportdisc.c
4713 ++++ b/drivers/scsi/lpfc/lpfc_nportdisc.c
4714 +@@ -662,7 +662,8 @@ lpfc_rcv_logo(struct lpfc_vport *vport, struct lpfc_nodelist *ndlp,
4715 + else
4716 + lpfc_els_rsp_acc(vport, ELS_CMD_ACC, cmdiocb, ndlp, NULL);
4717 + if (ndlp->nlp_DID == Fabric_DID) {
4718 +- if (vport->port_state <= LPFC_FDISC)
4719 ++ if (vport->port_state <= LPFC_FDISC ||
4720 ++ vport->fc_flag & FC_PT2PT)
4721 + goto out;
4722 + lpfc_linkdown_port(vport);
4723 + spin_lock_irq(shost->host_lock);
4724 +diff --git a/drivers/scsi/pmcraid.c b/drivers/scsi/pmcraid.c
4725 +index b4d6cd8cd1ad5..d27ae937ff68a 100644
4726 +--- a/drivers/scsi/pmcraid.c
4727 ++++ b/drivers/scsi/pmcraid.c
4728 +@@ -4601,7 +4601,7 @@ pmcraid_register_interrupt_handler(struct pmcraid_instance *pinstance)
4729 + return 0;
4730 +
4731 + out_unwind:
4732 +- while (--i > 0)
4733 ++ while (--i >= 0)
4734 + free_irq(pci_irq_vector(pdev, i), &pinstance->hrrq_vector[i]);
4735 + pci_free_irq_vectors(pdev);
4736 + return rc;
4737 +diff --git a/drivers/scsi/vmw_pvscsi.h b/drivers/scsi/vmw_pvscsi.h
4738 +index 75966d3f326e0..d87c12324c032 100644
4739 +--- a/drivers/scsi/vmw_pvscsi.h
4740 ++++ b/drivers/scsi/vmw_pvscsi.h
4741 +@@ -333,8 +333,8 @@ struct PVSCSIRingReqDesc {
4742 + u8 tag;
4743 + u8 bus;
4744 + u8 target;
4745 +- u8 vcpuHint;
4746 +- u8 unused[59];
4747 ++ u16 vcpuHint;
4748 ++ u8 unused[58];
4749 + } __packed;
4750 +
4751 + /*
4752 +diff --git a/drivers/staging/comedi/drivers/vmk80xx.c b/drivers/staging/comedi/drivers/vmk80xx.c
4753 +index 36470ee065967..6010c047f4f86 100644
4754 +--- a/drivers/staging/comedi/drivers/vmk80xx.c
4755 ++++ b/drivers/staging/comedi/drivers/vmk80xx.c
4756 +@@ -694,7 +694,7 @@ static int vmk80xx_alloc_usb_buffers(struct comedi_device *dev)
4757 + if (!devpriv->usb_rx_buf)
4758 + return -ENOMEM;
4759 +
4760 +- size = max(usb_endpoint_maxp(devpriv->ep_rx), MIN_BUF_SIZE);
4761 ++ size = max(usb_endpoint_maxp(devpriv->ep_tx), MIN_BUF_SIZE);
4762 + devpriv->usb_tx_buf = kzalloc(size, GFP_KERNEL);
4763 + if (!devpriv->usb_tx_buf)
4764 + return -ENOMEM;
4765 +diff --git a/drivers/tty/goldfish.c b/drivers/tty/goldfish.c
4766 +index 1b72321f2d0b2..9f0b6b185be7f 100644
4767 +--- a/drivers/tty/goldfish.c
4768 ++++ b/drivers/tty/goldfish.c
4769 +@@ -435,7 +435,7 @@ static int goldfish_tty_remove(struct platform_device *pdev)
4770 + tty_unregister_device(goldfish_tty_driver, qtty->console.index);
4771 + iounmap(qtty->base);
4772 + qtty->base = NULL;
4773 +- free_irq(qtty->irq, pdev);
4774 ++ free_irq(qtty->irq, qtty);
4775 + tty_port_destroy(&qtty->port);
4776 + goldfish_tty_current_line_count--;
4777 + if (goldfish_tty_current_line_count == 0)
4778 +diff --git a/drivers/tty/serial/8250/8250_port.c b/drivers/tty/serial/8250/8250_port.c
4779 +index bbfcb220e1eb9..1aca30a3f716b 100644
4780 +--- a/drivers/tty/serial/8250/8250_port.c
4781 ++++ b/drivers/tty/serial/8250/8250_port.c
4782 +@@ -1524,6 +1524,8 @@ static inline void __stop_tx(struct uart_8250_port *p)
4783 +
4784 + if (em485) {
4785 + unsigned char lsr = serial_in(p, UART_LSR);
4786 ++ p->lsr_saved_flags |= lsr & LSR_SAVE_FLAGS;
4787 ++
4788 + /*
4789 + * To provide required timeing and allow FIFO transfer,
4790 + * __stop_tx_rs485() must be called only when both FIFO and
4791 +diff --git a/drivers/usb/dwc2/hcd.c b/drivers/usb/dwc2/hcd.c
4792 +index ba7528916da4c..50ec2cd36db0a 100644
4793 +--- a/drivers/usb/dwc2/hcd.c
4794 ++++ b/drivers/usb/dwc2/hcd.c
4795 +@@ -5231,7 +5231,7 @@ int dwc2_hcd_init(struct dwc2_hsotg *hsotg)
4796 + res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
4797 + if (!res) {
4798 + retval = -EINVAL;
4799 +- goto error1;
4800 ++ goto error2;
4801 + }
4802 + hcd->rsrc_start = res->start;
4803 + hcd->rsrc_len = resource_size(res);
4804 +diff --git a/drivers/usb/gadget/function/u_ether.c b/drivers/usb/gadget/function/u_ether.c
4805 +index f59c20457e658..2d45233ba027e 100644
4806 +--- a/drivers/usb/gadget/function/u_ether.c
4807 ++++ b/drivers/usb/gadget/function/u_ether.c
4808 +@@ -776,9 +776,13 @@ struct eth_dev *gether_setup_name(struct usb_gadget *g,
4809 + dev->qmult = qmult;
4810 + snprintf(net->name, sizeof(net->name), "%s%%d", netname);
4811 +
4812 +- if (get_ether_addr(dev_addr, net->dev_addr))
4813 ++ if (get_ether_addr(dev_addr, net->dev_addr)) {
4814 ++ net->addr_assign_type = NET_ADDR_RANDOM;
4815 + dev_warn(&g->dev,
4816 + "using random %s ethernet address\n", "self");
4817 ++ } else {
4818 ++ net->addr_assign_type = NET_ADDR_SET;
4819 ++ }
4820 + if (get_ether_addr(host_addr, dev->host_mac))
4821 + dev_warn(&g->dev,
4822 + "using random %s ethernet address\n", "host");
4823 +@@ -835,6 +839,9 @@ struct net_device *gether_setup_name_default(const char *netname)
4824 + INIT_LIST_HEAD(&dev->tx_reqs);
4825 + INIT_LIST_HEAD(&dev->rx_reqs);
4826 +
4827 ++ /* by default we always have a random MAC address */
4828 ++ net->addr_assign_type = NET_ADDR_RANDOM;
4829 ++
4830 + skb_queue_head_init(&dev->rx_frames);
4831 +
4832 + /* network device setup */
4833 +@@ -872,7 +879,6 @@ int gether_register_netdev(struct net_device *net)
4834 + g = dev->gadget;
4835 +
4836 + memcpy(net->dev_addr, dev->dev_mac, ETH_ALEN);
4837 +- net->addr_assign_type = NET_ADDR_RANDOM;
4838 +
4839 + status = register_netdev(net);
4840 + if (status < 0) {
4841 +@@ -912,6 +918,7 @@ int gether_set_dev_addr(struct net_device *net, const char *dev_addr)
4842 + if (get_ether_addr(dev_addr, new_addr))
4843 + return -EINVAL;
4844 + memcpy(dev->dev_mac, new_addr, ETH_ALEN);
4845 ++ net->addr_assign_type = NET_ADDR_SET;
4846 + return 0;
4847 + }
4848 + EXPORT_SYMBOL_GPL(gether_set_dev_addr);
4849 +diff --git a/drivers/usb/gadget/udc/lpc32xx_udc.c b/drivers/usb/gadget/udc/lpc32xx_udc.c
4850 +index 7107931617953..498a44ae2b925 100644
4851 +--- a/drivers/usb/gadget/udc/lpc32xx_udc.c
4852 ++++ b/drivers/usb/gadget/udc/lpc32xx_udc.c
4853 +@@ -3034,6 +3034,7 @@ static int lpc32xx_udc_probe(struct platform_device *pdev)
4854 + }
4855 +
4856 + udc->isp1301_i2c_client = isp1301_get_client(isp1301_node);
4857 ++ of_node_put(isp1301_node);
4858 + if (!udc->isp1301_i2c_client) {
4859 + retval = -EPROBE_DEFER;
4860 + goto phy_fail;
4861 +diff --git a/drivers/usb/serial/io_ti.c b/drivers/usb/serial/io_ti.c
4862 +index 6cefb9cb133d7..9bd24a7fe9e3c 100644
4863 +--- a/drivers/usb/serial/io_ti.c
4864 ++++ b/drivers/usb/serial/io_ti.c
4865 +@@ -172,6 +172,7 @@ static const struct usb_device_id edgeport_2port_id_table[] = {
4866 + { USB_DEVICE(USB_VENDOR_ID_ION, ION_DEVICE_ID_TI_EDGEPORT_8S) },
4867 + { USB_DEVICE(USB_VENDOR_ID_ION, ION_DEVICE_ID_TI_EDGEPORT_416) },
4868 + { USB_DEVICE(USB_VENDOR_ID_ION, ION_DEVICE_ID_TI_EDGEPORT_416B) },
4869 ++ { USB_DEVICE(USB_VENDOR_ID_ION, ION_DEVICE_ID_E5805A) },
4870 + { }
4871 + };
4872 +
4873 +@@ -210,6 +211,7 @@ static const struct usb_device_id id_table_combined[] = {
4874 + { USB_DEVICE(USB_VENDOR_ID_ION, ION_DEVICE_ID_TI_EDGEPORT_8S) },
4875 + { USB_DEVICE(USB_VENDOR_ID_ION, ION_DEVICE_ID_TI_EDGEPORT_416) },
4876 + { USB_DEVICE(USB_VENDOR_ID_ION, ION_DEVICE_ID_TI_EDGEPORT_416B) },
4877 ++ { USB_DEVICE(USB_VENDOR_ID_ION, ION_DEVICE_ID_E5805A) },
4878 + { }
4879 + };
4880 +
4881 +diff --git a/drivers/usb/serial/io_usbvend.h b/drivers/usb/serial/io_usbvend.h
4882 +index 6f6a856bc37cd..a470262dcf48b 100644
4883 +--- a/drivers/usb/serial/io_usbvend.h
4884 ++++ b/drivers/usb/serial/io_usbvend.h
4885 +@@ -215,6 +215,7 @@
4886 + //
4887 + // Definitions for other product IDs
4888 + #define ION_DEVICE_ID_MT4X56USB 0x1403 // OEM device
4889 ++#define ION_DEVICE_ID_E5805A 0x1A01 // OEM device (rebranded Edgeport/4)
4890 +
4891 +
4892 + #define GENERATION_ID_FROM_USB_PRODUCT_ID(ProductId) \
4893 +diff --git a/drivers/usb/serial/option.c b/drivers/usb/serial/option.c
4894 +index 6facdd77c138e..bbd20defaae66 100644
4895 +--- a/drivers/usb/serial/option.c
4896 ++++ b/drivers/usb/serial/option.c
4897 +@@ -435,6 +435,8 @@ static void option_instat_callback(struct urb *urb);
4898 + #define CINTERION_PRODUCT_CLS8 0x00b0
4899 + #define CINTERION_PRODUCT_MV31_MBIM 0x00b3
4900 + #define CINTERION_PRODUCT_MV31_RMNET 0x00b7
4901 ++#define CINTERION_PRODUCT_MV31_2_MBIM 0x00b8
4902 ++#define CINTERION_PRODUCT_MV31_2_RMNET 0x00b9
4903 + #define CINTERION_PRODUCT_MV32_WA 0x00f1
4904 + #define CINTERION_PRODUCT_MV32_WB 0x00f2
4905 +
4906 +@@ -1982,6 +1984,10 @@ static const struct usb_device_id option_ids[] = {
4907 + .driver_info = RSVD(3)},
4908 + { USB_DEVICE_INTERFACE_CLASS(CINTERION_VENDOR_ID, CINTERION_PRODUCT_MV31_RMNET, 0xff),
4909 + .driver_info = RSVD(0)},
4910 ++ { USB_DEVICE_INTERFACE_CLASS(CINTERION_VENDOR_ID, CINTERION_PRODUCT_MV31_2_MBIM, 0xff),
4911 ++ .driver_info = RSVD(3)},
4912 ++ { USB_DEVICE_INTERFACE_CLASS(CINTERION_VENDOR_ID, CINTERION_PRODUCT_MV31_2_RMNET, 0xff),
4913 ++ .driver_info = RSVD(0)},
4914 + { USB_DEVICE_INTERFACE_CLASS(CINTERION_VENDOR_ID, CINTERION_PRODUCT_MV32_WA, 0xff),
4915 + .driver_info = RSVD(3)},
4916 + { USB_DEVICE_INTERFACE_CLASS(CINTERION_VENDOR_ID, CINTERION_PRODUCT_MV32_WB, 0xff),
4917 +diff --git a/drivers/virtio/virtio_mmio.c b/drivers/virtio/virtio_mmio.c
4918 +index 74dc7170fd351..181386e06cb70 100644
4919 +--- a/drivers/virtio/virtio_mmio.c
4920 ++++ b/drivers/virtio/virtio_mmio.c
4921 +@@ -635,6 +635,7 @@ static int vm_cmdline_set(const char *device,
4922 + if (!vm_cmdline_parent_registered) {
4923 + err = device_register(&vm_cmdline_parent);
4924 + if (err) {
4925 ++ put_device(&vm_cmdline_parent);
4926 + pr_err("Failed to register parent device!\n");
4927 + return err;
4928 + }
4929 +diff --git a/drivers/virtio/virtio_pci_common.c b/drivers/virtio/virtio_pci_common.c
4930 +index b9c06885de6a1..924554b7010d9 100644
4931 +--- a/drivers/virtio/virtio_pci_common.c
4932 ++++ b/drivers/virtio/virtio_pci_common.c
4933 +@@ -256,8 +256,7 @@ void vp_del_vqs(struct virtio_device *vdev)
4934 +
4935 + if (vp_dev->msix_affinity_masks) {
4936 + for (i = 0; i < vp_dev->msix_vectors; i++)
4937 +- if (vp_dev->msix_affinity_masks[i])
4938 +- free_cpumask_var(vp_dev->msix_affinity_masks[i]);
4939 ++ free_cpumask_var(vp_dev->msix_affinity_masks[i]);
4940 + }
4941 +
4942 + if (vp_dev->msix_enabled) {
4943 +diff --git a/fs/9p/vfs_inode_dotl.c b/fs/9p/vfs_inode_dotl.c
4944 +index 3446ab1f44e7d..309d69c7ae5ce 100644
4945 +--- a/fs/9p/vfs_inode_dotl.c
4946 ++++ b/fs/9p/vfs_inode_dotl.c
4947 +@@ -657,14 +657,10 @@ v9fs_stat2inode_dotl(struct p9_stat_dotl *stat, struct inode *inode,
4948 + if (stat->st_result_mask & P9_STATS_NLINK)
4949 + set_nlink(inode, stat->st_nlink);
4950 + if (stat->st_result_mask & P9_STATS_MODE) {
4951 +- inode->i_mode = stat->st_mode;
4952 +- if ((S_ISBLK(inode->i_mode)) ||
4953 +- (S_ISCHR(inode->i_mode)))
4954 +- init_special_inode(inode, inode->i_mode,
4955 +- inode->i_rdev);
4956 ++ mode = stat->st_mode & S_IALLUGO;
4957 ++ mode |= inode->i_mode & ~S_IALLUGO;
4958 ++ inode->i_mode = mode;
4959 + }
4960 +- if (stat->st_result_mask & P9_STATS_RDEV)
4961 +- inode->i_rdev = new_decode_dev(stat->st_rdev);
4962 + if (!(flags & V9FS_STAT2INODE_KEEP_ISIZE) &&
4963 + stat->st_result_mask & P9_STATS_SIZE)
4964 + v9fs_i_size_write(inode, stat->st_size);
4965 +diff --git a/fs/ext4/mballoc.c b/fs/ext4/mballoc.c
4966 +index 28bee66c5fbf0..a93bd342e0ae6 100644
4967 +--- a/fs/ext4/mballoc.c
4968 ++++ b/fs/ext4/mballoc.c
4969 +@@ -3197,6 +3197,15 @@ ext4_mb_normalize_request(struct ext4_allocation_context *ac,
4970 + size = size >> bsbits;
4971 + start = start_off >> bsbits;
4972 +
4973 ++ /*
4974 ++ * For tiny groups (smaller than 8MB) the chosen allocation
4975 ++ * alignment may be larger than group size. Make sure the
4976 ++ * alignment does not move allocation to a different group which
4977 ++ * makes mballoc fail assertions later.
4978 ++ */
4979 ++ start = max(start, rounddown(ac->ac_o_ex.fe_logical,
4980 ++ (ext4_lblk_t)EXT4_BLOCKS_PER_GROUP(ac->ac_sb)));
4981 ++
4982 + /* don't cover already allocated blocks in selected range */
4983 + if (ar->pleft && start <= ar->lleft) {
4984 + size -= ar->lleft + 1 - start;
4985 +diff --git a/fs/ext4/namei.c b/fs/ext4/namei.c
4986 +index eb4de706cc9c5..c5f289737d0f9 100644
4987 +--- a/fs/ext4/namei.c
4988 ++++ b/fs/ext4/namei.c
4989 +@@ -1724,7 +1724,8 @@ static struct ext4_dir_entry_2 *do_split(handle_t *handle, struct inode *dir,
4990 + struct dx_hash_info *hinfo)
4991 + {
4992 + unsigned blocksize = dir->i_sb->s_blocksize;
4993 +- unsigned count, continued;
4994 ++ unsigned continued;
4995 ++ int count;
4996 + struct buffer_head *bh2;
4997 + ext4_lblk_t newblock;
4998 + u32 hash2;
4999 +diff --git a/fs/ext4/resize.c b/fs/ext4/resize.c
5000 +index e0f4a9e3114e9..721a00767d0e8 100644
5001 +--- a/fs/ext4/resize.c
5002 ++++ b/fs/ext4/resize.c
5003 +@@ -52,6 +52,16 @@ int ext4_resize_begin(struct super_block *sb)
5004 + if (!capable(CAP_SYS_RESOURCE))
5005 + return -EPERM;
5006 +
5007 ++ /*
5008 ++ * If the reserved GDT blocks is non-zero, the resize_inode feature
5009 ++ * should always be set.
5010 ++ */
5011 ++ if (EXT4_SB(sb)->s_es->s_reserved_gdt_blocks &&
5012 ++ !ext4_has_feature_resize_inode(sb)) {
5013 ++ ext4_error(sb, "resize_inode disabled but reserved GDT blocks non-zero");
5014 ++ return -EFSCORRUPTED;
5015 ++ }
5016 ++
5017 + /*
5018 + * If we are not using the primary superblock/GDT copy don't resize,
5019 + * because the user tools have no way of handling this. Probably a
5020 +diff --git a/fs/nfs/pnfs.c b/fs/nfs/pnfs.c
5021 +index 18bbdaefd9405..962585e00c868 100644
5022 +--- a/fs/nfs/pnfs.c
5023 ++++ b/fs/nfs/pnfs.c
5024 +@@ -1878,6 +1878,12 @@ lookup_again:
5025 + /* Fallthrough */
5026 + case -EAGAIN:
5027 + break;
5028 ++ case -ENODATA:
5029 ++ /* The server returned NFS4ERR_LAYOUTUNAVAILABLE */
5030 ++ pnfs_layout_set_fail_bit(
5031 ++ lo, pnfs_iomode_to_fail_bit(iomode));
5032 ++ lseg = NULL;
5033 ++ goto out_put_layout_hdr;
5034 + default:
5035 + if (!nfs_error_is_fatal(PTR_ERR(lseg))) {
5036 + pnfs_layout_clear_fail_bit(lo, pnfs_iomode_to_fail_bit(iomode));
5037 +diff --git a/include/crypto/blake2s.h b/include/crypto/blake2s.h
5038 +new file mode 100644
5039 +index 0000000000000..d439496fa6baa
5040 +--- /dev/null
5041 ++++ b/include/crypto/blake2s.h
5042 +@@ -0,0 +1,102 @@
5043 ++/* SPDX-License-Identifier: GPL-2.0 OR MIT */
5044 ++/*
5045 ++ * Copyright (C) 2015-2019 Jason A. Donenfeld <Jason@×××××.com>. All Rights Reserved.
5046 ++ */
5047 ++
5048 ++#ifndef _CRYPTO_BLAKE2S_H
5049 ++#define _CRYPTO_BLAKE2S_H
5050 ++
5051 ++#include <linux/bug.h>
5052 ++#include <linux/types.h>
5053 ++#include <linux/kernel.h>
5054 ++#include <linux/string.h>
5055 ++
5056 ++enum blake2s_lengths {
5057 ++ BLAKE2S_BLOCK_SIZE = 64,
5058 ++ BLAKE2S_HASH_SIZE = 32,
5059 ++ BLAKE2S_KEY_SIZE = 32,
5060 ++
5061 ++ BLAKE2S_128_HASH_SIZE = 16,
5062 ++ BLAKE2S_160_HASH_SIZE = 20,
5063 ++ BLAKE2S_224_HASH_SIZE = 28,
5064 ++ BLAKE2S_256_HASH_SIZE = 32,
5065 ++};
5066 ++
5067 ++struct blake2s_state {
5068 ++ u32 h[8];
5069 ++ u32 t[2];
5070 ++ u32 f[2];
5071 ++ u8 buf[BLAKE2S_BLOCK_SIZE];
5072 ++ unsigned int buflen;
5073 ++ unsigned int outlen;
5074 ++};
5075 ++
5076 ++enum blake2s_iv {
5077 ++ BLAKE2S_IV0 = 0x6A09E667UL,
5078 ++ BLAKE2S_IV1 = 0xBB67AE85UL,
5079 ++ BLAKE2S_IV2 = 0x3C6EF372UL,
5080 ++ BLAKE2S_IV3 = 0xA54FF53AUL,
5081 ++ BLAKE2S_IV4 = 0x510E527FUL,
5082 ++ BLAKE2S_IV5 = 0x9B05688CUL,
5083 ++ BLAKE2S_IV6 = 0x1F83D9ABUL,
5084 ++ BLAKE2S_IV7 = 0x5BE0CD19UL,
5085 ++};
5086 ++
5087 ++void blake2s_update(struct blake2s_state *state, const u8 *in, size_t inlen);
5088 ++void blake2s_final(struct blake2s_state *state, u8 *out);
5089 ++
5090 ++static inline void blake2s_init_param(struct blake2s_state *state,
5091 ++ const u32 param)
5092 ++{
5093 ++ *state = (struct blake2s_state){{
5094 ++ BLAKE2S_IV0 ^ param,
5095 ++ BLAKE2S_IV1,
5096 ++ BLAKE2S_IV2,
5097 ++ BLAKE2S_IV3,
5098 ++ BLAKE2S_IV4,
5099 ++ BLAKE2S_IV5,
5100 ++ BLAKE2S_IV6,
5101 ++ BLAKE2S_IV7,
5102 ++ }};
5103 ++}
5104 ++
5105 ++static inline void blake2s_init(struct blake2s_state *state,
5106 ++ const size_t outlen)
5107 ++{
5108 ++ blake2s_init_param(state, 0x01010000 | outlen);
5109 ++ state->outlen = outlen;
5110 ++}
5111 ++
5112 ++static inline void blake2s_init_key(struct blake2s_state *state,
5113 ++ const size_t outlen, const void *key,
5114 ++ const size_t keylen)
5115 ++{
5116 ++ WARN_ON(IS_ENABLED(DEBUG) && (!outlen || outlen > BLAKE2S_HASH_SIZE ||
5117 ++ !key || !keylen || keylen > BLAKE2S_KEY_SIZE));
5118 ++
5119 ++ blake2s_init_param(state, 0x01010000 | keylen << 8 | outlen);
5120 ++ memcpy(state->buf, key, keylen);
5121 ++ state->buflen = BLAKE2S_BLOCK_SIZE;
5122 ++ state->outlen = outlen;
5123 ++}
5124 ++
5125 ++static inline void blake2s(u8 *out, const u8 *in, const u8 *key,
5126 ++ const size_t outlen, const size_t inlen,
5127 ++ const size_t keylen)
5128 ++{
5129 ++ struct blake2s_state state;
5130 ++
5131 ++ WARN_ON(IS_ENABLED(DEBUG) && ((!in && inlen > 0) || !out || !outlen ||
5132 ++ outlen > BLAKE2S_HASH_SIZE || keylen > BLAKE2S_KEY_SIZE ||
5133 ++ (!key && keylen)));
5134 ++
5135 ++ if (keylen)
5136 ++ blake2s_init_key(&state, outlen, key, keylen);
5137 ++ else
5138 ++ blake2s_init(&state, outlen);
5139 ++
5140 ++ blake2s_update(&state, in, inlen);
5141 ++ blake2s_final(&state, out);
5142 ++}
5143 ++
5144 ++#endif /* _CRYPTO_BLAKE2S_H */
5145 +diff --git a/include/crypto/chacha20.h b/include/crypto/chacha20.h
5146 +index caaa470389e0e..3dd5ab189543f 100644
5147 +--- a/include/crypto/chacha20.h
5148 ++++ b/include/crypto/chacha20.h
5149 +@@ -18,10 +18,25 @@ struct chacha20_ctx {
5150 + u32 key[8];
5151 + };
5152 +
5153 +-void chacha20_block(u32 *state, void *stream);
5154 ++void chacha20_block(u32 *state, u8 *stream);
5155 + void crypto_chacha20_init(u32 *state, struct chacha20_ctx *ctx, u8 *iv);
5156 + int crypto_chacha20_setkey(struct crypto_skcipher *tfm, const u8 *key,
5157 + unsigned int keysize);
5158 + int crypto_chacha20_crypt(struct skcipher_request *req);
5159 +
5160 ++enum chacha_constants { /* expand 32-byte k */
5161 ++ CHACHA_CONSTANT_EXPA = 0x61707865U,
5162 ++ CHACHA_CONSTANT_ND_3 = 0x3320646eU,
5163 ++ CHACHA_CONSTANT_2_BY = 0x79622d32U,
5164 ++ CHACHA_CONSTANT_TE_K = 0x6b206574U
5165 ++};
5166 ++
5167 ++static inline void chacha_init_consts(u32 *state)
5168 ++{
5169 ++ state[0] = CHACHA_CONSTANT_EXPA;
5170 ++ state[1] = CHACHA_CONSTANT_ND_3;
5171 ++ state[2] = CHACHA_CONSTANT_2_BY;
5172 ++ state[3] = CHACHA_CONSTANT_TE_K;
5173 ++}
5174 ++
5175 + #endif
5176 +diff --git a/include/crypto/drbg.h b/include/crypto/drbg.h
5177 +index 22f884c97387e..19b44179bb7c7 100644
5178 +--- a/include/crypto/drbg.h
5179 ++++ b/include/crypto/drbg.h
5180 +@@ -105,6 +105,12 @@ struct drbg_test_data {
5181 + struct drbg_string *testentropy; /* TEST PARAMETER: test entropy */
5182 + };
5183 +
5184 ++enum drbg_seed_state {
5185 ++ DRBG_SEED_STATE_UNSEEDED,
5186 ++ DRBG_SEED_STATE_PARTIAL, /* Seeded with !rng_is_initialized() */
5187 ++ DRBG_SEED_STATE_FULL,
5188 ++};
5189 ++
5190 + struct drbg_state {
5191 + struct mutex drbg_mutex; /* lock around DRBG */
5192 + unsigned char *V; /* internal state 10.1.1.1 1a) */
5193 +@@ -129,14 +135,14 @@ struct drbg_state {
5194 + struct completion ctr_completion; /* CTR mode async handler */
5195 + int ctr_async_err; /* CTR mode async error */
5196 +
5197 +- bool seeded; /* DRBG fully seeded? */
5198 ++ enum drbg_seed_state seeded; /* DRBG fully seeded? */
5199 + bool pr; /* Prediction resistance enabled? */
5200 +- struct work_struct seed_work; /* asynchronous seeding support */
5201 ++ bool fips_primed; /* Continuous test primed? */
5202 ++ unsigned char *prev; /* FIPS 140-2 continuous test value */
5203 + struct crypto_rng *jent;
5204 + const struct drbg_state_ops *d_ops;
5205 + const struct drbg_core *core;
5206 + struct drbg_string test_data;
5207 +- struct random_ready_callback random_ready;
5208 + };
5209 +
5210 + static inline __u8 drbg_statelen(struct drbg_state *drbg)
5211 +@@ -184,11 +190,7 @@ static inline size_t drbg_max_addtl(struct drbg_state *drbg)
5212 + static inline size_t drbg_max_requests(struct drbg_state *drbg)
5213 + {
5214 + /* SP800-90A requires 2**48 maximum requests before reseeding */
5215 +-#if (__BITS_PER_LONG == 32)
5216 +- return SIZE_MAX;
5217 +-#else
5218 +- return (1UL<<48);
5219 +-#endif
5220 ++ return (1<<20);
5221 + }
5222 +
5223 + /*
5224 +diff --git a/include/crypto/internal/blake2s.h b/include/crypto/internal/blake2s.h
5225 +new file mode 100644
5226 +index 0000000000000..3ba066845b699
5227 +--- /dev/null
5228 ++++ b/include/crypto/internal/blake2s.h
5229 +@@ -0,0 +1,19 @@
5230 ++/* SPDX-License-Identifier: GPL-2.0 OR MIT */
5231 ++
5232 ++#ifndef _CRYPTO_INTERNAL_BLAKE2S_H
5233 ++#define _CRYPTO_INTERNAL_BLAKE2S_H
5234 ++
5235 ++#include <crypto/blake2s.h>
5236 ++
5237 ++void blake2s_compress_generic(struct blake2s_state *state,const u8 *block,
5238 ++ size_t nblocks, const u32 inc);
5239 ++
5240 ++void blake2s_compress_arch(struct blake2s_state *state,const u8 *block,
5241 ++ size_t nblocks, const u32 inc);
5242 ++
5243 ++static inline void blake2s_set_lastblock(struct blake2s_state *state)
5244 ++{
5245 ++ state->f[0] = -1;
5246 ++}
5247 ++
5248 ++#endif /* _CRYPTO_INTERNAL_BLAKE2S_H */
5249 +diff --git a/include/linux/byteorder/generic.h b/include/linux/byteorder/generic.h
5250 +index 133b76d0d2c03..802f433e5a719 100644
5251 +--- a/include/linux/byteorder/generic.h
5252 ++++ b/include/linux/byteorder/generic.h
5253 +@@ -156,6 +156,23 @@ static inline void le64_add_cpu(__le64 *var, u64 val)
5254 + *var = cpu_to_le64(le64_to_cpu(*var) + val);
5255 + }
5256 +
5257 ++/* XXX: this stuff can be optimized */
5258 ++static inline void le32_to_cpu_array(u32 *buf, unsigned int words)
5259 ++{
5260 ++ while (words--) {
5261 ++ __le32_to_cpus(buf);
5262 ++ buf++;
5263 ++ }
5264 ++}
5265 ++
5266 ++static inline void cpu_to_le32_array(u32 *buf, unsigned int words)
5267 ++{
5268 ++ while (words--) {
5269 ++ __cpu_to_le32s(buf);
5270 ++ buf++;
5271 ++ }
5272 ++}
5273 ++
5274 + static inline void be16_add_cpu(__be16 *var, u16 val)
5275 + {
5276 + *var = cpu_to_be16(be16_to_cpu(*var) + val);
5277 +diff --git a/include/linux/cpuhotplug.h b/include/linux/cpuhotplug.h
5278 +index 0834eb5ea9e61..17a8a65f325f5 100644
5279 +--- a/include/linux/cpuhotplug.h
5280 ++++ b/include/linux/cpuhotplug.h
5281 +@@ -59,6 +59,7 @@ enum cpuhp_state {
5282 + CPUHP_PCI_XGENE_DEAD,
5283 + CPUHP_IOMMU_INTEL_DEAD,
5284 + CPUHP_LUSTRE_CFS_DEAD,
5285 ++ CPUHP_RANDOM_PREPARE,
5286 + CPUHP_WORKQUEUE_PREP,
5287 + CPUHP_POWER_NUMA_PREPARE,
5288 + CPUHP_HRTIMERS_PREPARE,
5289 +@@ -162,6 +163,7 @@ enum cpuhp_state {
5290 + CPUHP_AP_PERF_POWERPC_CORE_IMC_ONLINE,
5291 + CPUHP_AP_PERF_POWERPC_THREAD_IMC_ONLINE,
5292 + CPUHP_AP_WORKQUEUE_ONLINE,
5293 ++ CPUHP_AP_RANDOM_ONLINE,
5294 + CPUHP_AP_RCUTREE_ONLINE,
5295 + CPUHP_AP_BASE_CACHEINFO_ONLINE,
5296 + CPUHP_AP_ONLINE_DYN,
5297 +diff --git a/include/linux/hw_random.h b/include/linux/hw_random.h
5298 +index bee0827766a39..31587f36c06e5 100644
5299 +--- a/include/linux/hw_random.h
5300 ++++ b/include/linux/hw_random.h
5301 +@@ -59,7 +59,5 @@ extern int devm_hwrng_register(struct device *dev, struct hwrng *rng);
5302 + /** Unregister a Hardware Random Number Generator driver. */
5303 + extern void hwrng_unregister(struct hwrng *rng);
5304 + extern void devm_hwrng_unregister(struct device *dve, struct hwrng *rng);
5305 +-/** Feed random bits into the pool. */
5306 +-extern void add_hwgenerator_randomness(const char *buffer, size_t count, size_t entropy);
5307 +
5308 + #endif /* LINUX_HWRANDOM_H_ */
5309 +diff --git a/include/linux/mm.h b/include/linux/mm.h
5310 +index 94b138e4c8409..794f52e2540e2 100644
5311 +--- a/include/linux/mm.h
5312 ++++ b/include/linux/mm.h
5313 +@@ -2164,6 +2164,8 @@ extern int install_special_mapping(struct mm_struct *mm,
5314 + unsigned long addr, unsigned long len,
5315 + unsigned long flags, struct page **pages);
5316 +
5317 ++unsigned long randomize_page(unsigned long start, unsigned long range);
5318 ++
5319 + extern unsigned long get_unmapped_area(struct file *, unsigned long, unsigned long, unsigned long, unsigned long);
5320 +
5321 + extern unsigned long mmap_region(struct file *file, unsigned long addr,
5322 +diff --git a/include/linux/prandom.h b/include/linux/prandom.h
5323 +index e20339c78a84c..709e8e69fb39b 100644
5324 +--- a/include/linux/prandom.h
5325 ++++ b/include/linux/prandom.h
5326 +@@ -10,6 +10,7 @@
5327 +
5328 + #include <linux/types.h>
5329 + #include <linux/percpu.h>
5330 ++#include <linux/siphash.h>
5331 +
5332 + u32 prandom_u32(void);
5333 + void prandom_bytes(void *buf, size_t nbytes);
5334 +@@ -21,15 +22,10 @@ void prandom_reseed_late(void);
5335 + * The core SipHash round function. Each line can be executed in
5336 + * parallel given enough CPU resources.
5337 + */
5338 +-#define PRND_SIPROUND(v0, v1, v2, v3) ( \
5339 +- v0 += v1, v1 = rol64(v1, 13), v2 += v3, v3 = rol64(v3, 16), \
5340 +- v1 ^= v0, v0 = rol64(v0, 32), v3 ^= v2, \
5341 +- v0 += v3, v3 = rol64(v3, 21), v2 += v1, v1 = rol64(v1, 17), \
5342 +- v3 ^= v0, v1 ^= v2, v2 = rol64(v2, 32) \
5343 +-)
5344 ++#define PRND_SIPROUND(v0, v1, v2, v3) SIPHASH_PERMUTATION(v0, v1, v2, v3)
5345 +
5346 +-#define PRND_K0 (0x736f6d6570736575 ^ 0x6c7967656e657261)
5347 +-#define PRND_K1 (0x646f72616e646f6d ^ 0x7465646279746573)
5348 ++#define PRND_K0 (SIPHASH_CONST_0 ^ SIPHASH_CONST_2)
5349 ++#define PRND_K1 (SIPHASH_CONST_1 ^ SIPHASH_CONST_3)
5350 +
5351 + #elif BITS_PER_LONG == 32
5352 + /*
5353 +@@ -37,14 +33,9 @@ void prandom_reseed_late(void);
5354 + * This is weaker, but 32-bit machines are not used for high-traffic
5355 + * applications, so there is less output for an attacker to analyze.
5356 + */
5357 +-#define PRND_SIPROUND(v0, v1, v2, v3) ( \
5358 +- v0 += v1, v1 = rol32(v1, 5), v2 += v3, v3 = rol32(v3, 8), \
5359 +- v1 ^= v0, v0 = rol32(v0, 16), v3 ^= v2, \
5360 +- v0 += v3, v3 = rol32(v3, 7), v2 += v1, v1 = rol32(v1, 13), \
5361 +- v3 ^= v0, v1 ^= v2, v2 = rol32(v2, 16) \
5362 +-)
5363 +-#define PRND_K0 0x6c796765
5364 +-#define PRND_K1 0x74656462
5365 ++#define PRND_SIPROUND(v0, v1, v2, v3) HSIPHASH_PERMUTATION(v0, v1, v2, v3)
5366 ++#define PRND_K0 (HSIPHASH_CONST_0 ^ HSIPHASH_CONST_2)
5367 ++#define PRND_K1 (HSIPHASH_CONST_1 ^ HSIPHASH_CONST_3)
5368 +
5369 + #else
5370 + #error Unsupported BITS_PER_LONG
5371 +diff --git a/include/linux/random.h b/include/linux/random.h
5372 +index 8eda8c0cbba70..3feafab498ad9 100644
5373 +--- a/include/linux/random.h
5374 ++++ b/include/linux/random.h
5375 +@@ -1,49 +1,35 @@
5376 + /* SPDX-License-Identifier: GPL-2.0 */
5377 +-/*
5378 +- * include/linux/random.h
5379 +- *
5380 +- * Include file for the random number generator.
5381 +- */
5382 ++
5383 + #ifndef _LINUX_RANDOM_H
5384 + #define _LINUX_RANDOM_H
5385 +
5386 ++#include <linux/bug.h>
5387 ++#include <linux/kernel.h>
5388 + #include <linux/list.h>
5389 + #include <linux/once.h>
5390 +
5391 + #include <uapi/linux/random.h>
5392 +
5393 +-struct random_ready_callback {
5394 +- struct list_head list;
5395 +- void (*func)(struct random_ready_callback *rdy);
5396 +- struct module *owner;
5397 +-};
5398 ++struct notifier_block;
5399 +
5400 +-extern void add_device_randomness(const void *, unsigned int);
5401 ++void add_device_randomness(const void *buf, size_t len);
5402 ++void __init add_bootloader_randomness(const void *buf, size_t len);
5403 ++void add_input_randomness(unsigned int type, unsigned int code,
5404 ++ unsigned int value) __latent_entropy;
5405 ++void add_interrupt_randomness(int irq) __latent_entropy;
5406 ++void add_hwgenerator_randomness(const void *buf, size_t len, size_t entropy);
5407 +
5408 +-#if defined(CONFIG_GCC_PLUGIN_LATENT_ENTROPY) && !defined(__CHECKER__)
5409 ++#if defined(LATENT_ENTROPY_PLUGIN) && !defined(__CHECKER__)
5410 + static inline void add_latent_entropy(void)
5411 + {
5412 +- add_device_randomness((const void *)&latent_entropy,
5413 +- sizeof(latent_entropy));
5414 ++ add_device_randomness((const void *)&latent_entropy, sizeof(latent_entropy));
5415 + }
5416 + #else
5417 +-static inline void add_latent_entropy(void) {}
5418 +-#endif
5419 +-
5420 +-extern void add_input_randomness(unsigned int type, unsigned int code,
5421 +- unsigned int value) __latent_entropy;
5422 +-extern void add_interrupt_randomness(int irq, int irq_flags) __latent_entropy;
5423 +-
5424 +-extern void get_random_bytes(void *buf, int nbytes);
5425 +-extern int wait_for_random_bytes(void);
5426 +-extern int add_random_ready_callback(struct random_ready_callback *rdy);
5427 +-extern void del_random_ready_callback(struct random_ready_callback *rdy);
5428 +-extern void get_random_bytes_arch(void *buf, int nbytes);
5429 +-
5430 +-#ifndef MODULE
5431 +-extern const struct file_operations random_fops, urandom_fops;
5432 ++static inline void add_latent_entropy(void) { }
5433 + #endif
5434 +
5435 ++void get_random_bytes(void *buf, size_t len);
5436 ++size_t __must_check get_random_bytes_arch(void *buf, size_t len);
5437 + u32 get_random_u32(void);
5438 + u64 get_random_u64(void);
5439 + static inline unsigned int get_random_int(void)
5440 +@@ -75,38 +61,38 @@ static inline unsigned long get_random_long(void)
5441 +
5442 + static inline unsigned long get_random_canary(void)
5443 + {
5444 +- unsigned long val = get_random_long();
5445 +-
5446 +- return val & CANARY_MASK;
5447 ++ return get_random_long() & CANARY_MASK;
5448 + }
5449 +
5450 ++int __init random_init(const char *command_line);
5451 ++bool rng_is_initialized(void);
5452 ++int wait_for_random_bytes(void);
5453 ++int register_random_ready_notifier(struct notifier_block *nb);
5454 ++int unregister_random_ready_notifier(struct notifier_block *nb);
5455 ++
5456 + /* Calls wait_for_random_bytes() and then calls get_random_bytes(buf, nbytes).
5457 + * Returns the result of the call to wait_for_random_bytes. */
5458 +-static inline int get_random_bytes_wait(void *buf, int nbytes)
5459 ++static inline int get_random_bytes_wait(void *buf, size_t nbytes)
5460 + {
5461 + int ret = wait_for_random_bytes();
5462 +- if (unlikely(ret))
5463 +- return ret;
5464 + get_random_bytes(buf, nbytes);
5465 +- return 0;
5466 ++ return ret;
5467 + }
5468 +
5469 +-#define declare_get_random_var_wait(var) \
5470 +- static inline int get_random_ ## var ## _wait(var *out) { \
5471 ++#define declare_get_random_var_wait(name, ret_type) \
5472 ++ static inline int get_random_ ## name ## _wait(ret_type *out) { \
5473 + int ret = wait_for_random_bytes(); \
5474 + if (unlikely(ret)) \
5475 + return ret; \
5476 +- *out = get_random_ ## var(); \
5477 ++ *out = get_random_ ## name(); \
5478 + return 0; \
5479 + }
5480 +-declare_get_random_var_wait(u32)
5481 +-declare_get_random_var_wait(u64)
5482 +-declare_get_random_var_wait(int)
5483 +-declare_get_random_var_wait(long)
5484 ++declare_get_random_var_wait(u32, u32)
5485 ++declare_get_random_var_wait(u64, u32)
5486 ++declare_get_random_var_wait(int, unsigned int)
5487 ++declare_get_random_var_wait(long, unsigned long)
5488 + #undef declare_get_random_var
5489 +
5490 +-unsigned long randomize_page(unsigned long start, unsigned long range);
5491 +-
5492 + /*
5493 + * This is designed to be standalone for just prandom
5494 + * users, but for now we include it from <linux/random.h>
5495 +@@ -117,30 +103,39 @@ unsigned long randomize_page(unsigned long start, unsigned long range);
5496 + #ifdef CONFIG_ARCH_RANDOM
5497 + # include <asm/archrandom.h>
5498 + #else
5499 +-static inline bool arch_get_random_long(unsigned long *v)
5500 +-{
5501 +- return 0;
5502 +-}
5503 +-static inline bool arch_get_random_int(unsigned int *v)
5504 +-{
5505 +- return 0;
5506 +-}
5507 +-static inline bool arch_has_random(void)
5508 +-{
5509 +- return 0;
5510 +-}
5511 +-static inline bool arch_get_random_seed_long(unsigned long *v)
5512 +-{
5513 +- return 0;
5514 +-}
5515 +-static inline bool arch_get_random_seed_int(unsigned int *v)
5516 ++static inline bool __must_check arch_get_random_long(unsigned long *v) { return false; }
5517 ++static inline bool __must_check arch_get_random_int(unsigned int *v) { return false; }
5518 ++static inline bool __must_check arch_get_random_seed_long(unsigned long *v) { return false; }
5519 ++static inline bool __must_check arch_get_random_seed_int(unsigned int *v) { return false; }
5520 ++#endif
5521 ++
5522 ++/*
5523 ++ * Called from the boot CPU during startup; not valid to call once
5524 ++ * secondary CPUs are up and preemption is possible.
5525 ++ */
5526 ++#ifndef arch_get_random_seed_long_early
5527 ++static inline bool __init arch_get_random_seed_long_early(unsigned long *v)
5528 + {
5529 +- return 0;
5530 ++ WARN_ON(system_state != SYSTEM_BOOTING);
5531 ++ return arch_get_random_seed_long(v);
5532 + }
5533 +-static inline bool arch_has_random_seed(void)
5534 ++#endif
5535 ++
5536 ++#ifndef arch_get_random_long_early
5537 ++static inline bool __init arch_get_random_long_early(unsigned long *v)
5538 + {
5539 +- return 0;
5540 ++ WARN_ON(system_state != SYSTEM_BOOTING);
5541 ++ return arch_get_random_long(v);
5542 + }
5543 + #endif
5544 +
5545 ++#ifdef CONFIG_SMP
5546 ++int random_prepare_cpu(unsigned int cpu);
5547 ++int random_online_cpu(unsigned int cpu);
5548 ++#endif
5549 ++
5550 ++#ifndef MODULE
5551 ++extern const struct file_operations random_fops, urandom_fops;
5552 ++#endif
5553 ++
5554 + #endif /* _LINUX_RANDOM_H */
5555 +diff --git a/include/linux/siphash.h b/include/linux/siphash.h
5556 +index 0cda61855d907..0bb5ecd507bef 100644
5557 +--- a/include/linux/siphash.h
5558 ++++ b/include/linux/siphash.h
5559 +@@ -136,4 +136,32 @@ static inline u32 hsiphash(const void *data, size_t len,
5560 + return ___hsiphash_aligned(data, len, key);
5561 + }
5562 +
5563 ++/*
5564 ++ * These macros expose the raw SipHash and HalfSipHash permutations.
5565 ++ * Do not use them directly! If you think you have a use for them,
5566 ++ * be sure to CC the maintainer of this file explaining why.
5567 ++ */
5568 ++
5569 ++#define SIPHASH_PERMUTATION(a, b, c, d) ( \
5570 ++ (a) += (b), (b) = rol64((b), 13), (b) ^= (a), (a) = rol64((a), 32), \
5571 ++ (c) += (d), (d) = rol64((d), 16), (d) ^= (c), \
5572 ++ (a) += (d), (d) = rol64((d), 21), (d) ^= (a), \
5573 ++ (c) += (b), (b) = rol64((b), 17), (b) ^= (c), (c) = rol64((c), 32))
5574 ++
5575 ++#define SIPHASH_CONST_0 0x736f6d6570736575ULL
5576 ++#define SIPHASH_CONST_1 0x646f72616e646f6dULL
5577 ++#define SIPHASH_CONST_2 0x6c7967656e657261ULL
5578 ++#define SIPHASH_CONST_3 0x7465646279746573ULL
5579 ++
5580 ++#define HSIPHASH_PERMUTATION(a, b, c, d) ( \
5581 ++ (a) += (b), (b) = rol32((b), 5), (b) ^= (a), (a) = rol32((a), 16), \
5582 ++ (c) += (d), (d) = rol32((d), 8), (d) ^= (c), \
5583 ++ (a) += (d), (d) = rol32((d), 7), (d) ^= (a), \
5584 ++ (c) += (b), (b) = rol32((b), 13), (b) ^= (c), (c) = rol32((c), 16))
5585 ++
5586 ++#define HSIPHASH_CONST_0 0U
5587 ++#define HSIPHASH_CONST_1 0U
5588 ++#define HSIPHASH_CONST_2 0x6c796765U
5589 ++#define HSIPHASH_CONST_3 0x74656462U
5590 ++
5591 + #endif /* _LINUX_SIPHASH_H */
5592 +diff --git a/include/linux/timex.h b/include/linux/timex.h
5593 +index 39c25dbebfe80..c7616cfb48d20 100644
5594 +--- a/include/linux/timex.h
5595 ++++ b/include/linux/timex.h
5596 +@@ -62,6 +62,8 @@
5597 + #include <linux/types.h>
5598 + #include <linux/param.h>
5599 +
5600 ++unsigned long random_get_entropy_fallback(void);
5601 ++
5602 + #include <asm/timex.h>
5603 +
5604 + #ifndef random_get_entropy
5605 +@@ -74,8 +76,14 @@
5606 + *
5607 + * By default we use get_cycles() for this purpose, but individual
5608 + * architectures may override this in their asm/timex.h header file.
5609 ++ * If a given arch does not have get_cycles(), then we fallback to
5610 ++ * using random_get_entropy_fallback().
5611 + */
5612 +-#define random_get_entropy() get_cycles()
5613 ++#ifdef get_cycles
5614 ++#define random_get_entropy() ((unsigned long)get_cycles())
5615 ++#else
5616 ++#define random_get_entropy() random_get_entropy_fallback()
5617 ++#endif
5618 + #endif
5619 +
5620 + /*
5621 +diff --git a/include/trace/events/random.h b/include/trace/events/random.h
5622 +deleted file mode 100644
5623 +index 0560dfc33f1c3..0000000000000
5624 +--- a/include/trace/events/random.h
5625 ++++ /dev/null
5626 +@@ -1,316 +0,0 @@
5627 +-/* SPDX-License-Identifier: GPL-2.0 */
5628 +-#undef TRACE_SYSTEM
5629 +-#define TRACE_SYSTEM random
5630 +-
5631 +-#if !defined(_TRACE_RANDOM_H) || defined(TRACE_HEADER_MULTI_READ)
5632 +-#define _TRACE_RANDOM_H
5633 +-
5634 +-#include <linux/writeback.h>
5635 +-#include <linux/tracepoint.h>
5636 +-
5637 +-TRACE_EVENT(add_device_randomness,
5638 +- TP_PROTO(int bytes, unsigned long IP),
5639 +-
5640 +- TP_ARGS(bytes, IP),
5641 +-
5642 +- TP_STRUCT__entry(
5643 +- __field( int, bytes )
5644 +- __field(unsigned long, IP )
5645 +- ),
5646 +-
5647 +- TP_fast_assign(
5648 +- __entry->bytes = bytes;
5649 +- __entry->IP = IP;
5650 +- ),
5651 +-
5652 +- TP_printk("bytes %d caller %pS",
5653 +- __entry->bytes, (void *)__entry->IP)
5654 +-);
5655 +-
5656 +-DECLARE_EVENT_CLASS(random__mix_pool_bytes,
5657 +- TP_PROTO(const char *pool_name, int bytes, unsigned long IP),
5658 +-
5659 +- TP_ARGS(pool_name, bytes, IP),
5660 +-
5661 +- TP_STRUCT__entry(
5662 +- __field( const char *, pool_name )
5663 +- __field( int, bytes )
5664 +- __field(unsigned long, IP )
5665 +- ),
5666 +-
5667 +- TP_fast_assign(
5668 +- __entry->pool_name = pool_name;
5669 +- __entry->bytes = bytes;
5670 +- __entry->IP = IP;
5671 +- ),
5672 +-
5673 +- TP_printk("%s pool: bytes %d caller %pS",
5674 +- __entry->pool_name, __entry->bytes, (void *)__entry->IP)
5675 +-);
5676 +-
5677 +-DEFINE_EVENT(random__mix_pool_bytes, mix_pool_bytes,
5678 +- TP_PROTO(const char *pool_name, int bytes, unsigned long IP),
5679 +-
5680 +- TP_ARGS(pool_name, bytes, IP)
5681 +-);
5682 +-
5683 +-DEFINE_EVENT(random__mix_pool_bytes, mix_pool_bytes_nolock,
5684 +- TP_PROTO(const char *pool_name, int bytes, unsigned long IP),
5685 +-
5686 +- TP_ARGS(pool_name, bytes, IP)
5687 +-);
5688 +-
5689 +-TRACE_EVENT(credit_entropy_bits,
5690 +- TP_PROTO(const char *pool_name, int bits, int entropy_count,
5691 +- int entropy_total, unsigned long IP),
5692 +-
5693 +- TP_ARGS(pool_name, bits, entropy_count, entropy_total, IP),
5694 +-
5695 +- TP_STRUCT__entry(
5696 +- __field( const char *, pool_name )
5697 +- __field( int, bits )
5698 +- __field( int, entropy_count )
5699 +- __field( int, entropy_total )
5700 +- __field(unsigned long, IP )
5701 +- ),
5702 +-
5703 +- TP_fast_assign(
5704 +- __entry->pool_name = pool_name;
5705 +- __entry->bits = bits;
5706 +- __entry->entropy_count = entropy_count;
5707 +- __entry->entropy_total = entropy_total;
5708 +- __entry->IP = IP;
5709 +- ),
5710 +-
5711 +- TP_printk("%s pool: bits %d entropy_count %d entropy_total %d "
5712 +- "caller %pS", __entry->pool_name, __entry->bits,
5713 +- __entry->entropy_count, __entry->entropy_total,
5714 +- (void *)__entry->IP)
5715 +-);
5716 +-
5717 +-TRACE_EVENT(push_to_pool,
5718 +- TP_PROTO(const char *pool_name, int pool_bits, int input_bits),
5719 +-
5720 +- TP_ARGS(pool_name, pool_bits, input_bits),
5721 +-
5722 +- TP_STRUCT__entry(
5723 +- __field( const char *, pool_name )
5724 +- __field( int, pool_bits )
5725 +- __field( int, input_bits )
5726 +- ),
5727 +-
5728 +- TP_fast_assign(
5729 +- __entry->pool_name = pool_name;
5730 +- __entry->pool_bits = pool_bits;
5731 +- __entry->input_bits = input_bits;
5732 +- ),
5733 +-
5734 +- TP_printk("%s: pool_bits %d input_pool_bits %d",
5735 +- __entry->pool_name, __entry->pool_bits,
5736 +- __entry->input_bits)
5737 +-);
5738 +-
5739 +-TRACE_EVENT(debit_entropy,
5740 +- TP_PROTO(const char *pool_name, int debit_bits),
5741 +-
5742 +- TP_ARGS(pool_name, debit_bits),
5743 +-
5744 +- TP_STRUCT__entry(
5745 +- __field( const char *, pool_name )
5746 +- __field( int, debit_bits )
5747 +- ),
5748 +-
5749 +- TP_fast_assign(
5750 +- __entry->pool_name = pool_name;
5751 +- __entry->debit_bits = debit_bits;
5752 +- ),
5753 +-
5754 +- TP_printk("%s: debit_bits %d", __entry->pool_name,
5755 +- __entry->debit_bits)
5756 +-);
5757 +-
5758 +-TRACE_EVENT(add_input_randomness,
5759 +- TP_PROTO(int input_bits),
5760 +-
5761 +- TP_ARGS(input_bits),
5762 +-
5763 +- TP_STRUCT__entry(
5764 +- __field( int, input_bits )
5765 +- ),
5766 +-
5767 +- TP_fast_assign(
5768 +- __entry->input_bits = input_bits;
5769 +- ),
5770 +-
5771 +- TP_printk("input_pool_bits %d", __entry->input_bits)
5772 +-);
5773 +-
5774 +-TRACE_EVENT(add_disk_randomness,
5775 +- TP_PROTO(dev_t dev, int input_bits),
5776 +-
5777 +- TP_ARGS(dev, input_bits),
5778 +-
5779 +- TP_STRUCT__entry(
5780 +- __field( dev_t, dev )
5781 +- __field( int, input_bits )
5782 +- ),
5783 +-
5784 +- TP_fast_assign(
5785 +- __entry->dev = dev;
5786 +- __entry->input_bits = input_bits;
5787 +- ),
5788 +-
5789 +- TP_printk("dev %d,%d input_pool_bits %d", MAJOR(__entry->dev),
5790 +- MINOR(__entry->dev), __entry->input_bits)
5791 +-);
5792 +-
5793 +-TRACE_EVENT(xfer_secondary_pool,
5794 +- TP_PROTO(const char *pool_name, int xfer_bits, int request_bits,
5795 +- int pool_entropy, int input_entropy),
5796 +-
5797 +- TP_ARGS(pool_name, xfer_bits, request_bits, pool_entropy,
5798 +- input_entropy),
5799 +-
5800 +- TP_STRUCT__entry(
5801 +- __field( const char *, pool_name )
5802 +- __field( int, xfer_bits )
5803 +- __field( int, request_bits )
5804 +- __field( int, pool_entropy )
5805 +- __field( int, input_entropy )
5806 +- ),
5807 +-
5808 +- TP_fast_assign(
5809 +- __entry->pool_name = pool_name;
5810 +- __entry->xfer_bits = xfer_bits;
5811 +- __entry->request_bits = request_bits;
5812 +- __entry->pool_entropy = pool_entropy;
5813 +- __entry->input_entropy = input_entropy;
5814 +- ),
5815 +-
5816 +- TP_printk("pool %s xfer_bits %d request_bits %d pool_entropy %d "
5817 +- "input_entropy %d", __entry->pool_name, __entry->xfer_bits,
5818 +- __entry->request_bits, __entry->pool_entropy,
5819 +- __entry->input_entropy)
5820 +-);
5821 +-
5822 +-DECLARE_EVENT_CLASS(random__get_random_bytes,
5823 +- TP_PROTO(int nbytes, unsigned long IP),
5824 +-
5825 +- TP_ARGS(nbytes, IP),
5826 +-
5827 +- TP_STRUCT__entry(
5828 +- __field( int, nbytes )
5829 +- __field(unsigned long, IP )
5830 +- ),
5831 +-
5832 +- TP_fast_assign(
5833 +- __entry->nbytes = nbytes;
5834 +- __entry->IP = IP;
5835 +- ),
5836 +-
5837 +- TP_printk("nbytes %d caller %pS", __entry->nbytes, (void *)__entry->IP)
5838 +-);
5839 +-
5840 +-DEFINE_EVENT(random__get_random_bytes, get_random_bytes,
5841 +- TP_PROTO(int nbytes, unsigned long IP),
5842 +-
5843 +- TP_ARGS(nbytes, IP)
5844 +-);
5845 +-
5846 +-DEFINE_EVENT(random__get_random_bytes, get_random_bytes_arch,
5847 +- TP_PROTO(int nbytes, unsigned long IP),
5848 +-
5849 +- TP_ARGS(nbytes, IP)
5850 +-);
5851 +-
5852 +-DECLARE_EVENT_CLASS(random__extract_entropy,
5853 +- TP_PROTO(const char *pool_name, int nbytes, int entropy_count,
5854 +- unsigned long IP),
5855 +-
5856 +- TP_ARGS(pool_name, nbytes, entropy_count, IP),
5857 +-
5858 +- TP_STRUCT__entry(
5859 +- __field( const char *, pool_name )
5860 +- __field( int, nbytes )
5861 +- __field( int, entropy_count )
5862 +- __field(unsigned long, IP )
5863 +- ),
5864 +-
5865 +- TP_fast_assign(
5866 +- __entry->pool_name = pool_name;
5867 +- __entry->nbytes = nbytes;
5868 +- __entry->entropy_count = entropy_count;
5869 +- __entry->IP = IP;
5870 +- ),
5871 +-
5872 +- TP_printk("%s pool: nbytes %d entropy_count %d caller %pS",
5873 +- __entry->pool_name, __entry->nbytes, __entry->entropy_count,
5874 +- (void *)__entry->IP)
5875 +-);
5876 +-
5877 +-
5878 +-DEFINE_EVENT(random__extract_entropy, extract_entropy,
5879 +- TP_PROTO(const char *pool_name, int nbytes, int entropy_count,
5880 +- unsigned long IP),
5881 +-
5882 +- TP_ARGS(pool_name, nbytes, entropy_count, IP)
5883 +-);
5884 +-
5885 +-DEFINE_EVENT(random__extract_entropy, extract_entropy_user,
5886 +- TP_PROTO(const char *pool_name, int nbytes, int entropy_count,
5887 +- unsigned long IP),
5888 +-
5889 +- TP_ARGS(pool_name, nbytes, entropy_count, IP)
5890 +-);
5891 +-
5892 +-TRACE_EVENT(random_read,
5893 +- TP_PROTO(int got_bits, int need_bits, int pool_left, int input_left),
5894 +-
5895 +- TP_ARGS(got_bits, need_bits, pool_left, input_left),
5896 +-
5897 +- TP_STRUCT__entry(
5898 +- __field( int, got_bits )
5899 +- __field( int, need_bits )
5900 +- __field( int, pool_left )
5901 +- __field( int, input_left )
5902 +- ),
5903 +-
5904 +- TP_fast_assign(
5905 +- __entry->got_bits = got_bits;
5906 +- __entry->need_bits = need_bits;
5907 +- __entry->pool_left = pool_left;
5908 +- __entry->input_left = input_left;
5909 +- ),
5910 +-
5911 +- TP_printk("got_bits %d still_needed_bits %d "
5912 +- "blocking_pool_entropy_left %d input_entropy_left %d",
5913 +- __entry->got_bits, __entry->got_bits, __entry->pool_left,
5914 +- __entry->input_left)
5915 +-);
5916 +-
5917 +-TRACE_EVENT(urandom_read,
5918 +- TP_PROTO(int got_bits, int pool_left, int input_left),
5919 +-
5920 +- TP_ARGS(got_bits, pool_left, input_left),
5921 +-
5922 +- TP_STRUCT__entry(
5923 +- __field( int, got_bits )
5924 +- __field( int, pool_left )
5925 +- __field( int, input_left )
5926 +- ),
5927 +-
5928 +- TP_fast_assign(
5929 +- __entry->got_bits = got_bits;
5930 +- __entry->pool_left = pool_left;
5931 +- __entry->input_left = input_left;
5932 +- ),
5933 +-
5934 +- TP_printk("got_bits %d nonblocking_pool_entropy_left %d "
5935 +- "input_entropy_left %d", __entry->got_bits,
5936 +- __entry->pool_left, __entry->input_left)
5937 +-);
5938 +-
5939 +-#endif /* _TRACE_RANDOM_H */
5940 +-
5941 +-/* This part must be outside protection */
5942 +-#include <trace/define_trace.h>
5943 +diff --git a/include/uapi/linux/random.h b/include/uapi/linux/random.h
5944 +index 26ee91300e3ec..dcc1b3e6106fe 100644
5945 +--- a/include/uapi/linux/random.h
5946 ++++ b/include/uapi/linux/random.h
5947 +@@ -48,9 +48,11 @@ struct rand_pool_info {
5948 + * Flags for getrandom(2)
5949 + *
5950 + * GRND_NONBLOCK Don't block and return EAGAIN instead
5951 +- * GRND_RANDOM Use the /dev/random pool instead of /dev/urandom
5952 ++ * GRND_RANDOM No effect
5953 ++ * GRND_INSECURE Return non-cryptographic random bytes
5954 + */
5955 + #define GRND_NONBLOCK 0x0001
5956 + #define GRND_RANDOM 0x0002
5957 ++#define GRND_INSECURE 0x0004
5958 +
5959 + #endif /* _UAPI_LINUX_RANDOM_H */
5960 +diff --git a/init/main.c b/init/main.c
5961 +index 621cedd9173a3..bf27f802d0a13 100644
5962 +--- a/init/main.c
5963 ++++ b/init/main.c
5964 +@@ -531,13 +531,6 @@ asmlinkage __visible void __init start_kernel(void)
5965 + page_address_init();
5966 + pr_notice("%s", linux_banner);
5967 + setup_arch(&command_line);
5968 +- /*
5969 +- * Set up the the initial canary and entropy after arch
5970 +- * and after adding latent and command line entropy.
5971 +- */
5972 +- add_latent_entropy();
5973 +- add_device_randomness(command_line, strlen(command_line));
5974 +- boot_init_stack_canary();
5975 + mm_init_cpumask(&init_mm);
5976 + setup_command_line(command_line);
5977 + setup_nr_cpu_ids();
5978 +@@ -615,6 +608,17 @@ asmlinkage __visible void __init start_kernel(void)
5979 + softirq_init();
5980 + timekeeping_init();
5981 + time_init();
5982 ++
5983 ++ /*
5984 ++ * For best initial stack canary entropy, prepare it after:
5985 ++ * - setup_arch() for any UEFI RNG entropy and boot cmdline access
5986 ++ * - timekeeping_init() for ktime entropy used in random_init()
5987 ++ * - time_init() for making random_get_entropy() work on some platforms
5988 ++ * - random_init() to initialize the RNG from from early entropy sources
5989 ++ */
5990 ++ random_init(command_line);
5991 ++ boot_init_stack_canary();
5992 ++
5993 + sched_clock_postinit();
5994 + printk_safe_init();
5995 + perf_event_init();
5996 +diff --git a/kernel/cpu.c b/kernel/cpu.c
5997 +index f3b231cb6b657..281efe4b6cb0e 100644
5998 +--- a/kernel/cpu.c
5999 ++++ b/kernel/cpu.c
6000 +@@ -31,6 +31,7 @@
6001 + #include <linux/slab.h>
6002 + #include <linux/percpu-rwsem.h>
6003 + #include <linux/cpuset.h>
6004 ++#include <linux/random.h>
6005 +
6006 + #include <trace/events/power.h>
6007 + #define CREATE_TRACE_POINTS
6008 +@@ -1404,6 +1405,11 @@ static struct cpuhp_step cpuhp_bp_states[] = {
6009 + .startup.single = perf_event_init_cpu,
6010 + .teardown.single = perf_event_exit_cpu,
6011 + },
6012 ++ [CPUHP_RANDOM_PREPARE] = {
6013 ++ .name = "random:prepare",
6014 ++ .startup.single = random_prepare_cpu,
6015 ++ .teardown.single = NULL,
6016 ++ },
6017 + [CPUHP_WORKQUEUE_PREP] = {
6018 + .name = "workqueue:prepare",
6019 + .startup.single = workqueue_prepare_cpu,
6020 +@@ -1523,6 +1529,11 @@ static struct cpuhp_step cpuhp_ap_states[] = {
6021 + .startup.single = workqueue_online_cpu,
6022 + .teardown.single = workqueue_offline_cpu,
6023 + },
6024 ++ [CPUHP_AP_RANDOM_ONLINE] = {
6025 ++ .name = "random:online",
6026 ++ .startup.single = random_online_cpu,
6027 ++ .teardown.single = NULL,
6028 ++ },
6029 + [CPUHP_AP_RCUTREE_ONLINE] = {
6030 + .name = "RCU/tree:online",
6031 + .startup.single = rcutree_online_cpu,
6032 +diff --git a/kernel/irq/handle.c b/kernel/irq/handle.c
6033 +index 79f987b942b84..1bc4dcc489d0b 100644
6034 +--- a/kernel/irq/handle.c
6035 ++++ b/kernel/irq/handle.c
6036 +@@ -186,7 +186,7 @@ irqreturn_t handle_irq_event_percpu(struct irq_desc *desc)
6037 +
6038 + retval = __handle_irq_event_percpu(desc, &flags);
6039 +
6040 +- add_interrupt_randomness(desc->irq_data.irq, flags);
6041 ++ add_interrupt_randomness(desc->irq_data.irq);
6042 +
6043 + if (!noirqdebug)
6044 + note_interrupt(desc, retval);
6045 +diff --git a/kernel/time/timekeeping.c b/kernel/time/timekeeping.c
6046 +index 602b476627dd5..1d6a476fdfbad 100644
6047 +--- a/kernel/time/timekeeping.c
6048 ++++ b/kernel/time/timekeeping.c
6049 +@@ -21,6 +21,7 @@
6050 + #include <linux/clocksource.h>
6051 + #include <linux/jiffies.h>
6052 + #include <linux/time.h>
6053 ++#include <linux/timex.h>
6054 + #include <linux/tick.h>
6055 + #include <linux/stop_machine.h>
6056 + #include <linux/pvclock_gtod.h>
6057 +@@ -2247,6 +2248,21 @@ ktime_t ktime_get_update_offsets_now(unsigned int *cwsseq, ktime_t *offs_real,
6058 + return base;
6059 + }
6060 +
6061 ++/**
6062 ++ * random_get_entropy_fallback - Returns the raw clock source value,
6063 ++ * used by random.c for platforms with no valid random_get_entropy().
6064 ++ */
6065 ++unsigned long random_get_entropy_fallback(void)
6066 ++{
6067 ++ struct tk_read_base *tkr = &tk_core.timekeeper.tkr_mono;
6068 ++ struct clocksource *clock = READ_ONCE(tkr->clock);
6069 ++
6070 ++ if (unlikely(timekeeping_suspended || !clock))
6071 ++ return 0;
6072 ++ return clock->read(clock);
6073 ++}
6074 ++EXPORT_SYMBOL_GPL(random_get_entropy_fallback);
6075 ++
6076 + /**
6077 + * do_adjtimex() - Accessor function to NTP __do_adjtimex function
6078 + */
6079 +diff --git a/lib/Kconfig.debug b/lib/Kconfig.debug
6080 +index f63a4faf244e8..a152e907e813b 100644
6081 +--- a/lib/Kconfig.debug
6082 ++++ b/lib/Kconfig.debug
6083 +@@ -1267,9 +1267,8 @@ config WARN_ALL_UNSEEDED_RANDOM
6084 + time. This is really bad from a security perspective, and
6085 + so architecture maintainers really need to do what they can
6086 + to get the CRNG seeded sooner after the system is booted.
6087 +- However, since users can not do anything actionble to
6088 +- address this, by default the kernel will issue only a single
6089 +- warning for the first use of unseeded randomness.
6090 ++ However, since users cannot do anything actionable to
6091 ++ address this, by default this option is disabled.
6092 +
6093 + Say Y here if you want to receive warnings for all uses of
6094 + unseeded randomness. This will be of use primarily for
6095 +diff --git a/lib/Makefile b/lib/Makefile
6096 +index ae2d261daec9a..4b74298a3aec3 100644
6097 +--- a/lib/Makefile
6098 ++++ b/lib/Makefile
6099 +@@ -262,3 +262,5 @@ CFLAGS_ubsan.o := $(call cc-option, -fno-stack-protector) $(DISABLE_STACKLEAK_PL
6100 + obj-$(CONFIG_SBITMAP) += sbitmap.o
6101 +
6102 + obj-$(CONFIG_PARMAN) += parman.o
6103 ++
6104 ++obj-y += crypto/
6105 +diff --git a/lib/chacha20.c b/lib/chacha20.c
6106 +index 250ceed9ec9a8..5f6c222e939a9 100644
6107 +--- a/lib/chacha20.c
6108 ++++ b/lib/chacha20.c
6109 +@@ -21,9 +21,9 @@ static inline u32 rotl32(u32 v, u8 n)
6110 + return (v << n) | (v >> (sizeof(v) * 8 - n));
6111 + }
6112 +
6113 +-extern void chacha20_block(u32 *state, void *stream)
6114 ++void chacha20_block(u32 *state, u8 *stream)
6115 + {
6116 +- u32 x[16], *out = stream;
6117 ++ u32 x[16];
6118 + int i;
6119 +
6120 + for (i = 0; i < ARRAY_SIZE(x); i++)
6121 +@@ -72,7 +72,7 @@ extern void chacha20_block(u32 *state, void *stream)
6122 + }
6123 +
6124 + for (i = 0; i < ARRAY_SIZE(x); i++)
6125 +- out[i] = cpu_to_le32(x[i] + state[i]);
6126 ++ put_unaligned_le32(x[i] + state[i], &stream[i * sizeof(u32)]);
6127 +
6128 + state[12]++;
6129 + }
6130 +diff --git a/lib/crypto/Makefile b/lib/crypto/Makefile
6131 +new file mode 100644
6132 +index 0000000000000..d0bca68618f03
6133 +--- /dev/null
6134 ++++ b/lib/crypto/Makefile
6135 +@@ -0,0 +1,7 @@
6136 ++# SPDX-License-Identifier: GPL-2.0
6137 ++
6138 ++obj-y += libblake2s.o
6139 ++libblake2s-y += blake2s.o blake2s-generic.o
6140 ++ifneq ($(CONFIG_CRYPTO_MANAGER_DISABLE_TESTS),y)
6141 ++libblake2s-y += blake2s-selftest.o
6142 ++endif
6143 +diff --git a/lib/crypto/blake2s-generic.c b/lib/crypto/blake2s-generic.c
6144 +new file mode 100644
6145 +index 0000000000000..04ff8df245136
6146 +--- /dev/null
6147 ++++ b/lib/crypto/blake2s-generic.c
6148 +@@ -0,0 +1,111 @@
6149 ++// SPDX-License-Identifier: GPL-2.0 OR MIT
6150 ++/*
6151 ++ * Copyright (C) 2015-2019 Jason A. Donenfeld <Jason@×××××.com>. All Rights Reserved.
6152 ++ *
6153 ++ * This is an implementation of the BLAKE2s hash and PRF functions.
6154 ++ *
6155 ++ * Information: https://blake2.net/
6156 ++ *
6157 ++ */
6158 ++
6159 ++#include <crypto/internal/blake2s.h>
6160 ++#include <linux/types.h>
6161 ++#include <linux/string.h>
6162 ++#include <linux/kernel.h>
6163 ++#include <linux/module.h>
6164 ++#include <linux/init.h>
6165 ++#include <linux/bug.h>
6166 ++#include <asm/unaligned.h>
6167 ++
6168 ++static const u8 blake2s_sigma[10][16] = {
6169 ++ { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 },
6170 ++ { 14, 10, 4, 8, 9, 15, 13, 6, 1, 12, 0, 2, 11, 7, 5, 3 },
6171 ++ { 11, 8, 12, 0, 5, 2, 15, 13, 10, 14, 3, 6, 7, 1, 9, 4 },
6172 ++ { 7, 9, 3, 1, 13, 12, 11, 14, 2, 6, 5, 10, 4, 0, 15, 8 },
6173 ++ { 9, 0, 5, 7, 2, 4, 10, 15, 14, 1, 11, 12, 6, 8, 3, 13 },
6174 ++ { 2, 12, 6, 10, 0, 11, 8, 3, 4, 13, 7, 5, 15, 14, 1, 9 },
6175 ++ { 12, 5, 1, 15, 14, 13, 4, 10, 0, 7, 6, 3, 9, 2, 8, 11 },
6176 ++ { 13, 11, 7, 14, 12, 1, 3, 9, 5, 0, 15, 4, 8, 6, 2, 10 },
6177 ++ { 6, 15, 14, 9, 11, 3, 0, 8, 12, 2, 13, 7, 1, 4, 10, 5 },
6178 ++ { 10, 2, 8, 4, 7, 6, 1, 5, 15, 11, 9, 14, 3, 12, 13, 0 },
6179 ++};
6180 ++
6181 ++static inline void blake2s_increment_counter(struct blake2s_state *state,
6182 ++ const u32 inc)
6183 ++{
6184 ++ state->t[0] += inc;
6185 ++ state->t[1] += (state->t[0] < inc);
6186 ++}
6187 ++
6188 ++void blake2s_compress_generic(struct blake2s_state *state,const u8 *block,
6189 ++ size_t nblocks, const u32 inc)
6190 ++{
6191 ++ u32 m[16];
6192 ++ u32 v[16];
6193 ++ int i;
6194 ++
6195 ++ WARN_ON(IS_ENABLED(DEBUG) &&
6196 ++ (nblocks > 1 && inc != BLAKE2S_BLOCK_SIZE));
6197 ++
6198 ++ while (nblocks > 0) {
6199 ++ blake2s_increment_counter(state, inc);
6200 ++ memcpy(m, block, BLAKE2S_BLOCK_SIZE);
6201 ++ le32_to_cpu_array(m, ARRAY_SIZE(m));
6202 ++ memcpy(v, state->h, 32);
6203 ++ v[ 8] = BLAKE2S_IV0;
6204 ++ v[ 9] = BLAKE2S_IV1;
6205 ++ v[10] = BLAKE2S_IV2;
6206 ++ v[11] = BLAKE2S_IV3;
6207 ++ v[12] = BLAKE2S_IV4 ^ state->t[0];
6208 ++ v[13] = BLAKE2S_IV5 ^ state->t[1];
6209 ++ v[14] = BLAKE2S_IV6 ^ state->f[0];
6210 ++ v[15] = BLAKE2S_IV7 ^ state->f[1];
6211 ++
6212 ++#define G(r, i, a, b, c, d) do { \
6213 ++ a += b + m[blake2s_sigma[r][2 * i + 0]]; \
6214 ++ d = ror32(d ^ a, 16); \
6215 ++ c += d; \
6216 ++ b = ror32(b ^ c, 12); \
6217 ++ a += b + m[blake2s_sigma[r][2 * i + 1]]; \
6218 ++ d = ror32(d ^ a, 8); \
6219 ++ c += d; \
6220 ++ b = ror32(b ^ c, 7); \
6221 ++} while (0)
6222 ++
6223 ++#define ROUND(r) do { \
6224 ++ G(r, 0, v[0], v[ 4], v[ 8], v[12]); \
6225 ++ G(r, 1, v[1], v[ 5], v[ 9], v[13]); \
6226 ++ G(r, 2, v[2], v[ 6], v[10], v[14]); \
6227 ++ G(r, 3, v[3], v[ 7], v[11], v[15]); \
6228 ++ G(r, 4, v[0], v[ 5], v[10], v[15]); \
6229 ++ G(r, 5, v[1], v[ 6], v[11], v[12]); \
6230 ++ G(r, 6, v[2], v[ 7], v[ 8], v[13]); \
6231 ++ G(r, 7, v[3], v[ 4], v[ 9], v[14]); \
6232 ++} while (0)
6233 ++ ROUND(0);
6234 ++ ROUND(1);
6235 ++ ROUND(2);
6236 ++ ROUND(3);
6237 ++ ROUND(4);
6238 ++ ROUND(5);
6239 ++ ROUND(6);
6240 ++ ROUND(7);
6241 ++ ROUND(8);
6242 ++ ROUND(9);
6243 ++
6244 ++#undef G
6245 ++#undef ROUND
6246 ++
6247 ++ for (i = 0; i < 8; ++i)
6248 ++ state->h[i] ^= v[i] ^ v[i + 8];
6249 ++
6250 ++ block += BLAKE2S_BLOCK_SIZE;
6251 ++ --nblocks;
6252 ++ }
6253 ++}
6254 ++
6255 ++EXPORT_SYMBOL(blake2s_compress_generic);
6256 ++
6257 ++MODULE_LICENSE("GPL v2");
6258 ++MODULE_DESCRIPTION("BLAKE2s hash function");
6259 ++MODULE_AUTHOR("Jason A. Donenfeld <Jason@×××××.com>");
6260 +diff --git a/lib/crypto/blake2s-selftest.c b/lib/crypto/blake2s-selftest.c
6261 +new file mode 100644
6262 +index 0000000000000..7a9edc96ddddf
6263 +--- /dev/null
6264 ++++ b/lib/crypto/blake2s-selftest.c
6265 +@@ -0,0 +1,591 @@
6266 ++// SPDX-License-Identifier: GPL-2.0 OR MIT
6267 ++/*
6268 ++ * Copyright (C) 2015-2019 Jason A. Donenfeld <Jason@×××××.com>. All Rights Reserved.
6269 ++ */
6270 ++
6271 ++#include <crypto/blake2s.h>
6272 ++#include <linux/string.h>
6273 ++
6274 ++/*
6275 ++ * blake2s_testvecs[] generated with the program below (using libb2-dev and
6276 ++ * libssl-dev [OpenSSL])
6277 ++ *
6278 ++ * #include <blake2.h>
6279 ++ * #include <stdint.h>
6280 ++ * #include <stdio.h>
6281 ++ *
6282 ++ * #include <openssl/evp.h>
6283 ++ *
6284 ++ * #define BLAKE2S_TESTVEC_COUNT 256
6285 ++ *
6286 ++ * static void print_vec(const uint8_t vec[], int len)
6287 ++ * {
6288 ++ * int i;
6289 ++ *
6290 ++ * printf(" { ");
6291 ++ * for (i = 0; i < len; i++) {
6292 ++ * if (i && (i % 12) == 0)
6293 ++ * printf("\n ");
6294 ++ * printf("0x%02x, ", vec[i]);
6295 ++ * }
6296 ++ * printf("},\n");
6297 ++ * }
6298 ++ *
6299 ++ * int main(void)
6300 ++ * {
6301 ++ * uint8_t key[BLAKE2S_KEYBYTES];
6302 ++ * uint8_t buf[BLAKE2S_TESTVEC_COUNT];
6303 ++ * uint8_t hash[BLAKE2S_OUTBYTES];
6304 ++ * int i, j;
6305 ++ *
6306 ++ * key[0] = key[1] = 1;
6307 ++ * for (i = 2; i < BLAKE2S_KEYBYTES; ++i)
6308 ++ * key[i] = key[i - 2] + key[i - 1];
6309 ++ *
6310 ++ * for (i = 0; i < BLAKE2S_TESTVEC_COUNT; ++i)
6311 ++ * buf[i] = (uint8_t)i;
6312 ++ *
6313 ++ * printf("static const u8 blake2s_testvecs[][BLAKE2S_HASH_SIZE] __initconst = {\n");
6314 ++ *
6315 ++ * for (i = 0; i < BLAKE2S_TESTVEC_COUNT; ++i) {
6316 ++ * int outlen = 1 + i % BLAKE2S_OUTBYTES;
6317 ++ * int keylen = (13 * i) % (BLAKE2S_KEYBYTES + 1);
6318 ++ *
6319 ++ * blake2s(hash, buf, key + BLAKE2S_KEYBYTES - keylen, outlen, i,
6320 ++ * keylen);
6321 ++ * print_vec(hash, outlen);
6322 ++ * }
6323 ++ * printf("};\n\n");
6324 ++ *
6325 ++ * return 0;
6326 ++ *}
6327 ++ */
6328 ++static const u8 blake2s_testvecs[][BLAKE2S_HASH_SIZE] __initconst = {
6329 ++ { 0xa1, },
6330 ++ { 0x7c, 0x89, },
6331 ++ { 0x74, 0x0e, 0xd4, },
6332 ++ { 0x47, 0x0c, 0x21, 0x15, },
6333 ++ { 0x18, 0xd6, 0x9c, 0xa6, 0xc4, },
6334 ++ { 0x13, 0x5d, 0x16, 0x63, 0x2e, 0xf9, },
6335 ++ { 0x2c, 0xb5, 0x04, 0xb7, 0x99, 0xe2, 0x73, },
6336 ++ { 0x9a, 0x0f, 0xd2, 0x39, 0xd6, 0x68, 0x1b, 0x92, },
6337 ++ { 0xc8, 0xde, 0x7a, 0xea, 0x2f, 0xf4, 0xd2, 0xe3, 0x2b, },
6338 ++ { 0x5b, 0xf9, 0x43, 0x52, 0x0c, 0x12, 0xba, 0xb5, 0x93, 0x9f, },
6339 ++ { 0xc6, 0x2c, 0x4e, 0x80, 0xfc, 0x32, 0x5b, 0x33, 0xb8, 0xb8, 0x0a, },
6340 ++ { 0xa7, 0x5c, 0xfd, 0x3a, 0xcc, 0xbf, 0x90, 0xca, 0xb7, 0x97, 0xde, 0xd8, },
6341 ++ { 0x66, 0xca, 0x3c, 0xc4, 0x19, 0xef, 0x92, 0x66, 0x3f, 0x21, 0x8f, 0xda,
6342 ++ 0xb7, },
6343 ++ { 0xba, 0xe5, 0xbb, 0x30, 0x25, 0x94, 0x6d, 0xc3, 0x89, 0x09, 0xc4, 0x25,
6344 ++ 0x52, 0x3e, },
6345 ++ { 0xa2, 0xef, 0x0e, 0x52, 0x0b, 0x5f, 0xa2, 0x01, 0x6d, 0x0a, 0x25, 0xbc,
6346 ++ 0x57, 0xe2, 0x27, },
6347 ++ { 0x4f, 0xe0, 0xf9, 0x52, 0x12, 0xda, 0x84, 0xb7, 0xab, 0xae, 0xb0, 0xa6,
6348 ++ 0x47, 0x2a, 0xc7, 0xf5, },
6349 ++ { 0x56, 0xe7, 0xa8, 0x1c, 0x4c, 0xca, 0xed, 0x90, 0x31, 0xec, 0x87, 0x43,
6350 ++ 0xe7, 0x72, 0x08, 0xec, 0xbe, },
6351 ++ { 0x7e, 0xdf, 0x80, 0x1c, 0x93, 0x33, 0xfd, 0x53, 0x44, 0xba, 0xfd, 0x96,
6352 ++ 0xe1, 0xbb, 0xb5, 0x65, 0xa5, 0x00, },
6353 ++ { 0xec, 0x6b, 0xed, 0xf7, 0x7b, 0x62, 0x1d, 0x7d, 0xf4, 0x82, 0xf3, 0x1e,
6354 ++ 0x18, 0xff, 0x2b, 0xc4, 0x06, 0x20, 0x2a, },
6355 ++ { 0x74, 0x98, 0xd7, 0x68, 0x63, 0xed, 0x87, 0xe4, 0x5d, 0x8d, 0x9e, 0x1d,
6356 ++ 0xfd, 0x2a, 0xbb, 0x86, 0xac, 0xe9, 0x2a, 0x89, },
6357 ++ { 0x89, 0xc3, 0x88, 0xce, 0x2b, 0x33, 0x1e, 0x10, 0xd1, 0x37, 0x20, 0x86,
6358 ++ 0x28, 0x43, 0x70, 0xd9, 0xfb, 0x96, 0xd9, 0xb5, 0xd3, },
6359 ++ { 0xcb, 0x56, 0x74, 0x41, 0x8d, 0x80, 0x01, 0x9a, 0x6b, 0x38, 0xe1, 0x41,
6360 ++ 0xad, 0x9c, 0x62, 0x74, 0xce, 0x35, 0xd5, 0x6c, 0x89, 0x6e, },
6361 ++ { 0x79, 0xaf, 0x94, 0x59, 0x99, 0x26, 0xe1, 0xc9, 0x34, 0xfe, 0x7c, 0x22,
6362 ++ 0xf7, 0x43, 0xd7, 0x65, 0xd4, 0x48, 0x18, 0xac, 0x3d, 0xfd, 0x93, },
6363 ++ { 0x85, 0x0d, 0xff, 0xb8, 0x3e, 0x87, 0x41, 0xb0, 0x95, 0xd3, 0x3d, 0x00,
6364 ++ 0x47, 0x55, 0x9e, 0xd2, 0x69, 0xea, 0xbf, 0xe9, 0x7a, 0x2d, 0x61, 0x45, },
6365 ++ { 0x03, 0xe0, 0x85, 0xec, 0x54, 0xb5, 0x16, 0x53, 0xa8, 0xc4, 0x71, 0xe9,
6366 ++ 0x6a, 0xe7, 0xcb, 0xc4, 0x15, 0x02, 0xfc, 0x34, 0xa4, 0xa4, 0x28, 0x13,
6367 ++ 0xd1, },
6368 ++ { 0xe3, 0x34, 0x4b, 0xe1, 0xd0, 0x4b, 0x55, 0x61, 0x8f, 0xc0, 0x24, 0x05,
6369 ++ 0xe6, 0xe0, 0x3d, 0x70, 0x24, 0x4d, 0xda, 0xb8, 0x91, 0x05, 0x29, 0x07,
6370 ++ 0x01, 0x3e, },
6371 ++ { 0x61, 0xff, 0x01, 0x72, 0xb1, 0x4d, 0xf6, 0xfe, 0xd1, 0xd1, 0x08, 0x74,
6372 ++ 0xe6, 0x91, 0x44, 0xeb, 0x61, 0xda, 0x40, 0xaf, 0xfc, 0x8c, 0x91, 0x6b,
6373 ++ 0xec, 0x13, 0xed, },
6374 ++ { 0xd4, 0x40, 0xd2, 0xa0, 0x7f, 0xc1, 0x58, 0x0c, 0x85, 0xa0, 0x86, 0xc7,
6375 ++ 0x86, 0xb9, 0x61, 0xc9, 0xea, 0x19, 0x86, 0x1f, 0xab, 0x07, 0xce, 0x37,
6376 ++ 0x72, 0x67, 0x09, 0xfc, },
6377 ++ { 0x9e, 0xf8, 0x18, 0x67, 0x93, 0x10, 0x9b, 0x39, 0x75, 0xe8, 0x8b, 0x38,
6378 ++ 0x82, 0x7d, 0xb8, 0xb7, 0xa5, 0xaf, 0xe6, 0x6a, 0x22, 0x5e, 0x1f, 0x9c,
6379 ++ 0x95, 0x29, 0x19, 0xf2, 0x4b, },
6380 ++ { 0xc8, 0x62, 0x25, 0xf5, 0x98, 0xc9, 0xea, 0xe5, 0x29, 0x3a, 0xd3, 0x22,
6381 ++ 0xeb, 0xeb, 0x07, 0x7c, 0x15, 0x07, 0xee, 0x15, 0x61, 0xbb, 0x05, 0x30,
6382 ++ 0x99, 0x7f, 0x11, 0xf6, 0x0a, 0x1d, },
6383 ++ { 0x68, 0x70, 0xf7, 0x90, 0xa1, 0x8b, 0x1f, 0x0f, 0xbb, 0xce, 0xd2, 0x0e,
6384 ++ 0x33, 0x1f, 0x7f, 0xa9, 0x78, 0xa8, 0xa6, 0x81, 0x66, 0xab, 0x8d, 0xcd,
6385 ++ 0x58, 0x55, 0x3a, 0x0b, 0x7a, 0xdb, 0xb5, },
6386 ++ { 0xdd, 0x35, 0xd2, 0xb4, 0xf6, 0xc7, 0xea, 0xab, 0x64, 0x24, 0x4e, 0xfe,
6387 ++ 0xe5, 0x3d, 0x4e, 0x95, 0x8b, 0x6d, 0x6c, 0xbc, 0xb0, 0xf8, 0x88, 0x61,
6388 ++ 0x09, 0xb7, 0x78, 0xa3, 0x31, 0xfe, 0xd9, 0x2f, },
6389 ++ { 0x0a, },
6390 ++ { 0x6e, 0xd4, },
6391 ++ { 0x64, 0xe9, 0xd1, },
6392 ++ { 0x30, 0xdd, 0x71, 0xef, },
6393 ++ { 0x11, 0xb5, 0x0c, 0x87, 0xc9, },
6394 ++ { 0x06, 0x1c, 0x6d, 0x04, 0x82, 0xd0, },
6395 ++ { 0x5c, 0x42, 0x0b, 0xee, 0xc5, 0x9c, 0xb2, },
6396 ++ { 0xe8, 0x29, 0xd6, 0xb4, 0x5d, 0xf7, 0x2b, 0x93, },
6397 ++ { 0x18, 0xca, 0x27, 0x72, 0x43, 0x39, 0x16, 0xbc, 0x6a, },
6398 ++ { 0x39, 0x8f, 0xfd, 0x64, 0xf5, 0x57, 0x23, 0xb0, 0x45, 0xf8, },
6399 ++ { 0xbb, 0x3a, 0x78, 0x6b, 0x02, 0x1d, 0x0b, 0x16, 0xe3, 0xb2, 0x9a, },
6400 ++ { 0xb8, 0xb4, 0x0b, 0xe5, 0xd4, 0x1d, 0x0d, 0x85, 0x49, 0x91, 0x35, 0xfa, },
6401 ++ { 0x6d, 0x48, 0x2a, 0x0c, 0x42, 0x08, 0xbd, 0xa9, 0x78, 0x6f, 0x18, 0xaf,
6402 ++ 0xe2, },
6403 ++ { 0x10, 0x45, 0xd4, 0x58, 0x88, 0xec, 0x4e, 0x1e, 0xf6, 0x14, 0x92, 0x64,
6404 ++ 0x7e, 0xb0, },
6405 ++ { 0x8b, 0x0b, 0x95, 0xee, 0x92, 0xc6, 0x3b, 0x91, 0xf1, 0x1e, 0xeb, 0x51,
6406 ++ 0x98, 0x0a, 0x8d, },
6407 ++ { 0xa3, 0x50, 0x4d, 0xa5, 0x1d, 0x03, 0x68, 0xe9, 0x57, 0x78, 0xd6, 0x04,
6408 ++ 0xf1, 0xc3, 0x94, 0xd8, },
6409 ++ { 0xb8, 0x66, 0x6e, 0xdd, 0x46, 0x15, 0xae, 0x3d, 0x83, 0x7e, 0xcf, 0xe7,
6410 ++ 0x2c, 0xe8, 0x8f, 0xc7, 0x34, },
6411 ++ { 0x2e, 0xc0, 0x1f, 0x29, 0xea, 0xf6, 0xb9, 0xe2, 0xc2, 0x93, 0xeb, 0x41,
6412 ++ 0x0d, 0xf0, 0x0a, 0x13, 0x0e, 0xa2, },
6413 ++ { 0x71, 0xb8, 0x33, 0xa9, 0x1b, 0xac, 0xf1, 0xb5, 0x42, 0x8f, 0x5e, 0x81,
6414 ++ 0x34, 0x43, 0xb7, 0xa4, 0x18, 0x5c, 0x47, },
6415 ++ { 0xda, 0x45, 0xb8, 0x2e, 0x82, 0x1e, 0xc0, 0x59, 0x77, 0x9d, 0xfa, 0xb4,
6416 ++ 0x1c, 0x5e, 0xa0, 0x2b, 0x33, 0x96, 0x5a, 0x58, },
6417 ++ { 0xe3, 0x09, 0x05, 0xa9, 0xeb, 0x48, 0x13, 0xad, 0x71, 0x88, 0x81, 0x9a,
6418 ++ 0x3e, 0x2c, 0xe1, 0x23, 0x99, 0x13, 0x35, 0x9f, 0xb5, },
6419 ++ { 0xb7, 0x86, 0x2d, 0x16, 0xe1, 0x04, 0x00, 0x47, 0x47, 0x61, 0x31, 0xfb,
6420 ++ 0x14, 0xac, 0xd8, 0xe9, 0xe3, 0x49, 0xbd, 0xf7, 0x9c, 0x3f, },
6421 ++ { 0x7f, 0xd9, 0x95, 0xa8, 0xa7, 0xa0, 0xcc, 0xba, 0xef, 0xb1, 0x0a, 0xa9,
6422 ++ 0x21, 0x62, 0x08, 0x0f, 0x1b, 0xff, 0x7b, 0x9d, 0xae, 0xb2, 0x95, },
6423 ++ { 0x85, 0x99, 0xea, 0x33, 0xe0, 0x56, 0xff, 0x13, 0xc6, 0x61, 0x8c, 0xf9,
6424 ++ 0x57, 0x05, 0x03, 0x11, 0xf9, 0xfb, 0x3a, 0xf7, 0xce, 0xbb, 0x52, 0x30, },
6425 ++ { 0xb2, 0x72, 0x9c, 0xf8, 0x77, 0x4e, 0x8f, 0x6b, 0x01, 0x6c, 0xff, 0x4e,
6426 ++ 0x4f, 0x02, 0xd2, 0xbc, 0xeb, 0x51, 0x28, 0x99, 0x50, 0xab, 0xc4, 0x42,
6427 ++ 0xe3, },
6428 ++ { 0x8b, 0x0a, 0xb5, 0x90, 0x8f, 0xf5, 0x7b, 0xdd, 0xba, 0x47, 0x37, 0xc9,
6429 ++ 0x2a, 0xd5, 0x4b, 0x25, 0x08, 0x8b, 0x02, 0x17, 0xa7, 0x9e, 0x6b, 0x6e,
6430 ++ 0xe3, 0x90, },
6431 ++ { 0x90, 0xdd, 0xf7, 0x75, 0xa7, 0xa3, 0x99, 0x5e, 0x5b, 0x7d, 0x75, 0xc3,
6432 ++ 0x39, 0x6b, 0xa0, 0xe2, 0x44, 0x53, 0xb1, 0x9e, 0xc8, 0xf1, 0x77, 0x10,
6433 ++ 0x58, 0x06, 0x9a, },
6434 ++ { 0x99, 0x52, 0xf0, 0x49, 0xa8, 0x8c, 0xec, 0xa6, 0x97, 0x32, 0x13, 0xb5,
6435 ++ 0xf7, 0xa3, 0x8e, 0xfb, 0x4b, 0x59, 0x31, 0x3d, 0x01, 0x59, 0x98, 0x5d,
6436 ++ 0x53, 0x03, 0x1a, 0x39, },
6437 ++ { 0x9f, 0xe0, 0xc2, 0xe5, 0x5d, 0x93, 0xd6, 0x9b, 0x47, 0x8f, 0x9b, 0xe0,
6438 ++ 0x26, 0x35, 0x84, 0x20, 0x1d, 0xc5, 0x53, 0x10, 0x0f, 0x22, 0xb9, 0xb5,
6439 ++ 0xd4, 0x36, 0xb1, 0xac, 0x73, },
6440 ++ { 0x30, 0x32, 0x20, 0x3b, 0x10, 0x28, 0xec, 0x1f, 0x4f, 0x9b, 0x47, 0x59,
6441 ++ 0xeb, 0x7b, 0xee, 0x45, 0xfb, 0x0c, 0x49, 0xd8, 0x3d, 0x69, 0xbd, 0x90,
6442 ++ 0x2c, 0xf0, 0x9e, 0x8d, 0xbf, 0xd5, },
6443 ++ { 0x2a, 0x37, 0x73, 0x7f, 0xf9, 0x96, 0x19, 0xaa, 0x25, 0xd8, 0x13, 0x28,
6444 ++ 0x01, 0x29, 0x89, 0xdf, 0x6e, 0x0c, 0x9b, 0x43, 0x44, 0x51, 0xe9, 0x75,
6445 ++ 0x26, 0x0c, 0xb7, 0x87, 0x66, 0x0b, 0x5f, },
6446 ++ { 0x23, 0xdf, 0x96, 0x68, 0x91, 0x86, 0xd0, 0x93, 0x55, 0x33, 0x24, 0xf6,
6447 ++ 0xba, 0x08, 0x75, 0x5b, 0x59, 0x11, 0x69, 0xb8, 0xb9, 0xe5, 0x2c, 0x77,
6448 ++ 0x02, 0xf6, 0x47, 0xee, 0x81, 0xdd, 0xb9, 0x06, },
6449 ++ { 0x9d, },
6450 ++ { 0x9d, 0x7d, },
6451 ++ { 0xfd, 0xc3, 0xda, },
6452 ++ { 0xe8, 0x82, 0xcd, 0x21, },
6453 ++ { 0xc3, 0x1d, 0x42, 0x4c, 0x74, },
6454 ++ { 0xe9, 0xda, 0xf1, 0xa2, 0xe5, 0x7c, },
6455 ++ { 0x52, 0xb8, 0x6f, 0x81, 0x5c, 0x3a, 0x4c, },
6456 ++ { 0x5b, 0x39, 0x26, 0xfc, 0x92, 0x5e, 0xe0, 0x49, },
6457 ++ { 0x59, 0xe4, 0x7c, 0x93, 0x1c, 0xf9, 0x28, 0x93, 0xde, },
6458 ++ { 0xde, 0xdf, 0xb2, 0x43, 0x61, 0x0b, 0x86, 0x16, 0x4c, 0x2e, },
6459 ++ { 0x14, 0x8f, 0x75, 0x51, 0xaf, 0xb9, 0xee, 0x51, 0x5a, 0xae, 0x23, },
6460 ++ { 0x43, 0x5f, 0x50, 0xd5, 0x70, 0xb0, 0x5b, 0x87, 0xf5, 0xd9, 0xb3, 0x6d, },
6461 ++ { 0x66, 0x0a, 0x64, 0x93, 0x79, 0x71, 0x94, 0x40, 0xb7, 0x68, 0x2d, 0xd3,
6462 ++ 0x63, },
6463 ++ { 0x15, 0x00, 0xc4, 0x0c, 0x7d, 0x1b, 0x10, 0xa9, 0x73, 0x1b, 0x90, 0x6f,
6464 ++ 0xe6, 0xa9, },
6465 ++ { 0x34, 0x75, 0xf3, 0x86, 0x8f, 0x56, 0xcf, 0x2a, 0x0a, 0xf2, 0x62, 0x0a,
6466 ++ 0xf6, 0x0e, 0x20, },
6467 ++ { 0xb1, 0xde, 0xc9, 0xf5, 0xdb, 0xf3, 0x2f, 0x4c, 0xd6, 0x41, 0x7d, 0x39,
6468 ++ 0x18, 0x3e, 0xc7, 0xc3, },
6469 ++ { 0xc5, 0x89, 0xb2, 0xf8, 0xb8, 0xc0, 0xa3, 0xb9, 0x3b, 0x10, 0x6d, 0x7c,
6470 ++ 0x92, 0xfc, 0x7f, 0x34, 0x41, },
6471 ++ { 0xc4, 0xd8, 0xef, 0xba, 0xef, 0xd2, 0xaa, 0xc5, 0x6c, 0x8e, 0x3e, 0xbb,
6472 ++ 0x12, 0xfc, 0x0f, 0x72, 0xbf, 0x0f, },
6473 ++ { 0xdd, 0x91, 0xd1, 0x15, 0x9e, 0x7d, 0xf8, 0xc1, 0xb9, 0x14, 0x63, 0x96,
6474 ++ 0xb5, 0xcb, 0x83, 0x1d, 0x35, 0x1c, 0xec, },
6475 ++ { 0xa9, 0xf8, 0x52, 0xc9, 0x67, 0x76, 0x2b, 0xad, 0xfb, 0xd8, 0x3a, 0xa6,
6476 ++ 0x74, 0x02, 0xae, 0xb8, 0x25, 0x2c, 0x63, 0x49, },
6477 ++ { 0x77, 0x1f, 0x66, 0x70, 0xfd, 0x50, 0x29, 0xaa, 0xeb, 0xdc, 0xee, 0xba,
6478 ++ 0x75, 0x98, 0xdc, 0x93, 0x12, 0x3f, 0xdc, 0x7c, 0x38, },
6479 ++ { 0xe2, 0xe1, 0x89, 0x5c, 0x37, 0x38, 0x6a, 0xa3, 0x40, 0xac, 0x3f, 0xb0,
6480 ++ 0xca, 0xfc, 0xa7, 0xf3, 0xea, 0xf9, 0x0f, 0x5d, 0x8e, 0x39, },
6481 ++ { 0x0f, 0x67, 0xc8, 0x38, 0x01, 0xb1, 0xb7, 0xb8, 0xa2, 0xe7, 0x0a, 0x6d,
6482 ++ 0xd2, 0x63, 0x69, 0x9e, 0xcc, 0xf0, 0xf2, 0xbe, 0x9b, 0x98, 0xdd, },
6483 ++ { 0x13, 0xe1, 0x36, 0x30, 0xfe, 0xc6, 0x01, 0x8a, 0xa1, 0x63, 0x96, 0x59,
6484 ++ 0xc2, 0xa9, 0x68, 0x3f, 0x58, 0xd4, 0x19, 0x0c, 0x40, 0xf3, 0xde, 0x02, },
6485 ++ { 0xa3, 0x9e, 0xce, 0xda, 0x42, 0xee, 0x8c, 0x6c, 0x5a, 0x7d, 0xdc, 0x89,
6486 ++ 0x02, 0x77, 0xdd, 0xe7, 0x95, 0xbb, 0xff, 0x0d, 0xa4, 0xb5, 0x38, 0x1e,
6487 ++ 0xaf, },
6488 ++ { 0x9a, 0xf6, 0xb5, 0x9a, 0x4f, 0xa9, 0x4f, 0x2c, 0x35, 0x3c, 0x24, 0xdc,
6489 ++ 0x97, 0x6f, 0xd9, 0xa1, 0x7d, 0x1a, 0x85, 0x0b, 0xf5, 0xda, 0x2e, 0xe7,
6490 ++ 0xb1, 0x1d, },
6491 ++ { 0x84, 0x1e, 0x8e, 0x3d, 0x45, 0xa5, 0xf2, 0x27, 0xf3, 0x31, 0xfe, 0xb9,
6492 ++ 0xfb, 0xc5, 0x45, 0x99, 0x99, 0xdd, 0x93, 0x43, 0x02, 0xee, 0x58, 0xaf,
6493 ++ 0xee, 0x6a, 0xbe, },
6494 ++ { 0x07, 0x2f, 0xc0, 0xa2, 0x04, 0xc4, 0xab, 0x7c, 0x26, 0xbb, 0xa8, 0xd8,
6495 ++ 0xe3, 0x1c, 0x75, 0x15, 0x64, 0x5d, 0x02, 0x6a, 0xf0, 0x86, 0xe9, 0xcd,
6496 ++ 0x5c, 0xef, 0xa3, 0x25, },
6497 ++ { 0x2f, 0x3b, 0x1f, 0xb5, 0x91, 0x8f, 0x86, 0xe0, 0xdc, 0x31, 0x48, 0xb6,
6498 ++ 0xa1, 0x8c, 0xfd, 0x75, 0xbb, 0x7d, 0x3d, 0xc1, 0xf0, 0x10, 0x9a, 0xd8,
6499 ++ 0x4b, 0x0e, 0xe3, 0x94, 0x9f, },
6500 ++ { 0x29, 0xbb, 0x8f, 0x6c, 0xd1, 0xf2, 0xb6, 0xaf, 0xe5, 0xe3, 0x2d, 0xdc,
6501 ++ 0x6f, 0xa4, 0x53, 0x88, 0xd8, 0xcf, 0x4d, 0x45, 0x42, 0x62, 0xdb, 0xdf,
6502 ++ 0xf8, 0x45, 0xc2, 0x13, 0xec, 0x35, },
6503 ++ { 0x06, 0x3c, 0xe3, 0x2c, 0x15, 0xc6, 0x43, 0x03, 0x81, 0xfb, 0x08, 0x76,
6504 ++ 0x33, 0xcb, 0x02, 0xc1, 0xba, 0x33, 0xe5, 0xe0, 0xd1, 0x92, 0xa8, 0x46,
6505 ++ 0x28, 0x3f, 0x3e, 0x9d, 0x2c, 0x44, 0x54, },
6506 ++ { 0xea, 0xbb, 0x96, 0xf8, 0xd1, 0x8b, 0x04, 0x11, 0x40, 0x78, 0x42, 0x02,
6507 ++ 0x19, 0xd1, 0xbc, 0x65, 0x92, 0xd3, 0xc3, 0xd6, 0xd9, 0x19, 0xe7, 0xc3,
6508 ++ 0x40, 0x97, 0xbd, 0xd4, 0xed, 0xfa, 0x5e, 0x28, },
6509 ++ { 0x02, },
6510 ++ { 0x52, 0xa8, },
6511 ++ { 0x38, 0x25, 0x0d, },
6512 ++ { 0xe3, 0x04, 0xd4, 0x92, },
6513 ++ { 0x97, 0xdb, 0xf7, 0x81, 0xca, },
6514 ++ { 0x8a, 0x56, 0x9d, 0x62, 0x56, 0xcc, },
6515 ++ { 0xa1, 0x8e, 0x3c, 0x72, 0x8f, 0x63, 0x03, },
6516 ++ { 0xf7, 0xf3, 0x39, 0x09, 0x0a, 0xa1, 0xbb, 0x23, },
6517 ++ { 0x6b, 0x03, 0xc0, 0xe9, 0xd9, 0x83, 0x05, 0x22, 0x01, },
6518 ++ { 0x1b, 0x4b, 0xf5, 0xd6, 0x4f, 0x05, 0x75, 0x91, 0x4c, 0x7f, },
6519 ++ { 0x4c, 0x8c, 0x25, 0x20, 0x21, 0xcb, 0xc2, 0x4b, 0x3a, 0x5b, 0x8d, },
6520 ++ { 0x56, 0xe2, 0x77, 0xa0, 0xb6, 0x9f, 0x81, 0xec, 0x83, 0x75, 0xc4, 0xf9, },
6521 ++ { 0x71, 0x70, 0x0f, 0xad, 0x4d, 0x35, 0x81, 0x9d, 0x88, 0x69, 0xf9, 0xaa,
6522 ++ 0xd3, },
6523 ++ { 0x50, 0x6e, 0x86, 0x6e, 0x43, 0xc0, 0xc2, 0x44, 0xc2, 0xe2, 0xa0, 0x1c,
6524 ++ 0xb7, 0x9a, },
6525 ++ { 0xe4, 0x7e, 0x72, 0xc6, 0x12, 0x8e, 0x7c, 0xfc, 0xbd, 0xe2, 0x08, 0x31,
6526 ++ 0x3d, 0x47, 0x3d, },
6527 ++ { 0x08, 0x97, 0x5b, 0x80, 0xae, 0xc4, 0x1d, 0x50, 0x77, 0xdf, 0x1f, 0xd0,
6528 ++ 0x24, 0xf0, 0x17, 0xc0, },
6529 ++ { 0x01, 0xb6, 0x29, 0xf4, 0xaf, 0x78, 0x5f, 0xb6, 0x91, 0xdd, 0x76, 0x76,
6530 ++ 0xd2, 0xfd, 0x0c, 0x47, 0x40, },
6531 ++ { 0xa1, 0xd8, 0x09, 0x97, 0x7a, 0xa6, 0xc8, 0x94, 0xf6, 0x91, 0x7b, 0xae,
6532 ++ 0x2b, 0x9f, 0x0d, 0x83, 0x48, 0xf7, },
6533 ++ { 0x12, 0xd5, 0x53, 0x7d, 0x9a, 0xb0, 0xbe, 0xd9, 0xed, 0xe9, 0x9e, 0xee,
6534 ++ 0x61, 0x5b, 0x42, 0xf2, 0xc0, 0x73, 0xc0, },
6535 ++ { 0xd5, 0x77, 0xd6, 0x5c, 0x6e, 0xa5, 0x69, 0x2b, 0x3b, 0x8c, 0xd6, 0x7d,
6536 ++ 0x1d, 0xbe, 0x2c, 0xa1, 0x02, 0x21, 0xcd, 0x29, },
6537 ++ { 0xa4, 0x98, 0x80, 0xca, 0x22, 0xcf, 0x6a, 0xab, 0x5e, 0x40, 0x0d, 0x61,
6538 ++ 0x08, 0x21, 0xef, 0xc0, 0x6c, 0x52, 0xb4, 0xb0, 0x53, },
6539 ++ { 0xbf, 0xaf, 0x8f, 0x3b, 0x7a, 0x97, 0x33, 0xe5, 0xca, 0x07, 0x37, 0xfd,
6540 ++ 0x15, 0xdf, 0xce, 0x26, 0x2a, 0xb1, 0xa7, 0x0b, 0xb3, 0xac, },
6541 ++ { 0x16, 0x22, 0xe1, 0xbc, 0x99, 0x4e, 0x01, 0xf0, 0xfa, 0xff, 0x8f, 0xa5,
6542 ++ 0x0c, 0x61, 0xb0, 0xad, 0xcc, 0xb1, 0xe1, 0x21, 0x46, 0xfa, 0x2e, },
6543 ++ { 0x11, 0x5b, 0x0b, 0x2b, 0xe6, 0x14, 0xc1, 0xd5, 0x4d, 0x71, 0x5e, 0x17,
6544 ++ 0xea, 0x23, 0xdd, 0x6c, 0xbd, 0x1d, 0xbe, 0x12, 0x1b, 0xee, 0x4c, 0x1a, },
6545 ++ { 0x40, 0x88, 0x22, 0xf3, 0x20, 0x6c, 0xed, 0xe1, 0x36, 0x34, 0x62, 0x2c,
6546 ++ 0x98, 0x83, 0x52, 0xe2, 0x25, 0xee, 0xe9, 0xf5, 0xe1, 0x17, 0xf0, 0x5c,
6547 ++ 0xae, },
6548 ++ { 0xc3, 0x76, 0x37, 0xde, 0x95, 0x8c, 0xca, 0x2b, 0x0c, 0x23, 0xe7, 0xb5,
6549 ++ 0x38, 0x70, 0x61, 0xcc, 0xff, 0xd3, 0x95, 0x7b, 0xf3, 0xff, 0x1f, 0x9d,
6550 ++ 0x59, 0x00, },
6551 ++ { 0x0c, 0x19, 0x52, 0x05, 0x22, 0x53, 0xcb, 0x48, 0xd7, 0x10, 0x0e, 0x7e,
6552 ++ 0x14, 0x69, 0xb5, 0xa2, 0x92, 0x43, 0xa3, 0x9e, 0x4b, 0x8f, 0x51, 0x2c,
6553 ++ 0x5a, 0x2c, 0x3b, },
6554 ++ { 0xe1, 0x9d, 0x70, 0x70, 0x28, 0xec, 0x86, 0x40, 0x55, 0x33, 0x56, 0xda,
6555 ++ 0x88, 0xca, 0xee, 0xc8, 0x6a, 0x20, 0xb1, 0xe5, 0x3d, 0x57, 0xf8, 0x3c,
6556 ++ 0x10, 0x07, 0x2a, 0xc4, },
6557 ++ { 0x0b, 0xae, 0xf1, 0xc4, 0x79, 0xee, 0x1b, 0x3d, 0x27, 0x35, 0x8d, 0x14,
6558 ++ 0xd6, 0xae, 0x4e, 0x3c, 0xe9, 0x53, 0x50, 0xb5, 0xcc, 0x0c, 0xf7, 0xdf,
6559 ++ 0xee, 0xa1, 0x74, 0xd6, 0x71, },
6560 ++ { 0xe6, 0xa4, 0xf4, 0x99, 0x98, 0xb9, 0x80, 0xea, 0x96, 0x7f, 0x4f, 0x33,
6561 ++ 0xcf, 0x74, 0x25, 0x6f, 0x17, 0x6c, 0xbf, 0xf5, 0x5c, 0x38, 0xd0, 0xff,
6562 ++ 0x96, 0xcb, 0x13, 0xf9, 0xdf, 0xfd, },
6563 ++ { 0xbe, 0x92, 0xeb, 0xba, 0x44, 0x2c, 0x24, 0x74, 0xd4, 0x03, 0x27, 0x3c,
6564 ++ 0x5d, 0x5b, 0x03, 0x30, 0x87, 0x63, 0x69, 0xe0, 0xb8, 0x94, 0xf4, 0x44,
6565 ++ 0x7e, 0xad, 0xcd, 0x20, 0x12, 0x16, 0x79, },
6566 ++ { 0x30, 0xf1, 0xc4, 0x8e, 0x05, 0x90, 0x2a, 0x97, 0x63, 0x94, 0x46, 0xff,
6567 ++ 0xce, 0xd8, 0x67, 0xa7, 0xac, 0x33, 0x8c, 0x95, 0xb7, 0xcd, 0xa3, 0x23,
6568 ++ 0x98, 0x9d, 0x76, 0x6c, 0x9d, 0xa8, 0xd6, 0x8a, },
6569 ++ { 0xbe, },
6570 ++ { 0x17, 0x6c, },
6571 ++ { 0x1a, 0x42, 0x4f, },
6572 ++ { 0xba, 0xaf, 0xb7, 0x65, },
6573 ++ { 0xc2, 0x63, 0x43, 0x6a, 0xea, },
6574 ++ { 0xe4, 0x4d, 0xad, 0xf2, 0x0b, 0x02, },
6575 ++ { 0x04, 0xc7, 0xc4, 0x7f, 0xa9, 0x2b, 0xce, },
6576 ++ { 0x66, 0xf6, 0x67, 0xcb, 0x03, 0x53, 0xc8, 0xf1, },
6577 ++ { 0x56, 0xa3, 0x60, 0x78, 0xc9, 0x5f, 0x70, 0x1b, 0x5e, },
6578 ++ { 0x99, 0xff, 0x81, 0x7c, 0x13, 0x3c, 0x29, 0x79, 0x4b, 0x65, },
6579 ++ { 0x51, 0x10, 0x50, 0x93, 0x01, 0x93, 0xb7, 0x01, 0xc9, 0x18, 0xb7, },
6580 ++ { 0x8e, 0x3c, 0x42, 0x1e, 0x5e, 0x7d, 0xc1, 0x50, 0x70, 0x1f, 0x00, 0x98, },
6581 ++ { 0x5f, 0xd9, 0x9b, 0xc8, 0xd7, 0xb2, 0x72, 0x62, 0x1a, 0x1e, 0xba, 0x92,
6582 ++ 0xe9, },
6583 ++ { 0x70, 0x2b, 0xba, 0xfe, 0xad, 0x5d, 0x96, 0x3f, 0x27, 0xc2, 0x41, 0x6d,
6584 ++ 0xc4, 0xb3, },
6585 ++ { 0xae, 0xe0, 0xd5, 0xd4, 0xc7, 0xae, 0x15, 0x5e, 0xdc, 0xdd, 0x33, 0x60,
6586 ++ 0xd7, 0xd3, 0x5e, },
6587 ++ { 0x79, 0x8e, 0xbc, 0x9e, 0x20, 0xb9, 0x19, 0x4b, 0x63, 0x80, 0xf3, 0x16,
6588 ++ 0xaf, 0x39, 0xbd, 0x92, },
6589 ++ { 0xc2, 0x0e, 0x85, 0xa0, 0x0b, 0x9a, 0xb0, 0xec, 0xde, 0x38, 0xd3, 0x10,
6590 ++ 0xd9, 0xa7, 0x66, 0x27, 0xcf, },
6591 ++ { 0x0e, 0x3b, 0x75, 0x80, 0x67, 0x14, 0x0c, 0x02, 0x90, 0xd6, 0xb3, 0x02,
6592 ++ 0x81, 0xf6, 0xa6, 0x87, 0xce, 0x58, },
6593 ++ { 0x79, 0xb5, 0xe9, 0x5d, 0x52, 0x4d, 0xf7, 0x59, 0xf4, 0x2e, 0x27, 0xdd,
6594 ++ 0xb3, 0xed, 0x57, 0x5b, 0x82, 0xea, 0x6f, },
6595 ++ { 0xa2, 0x97, 0xf5, 0x80, 0x02, 0x3d, 0xde, 0xa3, 0xf9, 0xf6, 0xab, 0xe3,
6596 ++ 0x57, 0x63, 0x7b, 0x9b, 0x10, 0x42, 0x6f, 0xf2, },
6597 ++ { 0x12, 0x7a, 0xfc, 0xb7, 0x67, 0x06, 0x0c, 0x78, 0x1a, 0xfe, 0x88, 0x4f,
6598 ++ 0xc6, 0xac, 0x52, 0x96, 0x64, 0x28, 0x97, 0x84, 0x06, },
6599 ++ { 0xc5, 0x04, 0x44, 0x6b, 0xb2, 0xa5, 0xa4, 0x66, 0xe1, 0x76, 0xa2, 0x51,
6600 ++ 0xf9, 0x59, 0x69, 0x97, 0x56, 0x0b, 0xbf, 0x50, 0xb3, 0x34, },
6601 ++ { 0x21, 0x32, 0x6b, 0x42, 0xb5, 0xed, 0x71, 0x8d, 0xf7, 0x5a, 0x35, 0xe3,
6602 ++ 0x90, 0xe2, 0xee, 0xaa, 0x89, 0xf6, 0xc9, 0x9c, 0x4d, 0x73, 0xf4, },
6603 ++ { 0x4c, 0xa6, 0x09, 0xf4, 0x48, 0xe7, 0x46, 0xbc, 0x49, 0xfc, 0xe5, 0xda,
6604 ++ 0xd1, 0x87, 0x13, 0x17, 0x4c, 0x59, 0x71, 0x26, 0x5b, 0x2c, 0x42, 0xb7, },
6605 ++ { 0x13, 0x63, 0xf3, 0x40, 0x02, 0xe5, 0xa3, 0x3a, 0x5e, 0x8e, 0xf8, 0xb6,
6606 ++ 0x8a, 0x49, 0x60, 0x76, 0x34, 0x72, 0x94, 0x73, 0xf6, 0xd9, 0x21, 0x6a,
6607 ++ 0x26, },
6608 ++ { 0xdf, 0x75, 0x16, 0x10, 0x1b, 0x5e, 0x81, 0xc3, 0xc8, 0xde, 0x34, 0x24,
6609 ++ 0xb0, 0x98, 0xeb, 0x1b, 0x8f, 0xa1, 0x9b, 0x05, 0xee, 0xa5, 0xe9, 0x35,
6610 ++ 0xf4, 0x1d, },
6611 ++ { 0xcd, 0x21, 0x93, 0x6e, 0x5b, 0xa0, 0x26, 0x2b, 0x21, 0x0e, 0xa0, 0xb9,
6612 ++ 0x1c, 0xb5, 0xbb, 0xb8, 0xf8, 0x1e, 0xff, 0x5c, 0xa8, 0xf9, 0x39, 0x46,
6613 ++ 0x4e, 0x29, 0x26, },
6614 ++ { 0x73, 0x7f, 0x0e, 0x3b, 0x0b, 0x5c, 0xf9, 0x60, 0xaa, 0x88, 0xa1, 0x09,
6615 ++ 0xb1, 0x5d, 0x38, 0x7b, 0x86, 0x8f, 0x13, 0x7a, 0x8d, 0x72, 0x7a, 0x98,
6616 ++ 0x1a, 0x5b, 0xff, 0xc9, },
6617 ++ { 0xd3, 0x3c, 0x61, 0x71, 0x44, 0x7e, 0x31, 0x74, 0x98, 0x9d, 0x9a, 0xd2,
6618 ++ 0x27, 0xf3, 0x46, 0x43, 0x42, 0x51, 0xd0, 0x5f, 0xe9, 0x1c, 0x5c, 0x69,
6619 ++ 0xbf, 0xf6, 0xbe, 0x3c, 0x40, },
6620 ++ { 0x31, 0x99, 0x31, 0x9f, 0xaa, 0x43, 0x2e, 0x77, 0x3e, 0x74, 0x26, 0x31,
6621 ++ 0x5e, 0x61, 0xf1, 0x87, 0xe2, 0xeb, 0x9b, 0xcd, 0xd0, 0x3a, 0xee, 0x20,
6622 ++ 0x7e, 0x10, 0x0a, 0x0b, 0x7e, 0xfa, },
6623 ++ { 0xa4, 0x27, 0x80, 0x67, 0x81, 0x2a, 0xa7, 0x62, 0xf7, 0x6e, 0xda, 0xd4,
6624 ++ 0x5c, 0x39, 0x74, 0xad, 0x7e, 0xbe, 0xad, 0xa5, 0x84, 0x7f, 0xa9, 0x30,
6625 ++ 0x5d, 0xdb, 0xe2, 0x05, 0x43, 0xf7, 0x1b, },
6626 ++ { 0x0b, 0x37, 0xd8, 0x02, 0xe1, 0x83, 0xd6, 0x80, 0xf2, 0x35, 0xc2, 0xb0,
6627 ++ 0x37, 0xef, 0xef, 0x5e, 0x43, 0x93, 0xf0, 0x49, 0x45, 0x0a, 0xef, 0xb5,
6628 ++ 0x76, 0x70, 0x12, 0x44, 0xc4, 0xdb, 0xf5, 0x7a, },
6629 ++ { 0x1f, },
6630 ++ { 0x82, 0x60, },
6631 ++ { 0xcc, 0xe3, 0x08, },
6632 ++ { 0x56, 0x17, 0xe4, 0x59, },
6633 ++ { 0xe2, 0xd7, 0x9e, 0xc4, 0x4c, },
6634 ++ { 0xb2, 0xad, 0xd3, 0x78, 0x58, 0x5a, },
6635 ++ { 0xce, 0x43, 0xb4, 0x02, 0x96, 0xab, 0x3c, },
6636 ++ { 0xe6, 0x05, 0x1a, 0x73, 0x22, 0x32, 0xbb, 0x77, },
6637 ++ { 0x23, 0xe7, 0xda, 0xfe, 0x2c, 0xef, 0x8c, 0x22, 0xec, },
6638 ++ { 0xe9, 0x8e, 0x55, 0x38, 0xd1, 0xd7, 0x35, 0x23, 0x98, 0xc7, },
6639 ++ { 0xb5, 0x81, 0x1a, 0xe5, 0xb5, 0xa5, 0xd9, 0x4d, 0xca, 0x41, 0xe7, },
6640 ++ { 0x41, 0x16, 0x16, 0x95, 0x8d, 0x9e, 0x0c, 0xea, 0x8c, 0x71, 0x9a, 0xc1, },
6641 ++ { 0x7c, 0x33, 0xc0, 0xa4, 0x00, 0x62, 0xea, 0x60, 0x67, 0xe4, 0x20, 0xbc,
6642 ++ 0x5b, },
6643 ++ { 0xdb, 0xb1, 0xdc, 0xfd, 0x08, 0xc0, 0xde, 0x82, 0xd1, 0xde, 0x38, 0xc0,
6644 ++ 0x90, 0x48, },
6645 ++ { 0x37, 0x18, 0x2e, 0x0d, 0x61, 0xaa, 0x61, 0xd7, 0x86, 0x20, 0x16, 0x60,
6646 ++ 0x04, 0xd9, 0xd5, },
6647 ++ { 0xb0, 0xcf, 0x2c, 0x4c, 0x5e, 0x5b, 0x4f, 0x2a, 0x23, 0x25, 0x58, 0x47,
6648 ++ 0xe5, 0x31, 0x06, 0x70, },
6649 ++ { 0x91, 0xa0, 0xa3, 0x86, 0x4e, 0xe0, 0x72, 0x38, 0x06, 0x67, 0x59, 0x5c,
6650 ++ 0x70, 0x25, 0xdb, 0x33, 0x27, },
6651 ++ { 0x44, 0x58, 0x66, 0xb8, 0x58, 0xc7, 0x13, 0xed, 0x4c, 0xc0, 0xf4, 0x9a,
6652 ++ 0x1e, 0x67, 0x75, 0x33, 0xb6, 0xb8, },
6653 ++ { 0x7f, 0x98, 0x4a, 0x8e, 0x50, 0xa2, 0x5c, 0xcd, 0x59, 0xde, 0x72, 0xb3,
6654 ++ 0x9d, 0xc3, 0x09, 0x8a, 0xab, 0x56, 0xf1, },
6655 ++ { 0x80, 0x96, 0x49, 0x1a, 0x59, 0xa2, 0xc5, 0xd5, 0xa7, 0x20, 0x8a, 0xb7,
6656 ++ 0x27, 0x62, 0x84, 0x43, 0xc6, 0xe1, 0x1b, 0x5d, },
6657 ++ { 0x6b, 0xb7, 0x2b, 0x26, 0x62, 0x14, 0x70, 0x19, 0x3d, 0x4d, 0xac, 0xac,
6658 ++ 0x63, 0x58, 0x5e, 0x94, 0xb5, 0xb7, 0xe8, 0xe8, 0xa2, },
6659 ++ { 0x20, 0xa8, 0xc0, 0xfd, 0x63, 0x3d, 0x6e, 0x98, 0xcf, 0x0c, 0x49, 0x98,
6660 ++ 0xe4, 0x5a, 0xfe, 0x8c, 0xaa, 0x70, 0x82, 0x1c, 0x7b, 0x74, },
6661 ++ { 0xc8, 0xe8, 0xdd, 0xdf, 0x69, 0x30, 0x01, 0xc2, 0x0f, 0x7e, 0x2f, 0x11,
6662 ++ 0xcc, 0x3e, 0x17, 0xa5, 0x69, 0x40, 0x3f, 0x0e, 0x79, 0x7f, 0xcf, },
6663 ++ { 0xdb, 0x61, 0xc0, 0xe2, 0x2e, 0x49, 0x07, 0x31, 0x1d, 0x91, 0x42, 0x8a,
6664 ++ 0xfc, 0x5e, 0xd3, 0xf8, 0x56, 0x1f, 0x2b, 0x73, 0xfd, 0x9f, 0xb2, 0x8e, },
6665 ++ { 0x0c, 0x89, 0x55, 0x0c, 0x1f, 0x59, 0x2c, 0x9d, 0x1b, 0x29, 0x1d, 0x41,
6666 ++ 0x1d, 0xe6, 0x47, 0x8f, 0x8c, 0x2b, 0xea, 0x8f, 0xf0, 0xff, 0x21, 0x70,
6667 ++ 0x88, },
6668 ++ { 0x12, 0x18, 0x95, 0xa6, 0x59, 0xb1, 0x31, 0x24, 0x45, 0x67, 0x55, 0xa4,
6669 ++ 0x1a, 0x2d, 0x48, 0x67, 0x1b, 0x43, 0x88, 0x2d, 0x8e, 0xa0, 0x70, 0xb3,
6670 ++ 0xc6, 0xbb, },
6671 ++ { 0xe7, 0xb1, 0x1d, 0xb2, 0x76, 0x4d, 0x68, 0x68, 0x68, 0x23, 0x02, 0x55,
6672 ++ 0x3a, 0xe2, 0xe5, 0xd5, 0x4b, 0x43, 0xf9, 0x34, 0x77, 0x5c, 0xa1, 0xf5,
6673 ++ 0x55, 0xfd, 0x4f, },
6674 ++ { 0x8c, 0x87, 0x5a, 0x08, 0x3a, 0x73, 0xad, 0x61, 0xe1, 0xe7, 0x99, 0x7e,
6675 ++ 0xf0, 0x5d, 0xe9, 0x5d, 0x16, 0x43, 0x80, 0x2f, 0xd0, 0x66, 0x34, 0xe2,
6676 ++ 0x42, 0x64, 0x3b, 0x1a, },
6677 ++ { 0x39, 0xc1, 0x99, 0xcf, 0x22, 0xbf, 0x16, 0x8f, 0x9f, 0x80, 0x7f, 0x95,
6678 ++ 0x0a, 0x05, 0x67, 0x27, 0xe7, 0x15, 0xdf, 0x9d, 0xb2, 0xfe, 0x1c, 0xb5,
6679 ++ 0x1d, 0x60, 0x8f, 0x8a, 0x1d, },
6680 ++ { 0x9b, 0x6e, 0x08, 0x09, 0x06, 0x73, 0xab, 0x68, 0x02, 0x62, 0x1a, 0xe4,
6681 ++ 0xd4, 0xdf, 0xc7, 0x02, 0x4c, 0x6a, 0x5f, 0xfd, 0x23, 0xac, 0xae, 0x6d,
6682 ++ 0x43, 0xa4, 0x7a, 0x50, 0x60, 0x3c, },
6683 ++ { 0x1d, 0xb4, 0xc6, 0xe1, 0xb1, 0x4b, 0xe3, 0xf2, 0xe2, 0x1a, 0x73, 0x1b,
6684 ++ 0xa0, 0x92, 0xa7, 0xf5, 0xff, 0x8f, 0x8b, 0x5d, 0xdf, 0xa8, 0x04, 0xb3,
6685 ++ 0xb0, 0xf7, 0xcc, 0x12, 0xfa, 0x35, 0x46, },
6686 ++ { 0x49, 0x45, 0x97, 0x11, 0x0f, 0x1c, 0x60, 0x8e, 0xe8, 0x47, 0x30, 0xcf,
6687 ++ 0x60, 0xa8, 0x71, 0xc5, 0x1b, 0xe9, 0x39, 0x4d, 0x49, 0xb6, 0x12, 0x1f,
6688 ++ 0x24, 0xab, 0x37, 0xff, 0x83, 0xc2, 0xe1, 0x3a, },
6689 ++ { 0x60, },
6690 ++ { 0x24, 0x26, },
6691 ++ { 0x47, 0xeb, 0xc9, },
6692 ++ { 0x4a, 0xd0, 0xbc, 0xf0, },
6693 ++ { 0x8e, 0x2b, 0xc9, 0x85, 0x3c, },
6694 ++ { 0xa2, 0x07, 0x15, 0xb8, 0x12, 0x74, },
6695 ++ { 0x0f, 0xdb, 0x5b, 0x33, 0x69, 0xfe, 0x4b, },
6696 ++ { 0xa2, 0x86, 0x54, 0xf4, 0xfd, 0xb2, 0xd4, 0xe6, },
6697 ++ { 0xbb, 0x84, 0x78, 0x49, 0x27, 0x8e, 0x61, 0xda, 0x60, },
6698 ++ { 0x04, 0xc3, 0xcd, 0xaa, 0x8f, 0xa7, 0x03, 0xc9, 0xf9, 0xb6, },
6699 ++ { 0xf8, 0x27, 0x1d, 0x61, 0xdc, 0x21, 0x42, 0xdd, 0xad, 0x92, 0x40, },
6700 ++ { 0x12, 0x87, 0xdf, 0xc2, 0x41, 0x45, 0x5a, 0x36, 0x48, 0x5b, 0x51, 0x2b, },
6701 ++ { 0xbb, 0x37, 0x5d, 0x1f, 0xf1, 0x68, 0x7a, 0xc4, 0xa5, 0xd2, 0xa4, 0x91,
6702 ++ 0x8d, },
6703 ++ { 0x5b, 0x27, 0xd1, 0x04, 0x54, 0x52, 0x9f, 0xa3, 0x47, 0x86, 0x33, 0x33,
6704 ++ 0xbf, 0xa0, },
6705 ++ { 0xcf, 0x04, 0xea, 0xf8, 0x03, 0x2a, 0x43, 0xff, 0xa6, 0x68, 0x21, 0x4c,
6706 ++ 0xd5, 0x4b, 0xed, },
6707 ++ { 0xaf, 0xb8, 0xbc, 0x63, 0x0f, 0x18, 0x4d, 0xe2, 0x7a, 0xdd, 0x46, 0x44,
6708 ++ 0xc8, 0x24, 0x0a, 0xb7, },
6709 ++ { 0x3e, 0xdc, 0x36, 0xe4, 0x89, 0xb1, 0xfa, 0xc6, 0x40, 0x93, 0x2e, 0x75,
6710 ++ 0xb2, 0x15, 0xd1, 0xb1, 0x10, },
6711 ++ { 0x6c, 0xd8, 0x20, 0x3b, 0x82, 0x79, 0xf9, 0xc8, 0xbc, 0x9d, 0xe0, 0x35,
6712 ++ 0xbe, 0x1b, 0x49, 0x1a, 0xbc, 0x3a, },
6713 ++ { 0x78, 0x65, 0x2c, 0xbe, 0x35, 0x67, 0xdc, 0x78, 0xd4, 0x41, 0xf6, 0xc9,
6714 ++ 0xde, 0xde, 0x1f, 0x18, 0x13, 0x31, 0x11, },
6715 ++ { 0x8a, 0x7f, 0xb1, 0x33, 0x8f, 0x0c, 0x3c, 0x0a, 0x06, 0x61, 0xf0, 0x47,
6716 ++ 0x29, 0x1b, 0x29, 0xbc, 0x1c, 0x47, 0xef, 0x7a, },
6717 ++ { 0x65, 0x91, 0xf1, 0xe6, 0xb3, 0x96, 0xd3, 0x8c, 0xc2, 0x4a, 0x59, 0x35,
6718 ++ 0x72, 0x8e, 0x0b, 0x9a, 0x87, 0xca, 0x34, 0x7b, 0x63, },
6719 ++ { 0x5f, 0x08, 0x87, 0x80, 0x56, 0x25, 0x89, 0x77, 0x61, 0x8c, 0x64, 0xa1,
6720 ++ 0x59, 0x6d, 0x59, 0x62, 0xe8, 0x4a, 0xc8, 0x58, 0x99, 0xd1, },
6721 ++ { 0x23, 0x87, 0x1d, 0xed, 0x6f, 0xf2, 0x91, 0x90, 0xe2, 0xfe, 0x43, 0x21,
6722 ++ 0xaf, 0x97, 0xc6, 0xbc, 0xd7, 0x15, 0xc7, 0x2d, 0x08, 0x77, 0x91, },
6723 ++ { 0x90, 0x47, 0x9a, 0x9e, 0x3a, 0xdf, 0xf3, 0xc9, 0x4c, 0x1e, 0xa7, 0xd4,
6724 ++ 0x6a, 0x32, 0x90, 0xfe, 0xb7, 0xb6, 0x7b, 0xfa, 0x96, 0x61, 0xfb, 0xa4, },
6725 ++ { 0xb1, 0x67, 0x60, 0x45, 0xb0, 0x96, 0xc5, 0x15, 0x9f, 0x4d, 0x26, 0xd7,
6726 ++ 0x9d, 0xf1, 0xf5, 0x6d, 0x21, 0x00, 0x94, 0x31, 0x64, 0x94, 0xd3, 0xa7,
6727 ++ 0xd3, },
6728 ++ { 0x02, 0x3e, 0xaf, 0xf3, 0x79, 0x73, 0xa5, 0xf5, 0xcc, 0x7a, 0x7f, 0xfb,
6729 ++ 0x79, 0x2b, 0x85, 0x8c, 0x88, 0x72, 0x06, 0xbe, 0xfe, 0xaf, 0xc1, 0x16,
6730 ++ 0xa6, 0xd6, },
6731 ++ { 0x2a, 0xb0, 0x1a, 0xe5, 0xaa, 0x6e, 0xb3, 0xae, 0x53, 0x85, 0x33, 0x80,
6732 ++ 0x75, 0xae, 0x30, 0xe6, 0xb8, 0x72, 0x42, 0xf6, 0x25, 0x4f, 0x38, 0x88,
6733 ++ 0x55, 0xd1, 0xa9, },
6734 ++ { 0x90, 0xd8, 0x0c, 0xc0, 0x93, 0x4b, 0x4f, 0x9e, 0x65, 0x6c, 0xa1, 0x54,
6735 ++ 0xa6, 0xf6, 0x6e, 0xca, 0xd2, 0xbb, 0x7e, 0x6a, 0x1c, 0xd3, 0xce, 0x46,
6736 ++ 0xef, 0xb0, 0x00, 0x8d, },
6737 ++ { 0xed, 0x9c, 0x49, 0xcd, 0xc2, 0xde, 0x38, 0x0e, 0xe9, 0x98, 0x6c, 0xc8,
6738 ++ 0x90, 0x9e, 0x3c, 0xd4, 0xd3, 0xeb, 0x88, 0x32, 0xc7, 0x28, 0xe3, 0x94,
6739 ++ 0x1c, 0x9f, 0x8b, 0xf3, 0xcb, },
6740 ++ { 0xac, 0xe7, 0x92, 0x16, 0xb4, 0x14, 0xa0, 0xe4, 0x04, 0x79, 0xa2, 0xf4,
6741 ++ 0x31, 0xe6, 0x0c, 0x26, 0xdc, 0xbf, 0x2f, 0x69, 0x1b, 0x55, 0x94, 0x67,
6742 ++ 0xda, 0x0c, 0xd7, 0x32, 0x1f, 0xef, },
6743 ++ { 0x68, 0x63, 0x85, 0x57, 0x95, 0x9e, 0x42, 0x27, 0x41, 0x43, 0x42, 0x02,
6744 ++ 0xa5, 0x78, 0xa7, 0xc6, 0x43, 0xc1, 0x6a, 0xba, 0x70, 0x80, 0xcd, 0x04,
6745 ++ 0xb6, 0x78, 0x76, 0x29, 0xf3, 0xe8, 0xa0, },
6746 ++ { 0xe6, 0xac, 0x8d, 0x9d, 0xf0, 0xc0, 0xf7, 0xf7, 0xe3, 0x3e, 0x4e, 0x28,
6747 ++ 0x0f, 0x59, 0xb2, 0x67, 0x9e, 0x84, 0x34, 0x42, 0x96, 0x30, 0x2b, 0xca,
6748 ++ 0x49, 0xb6, 0xc5, 0x9a, 0x84, 0x59, 0xa7, 0x81, },
6749 ++ { 0x7e, },
6750 ++ { 0x1e, 0x21, },
6751 ++ { 0x26, 0xd3, 0xdd, },
6752 ++ { 0x2c, 0xd4, 0xb3, 0x3d, },
6753 ++ { 0x86, 0x7b, 0x76, 0x3c, 0xf0, },
6754 ++ { 0x12, 0xc3, 0x70, 0x1d, 0x55, 0x18, },
6755 ++ { 0x96, 0xc2, 0xbd, 0x61, 0x55, 0xf4, 0x24, },
6756 ++ { 0x20, 0x51, 0xf7, 0x86, 0x58, 0x8f, 0x07, 0x2a, },
6757 ++ { 0x93, 0x15, 0xa8, 0x1d, 0xda, 0x97, 0xee, 0x0e, 0x6c, },
6758 ++ { 0x39, 0x93, 0xdf, 0xd5, 0x0e, 0xca, 0xdc, 0x7a, 0x92, 0xce, },
6759 ++ { 0x60, 0xd5, 0xfd, 0xf5, 0x1b, 0x26, 0x82, 0x26, 0x73, 0x02, 0xbc, },
6760 ++ { 0x98, 0xf2, 0x34, 0xe1, 0xf5, 0xfb, 0x00, 0xac, 0x10, 0x4a, 0x38, 0x9f, },
6761 ++ { 0xda, 0x3a, 0x92, 0x8a, 0xd0, 0xcd, 0x12, 0xcd, 0x15, 0xbb, 0xab, 0x77,
6762 ++ 0x66, },
6763 ++ { 0xa2, 0x92, 0x1a, 0xe5, 0xca, 0x0c, 0x30, 0x75, 0xeb, 0xaf, 0x00, 0x31,
6764 ++ 0x55, 0x66, },
6765 ++ { 0x06, 0xea, 0xfd, 0x3e, 0x86, 0x38, 0x62, 0x4e, 0xa9, 0x12, 0xa4, 0x12,
6766 ++ 0x43, 0xbf, 0xa1, },
6767 ++ { 0xe4, 0x71, 0x7b, 0x94, 0xdb, 0xa0, 0xd2, 0xff, 0x9b, 0xeb, 0xad, 0x8e,
6768 ++ 0x95, 0x8a, 0xc5, 0xed, },
6769 ++ { 0x25, 0x5a, 0x77, 0x71, 0x41, 0x0e, 0x7a, 0xe9, 0xed, 0x0c, 0x10, 0xef,
6770 ++ 0xf6, 0x2b, 0x3a, 0xba, 0x60, },
6771 ++ { 0xee, 0xe2, 0xa3, 0x67, 0x64, 0x1d, 0xc6, 0x04, 0xc4, 0xe1, 0x68, 0xd2,
6772 ++ 0x6e, 0xd2, 0x91, 0x75, 0x53, 0x07, },
6773 ++ { 0xe0, 0xf6, 0x4d, 0x8f, 0x68, 0xfc, 0x06, 0x7e, 0x18, 0x79, 0x7f, 0x2b,
6774 ++ 0x6d, 0xef, 0x46, 0x7f, 0xab, 0xb2, 0xad, },
6775 ++ { 0x3d, 0x35, 0x88, 0x9f, 0x2e, 0xcf, 0x96, 0x45, 0x07, 0x60, 0x71, 0x94,
6776 ++ 0x00, 0x8d, 0xbf, 0xf4, 0xef, 0x46, 0x2e, 0x3c, },
6777 ++ { 0x43, 0xcf, 0x98, 0xf7, 0x2d, 0xf4, 0x17, 0xe7, 0x8c, 0x05, 0x2d, 0x9b,
6778 ++ 0x24, 0xfb, 0x4d, 0xea, 0x4a, 0xec, 0x01, 0x25, 0x29, },
6779 ++ { 0x8e, 0x73, 0x9a, 0x78, 0x11, 0xfe, 0x48, 0xa0, 0x3b, 0x1a, 0x26, 0xdf,
6780 ++ 0x25, 0xe9, 0x59, 0x1c, 0x70, 0x07, 0x9f, 0xdc, 0xa0, 0xa6, },
6781 ++ { 0xe8, 0x47, 0x71, 0xc7, 0x3e, 0xdf, 0xb5, 0x13, 0xb9, 0x85, 0x13, 0xa8,
6782 ++ 0x54, 0x47, 0x6e, 0x59, 0x96, 0x09, 0x13, 0x5f, 0x82, 0x16, 0x0b, },
6783 ++ { 0xfb, 0xc0, 0x8c, 0x03, 0x21, 0xb3, 0xc4, 0xb5, 0x43, 0x32, 0x6c, 0xea,
6784 ++ 0x7f, 0xa8, 0x43, 0x91, 0xe8, 0x4e, 0x3f, 0xbf, 0x45, 0x58, 0x6a, 0xa3, },
6785 ++ { 0x55, 0xf8, 0xf3, 0x00, 0x76, 0x09, 0xef, 0x69, 0x5d, 0xd2, 0x8a, 0xf2,
6786 ++ 0x65, 0xc3, 0xcb, 0x9b, 0x43, 0xfd, 0xb1, 0x7e, 0x7f, 0xa1, 0x94, 0xb0,
6787 ++ 0xd7, },
6788 ++ { 0xaa, 0x13, 0xc1, 0x51, 0x40, 0x6d, 0x8d, 0x4c, 0x0a, 0x95, 0x64, 0x7b,
6789 ++ 0xd1, 0x96, 0xb6, 0x56, 0xb4, 0x5b, 0xcf, 0xd6, 0xd9, 0x15, 0x97, 0xdd,
6790 ++ 0xb6, 0xef, },
6791 ++ { 0xaf, 0xb7, 0x36, 0xb0, 0x04, 0xdb, 0xd7, 0x9c, 0x9a, 0x44, 0xc4, 0xf6,
6792 ++ 0x1f, 0x12, 0x21, 0x2d, 0x59, 0x30, 0x54, 0xab, 0x27, 0x61, 0xa3, 0x57,
6793 ++ 0xef, 0xf8, 0x53, },
6794 ++ { 0x97, 0x34, 0x45, 0x3e, 0xce, 0x7c, 0x35, 0xa2, 0xda, 0x9f, 0x4b, 0x46,
6795 ++ 0x6c, 0x11, 0x67, 0xff, 0x2f, 0x76, 0x58, 0x15, 0x71, 0xfa, 0x44, 0x89,
6796 ++ 0x89, 0xfd, 0xf7, 0x99, },
6797 ++ { 0x1f, 0xb1, 0x62, 0xeb, 0x83, 0xc5, 0x9c, 0x89, 0xf9, 0x2c, 0xd2, 0x03,
6798 ++ 0x61, 0xbc, 0xbb, 0xa5, 0x74, 0x0e, 0x9b, 0x7e, 0x82, 0x3e, 0x70, 0x0a,
6799 ++ 0xa9, 0x8f, 0x2b, 0x59, 0xfb, },
6800 ++ { 0xf8, 0xca, 0x5e, 0x3a, 0x4f, 0x9e, 0x10, 0x69, 0x10, 0xd5, 0x4c, 0xeb,
6801 ++ 0x1a, 0x0f, 0x3c, 0x6a, 0x98, 0xf5, 0xb0, 0x97, 0x5b, 0x37, 0x2f, 0x0d,
6802 ++ 0xbd, 0x42, 0x4b, 0x69, 0xa1, 0x82, },
6803 ++ { 0x12, 0x8c, 0x6d, 0x52, 0x08, 0xef, 0x74, 0xb2, 0xe6, 0xaa, 0xd3, 0xb0,
6804 ++ 0x26, 0xb0, 0xd9, 0x94, 0xb6, 0x11, 0x45, 0x0e, 0x36, 0x71, 0x14, 0x2d,
6805 ++ 0x41, 0x8c, 0x21, 0x53, 0x31, 0xe9, 0x68, },
6806 ++ { 0xee, 0xea, 0x0d, 0x89, 0x47, 0x7e, 0x72, 0xd1, 0xd8, 0xce, 0x58, 0x4c,
6807 ++ 0x94, 0x1f, 0x0d, 0x51, 0x08, 0xa3, 0xb6, 0x3d, 0xe7, 0x82, 0x46, 0x92,
6808 ++ 0xd6, 0x98, 0x6b, 0x07, 0x10, 0x65, 0x52, 0x65, },
6809 ++};
6810 ++
6811 ++bool __init blake2s_selftest(void)
6812 ++{
6813 ++ u8 key[BLAKE2S_KEY_SIZE];
6814 ++ u8 buf[ARRAY_SIZE(blake2s_testvecs)];
6815 ++ u8 hash[BLAKE2S_HASH_SIZE];
6816 ++ struct blake2s_state state;
6817 ++ bool success = true;
6818 ++ int i, l;
6819 ++
6820 ++ key[0] = key[1] = 1;
6821 ++ for (i = 2; i < sizeof(key); ++i)
6822 ++ key[i] = key[i - 2] + key[i - 1];
6823 ++
6824 ++ for (i = 0; i < sizeof(buf); ++i)
6825 ++ buf[i] = (u8)i;
6826 ++
6827 ++ for (i = l = 0; i < ARRAY_SIZE(blake2s_testvecs); l = (l + 37) % ++i) {
6828 ++ int outlen = 1 + i % BLAKE2S_HASH_SIZE;
6829 ++ int keylen = (13 * i) % (BLAKE2S_KEY_SIZE + 1);
6830 ++
6831 ++ blake2s(hash, buf, key + BLAKE2S_KEY_SIZE - keylen, outlen, i,
6832 ++ keylen);
6833 ++ if (memcmp(hash, blake2s_testvecs[i], outlen)) {
6834 ++ pr_err("blake2s self-test %d: FAIL\n", i + 1);
6835 ++ success = false;
6836 ++ }
6837 ++
6838 ++ if (!keylen)
6839 ++ blake2s_init(&state, outlen);
6840 ++ else
6841 ++ blake2s_init_key(&state, outlen,
6842 ++ key + BLAKE2S_KEY_SIZE - keylen,
6843 ++ keylen);
6844 ++
6845 ++ blake2s_update(&state, buf, l);
6846 ++ blake2s_update(&state, buf + l, i - l);
6847 ++ blake2s_final(&state, hash);
6848 ++ if (memcmp(hash, blake2s_testvecs[i], outlen)) {
6849 ++ pr_err("blake2s init/update/final self-test %d: FAIL\n",
6850 ++ i + 1);
6851 ++ success = false;
6852 ++ }
6853 ++ }
6854 ++
6855 ++ return success;
6856 ++}
6857 +diff --git a/lib/crypto/blake2s.c b/lib/crypto/blake2s.c
6858 +new file mode 100644
6859 +index 0000000000000..536fce87555b3
6860 +--- /dev/null
6861 ++++ b/lib/crypto/blake2s.c
6862 +@@ -0,0 +1,78 @@
6863 ++// SPDX-License-Identifier: GPL-2.0 OR MIT
6864 ++/*
6865 ++ * Copyright (C) 2015-2019 Jason A. Donenfeld <Jason@×××××.com>. All Rights Reserved.
6866 ++ *
6867 ++ * This is an implementation of the BLAKE2s hash and PRF functions.
6868 ++ *
6869 ++ * Information: https://blake2.net/
6870 ++ *
6871 ++ */
6872 ++
6873 ++#include <crypto/internal/blake2s.h>
6874 ++#include <linux/types.h>
6875 ++#include <linux/string.h>
6876 ++#include <linux/kernel.h>
6877 ++#include <linux/module.h>
6878 ++#include <linux/init.h>
6879 ++#include <linux/bug.h>
6880 ++#include <asm/unaligned.h>
6881 ++
6882 ++bool blake2s_selftest(void);
6883 ++
6884 ++void blake2s_update(struct blake2s_state *state, const u8 *in, size_t inlen)
6885 ++{
6886 ++ const size_t fill = BLAKE2S_BLOCK_SIZE - state->buflen;
6887 ++
6888 ++ if (unlikely(!inlen))
6889 ++ return;
6890 ++ if (inlen > fill) {
6891 ++ memcpy(state->buf + state->buflen, in, fill);
6892 ++ blake2s_compress_generic(state, state->buf, 1,
6893 ++ BLAKE2S_BLOCK_SIZE);
6894 ++ state->buflen = 0;
6895 ++ in += fill;
6896 ++ inlen -= fill;
6897 ++ }
6898 ++ if (inlen > BLAKE2S_BLOCK_SIZE) {
6899 ++ const size_t nblocks = DIV_ROUND_UP(inlen, BLAKE2S_BLOCK_SIZE);
6900 ++ /* Hash one less (full) block than strictly possible */
6901 ++ blake2s_compress_generic(state, in, nblocks - 1,
6902 ++ BLAKE2S_BLOCK_SIZE);
6903 ++ in += BLAKE2S_BLOCK_SIZE * (nblocks - 1);
6904 ++ inlen -= BLAKE2S_BLOCK_SIZE * (nblocks - 1);
6905 ++ }
6906 ++ memcpy(state->buf + state->buflen, in, inlen);
6907 ++ state->buflen += inlen;
6908 ++}
6909 ++EXPORT_SYMBOL(blake2s_update);
6910 ++
6911 ++void blake2s_final(struct blake2s_state *state, u8 *out)
6912 ++{
6913 ++ WARN_ON(IS_ENABLED(DEBUG) && !out);
6914 ++ blake2s_set_lastblock(state);
6915 ++ memset(state->buf + state->buflen, 0,
6916 ++ BLAKE2S_BLOCK_SIZE - state->buflen); /* Padding */
6917 ++ blake2s_compress_generic(state, state->buf, 1, state->buflen);
6918 ++ cpu_to_le32_array(state->h, ARRAY_SIZE(state->h));
6919 ++ memcpy(out, state->h, state->outlen);
6920 ++ memzero_explicit(state, sizeof(*state));
6921 ++}
6922 ++EXPORT_SYMBOL(blake2s_final);
6923 ++
6924 ++static int __init mod_init(void)
6925 ++{
6926 ++ if (!IS_ENABLED(CONFIG_CRYPTO_MANAGER_DISABLE_TESTS) &&
6927 ++ WARN_ON(!blake2s_selftest()))
6928 ++ return -ENODEV;
6929 ++ return 0;
6930 ++}
6931 ++
6932 ++static void __exit mod_exit(void)
6933 ++{
6934 ++}
6935 ++
6936 ++module_init(mod_init);
6937 ++module_exit(mod_exit);
6938 ++MODULE_LICENSE("GPL v2");
6939 ++MODULE_DESCRIPTION("BLAKE2s hash function");
6940 ++MODULE_AUTHOR("Jason A. Donenfeld <Jason@×××××.com>");
6941 +diff --git a/lib/random32.c b/lib/random32.c
6942 +index f5e967f4adfa7..357b1ae563ce1 100644
6943 +--- a/lib/random32.c
6944 ++++ b/lib/random32.c
6945 +@@ -38,6 +38,9 @@
6946 + #include <linux/jiffies.h>
6947 + #include <linux/random.h>
6948 + #include <linux/sched.h>
6949 ++#include <linux/bitops.h>
6950 ++#include <linux/slab.h>
6951 ++#include <linux/notifier.h>
6952 + #include <asm/unaligned.h>
6953 +
6954 + /**
6955 +@@ -544,9 +547,11 @@ static void prandom_reseed(unsigned long dontcare)
6956 + * To avoid worrying about whether it's safe to delay that interrupt
6957 + * long enough to seed all CPUs, just schedule an immediate timer event.
6958 + */
6959 +-static void prandom_timer_start(struct random_ready_callback *unused)
6960 ++static int prandom_timer_start(struct notifier_block *nb,
6961 ++ unsigned long action, void *data)
6962 + {
6963 + mod_timer(&seed_timer, jiffies);
6964 ++ return 0;
6965 + }
6966 +
6967 + /*
6968 +@@ -555,13 +560,13 @@ static void prandom_timer_start(struct random_ready_callback *unused)
6969 + */
6970 + static int __init prandom_init_late(void)
6971 + {
6972 +- static struct random_ready_callback random_ready = {
6973 +- .func = prandom_timer_start
6974 ++ static struct notifier_block random_ready = {
6975 ++ .notifier_call = prandom_timer_start
6976 + };
6977 +- int ret = add_random_ready_callback(&random_ready);
6978 ++ int ret = register_random_ready_notifier(&random_ready);
6979 +
6980 + if (ret == -EALREADY) {
6981 +- prandom_timer_start(&random_ready);
6982 ++ prandom_timer_start(&random_ready, 0, NULL);
6983 + ret = 0;
6984 + }
6985 + return ret;
6986 +diff --git a/lib/sha1.c b/lib/sha1.c
6987 +index 1d96d2c02b826..bad46695476bd 100644
6988 +--- a/lib/sha1.c
6989 ++++ b/lib/sha1.c
6990 +@@ -10,6 +10,7 @@
6991 + #include <linux/export.h>
6992 + #include <linux/bitops.h>
6993 + #include <linux/cryptohash.h>
6994 ++#include <linux/string.h>
6995 + #include <asm/unaligned.h>
6996 +
6997 + /*
6998 +@@ -55,7 +56,8 @@
6999 + #define SHA_ROUND(t, input, fn, constant, A, B, C, D, E) do { \
7000 + __u32 TEMP = input(t); setW(t, TEMP); \
7001 + E += TEMP + rol32(A,5) + (fn) + (constant); \
7002 +- B = ror32(B, 2); } while (0)
7003 ++ B = ror32(B, 2); \
7004 ++ TEMP = E; E = D; D = C; C = B; B = A; A = TEMP; } while (0)
7005 +
7006 + #define T_0_15(t, A, B, C, D, E) SHA_ROUND(t, SHA_SRC, (((C^D)&B)^D) , 0x5a827999, A, B, C, D, E )
7007 + #define T_16_19(t, A, B, C, D, E) SHA_ROUND(t, SHA_MIX, (((C^D)&B)^D) , 0x5a827999, A, B, C, D, E )
7008 +@@ -82,6 +84,7 @@
7009 + void sha_transform(__u32 *digest, const char *data, __u32 *array)
7010 + {
7011 + __u32 A, B, C, D, E;
7012 ++ unsigned int i = 0;
7013 +
7014 + A = digest[0];
7015 + B = digest[1];
7016 +@@ -90,94 +93,24 @@ void sha_transform(__u32 *digest, const char *data, __u32 *array)
7017 + E = digest[4];
7018 +
7019 + /* Round 1 - iterations 0-16 take their input from 'data' */
7020 +- T_0_15( 0, A, B, C, D, E);
7021 +- T_0_15( 1, E, A, B, C, D);
7022 +- T_0_15( 2, D, E, A, B, C);
7023 +- T_0_15( 3, C, D, E, A, B);
7024 +- T_0_15( 4, B, C, D, E, A);
7025 +- T_0_15( 5, A, B, C, D, E);
7026 +- T_0_15( 6, E, A, B, C, D);
7027 +- T_0_15( 7, D, E, A, B, C);
7028 +- T_0_15( 8, C, D, E, A, B);
7029 +- T_0_15( 9, B, C, D, E, A);
7030 +- T_0_15(10, A, B, C, D, E);
7031 +- T_0_15(11, E, A, B, C, D);
7032 +- T_0_15(12, D, E, A, B, C);
7033 +- T_0_15(13, C, D, E, A, B);
7034 +- T_0_15(14, B, C, D, E, A);
7035 +- T_0_15(15, A, B, C, D, E);
7036 ++ for (; i < 16; ++i)
7037 ++ T_0_15(i, A, B, C, D, E);
7038 +
7039 + /* Round 1 - tail. Input from 512-bit mixing array */
7040 +- T_16_19(16, E, A, B, C, D);
7041 +- T_16_19(17, D, E, A, B, C);
7042 +- T_16_19(18, C, D, E, A, B);
7043 +- T_16_19(19, B, C, D, E, A);
7044 ++ for (; i < 20; ++i)
7045 ++ T_16_19(i, A, B, C, D, E);
7046 +
7047 + /* Round 2 */
7048 +- T_20_39(20, A, B, C, D, E);
7049 +- T_20_39(21, E, A, B, C, D);
7050 +- T_20_39(22, D, E, A, B, C);
7051 +- T_20_39(23, C, D, E, A, B);
7052 +- T_20_39(24, B, C, D, E, A);
7053 +- T_20_39(25, A, B, C, D, E);
7054 +- T_20_39(26, E, A, B, C, D);
7055 +- T_20_39(27, D, E, A, B, C);
7056 +- T_20_39(28, C, D, E, A, B);
7057 +- T_20_39(29, B, C, D, E, A);
7058 +- T_20_39(30, A, B, C, D, E);
7059 +- T_20_39(31, E, A, B, C, D);
7060 +- T_20_39(32, D, E, A, B, C);
7061 +- T_20_39(33, C, D, E, A, B);
7062 +- T_20_39(34, B, C, D, E, A);
7063 +- T_20_39(35, A, B, C, D, E);
7064 +- T_20_39(36, E, A, B, C, D);
7065 +- T_20_39(37, D, E, A, B, C);
7066 +- T_20_39(38, C, D, E, A, B);
7067 +- T_20_39(39, B, C, D, E, A);
7068 ++ for (; i < 40; ++i)
7069 ++ T_20_39(i, A, B, C, D, E);
7070 +
7071 + /* Round 3 */
7072 +- T_40_59(40, A, B, C, D, E);
7073 +- T_40_59(41, E, A, B, C, D);
7074 +- T_40_59(42, D, E, A, B, C);
7075 +- T_40_59(43, C, D, E, A, B);
7076 +- T_40_59(44, B, C, D, E, A);
7077 +- T_40_59(45, A, B, C, D, E);
7078 +- T_40_59(46, E, A, B, C, D);
7079 +- T_40_59(47, D, E, A, B, C);
7080 +- T_40_59(48, C, D, E, A, B);
7081 +- T_40_59(49, B, C, D, E, A);
7082 +- T_40_59(50, A, B, C, D, E);
7083 +- T_40_59(51, E, A, B, C, D);
7084 +- T_40_59(52, D, E, A, B, C);
7085 +- T_40_59(53, C, D, E, A, B);
7086 +- T_40_59(54, B, C, D, E, A);
7087 +- T_40_59(55, A, B, C, D, E);
7088 +- T_40_59(56, E, A, B, C, D);
7089 +- T_40_59(57, D, E, A, B, C);
7090 +- T_40_59(58, C, D, E, A, B);
7091 +- T_40_59(59, B, C, D, E, A);
7092 ++ for (; i < 60; ++i)
7093 ++ T_40_59(i, A, B, C, D, E);
7094 +
7095 + /* Round 4 */
7096 +- T_60_79(60, A, B, C, D, E);
7097 +- T_60_79(61, E, A, B, C, D);
7098 +- T_60_79(62, D, E, A, B, C);
7099 +- T_60_79(63, C, D, E, A, B);
7100 +- T_60_79(64, B, C, D, E, A);
7101 +- T_60_79(65, A, B, C, D, E);
7102 +- T_60_79(66, E, A, B, C, D);
7103 +- T_60_79(67, D, E, A, B, C);
7104 +- T_60_79(68, C, D, E, A, B);
7105 +- T_60_79(69, B, C, D, E, A);
7106 +- T_60_79(70, A, B, C, D, E);
7107 +- T_60_79(71, E, A, B, C, D);
7108 +- T_60_79(72, D, E, A, B, C);
7109 +- T_60_79(73, C, D, E, A, B);
7110 +- T_60_79(74, B, C, D, E, A);
7111 +- T_60_79(75, A, B, C, D, E);
7112 +- T_60_79(76, E, A, B, C, D);
7113 +- T_60_79(77, D, E, A, B, C);
7114 +- T_60_79(78, C, D, E, A, B);
7115 +- T_60_79(79, B, C, D, E, A);
7116 ++ for (; i < 80; ++i)
7117 ++ T_60_79(i, A, B, C, D, E);
7118 +
7119 + digest[0] += A;
7120 + digest[1] += B;
7121 +diff --git a/lib/siphash.c b/lib/siphash.c
7122 +index e632ee40aac1a..5b34b5c839887 100644
7123 +--- a/lib/siphash.c
7124 ++++ b/lib/siphash.c
7125 +@@ -18,19 +18,13 @@
7126 + #include <asm/word-at-a-time.h>
7127 + #endif
7128 +
7129 +-#define SIPROUND \
7130 +- do { \
7131 +- v0 += v1; v1 = rol64(v1, 13); v1 ^= v0; v0 = rol64(v0, 32); \
7132 +- v2 += v3; v3 = rol64(v3, 16); v3 ^= v2; \
7133 +- v0 += v3; v3 = rol64(v3, 21); v3 ^= v0; \
7134 +- v2 += v1; v1 = rol64(v1, 17); v1 ^= v2; v2 = rol64(v2, 32); \
7135 +- } while (0)
7136 ++#define SIPROUND SIPHASH_PERMUTATION(v0, v1, v2, v3)
7137 +
7138 + #define PREAMBLE(len) \
7139 +- u64 v0 = 0x736f6d6570736575ULL; \
7140 +- u64 v1 = 0x646f72616e646f6dULL; \
7141 +- u64 v2 = 0x6c7967656e657261ULL; \
7142 +- u64 v3 = 0x7465646279746573ULL; \
7143 ++ u64 v0 = SIPHASH_CONST_0; \
7144 ++ u64 v1 = SIPHASH_CONST_1; \
7145 ++ u64 v2 = SIPHASH_CONST_2; \
7146 ++ u64 v3 = SIPHASH_CONST_3; \
7147 + u64 b = ((u64)(len)) << 56; \
7148 + v3 ^= key->key[1]; \
7149 + v2 ^= key->key[0]; \
7150 +@@ -389,19 +383,13 @@ u32 hsiphash_4u32(const u32 first, const u32 second, const u32 third,
7151 + }
7152 + EXPORT_SYMBOL(hsiphash_4u32);
7153 + #else
7154 +-#define HSIPROUND \
7155 +- do { \
7156 +- v0 += v1; v1 = rol32(v1, 5); v1 ^= v0; v0 = rol32(v0, 16); \
7157 +- v2 += v3; v3 = rol32(v3, 8); v3 ^= v2; \
7158 +- v0 += v3; v3 = rol32(v3, 7); v3 ^= v0; \
7159 +- v2 += v1; v1 = rol32(v1, 13); v1 ^= v2; v2 = rol32(v2, 16); \
7160 +- } while (0)
7161 ++#define HSIPROUND HSIPHASH_PERMUTATION(v0, v1, v2, v3)
7162 +
7163 + #define HPREAMBLE(len) \
7164 +- u32 v0 = 0; \
7165 +- u32 v1 = 0; \
7166 +- u32 v2 = 0x6c796765U; \
7167 +- u32 v3 = 0x74656462U; \
7168 ++ u32 v0 = HSIPHASH_CONST_0; \
7169 ++ u32 v1 = HSIPHASH_CONST_1; \
7170 ++ u32 v2 = HSIPHASH_CONST_2; \
7171 ++ u32 v3 = HSIPHASH_CONST_3; \
7172 + u32 b = ((u32)(len)) << 24; \
7173 + v3 ^= key->key[1]; \
7174 + v2 ^= key->key[0]; \
7175 +diff --git a/mm/util.c b/mm/util.c
7176 +index f0d773c719a1b..84e775f5216fe 100644
7177 +--- a/mm/util.c
7178 ++++ b/mm/util.c
7179 +@@ -14,6 +14,7 @@
7180 + #include <linux/hugetlb.h>
7181 + #include <linux/vmalloc.h>
7182 + #include <linux/userfaultfd_k.h>
7183 ++#include <linux/random.h>
7184 +
7185 + #include <asm/sections.h>
7186 + #include <linux/uaccess.h>
7187 +@@ -264,6 +265,38 @@ int vma_is_stack_for_current(struct vm_area_struct *vma)
7188 + return (vma->vm_start <= KSTK_ESP(t) && vma->vm_end >= KSTK_ESP(t));
7189 + }
7190 +
7191 ++/**
7192 ++ * randomize_page - Generate a random, page aligned address
7193 ++ * @start: The smallest acceptable address the caller will take.
7194 ++ * @range: The size of the area, starting at @start, within which the
7195 ++ * random address must fall.
7196 ++ *
7197 ++ * If @start + @range would overflow, @range is capped.
7198 ++ *
7199 ++ * NOTE: Historical use of randomize_range, which this replaces, presumed that
7200 ++ * @start was already page aligned. We now align it regardless.
7201 ++ *
7202 ++ * Return: A page aligned address within [start, start + range). On error,
7203 ++ * @start is returned.
7204 ++ */
7205 ++unsigned long randomize_page(unsigned long start, unsigned long range)
7206 ++{
7207 ++ if (!PAGE_ALIGNED(start)) {
7208 ++ range -= PAGE_ALIGN(start) - start;
7209 ++ start = PAGE_ALIGN(start);
7210 ++ }
7211 ++
7212 ++ if (start > ULONG_MAX - range)
7213 ++ range = ULONG_MAX - start;
7214 ++
7215 ++ range >>= PAGE_SHIFT;
7216 ++
7217 ++ if (range == 0)
7218 ++ return start;
7219 ++
7220 ++ return start + (get_random_long() % range << PAGE_SHIFT);
7221 ++}
7222 ++
7223 + #if defined(CONFIG_MMU) && !defined(HAVE_ARCH_PICK_MMAP_LAYOUT)
7224 + void arch_pick_mmap_layout(struct mm_struct *mm)
7225 + {
7226 +diff --git a/net/ipv4/inet_hashtables.c b/net/ipv4/inet_hashtables.c
7227 +index 1ebad5a024a7d..19369fc9bcda8 100644
7228 +--- a/net/ipv4/inet_hashtables.c
7229 ++++ b/net/ipv4/inet_hashtables.c
7230 +@@ -591,12 +591,14 @@ EXPORT_SYMBOL_GPL(inet_unhash);
7231 + * Note that we use 32bit integers (vs RFC 'short integers')
7232 + * because 2^16 is not a multiple of num_ephemeral and this
7233 + * property might be used by clever attacker.
7234 +- * RFC claims using TABLE_LENGTH=10 buckets gives an improvement,
7235 +- * we use 256 instead to really give more isolation and
7236 +- * privacy, this only consumes 1 KB of kernel memory.
7237 ++ * RFC claims using TABLE_LENGTH=10 buckets gives an improvement, though
7238 ++ * attacks were since demonstrated, thus we use 65536 instead to really
7239 ++ * give more isolation and privacy, at the expense of 256kB of kernel
7240 ++ * memory.
7241 + */
7242 +-#define INET_TABLE_PERTURB_SHIFT 8
7243 +-static u32 table_perturb[1 << INET_TABLE_PERTURB_SHIFT];
7244 ++#define INET_TABLE_PERTURB_SHIFT 16
7245 ++#define INET_TABLE_PERTURB_SIZE (1 << INET_TABLE_PERTURB_SHIFT)
7246 ++static u32 *table_perturb;
7247 +
7248 + int __inet_hash_connect(struct inet_timewait_death_row *death_row,
7249 + struct sock *sk, u64 port_offset,
7250 +@@ -636,10 +638,11 @@ int __inet_hash_connect(struct inet_timewait_death_row *death_row,
7251 + if (likely(remaining > 1))
7252 + remaining &= ~1U;
7253 +
7254 +- net_get_random_once(table_perturb, sizeof(table_perturb));
7255 +- index = hash_32(port_offset, INET_TABLE_PERTURB_SHIFT);
7256 ++ net_get_random_once(table_perturb,
7257 ++ INET_TABLE_PERTURB_SIZE * sizeof(*table_perturb));
7258 ++ index = port_offset & (INET_TABLE_PERTURB_SIZE - 1);
7259 +
7260 +- offset = READ_ONCE(table_perturb[index]) + port_offset;
7261 ++ offset = READ_ONCE(table_perturb[index]) + (port_offset >> 32);
7262 + offset %= remaining;
7263 +
7264 + /* In first pass we try ports of @low parity.
7265 +@@ -694,6 +697,12 @@ next_port:
7266 + return -EADDRNOTAVAIL;
7267 +
7268 + ok:
7269 ++ /* Here we want to add a little bit of randomness to the next source
7270 ++ * port that will be chosen. We use a max() with a random here so that
7271 ++ * on low contention the randomness is maximal and on high contention
7272 ++ * it may be inexistent.
7273 ++ */
7274 ++ i = max_t(int, i, (prandom_u32() & 7) * 2);
7275 + WRITE_ONCE(table_perturb[index], READ_ONCE(table_perturb[index]) + i + 2);
7276 +
7277 + /* Head lock still held and bh's disabled */
7278 +@@ -735,6 +744,15 @@ void inet_hashinfo_init(struct inet_hashinfo *h)
7279 + INIT_HLIST_NULLS_HEAD(&h->listening_hash[i].nulls_head,
7280 + i + LISTENING_NULLS_BASE);
7281 + }
7282 ++
7283 ++ if (h != &tcp_hashinfo)
7284 ++ return;
7285 ++
7286 ++ /* this one is used for source ports of outgoing connections */
7287 ++ table_perturb = kmalloc_array(INET_TABLE_PERTURB_SIZE,
7288 ++ sizeof(*table_perturb), GFP_KERNEL);
7289 ++ if (!table_perturb)
7290 ++ panic("TCP: failed to alloc table_perturb");
7291 + }
7292 + EXPORT_SYMBOL_GPL(inet_hashinfo_init);
7293 +
7294 +diff --git a/net/l2tp/l2tp_ip6.c b/net/l2tp/l2tp_ip6.c
7295 +index 9c4670fb29d74..7b0963712c22f 100644
7296 +--- a/net/l2tp/l2tp_ip6.c
7297 ++++ b/net/l2tp/l2tp_ip6.c
7298 +@@ -526,14 +526,15 @@ static int l2tp_ip6_sendmsg(struct sock *sk, struct msghdr *msg, size_t len)
7299 + struct ipcm6_cookie ipc6;
7300 + int addr_len = msg->msg_namelen;
7301 + int transhdrlen = 4; /* zero session-id */
7302 +- int ulen = len + transhdrlen;
7303 ++ int ulen;
7304 + int err;
7305 +
7306 + /* Rough check on arithmetic overflow,
7307 + better check is made in ip6_append_data().
7308 + */
7309 +- if (len > INT_MAX)
7310 ++ if (len > INT_MAX - transhdrlen)
7311 + return -EMSGSIZE;
7312 ++ ulen = len + transhdrlen;
7313 +
7314 + /* Mirror BSD error message compatibility */
7315 + if (msg->msg_flags & MSG_OOB)
7316 +diff --git a/net/l2tp/l2tp_ppp.c b/net/l2tp/l2tp_ppp.c
7317 +index ad07b7cb65581..332fa7ec8a3d6 100644
7318 +--- a/net/l2tp/l2tp_ppp.c
7319 ++++ b/net/l2tp/l2tp_ppp.c
7320 +@@ -435,20 +435,28 @@ abort:
7321 + * Session (and tunnel control) socket create/destroy.
7322 + *****************************************************************************/
7323 +
7324 ++static void pppol2tp_put_sk(struct rcu_head *head)
7325 ++{
7326 ++ struct pppol2tp_session *ps;
7327 ++
7328 ++ ps = container_of(head, typeof(*ps), rcu);
7329 ++ sock_put(ps->__sk);
7330 ++}
7331 ++
7332 + /* Called by l2tp_core when a session socket is being closed.
7333 + */
7334 + static void pppol2tp_session_close(struct l2tp_session *session)
7335 + {
7336 +- struct sock *sk;
7337 +-
7338 +- BUG_ON(session->magic != L2TP_SESSION_MAGIC);
7339 ++ struct pppol2tp_session *ps;
7340 +
7341 +- sk = pppol2tp_session_get_sock(session);
7342 +- if (sk) {
7343 +- if (sk->sk_socket)
7344 +- inet_shutdown(sk->sk_socket, SEND_SHUTDOWN);
7345 +- sock_put(sk);
7346 +- }
7347 ++ ps = l2tp_session_priv(session);
7348 ++ mutex_lock(&ps->sk_lock);
7349 ++ ps->__sk = rcu_dereference_protected(ps->sk,
7350 ++ lockdep_is_held(&ps->sk_lock));
7351 ++ RCU_INIT_POINTER(ps->sk, NULL);
7352 ++ if (ps->__sk)
7353 ++ call_rcu(&ps->rcu, pppol2tp_put_sk);
7354 ++ mutex_unlock(&ps->sk_lock);
7355 + }
7356 +
7357 + /* Really kill the session socket. (Called from sock_put() if
7358 +@@ -468,14 +476,6 @@ static void pppol2tp_session_destruct(struct sock *sk)
7359 + }
7360 + }
7361 +
7362 +-static void pppol2tp_put_sk(struct rcu_head *head)
7363 +-{
7364 +- struct pppol2tp_session *ps;
7365 +-
7366 +- ps = container_of(head, typeof(*ps), rcu);
7367 +- sock_put(ps->__sk);
7368 +-}
7369 +-
7370 + /* Called when the PPPoX socket (session) is closed.
7371 + */
7372 + static int pppol2tp_release(struct socket *sock)
7373 +@@ -499,26 +499,17 @@ static int pppol2tp_release(struct socket *sock)
7374 + sock_orphan(sk);
7375 + sock->sk = NULL;
7376 +
7377 ++ /* If the socket is associated with a session,
7378 ++ * l2tp_session_delete will call pppol2tp_session_close which
7379 ++ * will drop the session's ref on the socket.
7380 ++ */
7381 + session = pppol2tp_sock_to_session(sk);
7382 +-
7383 +- if (session != NULL) {
7384 +- struct pppol2tp_session *ps;
7385 +-
7386 ++ if (session) {
7387 + l2tp_session_delete(session);
7388 +-
7389 +- ps = l2tp_session_priv(session);
7390 +- mutex_lock(&ps->sk_lock);
7391 +- ps->__sk = rcu_dereference_protected(ps->sk,
7392 +- lockdep_is_held(&ps->sk_lock));
7393 +- RCU_INIT_POINTER(ps->sk, NULL);
7394 +- mutex_unlock(&ps->sk_lock);
7395 +- call_rcu(&ps->rcu, pppol2tp_put_sk);
7396 +-
7397 +- /* Rely on the sock_put() call at the end of the function for
7398 +- * dropping the reference held by pppol2tp_sock_to_session().
7399 +- * The last reference will be dropped by pppol2tp_put_sk().
7400 +- */
7401 ++ /* drop the ref obtained by pppol2tp_sock_to_session */
7402 ++ sock_put(sk);
7403 + }
7404 ++
7405 + release_sock(sk);
7406 +
7407 + /* This will delete the session context via
7408 +@@ -830,6 +821,7 @@ static int pppol2tp_connect(struct socket *sock, struct sockaddr *uservaddr,
7409 +
7410 + out_no_ppp:
7411 + /* This is how we get the session context from the socket. */
7412 ++ sock_hold(sk);
7413 + sk->sk_user_data = session;
7414 + rcu_assign_pointer(ps->sk, sk);
7415 + mutex_unlock(&ps->sk_lock);
7416 +diff --git a/net/sunrpc/xprtrdma/rpc_rdma.c b/net/sunrpc/xprtrdma/rpc_rdma.c
7417 +index 030bf17a20b6a..0dae9260ba3c3 100644
7418 +--- a/net/sunrpc/xprtrdma/rpc_rdma.c
7419 ++++ b/net/sunrpc/xprtrdma/rpc_rdma.c
7420 +@@ -75,7 +75,7 @@ static unsigned int rpcrdma_max_call_header_size(unsigned int maxsegs)
7421 +
7422 + /* Maximum Read list size */
7423 + maxsegs += 2; /* segment for head and tail buffers */
7424 +- size = maxsegs * sizeof(struct rpcrdma_read_chunk);
7425 ++ size += maxsegs * sizeof(struct rpcrdma_read_chunk);
7426 +
7427 + /* Minimal Read chunk size */
7428 + size += sizeof(__be32); /* segment count */
7429 +@@ -101,7 +101,7 @@ static unsigned int rpcrdma_max_reply_header_size(unsigned int maxsegs)
7430 +
7431 + /* Maximum Write list size */
7432 + maxsegs += 2; /* segment for head and tail buffers */
7433 +- size = sizeof(__be32); /* segment count */
7434 ++ size += sizeof(__be32); /* segment count */
7435 + size += maxsegs * sizeof(struct rpcrdma_segment);
7436 + size += sizeof(__be32); /* list discriminator */
7437 +
7438 +diff --git a/sound/soc/codecs/cs42l52.c b/sound/soc/codecs/cs42l52.c
7439 +index 0d9c4a57301bb..47f2439fd7b08 100644
7440 +--- a/sound/soc/codecs/cs42l52.c
7441 ++++ b/sound/soc/codecs/cs42l52.c
7442 +@@ -141,7 +141,9 @@ static DECLARE_TLV_DB_SCALE(mic_tlv, 1600, 100, 0);
7443 +
7444 + static DECLARE_TLV_DB_SCALE(pga_tlv, -600, 50, 0);
7445 +
7446 +-static DECLARE_TLV_DB_SCALE(mix_tlv, -50, 50, 0);
7447 ++static DECLARE_TLV_DB_SCALE(pass_tlv, -6000, 50, 0);
7448 ++
7449 ++static DECLARE_TLV_DB_SCALE(mix_tlv, -5150, 50, 0);
7450 +
7451 + static DECLARE_TLV_DB_SCALE(beep_tlv, -56, 200, 0);
7452 +
7453 +@@ -355,7 +357,7 @@ static const struct snd_kcontrol_new cs42l52_snd_controls[] = {
7454 + CS42L52_SPKB_VOL, 0, 0x40, 0xC0, hl_tlv),
7455 +
7456 + SOC_DOUBLE_R_SX_TLV("Bypass Volume", CS42L52_PASSTHRUA_VOL,
7457 +- CS42L52_PASSTHRUB_VOL, 0, 0x88, 0x90, pga_tlv),
7458 ++ CS42L52_PASSTHRUB_VOL, 0, 0x88, 0x90, pass_tlv),
7459 +
7460 + SOC_DOUBLE("Bypass Mute", CS42L52_MISC_CTL, 4, 5, 1, 0),
7461 +
7462 +@@ -368,7 +370,7 @@ static const struct snd_kcontrol_new cs42l52_snd_controls[] = {
7463 + CS42L52_ADCB_VOL, 0, 0xA0, 0x78, ipd_tlv),
7464 + SOC_DOUBLE_R_SX_TLV("ADC Mixer Volume",
7465 + CS42L52_ADCA_MIXER_VOL, CS42L52_ADCB_MIXER_VOL,
7466 +- 0, 0x19, 0x7F, ipd_tlv),
7467 ++ 0, 0x19, 0x7F, mix_tlv),
7468 +
7469 + SOC_DOUBLE("ADC Switch", CS42L52_ADC_MISC_CTL, 0, 1, 1, 0),
7470 +
7471 +diff --git a/sound/soc/codecs/cs42l56.c b/sound/soc/codecs/cs42l56.c
7472 +index 52858b6c95a63..9274b52edc442 100644
7473 +--- a/sound/soc/codecs/cs42l56.c
7474 ++++ b/sound/soc/codecs/cs42l56.c
7475 +@@ -403,9 +403,9 @@ static const struct snd_kcontrol_new cs42l56_snd_controls[] = {
7476 + SOC_DOUBLE("ADC Boost Switch", CS42L56_GAIN_BIAS_CTL, 3, 2, 1, 1),
7477 +
7478 + SOC_DOUBLE_R_SX_TLV("Headphone Volume", CS42L56_HPA_VOLUME,
7479 +- CS42L56_HPB_VOLUME, 0, 0x84, 0x48, hl_tlv),
7480 ++ CS42L56_HPB_VOLUME, 0, 0x44, 0x48, hl_tlv),
7481 + SOC_DOUBLE_R_SX_TLV("LineOut Volume", CS42L56_LOA_VOLUME,
7482 +- CS42L56_LOB_VOLUME, 0, 0x84, 0x48, hl_tlv),
7483 ++ CS42L56_LOB_VOLUME, 0, 0x44, 0x48, hl_tlv),
7484 +
7485 + SOC_SINGLE_TLV("Bass Shelving Volume", CS42L56_TONE_CTL,
7486 + 0, 0x00, 1, tone_tlv),
7487 +diff --git a/sound/soc/codecs/cs53l30.c b/sound/soc/codecs/cs53l30.c
7488 +index c7edf2df5e36b..f30807b3705db 100644
7489 +--- a/sound/soc/codecs/cs53l30.c
7490 ++++ b/sound/soc/codecs/cs53l30.c
7491 +@@ -351,22 +351,22 @@ static const struct snd_kcontrol_new cs53l30_snd_controls[] = {
7492 + SOC_ENUM("ADC2 NG Delay", adc2_ng_delay_enum),
7493 +
7494 + SOC_SINGLE_SX_TLV("ADC1A PGA Volume",
7495 +- CS53L30_ADC1A_AFE_CTL, 0, 0x34, 0x18, pga_tlv),
7496 ++ CS53L30_ADC1A_AFE_CTL, 0, 0x34, 0x24, pga_tlv),
7497 + SOC_SINGLE_SX_TLV("ADC1B PGA Volume",
7498 +- CS53L30_ADC1B_AFE_CTL, 0, 0x34, 0x18, pga_tlv),
7499 ++ CS53L30_ADC1B_AFE_CTL, 0, 0x34, 0x24, pga_tlv),
7500 + SOC_SINGLE_SX_TLV("ADC2A PGA Volume",
7501 +- CS53L30_ADC2A_AFE_CTL, 0, 0x34, 0x18, pga_tlv),
7502 ++ CS53L30_ADC2A_AFE_CTL, 0, 0x34, 0x24, pga_tlv),
7503 + SOC_SINGLE_SX_TLV("ADC2B PGA Volume",
7504 +- CS53L30_ADC2B_AFE_CTL, 0, 0x34, 0x18, pga_tlv),
7505 ++ CS53L30_ADC2B_AFE_CTL, 0, 0x34, 0x24, pga_tlv),
7506 +
7507 + SOC_SINGLE_SX_TLV("ADC1A Digital Volume",
7508 +- CS53L30_ADC1A_DIG_VOL, 0, 0xA0, 0x0C, dig_tlv),
7509 ++ CS53L30_ADC1A_DIG_VOL, 0, 0xA0, 0x6C, dig_tlv),
7510 + SOC_SINGLE_SX_TLV("ADC1B Digital Volume",
7511 +- CS53L30_ADC1B_DIG_VOL, 0, 0xA0, 0x0C, dig_tlv),
7512 ++ CS53L30_ADC1B_DIG_VOL, 0, 0xA0, 0x6C, dig_tlv),
7513 + SOC_SINGLE_SX_TLV("ADC2A Digital Volume",
7514 +- CS53L30_ADC2A_DIG_VOL, 0, 0xA0, 0x0C, dig_tlv),
7515 ++ CS53L30_ADC2A_DIG_VOL, 0, 0xA0, 0x6C, dig_tlv),
7516 + SOC_SINGLE_SX_TLV("ADC2B Digital Volume",
7517 +- CS53L30_ADC2B_DIG_VOL, 0, 0xA0, 0x0C, dig_tlv),
7518 ++ CS53L30_ADC2B_DIG_VOL, 0, 0xA0, 0x6C, dig_tlv),
7519 + };
7520 +
7521 + static const struct snd_soc_dapm_widget cs53l30_dapm_widgets[] = {
7522 +diff --git a/sound/soc/codecs/wm8962.c b/sound/soc/codecs/wm8962.c
7523 +index 0e8008d381619..d46881f96c164 100644
7524 +--- a/sound/soc/codecs/wm8962.c
7525 ++++ b/sound/soc/codecs/wm8962.c
7526 +@@ -3861,6 +3861,7 @@ static int wm8962_runtime_suspend(struct device *dev)
7527 + #endif
7528 +
7529 + static const struct dev_pm_ops wm8962_pm = {
7530 ++ SET_SYSTEM_SLEEP_PM_OPS(pm_runtime_force_suspend, pm_runtime_force_resume)
7531 + SET_RUNTIME_PM_OPS(wm8962_runtime_suspend, wm8962_runtime_resume, NULL)
7532 + };
7533 +