1 /***********************license start***********************************
2 * Copyright (c) 2003-2017 Cavium Inc. ([email protected]). All rights
3 * reserved.
4 *
5 *
6 * Redistribution and use in source and binary forms, with or without
7 * modification, are permitted provided that the following conditions are
8 * met:
9 *
10 * * Redistributions of source code must retain the above copyright
11 * notice, this list of conditions and the following disclaimer.
12 *
13 * * Redistributions in binary form must reproduce the above
14 * copyright notice, this list of conditions and the following
15 * disclaimer in the documentation and/or other materials provided
16 * with the distribution.
17 *
18 * * Neither the name of Cavium Inc. nor the names of
19 * its contributors may be used to endorse or promote products
20 * derived from this software without specific prior written
21 * permission.
22 *
23 * This Software, including technical data, may be subject to U.S. export
24 * control laws, including the U.S. Export Administration Act and its
25 * associated regulations, and may be subject to export or import
26 * regulations in other countries.
27 *
28 * TO THE MAXIMUM EXTENT PERMITTED BY LAW, THE SOFTWARE IS PROVIDED "AS IS"
29 * AND WITH ALL FAULTS AND CAVIUM INC. MAKES NO PROMISES, REPRESENTATIONS OR
30 * WARRANTIES, EITHER EXPRESS, IMPLIED, STATUTORY, OR OTHERWISE, WITH RESPECT
31 * TO THE SOFTWARE, INCLUDING ITS CONDITION, ITS CONFORMITY TO ANY
32 * REPRESENTATION OR DESCRIPTION, OR THE EXISTENCE OF ANY LATENT OR PATENT
33 * DEFECTS, AND CAVIUM SPECIFICALLY DISCLAIMS ALL IMPLIED (IF ANY) WARRANTIES
34 * OF TITLE, MERCHANTABILITY, NONINFRINGEMENT, FITNESS FOR A PARTICULAR
35 * PURPOSE, LACK OF VIRUSES, ACCURACY OR COMPLETENESS, QUIET ENJOYMENT,
36 * QUIET POSSESSION OR CORRESPONDENCE TO DESCRIPTION. THE ENTIRE RISK
37 * ARISING OUT OF USE OR PERFORMANCE OF THE SOFTWARE LIES WITH YOU.
38 ***********************license end**************************************/
39 #include "bdk.h"
40 #include <libbdk-hal/bdk-rng.h>
41 #include <libbdk-hal/bdk-utils.h>
42
43 // choose prediction-based algorithms for mem_xor and mem_rows tests
44 #define USE_PREDICTION_CODE_VERSIONS 1 // change to 0 to go back to the original versions
45
46 /* Used for all memory reads/writes related to the test */
47 #define READ64(address) __bdk_dram_read64(address)
48 #define WRITE64(address, data) __bdk_dram_write64(address, data)
49
50 /**
51 * Fill an memory area with the address of each 64-bit word in the area.
52 * Reread to confirm the pattern.
53 *
54 * @param area Start of the physical memory area
55 * @param max_address
56 * End of the physical memory area (exclusive)
57 * @param bursts Number of time to repeat the test over the entire area
58 *
59 * @return Number of errors, zero on success
60 */
__bdk_dram_test_mem_self_addr(uint64_t area,uint64_t max_address,int bursts)61 int __bdk_dram_test_mem_self_addr(uint64_t area, uint64_t max_address, int bursts)
62 {
63 int failures = 0;
64
65 for (int burst = 0; burst < bursts; burst++)
66 {
67 /* Write the pattern to memory. Each location receives the address
68 * of the location.
69 */
70 for (uint64_t address = area; address < max_address; address+=8)
71 WRITE64(address, address);
72 __bdk_dram_flush_to_mem_range(area, max_address);
73 BDK_DCACHE_INVALIDATE;
74
75 /* Read by ascending address the written memory and confirm that it
76 * has the expected data pattern.
77 */
78 for (uint64_t address = area; address < max_address; )
79 {
80 if (address + 256 < max_address)
81 BDK_PREFETCH(address + 256, 0);
82 for (int i=0; i<16; i++)
83 {
84 uint64_t data = READ64(address);
85 if (bdk_unlikely(data != address))
86 failures += __bdk_dram_retry_failure(burst, address, data, address);
87 address += 8;
88 }
89 }
90 __bdk_dram_flush_to_mem_range(area, max_address);
91 BDK_DCACHE_INVALIDATE;
92
93 /* Read by descending address the written memory and confirm that it
94 * has the expected data pattern.
95 */
96 uint64_t end = max_address - sizeof(uint64_t);
97 for (uint64_t address = end; address >= area; )
98 {
99 if (address - 256 >= area)
100 BDK_PREFETCH(address - 256, 0);
101 for (int i=0; i<16; i++)
102 {
103 uint64_t data = READ64(address);
104 if (bdk_unlikely(data != address))
105 failures += __bdk_dram_retry_failure(burst, address, data, address);
106 address -= 8;
107 }
108 }
109 __bdk_dram_flush_to_mem_range(area, max_address);
110 BDK_DCACHE_INVALIDATE;
111
112 /* Read from random addresses within the memory area.
113 */
114 uint64_t probes = (max_address - area) / 128;
115 uint64_t address_ahead1 = area;
116 uint64_t address_ahead2 = area;
117 for (uint64_t i = 0; i < probes; i++)
118 {
119 /* Create a pipeline of prefetches:
120 address = address read this loop
121 address_ahead1 = prefetch started last loop
122 address_ahead2 = prefetch started this loop */
123 uint64_t address = address_ahead1;
124 address_ahead1 = address_ahead2;
125 address_ahead2 = bdk_rng_get_random64() % (max_address - area);
126 address_ahead2 += area;
127 address_ahead2 &= -8;
128 BDK_PREFETCH(address_ahead2, 0);
129
130 uint64_t data = READ64(address);
131 if (bdk_unlikely(data != address))
132 failures += __bdk_dram_retry_failure(burst, address, data, address);
133 }
134 }
135 return failures;
136 }
137
138 /**
139 * Write "pattern" and its compliment to memory and verify it was written
140 * properly. Memory will be filled with DWORDs pattern, ~pattern, pattern,
141 * ~pattern, ...
142 *
143 * @param area Start physical address of memory
144 * @param max_address
145 * End of physical memory region
146 * @param pattern Pattern to write
147 * @param passes Number of time to repeat the test
148 *
149 * @return Number of errors, zero on success
150 */
test_mem_pattern(uint64_t area,uint64_t max_address,uint64_t pattern,int passes)151 static uint32_t test_mem_pattern(uint64_t area, uint64_t max_address, uint64_t pattern,
152 int passes)
153 {
154 int failures = 0;
155
156 for (int pass = 0; pass < passes; pass++)
157 {
158 if (pass & 0x1)
159 pattern = ~pattern;
160
161 for (uint64_t address = area; address < max_address; address += 8)
162 WRITE64(address, pattern);
163 __bdk_dram_flush_to_mem_range(area, max_address);
164 BDK_DCACHE_INVALIDATE;
165
166 /* Read the written memory and confirm that it has the expected
167 * data pattern.
168 */
169 uint64_t address = area;
170 while (address < max_address)
171 {
172 if (address + 256 < max_address)
173 BDK_PREFETCH(address + 256, 0);
174 for (int i=0; i<16; i++)
175 {
176 uint64_t data = READ64(address);
177 if (bdk_unlikely(data != pattern))
178 failures += __bdk_dram_retry_failure(pass, address, data, pattern);
179 address += 8;
180 }
181 }
182 }
183 return failures;
184 }
185
186 /**
187 * Walking zero written to memory, left shift
188 *
189 * @param area Start of the physical memory area
190 * @param max_address
191 * End of the physical memory area
192 * @param bursts Number of time to repeat the test over the entire area
193 *
194 * @return Number of errors, zero on success
195 */
__bdk_dram_test_mem_leftwalk0(uint64_t area,uint64_t max_address,int bursts)196 int __bdk_dram_test_mem_leftwalk0(uint64_t area, uint64_t max_address, int bursts)
197 {
198 int failures = 0;
199 for (int burst = 0; burst < bursts; burst++)
200 {
201 for (uint64_t pattern = 1; pattern != 0; pattern = pattern << 1)
202 failures += test_mem_pattern(area, max_address, ~pattern, 1);
203 }
204 return failures;
205 }
206
207 /**
208 * Walking one written to memory, left shift
209 *
210 * @param area Start of the physical memory area
211 * @param max_address
212 * End of the physical memory area
213 * @param bursts Number of time to repeat the test over the entire area
214 *
215 * @return Number of errors, zero on success
216 */
__bdk_dram_test_mem_leftwalk1(uint64_t area,uint64_t max_address,int bursts)217 int __bdk_dram_test_mem_leftwalk1(uint64_t area, uint64_t max_address, int bursts)
218 {
219 int failures = 0;
220 for (int burst = 0; burst < bursts; burst++)
221 {
222 for (uint64_t pattern = 1; pattern != 0; pattern = pattern << 1)
223 failures += test_mem_pattern(area, max_address, pattern, 1);
224 }
225 return failures;
226 }
227
228 /**
229 * Walking zero written to memory, right shift
230 *
231 * @param area Start of the physical memory area
232 * @param max_address
233 * End of the physical memory area
234 * @param bursts Number of time to repeat the test over the entire area
235 *
236 * @return Number of errors, zero on success
237 */
__bdk_dram_test_mem_rightwalk0(uint64_t area,uint64_t max_address,int bursts)238 int __bdk_dram_test_mem_rightwalk0(uint64_t area, uint64_t max_address, int bursts)
239 {
240 int failures = 0;
241 for (int burst = 0; burst < bursts; burst++)
242 {
243 for (uint64_t pattern = 1ull << 63; pattern != 0; pattern = pattern >> 1)
244 failures += test_mem_pattern(area, max_address, ~pattern, 1);
245 }
246 return failures;
247 }
248
249 /**
250 * Walking one written to memory, right shift
251 *
252 * @param area Start of the physical memory area
253 * @param max_address
254 * End of the physical memory area
255 * @param bursts Number of time to repeat the test over the entire area
256 *
257 * @return Number of errors, zero on success
258 */
__bdk_dram_test_mem_rightwalk1(uint64_t area,uint64_t max_address,int bursts)259 int __bdk_dram_test_mem_rightwalk1(uint64_t area, uint64_t max_address, int bursts)
260 {
261 int failures = 0;
262 for (int burst = 0; burst < bursts; burst++)
263 {
264 for (uint64_t pattern = 1ull<<63; pattern != 0; pattern = pattern >> 1)
265 failures += test_mem_pattern(area, max_address, pattern, 1);
266 }
267 return failures;
268 }
269
270 /**
271 * Apply the March C- testing algorithm to the given memory area.
272 * 1) Write "pattern" to memory.
273 * 2) Verify "pattern" and write "~pattern".
274 * 3) Verify "~pattern" and write "pattern".
275 * 4) Verify "pattern" and write "~pattern".
276 * 5) Verify "~pattern" and write "pattern".
277 * 6) Verify "pattern".
278 *
279 * @param area Start of the physical memory area
280 * @param max_address
281 * End of the physical memory area
282 * @param pattern
283 *
284 * @return Number of errors, zero on success
285 */
test_mem_march_c(uint64_t area,uint64_t max_address,uint64_t pattern)286 static int test_mem_march_c(uint64_t area, uint64_t max_address, uint64_t pattern)
287 {
288 int failures = 0;
289
290 /* Pass 1 ascending addresses, fill memory with pattern. */
291 BDK_TRACE(DRAM_TEST, " [0x%016llx:0x%016llx] Phase1, address incrementing, pattern 0x%016llx\n", area, max_address-1, pattern);
292 for (uint64_t address = area; address < max_address; address += 8)
293 WRITE64(address, pattern);
294
295 __bdk_dram_flush_to_mem_range(area, max_address);
296 BDK_DCACHE_INVALIDATE;
297
298 /* Pass 2: ascending addresses, read pattern and write ~pattern */
299 BDK_TRACE(DRAM_TEST, " [0x%016llx:0x%016llx] Phase2, address incrementing, pattern 0x%016llx\n", area, max_address-1, ~pattern);
300 for (uint64_t address = area; address < max_address; address += 8)
301 {
302 uint64_t data = READ64(address);
303 if (bdk_unlikely(data != pattern))
304 failures += __bdk_dram_retry_failure(1, address, data, pattern);
305 WRITE64(address, ~pattern);
306 }
307
308 __bdk_dram_flush_to_mem_range(area, max_address);
309 BDK_DCACHE_INVALIDATE;
310
311 /* Pass 3: ascending addresses, read ~pattern and write pattern. */
312 BDK_TRACE(DRAM_TEST, " [0x%016llx:0x%016llx] Phase3, address incrementing, pattern 0x%016llx\n", area, max_address-1, pattern);
313 for (uint64_t address = area; address < max_address; address += 8)
314 {
315 uint64_t data = READ64(address);
316 if (bdk_unlikely(data != ~pattern))
317 failures += __bdk_dram_retry_failure(1, address, data, ~pattern);
318 WRITE64(address, pattern);
319 }
320
321 __bdk_dram_flush_to_mem_range(area, max_address);
322 BDK_DCACHE_INVALIDATE;
323
324 /* Pass 4: descending addresses, read pattern and write ~pattern. */
325 BDK_TRACE(DRAM_TEST, " [0x%016llx:0x%016llx] Phase4, address decrementing, pattern 0x%016llx\n", area, max_address-1, ~pattern);
326 uint64_t end = max_address - sizeof(uint64_t);
327 for (uint64_t address = end; address >= area; address -= 8)
328 {
329 uint64_t data = READ64(address);
330 if (bdk_unlikely(data != pattern))
331 failures += __bdk_dram_retry_failure(1, address, data, pattern);
332 WRITE64(address, ~pattern);
333 }
334
335 __bdk_dram_flush_to_mem_range(area, max_address);
336 BDK_DCACHE_INVALIDATE;
337
338 /* Pass 5: descending addresses, read ~pattern and write pattern. */
339 BDK_TRACE(DRAM_TEST, " [0x%016llx:0x%016llx] Phase5, address decrementing, pattern 0x%016llx\n", area, max_address-1, pattern);
340 for (uint64_t address = end; address >= area; address -= 8)
341 {
342 uint64_t data = READ64(address);
343 if (bdk_unlikely(data != ~pattern))
344 failures += __bdk_dram_retry_failure(1, address, data, ~pattern);
345 WRITE64(address, pattern);
346 }
347
348 __bdk_dram_flush_to_mem_range(area, max_address);
349 BDK_DCACHE_INVALIDATE;
350
351 /* Pass 6: ascending addresses, read pattern. */
352 BDK_TRACE(DRAM_TEST, " [0x%016llx:0x%016llx] Phase6, address incrementing\n", area, max_address-1);
353 for (uint64_t address = area; address < max_address; address += 8)
354 {
355 uint64_t data = READ64(address);
356 if (bdk_unlikely(data != pattern))
357 failures += __bdk_dram_retry_failure(1, address, data, pattern);
358 }
359
360 return failures;
361 }
362
363 /**
364 * Use test_mem_march_c() with a all ones pattern
365 *
366 * @param area Start of the physical memory area
367 * @param max_address
368 * End of the physical memory area
369 * @param bursts Number of time to repeat the test over the entire area
370 *
371 * @return Number of errors, zero on success
372 */
__bdk_dram_test_mem_solid(uint64_t area,uint64_t max_address,int bursts)373 int __bdk_dram_test_mem_solid(uint64_t area, uint64_t max_address, int bursts)
374 {
375 int failures = 0;
376 for (int burst = 0; burst < bursts; burst++)
377 failures += test_mem_march_c(area, max_address, -1);
378 return failures;
379 }
380
381 /**
382 * Use test_mem_march_c() with a 0x55 pattern
383 *
384 * @param area Start of the physical memory area
385 * @param max_address
386 * End of the physical memory area
387 * @param bursts Number of time to repeat the test over the entire area
388 *
389 * @return Number of errors, zero on success
390 */
__bdk_dram_test_mem_checkerboard(uint64_t area,uint64_t max_address,int bursts)391 int __bdk_dram_test_mem_checkerboard(uint64_t area, uint64_t max_address, int bursts)
392 {
393 int failures = 0;
394 for (int burst = 0; burst < bursts; burst++)
395 failures += test_mem_march_c(area, max_address, 0x5555555555555555L);
396 return failures;
397 }
398
399 /**
400 * Write a pseudo random pattern to memory and verify it
401 *
402 * @param area Start of the physical memory area
403 * @param max_address
404 * End of the physical memory area
405 * @param bursts Number of time to repeat the test over the entire area
406 *
407 * @return Number of errors, zero on success
408 */
__bdk_dram_test_mem_random(uint64_t area,uint64_t max_address,int bursts)409 int __bdk_dram_test_mem_random(uint64_t area, uint64_t max_address, int bursts)
410 {
411 /* This constant is used to increment the pattern after every DWORD. This
412 makes only the first DWORD truly random, but saves us processing
413 power generating the random values */
414 const uint64_t INC = 0x1010101010101010ULL;
415
416 int failures = 0;
417 for (int burst = 0; burst < bursts; burst++)
418 {
419 const uint64_t init_pattern = bdk_rng_get_random64();
420 uint64_t pattern = init_pattern;
421
422 /* Write the pattern to memory. Each location receives the address
423 * of the location. A second write pass is needed to force all of
424 * the cached memory out to the DDR.
425 */
426 for (uint64_t address = area; address < max_address; address += 8)
427 {
428 WRITE64(address, pattern);
429 pattern += INC;
430 }
431
432 __bdk_dram_flush_to_mem_range(area, max_address);
433 BDK_DCACHE_INVALIDATE;
434
435 /* Read the written memory and confirm that it has the expected
436 * data pattern.
437 */
438 pattern = init_pattern;
439 for (uint64_t address = area; address < max_address; address += 8)
440 {
441 uint64_t data = READ64(address);
442 if (bdk_unlikely(data != pattern))
443 failures += __bdk_dram_retry_failure(burst, address, data, pattern);
444 pattern += INC;
445 }
446 }
447 return failures;
448 }
449
450 #if !USE_PREDICTION_CODE_VERSIONS
451 /**
452 * test_mem_xor
453 *
454 * @param area Start of the physical memory area
455 * @param max_address
456 * End of the physical memory area
457 * @param bursts Number of time to repeat the test over the entire area
458 *
459 * @return Number of errors, zero on success
460 */
__bdk_dram_test_mem_xor(uint64_t area,uint64_t max_address,int bursts)461 int __bdk_dram_test_mem_xor(uint64_t area, uint64_t max_address, int bursts)
462 {
463 int failures = 0;
464
465 uint64_t extent = max_address - area;
466 uint64_t count = (extent / sizeof(uint64_t)) / 2;
467
468 /* Fill both halves of the memory area with identical randomized data.
469 */
470 uint64_t address1 = area;
471 uint64_t address2 = area + count * sizeof(uint64_t);
472
473 uint64_t pattern = bdk_rng_get_random64();
474
475 for (uint64_t j = 0; j < count; j++)
476 {
477 uint64_t p = pattern * address1;
478 WRITE64(address1, p);
479 WRITE64(address2, p);
480 address1 += 8;
481 address2 += 8;
482 }
483 __bdk_dram_flush_to_mem_range(area, max_address);
484 BDK_DCACHE_INVALIDATE;
485
486 /* Make a series of passes over the memory areas. */
487 for (int burst = 0; burst < bursts; burst++)
488 {
489 /* XOR the data with a random value, applying the change to both
490 * memory areas.
491 */
492 address1 = area;
493 address2 = area + count * sizeof(uint64_t);
494
495 pattern = bdk_rng_get_random64();
496
497 for (uint64_t j = 0; j < count; j++)
498 {
499 if ((address1 & BDK_CACHE_LINE_MASK) == 0)
500 BDK_PREFETCH(address1, BDK_CACHE_LINE_SIZE);
501 if ((address2 & BDK_CACHE_LINE_MASK) == 0)
502 BDK_PREFETCH(address2, BDK_CACHE_LINE_SIZE);
503 WRITE64(address1, READ64(address1) ^ pattern);
504 WRITE64(address2, READ64(address2) ^ pattern);
505 address1 += 8;
506 address2 += 8;
507 }
508
509 __bdk_dram_flush_to_mem_range(area, max_address);
510 BDK_DCACHE_INVALIDATE;
511
512 /* Look for differences in the areas. If there is a mismatch, reset
513 * both memory locations with the same pattern. Failing to do so
514 * means that on all subsequent passes the pair of locations remain
515 * out of sync giving spurious errors.
516 */
517 address1 = area;
518 address2 = area + count * sizeof(uint64_t);
519 for (uint64_t j = 0; j < count; j++)
520 {
521 if ((address1 & BDK_CACHE_LINE_MASK) == 0)
522 BDK_PREFETCH(address1, BDK_CACHE_LINE_SIZE);
523 if ((address2 & BDK_CACHE_LINE_MASK) == 0)
524 BDK_PREFETCH(address2, BDK_CACHE_LINE_SIZE);
525 uint64_t d1 = READ64(address1);
526 uint64_t d2 = READ64(address2);
527 if (bdk_unlikely(d1 != d2))
528 {
529 failures += __bdk_dram_retry_failure2(burst, address1, d1, address2, d2);
530
531 // Synchronize the two areas, adjusting for the error.
532 WRITE64(address1, d2);
533 WRITE64(address2, d2);
534 }
535 address1 += 8;
536 address2 += 8;
537 }
538 }
539 return failures;
540 }
541
542 /**
543 * test_mem_rows
544 *
545 * Write a pattern of alternating 64-bit words of all one bits and then all 0
546 * bits. This pattern generates the maximum amount of simultaneous switching
547 * activity on the memory channels. Each pass flips the pattern with words
548 * going from all ones to all zeros and vice versa.
549 *
550 * @param area Start of the physical memory area
551 * @param max_address
552 * End of the physical memory area
553 * @param bursts Number of times to repeat the test over the entire area
554 *
555 * @return Number of errors, zero on success
556 */
__bdk_dram_test_mem_rows(uint64_t area,uint64_t max_address,int bursts)557 int __bdk_dram_test_mem_rows(uint64_t area, uint64_t max_address, int bursts)
558 {
559 int failures = 0;
560 uint64_t pattern = 0x0;
561 uint64_t extent = (max_address - area);
562 uint64_t count = (extent / 2) / sizeof(uint64_t); // in terms of 64bit words
563
564 /* Fill both halves of the memory area with identical data pattern. Odd
565 * address 64-bit words get the pattern, while even address words get the
566 * inverted pattern.
567 */
568 uint64_t address1 = area;
569 uint64_t address2 = area + count * sizeof(uint64_t);
570
571 for (uint64_t j = 0; j < (count / 2); j++)
572 {
573 WRITE64(address1, pattern);
574 WRITE64(address2, pattern);
575 address1 += 8;
576 address2 += 8;
577 WRITE64(address1, ~pattern);
578 WRITE64(address2, ~pattern);
579 address1 += 8;
580 address2 += 8;
581 }
582 __bdk_dram_flush_to_mem_range(area, max_address);
583 BDK_DCACHE_INVALIDATE;
584
585 /* Make a series of passes over the memory areas. */
586 for (int burst = 0; burst < bursts; burst++)
587 {
588 /* Invert the data, applying the change to both memory areas. Thus on
589 * alternate passes, the data flips from 0 to 1 and vice versa.
590 */
591 address1 = area;
592 address2 = area + count * sizeof(uint64_t);
593 for (uint64_t j = 0; j < count; j++)
594 {
595 WRITE64(address1, ~READ64(address1));
596 WRITE64(address2, ~READ64(address2));
597 address1 += 8;
598 address2 += 8;
599 }
600 __bdk_dram_flush_to_mem_range(area, max_address);
601 BDK_DCACHE_INVALIDATE;
602
603 /* Look for differences in the areas. If there is a mismatch, reset
604 * both memory locations with the same pattern. Failing to do so
605 * means that on all subsequent passes the pair of locations remain
606 * out of sync giving spurious errors.
607 */
608 address1 = area;
609 address2 = area + count * sizeof(uint64_t);
610 for (uint64_t j = 0; j < count; j++)
611 {
612 uint64_t d1 = READ64(address1);
613 uint64_t d2 = READ64(address2);
614 if (bdk_unlikely(d1 != d2))
615 {
616 failures += __bdk_dram_retry_failure2(burst, address1, d1, address2, d2);
617
618 // Synchronize the two areas, adjusting for the error.
619 WRITE64(address1, d2);
620 WRITE64(address2, d2);
621 }
622 address1 += 8;
623 address2 += 8;
624 }
625 }
626 return failures;
627 }
628 #endif /* !USE_PREDICTION_CODE_VERSIONS */
629
630 #if USE_PREDICTION_CODE_VERSIONS
631 //////////////////////////// this is the new code...
632
__bdk_dram_test_mem_xor(uint64_t area,uint64_t max_address,int bursts)633 int __bdk_dram_test_mem_xor(uint64_t area, uint64_t max_address, int bursts)
634 {
635 int failures = 0;
636 int burst;
637
638 uint64_t extent = max_address - area;
639 uint64_t count = (extent / sizeof(uint64_t)) / 2;
640 uint64_t offset = count * sizeof(uint64_t);
641 uint64_t area2 = area + offset;
642
643 /* Fill both halves of the memory area with identical randomized data.
644 */
645 uint64_t address1 = area;
646
647 uint64_t pattern1 = bdk_rng_get_random64();
648 uint64_t pattern2 = 0;
649 uint64_t this_pattern;
650
651 uint64_t p;
652 uint64_t d1, d2;
653
654 // move the multiplies outside the loop
655 uint64_t pbase = address1 * pattern1;
656 uint64_t pincr = 8 * pattern1;
657 uint64_t ppred;
658
659 p = pbase;
660 while (address1 < area2)
661 {
662 WRITE64(address1 , p);
663 WRITE64(address1 + offset, p);
664 address1 += 8;
665 p += pincr;
666 }
667 __bdk_dram_flush_to_mem_range(area, max_address);
668 BDK_DCACHE_INVALIDATE;
669
670 /* Make a series of passes over the memory areas. */
671 for (burst = 0; burst < bursts; burst++)
672 {
673 /* XOR the data with a random value, applying the change to both
674 * memory areas.
675 */
676 address1 = area;
677
678 this_pattern = bdk_rng_get_random64();
679 pattern2 ^= this_pattern;
680
681 while (address1 < area2)
682 {
683 #if 1
684 if ((address1 & BDK_CACHE_LINE_MASK) == 0)
685 BDK_PREFETCH(address1, BDK_CACHE_LINE_SIZE);
686 if (((address1 + offset) & BDK_CACHE_LINE_MASK) == 0)
687 BDK_PREFETCH(address1 + offset, BDK_CACHE_LINE_SIZE);
688 #endif
689 WRITE64(address1 , READ64(address1 ) ^ this_pattern);
690 WRITE64(address1 + offset, READ64(address1 + offset) ^ this_pattern);
691 address1 += 8;
692 }
693
694 __bdk_dram_flush_to_mem_range(area, max_address);
695 BDK_DCACHE_INVALIDATE;
696
697 /* Look for differences from the expected pattern in both areas.
698 * If there is a mismatch, reset the appropriate memory location
699 * with the correct pattern. Failing to do so
700 * means that on all subsequent passes the erroring locations
701 * will be out of sync, giving spurious errors.
702 */
703 address1 = area;
704 ppred = pbase;
705
706 while (address1 < area2)
707 {
708 #if 1
709 if ((address1 & BDK_CACHE_LINE_MASK) == 0)
710 BDK_PREFETCH(address1, BDK_CACHE_LINE_SIZE);
711 if (((address1 + offset) & BDK_CACHE_LINE_MASK) == 0)
712 BDK_PREFETCH(address1 + offset, BDK_CACHE_LINE_SIZE);
713 #endif
714 d1 = READ64(address1 );
715 d2 = READ64(address1 + offset);
716
717 p = ppred ^ pattern2;
718
719 if (bdk_unlikely(d1 != p)) {
720 failures += __bdk_dram_retry_failure(burst, address1, d1, p);
721 // Synchronize the area, adjusting for the error.
722 //WRITE64(address1, p); // retries should do this
723 }
724 if (bdk_unlikely(d2 != p)) {
725 failures += __bdk_dram_retry_failure(burst, address1 + offset, d2, p);
726 // Synchronize the area, adjusting for the error.
727 //WRITE64(address1 + offset, p); // retries should do this
728 }
729
730 address1 += 8;
731 ppred += pincr;
732
733 } /* while (address1 < area2) */
734 } /* for (int burst = 0; burst < bursts; burst++) */
735 return failures;
736 }
737
738 //////////////// this is the new code...
739
__bdk_dram_test_mem_rows(uint64_t area,uint64_t max_address,int bursts)740 int __bdk_dram_test_mem_rows(uint64_t area, uint64_t max_address, int bursts)
741 {
742 int failures = 0;
743
744 uint64_t pattern1 = 0x0;
745 uint64_t extent = (max_address - area);
746 uint64_t count = (extent / 2) / sizeof(uint64_t); // in terms of 64bit words
747 uint64_t offset = count * sizeof(uint64_t);
748 uint64_t area2 = area + offset;
749 uint64_t pattern2;
750 uint64_t d1, d2;
751 int burst;
752
753 /* Fill both halves of the memory area with identical data pattern. Odd
754 * address 64-bit words get the pattern, while even address words get the
755 * inverted pattern.
756 */
757 uint64_t address1 = area;
758
759 pattern2 = pattern1; // start with original pattern
760
761 while (address1 < area2)
762 {
763 WRITE64(address1 , pattern2);
764 WRITE64(address1 + offset, pattern2);
765 address1 += 8;
766 pattern2 = ~pattern2; // flip for next slots
767 }
768
769 __bdk_dram_flush_to_mem_range(area, max_address);
770 BDK_DCACHE_INVALIDATE;
771
772 /* Make a series of passes over the memory areas. */
773 for (burst = 0; burst < bursts; burst++)
774 {
775 /* Invert the data, applying the change to both memory areas. Thus on
776 * alternate passes, the data flips from 0 to 1 and vice versa.
777 */
778 address1 = area;
779
780 while (address1 < area2)
781 {
782 if ((address1 & BDK_CACHE_LINE_MASK) == 0)
783 BDK_PREFETCH(address1 , BDK_CACHE_LINE_SIZE);
784 if (((address1 + offset) & BDK_CACHE_LINE_MASK) == 0)
785 BDK_PREFETCH(address1 + offset, BDK_CACHE_LINE_SIZE);
786
787 WRITE64(address1 , ~READ64(address1 ));
788 WRITE64(address1 + offset, ~READ64(address1 + offset));
789 address1 += 8;
790 }
791
792 __bdk_dram_flush_to_mem_range(area, max_address);
793 BDK_DCACHE_INVALIDATE;
794
795 /* Look for differences in the areas. If there is a mismatch, reset
796 * both memory locations with the same pattern. Failing to do so
797 * means that on all subsequent passes the pair of locations remain
798 * out of sync giving spurious errors.
799 */
800 address1 = area;
801 pattern1 = ~pattern1; // flip the starting pattern to match above loop
802 pattern2 = pattern1; // slots have been flipped by the above loop
803
804 while (address1 < area2)
805 {
806 if ((address1 & BDK_CACHE_LINE_MASK) == 0)
807 BDK_PREFETCH(address1 , BDK_CACHE_LINE_SIZE);
808 if (((address1 + offset) & BDK_CACHE_LINE_MASK) == 0)
809 BDK_PREFETCH(address1 + offset, BDK_CACHE_LINE_SIZE);
810
811 d1 = READ64(address1 );
812 d2 = READ64(address1 + offset);
813
814 if (bdk_unlikely(d1 != pattern2)) {
815 failures += __bdk_dram_retry_failure(burst, address1, d1, pattern2);
816 // Synchronize the area, adjusting for the error.
817 //WRITE64(address1, pattern2); // retries should do this
818 }
819 if (bdk_unlikely(d2 != pattern2)) {
820 failures += __bdk_dram_retry_failure(burst, address1 + offset, d2, pattern2);
821 // Synchronize the two areas, adjusting for the error.
822 //WRITE64(address1 + offset, pattern2); // retries should do this
823 }
824
825 address1 += 8;
826 pattern2 = ~pattern2; // flip for next pair of slots
827 }
828 }
829 return failures;
830 }
831 #endif /* USE_PREDICTION_CODE_VERSIONS */
832