Lines Matching +full:entry +full:- +full:address
1 // SPDX-License-Identifier: GPL-2.0-only
8 #define pr_fmt(fmt) "DMA-API: " fmt
12 #include <linux/dma-map-ops.h>
31 #define HASH_FN_MASK (HASH_SIZE - 1)
53 * struct dma_debug_entry - track a dma_map* or dma_alloc_coherent mapping
54 * @list: node on pre-allocated free_entries list
56 * @dev_addr: dma address
62 * @paddr: physical start address of the mapping
93 /* List of pre-allocated dma_debug_entry's */
98 /* Global disable flag - will be set in case of an error */
124 /* per-driver filter related state */
141 [dma_debug_sg] = "scatter-gather",
166 static inline void dump_entry_trace(struct dma_debug_entry *entry) in dump_entry_trace() argument
169 if (entry) { in dump_entry_trace()
171 stack_trace_print(entry->stack_entries, entry->stack_len, 0); in dump_entry_trace()
187 if (current_driver && dev && dev->driver == current_driver) in driver_filter()
198 drv = dev->driver; in driver_filter()
206 if (drv->name && in driver_filter()
207 strncmp(current_driver_name, drv->name, NAME_MAX_LEN - 1) == 0) { in driver_filter()
217 #define err_printk(dev, entry, format, arg...) do { \ argument
224 dump_entry_trace(entry); \
227 show_num_errors -= 1; \
233 * Every DMA-API request is saved into a struct dma_debug_entry. To
236 static int hash_fn(struct dma_debug_entry *entry) in hash_fn() argument
239 * Hash function is based on the dma address. in hash_fn()
240 * We use bits 20-27 here as the index into the hash in hash_fn()
242 return (entry->dev_addr >> HASH_FN_SHIFT) & HASH_FN_MASK; in hash_fn()
248 static struct hash_bucket *get_hash_bucket(struct dma_debug_entry *entry, in get_hash_bucket() argument
252 int idx = hash_fn(entry); in get_hash_bucket()
265 __releases(&bucket->lock) in put_hash_bucket()
267 spin_unlock_irqrestore(&bucket->lock, flags); in put_hash_bucket()
272 return ((a->dev_addr == b->dev_addr) && in exact_match()
273 (a->dev == b->dev)) ? true : false; in exact_match()
279 if (a->dev != b->dev) in containing_match()
282 if ((b->dev_addr <= a->dev_addr) && in containing_match()
283 ((b->dev_addr + b->size) >= (a->dev_addr + a->size))) in containing_match()
290 * Search a given entry in the hash bucket list
296 struct dma_debug_entry *entry, *ret = NULL; in __hash_bucket_find() local
297 int matches = 0, match_lvl, last_lvl = -1; in __hash_bucket_find()
299 list_for_each_entry(entry, &bucket->list, list) { in __hash_bucket_find()
300 if (!match(ref, entry)) in __hash_bucket_find()
304 * Some drivers map the same physical address multiple in __hash_bucket_find()
306 * same device addresses being put into the dma-debug in __hash_bucket_find()
309 * best-fit algorithm here which returns the entry from in __hash_bucket_find()
311 * instead of the first-fit. in __hash_bucket_find()
315 entry->size == ref->size ? ++match_lvl : 0; in __hash_bucket_find()
316 entry->type == ref->type ? ++match_lvl : 0; in __hash_bucket_find()
317 entry->direction == ref->direction ? ++match_lvl : 0; in __hash_bucket_find()
318 entry->sg_call_ents == ref->sg_call_ents ? ++match_lvl : 0; in __hash_bucket_find()
321 /* perfect-fit - return the result */ in __hash_bucket_find()
322 return entry; in __hash_bucket_find()
325 * We found an entry that fits better then the in __hash_bucket_find()
329 ret = entry; in __hash_bucket_find()
334 * If we have multiple matches but no perfect-fit, just return in __hash_bucket_find()
353 struct dma_debug_entry *entry, index = *ref; in bucket_find_contain() local
357 entry = __hash_bucket_find(*bucket, ref, containing_match); in bucket_find_contain()
359 if (entry) in bucket_find_contain()
360 return entry; in bucket_find_contain()
366 index.dev_addr -= (1 << HASH_FN_SHIFT); in bucket_find_contain()
374 * Add an entry to a hash bucket
377 struct dma_debug_entry *entry) in hash_bucket_add() argument
379 list_add_tail(&entry->list, &bucket->list); in hash_bucket_add()
383 * Remove entry from a hash bucket list
385 static void hash_bucket_del(struct dma_debug_entry *entry) in hash_bucket_del() argument
387 list_del(&entry->list); in hash_bucket_del()
395 * dma_unmap_{single|sg|page} or dma_free_coherent delete the entry. If
396 * the entry already exists at insertion time add a tag as a reference
403 * dma-debug entries in that we need a free dma_debug_entry before
406 * dma_active_cacheline entry to track per event. dma_map_sg(), on the
415 #define ACTIVE_CACHELINE_MAX_OVERLAP ((1 << RADIX_TREE_MAX_TAGS) - 1)
416 #define CACHELINE_PER_PAGE_SHIFT (PAGE_SHIFT - L1_CACHE_SHIFT)
419 static phys_addr_t to_cacheline_number(struct dma_debug_entry *entry) in to_cacheline_number() argument
421 return ((entry->paddr >> PAGE_SHIFT) << CACHELINE_PER_PAGE_SHIFT) + in to_cacheline_number()
422 (offset_in_page(entry->paddr) >> L1_CACHE_SHIFT); in to_cacheline_number()
429 for (i = RADIX_TREE_MAX_TAGS - 1; i >= 0; i--) in active_cacheline_read_overlap()
442 for (i = RADIX_TREE_MAX_TAGS - 1; i >= 0; i--) in active_cacheline_set_overlap()
458 * leaking dma-mappings. in active_cacheline_inc_overlap()
469 return active_cacheline_set_overlap(cln, --overlap); in active_cacheline_dec_overlap()
472 static int active_cacheline_insert(struct dma_debug_entry *entry) in active_cacheline_insert() argument
474 phys_addr_t cln = to_cacheline_number(entry); in active_cacheline_insert()
482 if (entry->direction == DMA_TO_DEVICE) in active_cacheline_insert()
486 rc = radix_tree_insert(&dma_active_cacheline, cln, entry); in active_cacheline_insert()
487 if (rc == -EEXIST) in active_cacheline_insert()
494 static void active_cacheline_remove(struct dma_debug_entry *entry) in active_cacheline_remove() argument
496 phys_addr_t cln = to_cacheline_number(entry); in active_cacheline_remove()
500 if (entry->direction == DMA_TO_DEVICE) in active_cacheline_remove()
506 * active_cacheline_dec_overlap() returns -1 in that case in active_cacheline_remove()
523 struct dma_debug_entry *entry; in debug_dma_dump_mappings() local
526 spin_lock_irqsave(&bucket->lock, flags); in debug_dma_dump_mappings()
527 list_for_each_entry(entry, &bucket->list, list) { in debug_dma_dump_mappings()
528 if (!dev || dev == entry->dev) { in debug_dma_dump_mappings()
529 cln = to_cacheline_number(entry); in debug_dma_dump_mappings()
530 dev_info(entry->dev, in debug_dma_dump_mappings()
532 type2name[entry->type], idx, in debug_dma_dump_mappings()
533 &entry->paddr, entry->dev_addr, in debug_dma_dump_mappings()
534 entry->size, &cln, in debug_dma_dump_mappings()
535 dir2name[entry->direction], in debug_dma_dump_mappings()
536 maperr2str[entry->map_err_type]); in debug_dma_dump_mappings()
539 spin_unlock_irqrestore(&bucket->lock, flags); in debug_dma_dump_mappings()
555 struct dma_debug_entry *entry; in dump_show() local
558 spin_lock_irqsave(&bucket->lock, flags); in dump_show()
559 list_for_each_entry(entry, &bucket->list, list) { in dump_show()
560 cln = to_cacheline_number(entry); in dump_show()
563 dev_driver_string(entry->dev), in dump_show()
564 dev_name(entry->dev), in dump_show()
565 type2name[entry->type], idx, in dump_show()
566 &entry->paddr, entry->dev_addr, in dump_show()
567 entry->size, &cln, in dump_show()
568 dir2name[entry->direction], in dump_show()
569 maperr2str[entry->map_err_type]); in dump_show()
571 spin_unlock_irqrestore(&bucket->lock, flags); in dump_show()
578 * Wrapper function for adding an entry to the hash.
581 static void add_dma_entry(struct dma_debug_entry *entry, unsigned long attrs) in add_dma_entry() argument
587 bucket = get_hash_bucket(entry, &flags); in add_dma_entry()
588 hash_bucket_add(bucket, entry); in add_dma_entry()
591 rc = active_cacheline_insert(entry); in add_dma_entry()
592 if (rc == -ENOMEM) { in add_dma_entry()
593 pr_err_once("cacheline tracking ENOMEM, dma-debug disabled\n"); in add_dma_entry()
595 } else if (rc == -EEXIST && !(attrs & DMA_ATTR_SKIP_CPU_SYNC)) { in add_dma_entry()
596 err_printk(entry->dev, entry, in add_dma_entry()
603 struct dma_debug_entry *entry; in dma_debug_create_entries() local
606 entry = (void *)get_zeroed_page(gfp); in dma_debug_create_entries()
607 if (!entry) in dma_debug_create_entries()
608 return -ENOMEM; in dma_debug_create_entries()
611 list_add_tail(&entry[i].list, &free_entries); in dma_debug_create_entries()
621 struct dma_debug_entry *entry; in __dma_entry_alloc() local
623 entry = list_entry(free_entries.next, struct dma_debug_entry, list); in __dma_entry_alloc()
624 list_del(&entry->list); in __dma_entry_alloc()
625 memset(entry, 0, sizeof(*entry)); in __dma_entry_alloc()
627 num_free_entries -= 1; in __dma_entry_alloc()
631 return entry; in __dma_entry_alloc()
658 struct dma_debug_entry *entry; in dma_entry_alloc() local
667 pr_err("debugging out of memory - disabling\n"); in dma_entry_alloc()
674 entry = __dma_entry_alloc(); in dma_entry_alloc()
682 entry->stack_len = stack_trace_save(entry->stack_entries, in dma_entry_alloc()
683 ARRAY_SIZE(entry->stack_entries), in dma_entry_alloc()
686 return entry; in dma_entry_alloc()
689 static void dma_entry_free(struct dma_debug_entry *entry) in dma_entry_free() argument
693 active_cacheline_remove(entry); in dma_entry_free()
696 * add to beginning of the list - this way the entries are in dma_entry_free()
700 list_add(&entry->list, &free_entries); in dma_entry_free()
706 * DMA-API debugging init code
749 len = min(count, (size_t)(NAME_MAX_LEN - 1)); in filter_write()
751 return -EFAULT; in filter_write()
760 * - only use the first token we got in filter_write()
761 * - token delimiter is everything looking like a space in filter_write()
772 pr_info("switching off dma-debug driver filter\n"); in filter_write()
782 for (i = 0; i < NAME_MAX_LEN - 1; ++i) { in filter_write()
807 struct dentry *dentry = debugfs_create_dir("dma-api", NULL); in dma_debug_fs_init()
825 struct dma_debug_entry *entry; in device_dma_allocations() local
831 list_for_each_entry(entry, &dma_entry_hash[i].list, list) { in device_dma_allocations()
832 if (entry->dev == dev) { in device_dma_allocations()
834 *out_entry = entry; in device_dma_allocations()
846 struct dma_debug_entry *entry; in dma_debug_device_change() local
854 count = device_dma_allocations(dev, &entry); in dma_debug_device_change()
857 err_printk(dev, entry, "device driver has pending " in dma_debug_device_change()
861 "[device address=0x%016llx] [size=%llu bytes] " in dma_debug_device_change()
863 count, entry->dev_addr, entry->size, in dma_debug_device_change()
864 dir2name[entry->direction], type2name[entry->type]); in dma_debug_device_change()
886 nb->notifier_call = dma_debug_device_change; in dma_debug_add_bus()
915 pr_err("debugging out of memory error - disabled\n"); in dma_debug_init()
932 return -EINVAL; in dma_debug_cmdline()
945 return -EINVAL; in dma_debug_entries_cmdline()
956 struct dma_debug_entry *entry; in check_unmap() local
961 entry = bucket_find_exact(bucket, ref); in check_unmap()
963 if (!entry) { in check_unmap()
967 if (dma_mapping_error(ref->dev, ref->dev_addr)) { in check_unmap()
968 err_printk(ref->dev, NULL, in check_unmap()
970 "invalid DMA memory address\n"); in check_unmap()
972 err_printk(ref->dev, NULL, in check_unmap()
975 "address=0x%016llx] [size=%llu bytes]\n", in check_unmap()
976 ref->dev_addr, ref->size); in check_unmap()
981 if (ref->size != entry->size) { in check_unmap()
982 err_printk(ref->dev, entry, "device driver frees " in check_unmap()
984 "[device address=0x%016llx] [map size=%llu bytes] " in check_unmap()
986 ref->dev_addr, entry->size, ref->size); in check_unmap()
989 if (ref->type != entry->type) { in check_unmap()
990 err_printk(ref->dev, entry, "device driver frees " in check_unmap()
992 "[device address=0x%016llx] [size=%llu bytes] " in check_unmap()
994 ref->dev_addr, ref->size, in check_unmap()
995 type2name[entry->type], type2name[ref->type]); in check_unmap()
996 } else if (entry->type == dma_debug_coherent && in check_unmap()
997 ref->paddr != entry->paddr) { in check_unmap()
998 err_printk(ref->dev, entry, "device driver frees " in check_unmap()
999 "DMA memory with different CPU address " in check_unmap()
1000 "[device address=0x%016llx] [size=%llu bytes] " in check_unmap()
1001 "[cpu alloc address=0x%pa] " in check_unmap()
1002 "[cpu free address=0x%pa]", in check_unmap()
1003 ref->dev_addr, ref->size, in check_unmap()
1004 &entry->paddr, in check_unmap()
1005 &ref->paddr); in check_unmap()
1008 if (ref->sg_call_ents && ref->type == dma_debug_sg && in check_unmap()
1009 ref->sg_call_ents != entry->sg_call_ents) { in check_unmap()
1010 err_printk(ref->dev, entry, "device driver frees " in check_unmap()
1011 "DMA sg list with different entry count " in check_unmap()
1013 entry->sg_call_ents, ref->sg_call_ents); in check_unmap()
1017 * This may be no bug in reality - but most implementations of the in check_unmap()
1020 if (ref->direction != entry->direction) { in check_unmap()
1021 err_printk(ref->dev, entry, "device driver frees " in check_unmap()
1023 "[device address=0x%016llx] [size=%llu bytes] " in check_unmap()
1025 ref->dev_addr, ref->size, in check_unmap()
1026 dir2name[entry->direction], in check_unmap()
1027 dir2name[ref->direction]); in check_unmap()
1033 * If not, print this warning message. See Documentation/core-api/dma-api.rst. in check_unmap()
1035 if (entry->map_err_type == MAP_ERR_NOT_CHECKED) { in check_unmap()
1036 err_printk(ref->dev, entry, in check_unmap()
1038 "[device address=0x%016llx] [size=%llu bytes] " in check_unmap()
1040 ref->dev_addr, ref->size, in check_unmap()
1041 type2name[entry->type]); in check_unmap()
1044 hash_bucket_del(entry); in check_unmap()
1048 * Free the entry outside of bucket_lock to avoid ABBA deadlocks in check_unmap()
1051 dma_entry_free(entry); in check_unmap()
1061 /* Stack is direct-mapped. */ in check_for_stack()
1071 for (i = 0; i < stack_vm_area->nr_pages; i++) { in check_for_stack()
1072 if (page != stack_vm_area->pages[i]) in check_for_stack()
1075 addr = (u8 *)current->stack + i * PAGE_SIZE + offset; in check_for_stack()
1093 struct dma_debug_entry *entry; in check_sync() local
1099 entry = bucket_find_contain(&bucket, ref, &flags); in check_sync()
1101 if (!entry) { in check_sync()
1104 "[device address=0x%016llx] [size=%llu bytes]\n", in check_sync()
1105 (unsigned long long)ref->dev_addr, ref->size); in check_sync()
1109 if (ref->size > entry->size) { in check_sync()
1110 err_printk(dev, entry, "device driver syncs" in check_sync()
1112 "[device address=0x%016llx] " in check_sync()
1115 entry->dev_addr, entry->size, in check_sync()
1116 ref->size); in check_sync()
1119 if (entry->direction == DMA_BIDIRECTIONAL) in check_sync()
1122 if (ref->direction != entry->direction) { in check_sync()
1123 err_printk(dev, entry, "device driver syncs " in check_sync()
1125 "[device address=0x%016llx] [size=%llu bytes] " in check_sync()
1127 (unsigned long long)ref->dev_addr, entry->size, in check_sync()
1128 dir2name[entry->direction], in check_sync()
1129 dir2name[ref->direction]); in check_sync()
1132 if (to_cpu && !(entry->direction == DMA_FROM_DEVICE) && in check_sync()
1133 !(ref->direction == DMA_TO_DEVICE)) in check_sync()
1134 err_printk(dev, entry, "device driver syncs " in check_sync()
1135 "device read-only DMA memory for cpu " in check_sync()
1136 "[device address=0x%016llx] [size=%llu bytes] " in check_sync()
1138 (unsigned long long)ref->dev_addr, entry->size, in check_sync()
1139 dir2name[entry->direction], in check_sync()
1140 dir2name[ref->direction]); in check_sync()
1142 if (!to_cpu && !(entry->direction == DMA_TO_DEVICE) && in check_sync()
1143 !(ref->direction == DMA_FROM_DEVICE)) in check_sync()
1144 err_printk(dev, entry, "device driver syncs " in check_sync()
1145 "device write-only DMA memory to device " in check_sync()
1146 "[device address=0x%016llx] [size=%llu bytes] " in check_sync()
1148 (unsigned long long)ref->dev_addr, entry->size, in check_sync()
1149 dir2name[entry->direction], in check_sync()
1150 dir2name[ref->direction]); in check_sync()
1152 if (ref->sg_call_ents && ref->type == dma_debug_sg && in check_sync()
1153 ref->sg_call_ents != entry->sg_call_ents) { in check_sync()
1154 err_printk(ref->dev, entry, "device driver syncs " in check_sync()
1155 "DMA sg list with different entry count " in check_sync()
1157 entry->sg_call_ents, ref->sg_call_ents); in check_sync()
1173 if (sg->length > max_seg) in check_sg_segment()
1175 sg->length, max_seg); in check_sg_segment()
1182 end = start + sg_dma_len(sg) - 1; in check_sg_segment()
1208 struct dma_debug_entry *entry; in debug_dma_map_page() local
1216 entry = dma_entry_alloc(); in debug_dma_map_page()
1217 if (!entry) in debug_dma_map_page()
1220 entry->dev = dev; in debug_dma_map_page()
1221 entry->type = dma_debug_single; in debug_dma_map_page()
1222 entry->paddr = page_to_phys(page) + offset; in debug_dma_map_page()
1223 entry->dev_addr = dma_addr; in debug_dma_map_page()
1224 entry->size = size; in debug_dma_map_page()
1225 entry->direction = direction; in debug_dma_map_page()
1226 entry->map_err_type = MAP_ERR_NOT_CHECKED; in debug_dma_map_page()
1236 add_dma_entry(entry, attrs); in debug_dma_map_page()
1242 struct dma_debug_entry *entry; in debug_dma_mapping_error() local
1253 list_for_each_entry(entry, &bucket->list, list) { in debug_dma_mapping_error()
1254 if (!exact_match(&ref, entry)) in debug_dma_mapping_error()
1258 * The same physical address can be mapped multiple in debug_dma_mapping_error()
1260 * same device addresses being put into the dma-debug in debug_dma_mapping_error()
1263 * best-fit algorithm here which updates the first entry in debug_dma_mapping_error()
1267 if (entry->map_err_type == MAP_ERR_NOT_CHECKED) { in debug_dma_mapping_error()
1268 entry->map_err_type = MAP_ERR_CHECKED; in debug_dma_mapping_error()
1297 struct dma_debug_entry *entry; in debug_dma_map_sg() local
1305 check_for_stack(dev, sg_page(s), s->offset); in debug_dma_map_sg()
1307 check_for_illegal_area(dev, sg_virt(s), s->length); in debug_dma_map_sg()
1311 entry = dma_entry_alloc(); in debug_dma_map_sg()
1312 if (!entry) in debug_dma_map_sg()
1315 entry->type = dma_debug_sg; in debug_dma_map_sg()
1316 entry->dev = dev; in debug_dma_map_sg()
1317 entry->paddr = sg_phys(s); in debug_dma_map_sg()
1318 entry->size = sg_dma_len(s); in debug_dma_map_sg()
1319 entry->dev_addr = sg_dma_address(s); in debug_dma_map_sg()
1320 entry->direction = direction; in debug_dma_map_sg()
1321 entry->sg_call_ents = nents; in debug_dma_map_sg()
1322 entry->sg_mapped_ents = mapped_ents; in debug_dma_map_sg()
1326 add_dma_entry(entry, attrs); in debug_dma_map_sg()
1333 struct dma_debug_entry *entry; in get_nr_mapped_entries() local
1339 entry = bucket_find_exact(bucket, ref); in get_nr_mapped_entries()
1342 if (entry) in get_nr_mapped_entries()
1343 mapped_ents = entry->sg_mapped_ents; in get_nr_mapped_entries()
1396 struct dma_debug_entry *entry; in debug_dma_alloc_coherent() local
1408 entry = dma_entry_alloc(); in debug_dma_alloc_coherent()
1409 if (!entry) in debug_dma_alloc_coherent()
1412 entry->type = dma_debug_coherent; in debug_dma_alloc_coherent()
1413 entry->dev = dev; in debug_dma_alloc_coherent()
1414 entry->paddr = virt_to_paddr(virt); in debug_dma_alloc_coherent()
1415 entry->size = size; in debug_dma_alloc_coherent()
1416 entry->dev_addr = dma_addr; in debug_dma_alloc_coherent()
1417 entry->direction = DMA_BIDIRECTIONAL; in debug_dma_alloc_coherent()
1419 add_dma_entry(entry, attrs); in debug_dma_alloc_coherent()
1449 struct dma_debug_entry *entry; in debug_dma_map_resource() local
1454 entry = dma_entry_alloc(); in debug_dma_map_resource()
1455 if (!entry) in debug_dma_map_resource()
1458 entry->type = dma_debug_resource; in debug_dma_map_resource()
1459 entry->dev = dev; in debug_dma_map_resource()
1460 entry->paddr = addr; in debug_dma_map_resource()
1461 entry->size = size; in debug_dma_map_resource()
1462 entry->dev_addr = dma_addr; in debug_dma_map_resource()
1463 entry->direction = direction; in debug_dma_map_resource()
1464 entry->map_err_type = MAP_ERR_NOT_CHECKED; in debug_dma_map_resource()
1466 add_dma_entry(entry, attrs); in debug_dma_map_resource()
1588 for (i = 0; i < NAME_MAX_LEN - 1; ++i, ++str) { in dma_debug_driver_setup()