Lines Matching full:object
17 * del_state modifications and accesses to the object trees
20 * kmemleak_object) for the allocated memory blocks. The object trees are
23 * the object_list and the object tree root in the create_object() function
38 * Note that the kmemleak_object.use_count is incremented when an object is
46 * scan_mutex [-> object->lock] -> kmemleak_lock -> other_object->lock (SINGLE_DEPTH_NESTING)
48 * No kmemleak_lock and object->lock nesting is allowed outside scan_mutex
110 #define MSECS_MIN_AGE 5000 /* minimum object age for reporting */
130 * object->lock. Insertions or deletions from object_list, gray_list or
137 unsigned int flags; /* object status flags */
142 /* object usage count; object freed when use_count == 0 */
151 /* the total number of pointers found pointing to this object */
156 /* memory ranges to be scanned inside an object (empty for all) */
165 /* flag set after the first reporting of an unreference object */
167 /* flag set to not scan the object */
169 /* flag set to fully scan the object when scan_area allocation failed */
171 /* flag set for object allocated with physical address */
199 /* search tree for object boundaries */
201 /* search tree for object (with OBJECT_PHYS flag) boundaries */
203 /* search tree for object (with OBJECT_PERCPU flag) boundaries */
293 * with the object->lock held.
296 struct kmemleak_object *object) in hex_dump_object() argument
298 const u8 *ptr = (const u8 *)object->pointer; in hex_dump_object()
301 if (WARN_ON_ONCE(object->flags & OBJECT_PHYS)) in hex_dump_object()
304 if (object->flags & OBJECT_PERCPU) in hex_dump_object()
305 ptr = (const u8 *)this_cpu_ptr((void __percpu *)object->pointer); in hex_dump_object()
308 len = min_t(size_t, object->size, HEX_MAX_LINES * HEX_ROW_SIZE); in hex_dump_object()
310 if (object->flags & OBJECT_PERCPU) in hex_dump_object()
322 * Object colors, encoded with count and min_count:
323 * - white - orphan object, not enough references to it (count < min_count)
328 * Newly created objects don't have any color assigned (object->count == -1)
331 static bool color_white(const struct kmemleak_object *object) in color_white() argument
333 return object->count != KMEMLEAK_BLACK && in color_white()
334 object->count < object->min_count; in color_white()
337 static bool color_gray(const struct kmemleak_object *object) in color_gray() argument
339 return object->min_count != KMEMLEAK_BLACK && in color_gray()
340 object->count >= object->min_count; in color_gray()
348 static bool unreferenced_object(struct kmemleak_object *object) in unreferenced_object() argument
350 return (color_white(object) && object->flags & OBJECT_ALLOCATED) && in unreferenced_object()
351 time_before_eq(object->jiffies + jiffies_min_age, in unreferenced_object()
357 * print_unreferenced function must be called with the object->lock held.
360 struct kmemleak_object *object) in print_unreferenced() argument
366 nr_entries = stack_depot_fetch(object->trace_handle, &entries); in print_unreferenced()
367 warn_or_seq_printf(seq, "unreferenced object 0x%08lx (size %zu):\n", in print_unreferenced()
368 object->pointer, object->size); in print_unreferenced()
370 object->comm, object->pid, object->jiffies); in print_unreferenced()
371 hex_dump_object(seq, object); in print_unreferenced()
372 warn_or_seq_printf(seq, " backtrace (crc %x):\n", object->checksum); in print_unreferenced()
383 * the object->lock held.
385 static void dump_object_info(struct kmemleak_object *object) in dump_object_info() argument
387 pr_notice("Object 0x%08lx (size %zu):\n", in dump_object_info()
388 object->pointer, object->size); in dump_object_info()
390 object->comm, object->pid, object->jiffies); in dump_object_info()
391 pr_notice(" min_count = %d\n", object->min_count); in dump_object_info()
392 pr_notice(" count = %d\n", object->count); in dump_object_info()
393 pr_notice(" flags = 0x%x\n", object->flags); in dump_object_info()
394 pr_notice(" checksum = %u\n", object->checksum); in dump_object_info()
396 if (object->trace_handle) in dump_object_info()
397 stack_depot_print(object->trace_handle); in dump_object_info()
410 * Look-up a memory block metadata (kmemleak_object) in the object search
422 struct kmemleak_object *object; in __lookup_object() local
425 object = rb_entry(rb, struct kmemleak_object, rb_node); in __lookup_object()
426 untagged_objp = (unsigned long)kasan_reset_tag((void *)object->pointer); in __lookup_object()
429 rb = object->rb_node.rb_left; in __lookup_object()
430 else if (untagged_objp + object->size <= untagged_ptr) in __lookup_object()
431 rb = object->rb_node.rb_right; in __lookup_object()
433 return object; in __lookup_object()
435 kmemleak_warn("Found object by alias at 0x%08lx\n", in __lookup_object()
437 dump_object_info(object); in __lookup_object()
444 /* Look-up a kmemleak object which allocated with virtual address. */
451 * Increment the object use_count. Return 1 if successful or 0 otherwise. Note
452 * that once an object's use_count reached 0, the RCU freeing was already
453 * registered and the object should no longer be used. This function must be
456 static int get_object(struct kmemleak_object *object) in get_object() argument
458 return atomic_inc_not_zero(&object->use_count); in get_object()
467 struct kmemleak_object *object; in mem_pool_alloc() local
471 object = kmem_cache_alloc_noprof(object_cache, in mem_pool_alloc()
473 if (object) in mem_pool_alloc()
474 return object; in mem_pool_alloc()
479 object = list_first_entry_or_null(&mem_pool_free_list, in mem_pool_alloc()
480 typeof(*object), object_list); in mem_pool_alloc()
481 if (object) in mem_pool_alloc()
482 list_del(&object->object_list); in mem_pool_alloc()
484 object = &mem_pool[--mem_pool_free_count]; in mem_pool_alloc()
489 return object; in mem_pool_alloc()
493 * Return the object to either the slab allocator or the memory pool.
495 static void mem_pool_free(struct kmemleak_object *object) in mem_pool_free() argument
499 if (object < mem_pool || object >= mem_pool + ARRAY_SIZE(mem_pool)) { in mem_pool_free()
500 kmem_cache_free(object_cache, object); in mem_pool_free()
504 /* add the object to the memory pool free list */ in mem_pool_free()
506 list_add(&object->object_list, &mem_pool_free_list); in mem_pool_free()
517 struct kmemleak_object *object = in free_object_rcu() local
522 * code accessing this object, hence no need for locking. in free_object_rcu()
524 hlist_for_each_entry_safe(area, tmp, &object->area_list, node) { in free_object_rcu()
528 mem_pool_free(object); in free_object_rcu()
532 * Decrement the object use_count. Once the count is 0, free the object using
538 static void put_object(struct kmemleak_object *object) in put_object() argument
540 if (!atomic_dec_and_test(&object->use_count)) in put_object()
544 WARN_ON(object->flags & OBJECT_ALLOCATED); in put_object()
549 * came from the memory pool. Free the object directly. in put_object()
552 call_rcu(&object->rcu, free_object_rcu); in put_object()
554 free_object_rcu(&object->rcu); in put_object()
558 * Look up an object in the object search tree and increase its use_count.
564 struct kmemleak_object *object; in __find_and_get_object() local
568 object = __lookup_object(ptr, alias, objflags); in __find_and_get_object()
571 /* check whether the object is still available */ in __find_and_get_object()
572 if (object && !get_object(object)) in __find_and_get_object()
573 object = NULL; in __find_and_get_object()
576 return object; in __find_and_get_object()
579 /* Look up and get an object which allocated with virtual address. */
586 * Remove an object from its object tree and object_list. Must be called with
589 static void __remove_object(struct kmemleak_object *object) in __remove_object() argument
591 rb_erase(&object->rb_node, object_tree(object->flags)); in __remove_object()
592 if (!(object->del_state & DELSTATE_NO_DELETE)) in __remove_object()
593 list_del_rcu(&object->object_list); in __remove_object()
594 object->del_state |= DELSTATE_REMOVED; in __remove_object()
601 struct kmemleak_object *object; in __find_and_remove_object() local
603 object = __lookup_object(ptr, alias, objflags); in __find_and_remove_object()
604 if (object) in __find_and_remove_object()
605 __remove_object(object); in __find_and_remove_object()
607 return object; in __find_and_remove_object()
611 * Look up an object in the object search tree and remove it from both object
612 * tree root and object_list. The returned object's use_count should be at
619 struct kmemleak_object *object; in find_and_remove_object() local
622 object = __find_and_remove_object(ptr, alias, objflags); in find_and_remove_object()
625 return object; in find_and_remove_object()
649 struct kmemleak_object *object; in __alloc_object() local
651 object = mem_pool_alloc(gfp); in __alloc_object()
652 if (!object) { in __alloc_object()
658 INIT_LIST_HEAD(&object->object_list); in __alloc_object()
659 INIT_LIST_HEAD(&object->gray_list); in __alloc_object()
660 INIT_HLIST_HEAD(&object->area_list); in __alloc_object()
661 raw_spin_lock_init(&object->lock); in __alloc_object()
662 atomic_set(&object->use_count, 1); in __alloc_object()
663 object->excess_ref = 0; in __alloc_object()
664 object->count = 0; /* white color initially */ in __alloc_object()
665 object->checksum = 0; in __alloc_object()
666 object->del_state = 0; in __alloc_object()
670 object->pid = 0; in __alloc_object()
671 strscpy(object->comm, "hardirq"); in __alloc_object()
673 object->pid = 0; in __alloc_object()
674 strscpy(object->comm, "softirq"); in __alloc_object()
676 object->pid = current->pid; in __alloc_object()
683 strscpy(object->comm, current->comm); in __alloc_object()
687 object->trace_handle = set_track_prepare(); in __alloc_object()
689 return object; in __alloc_object()
692 static int __link_object(struct kmemleak_object *object, unsigned long ptr, in __link_object() argument
701 object->flags = OBJECT_ALLOCATED | objflags; in __link_object()
702 object->pointer = ptr; in __link_object()
703 object->size = kfence_ksize((void *)ptr) ?: size; in __link_object()
704 object->min_count = min_count; in __link_object()
705 object->jiffies = jiffies; in __link_object()
709 * Only update min_addr and max_addr with object storing virtual in __link_object()
731 kmemleak_stop("Cannot insert 0x%lx into the object search tree (overlaps existing)\n", in __link_object()
741 rb_link_node(&object->rb_node, rb_parent, link); in __link_object()
742 rb_insert_color(&object->rb_node, object_tree(objflags)); in __link_object()
743 list_add_tail_rcu(&object->object_list, &object_list); in __link_object()
750 * memory block and add it to the object_list and object tree.
755 struct kmemleak_object *object; in __create_object() local
759 object = __alloc_object(gfp); in __create_object()
760 if (!object) in __create_object()
764 ret = __link_object(object, ptr, size, min_count, objflags); in __create_object()
767 mem_pool_free(object); in __create_object()
770 /* Create kmemleak object which allocated with virtual address. */
777 /* Create kmemleak object which allocated with physical address. */
784 /* Create kmemleak object corresponding to a per-CPU allocation. */
792 * Mark the object as not allocated and schedule RCU freeing via put_object().
794 static void __delete_object(struct kmemleak_object *object) in __delete_object() argument
798 WARN_ON(!(object->flags & OBJECT_ALLOCATED)); in __delete_object()
799 WARN_ON(atomic_read(&object->use_count) < 1); in __delete_object()
805 raw_spin_lock_irqsave(&object->lock, flags); in __delete_object()
806 object->flags &= ~OBJECT_ALLOCATED; in __delete_object()
807 raw_spin_unlock_irqrestore(&object->lock, flags); in __delete_object()
808 put_object(object); in __delete_object()
817 struct kmemleak_object *object; in delete_object_full() local
819 object = find_and_remove_object(ptr, 0, objflags); in delete_object_full()
820 if (!object) { in delete_object_full()
822 kmemleak_warn("Freeing unknown object at 0x%08lx\n", in delete_object_full()
827 __delete_object(object); in delete_object_full()
838 struct kmemleak_object *object, *object_l, *object_r; in delete_object_part() local
850 object = __find_and_remove_object(ptr, 1, objflags); in delete_object_part()
851 if (!object) { in delete_object_part()
853 kmemleak_warn("Partially freeing unknown object at 0x%08lx (size %zu)\n", in delete_object_part()
864 start = object->pointer; in delete_object_part()
865 end = object->pointer + object->size; in delete_object_part()
868 object->min_count, objflags)) in delete_object_part()
872 object->min_count, objflags)) in delete_object_part()
877 if (object) in delete_object_part()
878 __delete_object(object); in delete_object_part()
887 static void __paint_it(struct kmemleak_object *object, int color) in __paint_it() argument
889 object->min_count = color; in __paint_it()
891 object->flags |= OBJECT_NO_SCAN; in __paint_it()
894 static void paint_it(struct kmemleak_object *object, int color) in paint_it() argument
898 raw_spin_lock_irqsave(&object->lock, flags); in paint_it()
899 __paint_it(object, color); in paint_it()
900 raw_spin_unlock_irqrestore(&object->lock, flags); in paint_it()
905 struct kmemleak_object *object; in paint_ptr() local
907 object = __find_and_get_object(ptr, 0, objflags); in paint_ptr()
908 if (!object) { in paint_ptr()
909 kmemleak_warn("Trying to color unknown object at 0x%08lx as %s\n", in paint_ptr()
915 paint_it(object, color); in paint_ptr()
916 put_object(object); in paint_ptr()
920 * Mark an object permanently as gray-colored so that it can no longer be
929 * Mark the object as black-colored so that it is ignored from scans and
938 * Reset the checksum of an object. The immediate effect is that it will not
944 struct kmemleak_object *object; in reset_checksum() local
946 object = find_and_get_object(ptr, 0); in reset_checksum()
947 if (!object) { in reset_checksum()
948 kmemleak_warn("Not resetting the checksum of an unknown object at 0x%08lx\n", in reset_checksum()
953 raw_spin_lock_irqsave(&object->lock, flags); in reset_checksum()
954 object->checksum = 0; in reset_checksum()
955 raw_spin_unlock_irqrestore(&object->lock, flags); in reset_checksum()
956 put_object(object); in reset_checksum()
960 * Add a scanning area to the object. If at least one such area is added,
966 struct kmemleak_object *object; in add_scan_area() local
971 object = find_and_get_object(ptr, 1); in add_scan_area()
972 if (!object) { in add_scan_area()
973 kmemleak_warn("Adding scan area to unknown object at 0x%08lx\n", in add_scan_area()
979 untagged_objp = (unsigned long)kasan_reset_tag((void *)object->pointer); in add_scan_area()
985 raw_spin_lock_irqsave(&object->lock, flags); in add_scan_area()
987 pr_warn_once("Cannot allocate a scan area, scanning the full object\n"); in add_scan_area()
988 /* mark the object for full scan to avoid false positives */ in add_scan_area()
989 object->flags |= OBJECT_FULL_SCAN; in add_scan_area()
993 size = untagged_objp + object->size - untagged_ptr; in add_scan_area()
994 } else if (untagged_ptr + size > untagged_objp + object->size) { in add_scan_area()
995 kmemleak_warn("Scan area larger than object 0x%08lx\n", ptr); in add_scan_area()
996 dump_object_info(object); in add_scan_area()
1005 hlist_add_head(&area->node, &object->area_list); in add_scan_area()
1007 raw_spin_unlock_irqrestore(&object->lock, flags); in add_scan_area()
1008 put_object(object); in add_scan_area()
1012 * Any surplus references (object already gray) to 'ptr' are passed to
1014 * vm_struct may be used as an alternative reference to the vmalloc'ed object
1020 struct kmemleak_object *object; in object_set_excess_ref() local
1022 object = find_and_get_object(ptr, 0); in object_set_excess_ref()
1023 if (!object) { in object_set_excess_ref()
1024 kmemleak_warn("Setting excess_ref on unknown object at 0x%08lx\n", in object_set_excess_ref()
1029 raw_spin_lock_irqsave(&object->lock, flags); in object_set_excess_ref()
1030 object->excess_ref = excess_ref; in object_set_excess_ref()
1031 raw_spin_unlock_irqrestore(&object->lock, flags); in object_set_excess_ref()
1032 put_object(object); in object_set_excess_ref()
1036 * Set the OBJECT_NO_SCAN flag for the object corresponding to the given
1037 * pointer. Such object will not be scanned by kmemleak but references to it
1043 struct kmemleak_object *object; in object_no_scan() local
1045 object = find_and_get_object(ptr, 0); in object_no_scan()
1046 if (!object) { in object_no_scan()
1047 kmemleak_warn("Not scanning unknown object at 0x%08lx\n", ptr); in object_no_scan()
1051 raw_spin_lock_irqsave(&object->lock, flags); in object_no_scan()
1052 object->flags |= OBJECT_NO_SCAN; in object_no_scan()
1053 raw_spin_unlock_irqrestore(&object->lock, flags); in object_no_scan()
1054 put_object(object); in object_no_scan()
1058 * kmemleak_alloc - register a newly allocated object
1059 * @ptr: pointer to beginning of the object
1060 * @size: size of the object
1061 * @min_count: minimum number of references to this object. If during memory
1063 * the object is reported as a memory leak. If @min_count is 0,
1064 * the object is never reported as a leak. If @min_count is -1,
1065 * the object is ignored (not scanned and not reported as a leak)
1068 * This function is called from the kernel allocators when a new object
1082 * kmemleak_alloc_percpu - register a newly allocated __percpu object
1083 * @ptr: __percpu pointer to beginning of the object
1084 * @size: size of the object
1087 * This function is called from the kernel percpu allocator when a new object
1101 * kmemleak_vmalloc - register a newly vmalloc'ed object
1103 * @size: size of the object
1107 * object (memory block) is allocated.
1126 * kmemleak_free - unregister a previously registered object
1127 * @ptr: pointer to beginning of the object
1129 * This function is called from the kernel allocators when an object (memory
1142 * kmemleak_free_part - partially unregister a previously registered object
1143 * @ptr: pointer to the beginning or inside the object. This also
1160 * kmemleak_free_percpu - unregister a previously registered __percpu object
1161 * @ptr: __percpu pointer to beginning of the object
1163 * This function is called from the kernel percpu allocator when an object
1176 * kmemleak_update_trace - update object allocation stack trace
1177 * @ptr: pointer to beginning of the object
1179 * Override the object allocation stack trace for cases where the actual
1184 struct kmemleak_object *object; in kmemleak_update_trace() local
1193 object = find_and_get_object((unsigned long)ptr, 1); in kmemleak_update_trace()
1194 if (!object) { in kmemleak_update_trace()
1196 kmemleak_warn("Updating stack trace for unknown object at %p\n", in kmemleak_update_trace()
1203 raw_spin_lock_irqsave(&object->lock, flags); in kmemleak_update_trace()
1204 object->trace_handle = trace_handle; in kmemleak_update_trace()
1205 raw_spin_unlock_irqrestore(&object->lock, flags); in kmemleak_update_trace()
1207 put_object(object); in kmemleak_update_trace()
1212 * kmemleak_not_leak - mark an allocated object as false positive
1213 * @ptr: pointer to beginning of the object
1215 * Calling this function on an object will cause the memory block to no longer
1228 * kmemleak_transient_leak - mark an allocated object as transient false positive
1229 * @ptr: pointer to beginning of the object
1231 * Calling this function on an object will cause the memory block to not be
1232 * reported as a leak temporarily. This may happen, for example, if the object
1245 * kmemleak_ignore - ignore an allocated object
1246 * @ptr: pointer to beginning of the object
1248 * Calling this function on an object will cause the memory block to be
1263 * kmemleak_scan_area - limit the range to be scanned in an allocated object
1264 * @ptr: pointer to beginning or inside the object. This also
1269 * This function is used when it is known that only certain parts of an object
1283 * kmemleak_no_scan - do not scan an allocated object
1284 * @ptr: pointer to beginning of the object
1287 * in situations where it is known that the given object does not contain any
1303 * @phys: physical address of the object
1304 * @size: size of the object
1313 * Create object with OBJECT_PHYS flag and in kmemleak_alloc_phys()
1323 * @phys: physical address if the beginning or inside an object. This
1339 * @phys: physical address of the object
1351 * Update an object's checksum and return true if it was modified.
1353 static bool update_checksum(struct kmemleak_object *object) in update_checksum() argument
1355 u32 old_csum = object->checksum; in update_checksum()
1357 if (WARN_ON_ONCE(object->flags & OBJECT_PHYS)) in update_checksum()
1362 if (object->flags & OBJECT_PERCPU) { in update_checksum()
1365 object->checksum = 0; in update_checksum()
1367 void *ptr = per_cpu_ptr((void __percpu *)object->pointer, cpu); in update_checksum()
1369 object->checksum ^= crc32(0, kasan_reset_tag((void *)ptr), object->size); in update_checksum()
1372 object->checksum = crc32(0, kasan_reset_tag((void *)object->pointer), object->size); in update_checksum()
1377 return object->checksum != old_csum; in update_checksum()
1381 * Update an object's references. object->lock must be held by the caller.
1383 static void update_refs(struct kmemleak_object *object) in update_refs() argument
1385 if (!color_white(object)) { in update_refs()
1391 * Increase the object's reference count (number of pointers to the in update_refs()
1393 * object's color will become gray and it will be added to the in update_refs()
1396 object->count++; in update_refs()
1397 if (color_gray(object)) { in update_refs()
1399 WARN_ON(!get_object(object)); in update_refs()
1400 list_add_tail(&object->gray_list, &gray_list); in update_refs()
1407 struct kmemleak_object *object; in pointer_update_refs() local
1422 * object->use_count cannot be dropped to 0 while the object in pointer_update_refs()
1426 object = __lookup_object(pointer, 1, objflags); in pointer_update_refs()
1427 if (!object) in pointer_update_refs()
1429 if (object == scanned) in pointer_update_refs()
1434 * Avoid the lockdep recursive warning on object->lock being in pointer_update_refs()
1438 raw_spin_lock_nested(&object->lock, SINGLE_DEPTH_NESTING); in pointer_update_refs()
1439 /* only pass surplus references (object already gray) */ in pointer_update_refs()
1440 if (color_gray(object)) { in pointer_update_refs()
1441 excess_ref = object->excess_ref; in pointer_update_refs()
1442 /* no need for update_refs() if object already gray */ in pointer_update_refs()
1445 update_refs(object); in pointer_update_refs()
1447 raw_spin_unlock(&object->lock); in pointer_update_refs()
1450 object = lookup_object(excess_ref, 0); in pointer_update_refs()
1451 if (!object) in pointer_update_refs()
1453 if (object == scanned) in pointer_update_refs()
1456 raw_spin_lock_nested(&object->lock, SINGLE_DEPTH_NESTING); in pointer_update_refs()
1457 update_refs(object); in pointer_update_refs()
1458 raw_spin_unlock(&object->lock); in pointer_update_refs()
1531 * that object->use_count >= 1.
1533 static void scan_object(struct kmemleak_object *object) in scan_object() argument
1539 * Once the object->lock is acquired, the corresponding memory block in scan_object()
1542 raw_spin_lock_irqsave(&object->lock, flags); in scan_object()
1543 if (object->flags & OBJECT_NO_SCAN) in scan_object()
1545 if (!(object->flags & OBJECT_ALLOCATED)) in scan_object()
1546 /* already freed object */ in scan_object()
1549 if (object->flags & OBJECT_PERCPU) { in scan_object()
1553 void *start = per_cpu_ptr((void __percpu *)object->pointer, cpu); in scan_object()
1554 void *end = start + object->size; in scan_object()
1556 scan_block(start, end, object); in scan_object()
1558 raw_spin_unlock_irqrestore(&object->lock, flags); in scan_object()
1560 raw_spin_lock_irqsave(&object->lock, flags); in scan_object()
1561 if (!(object->flags & OBJECT_ALLOCATED)) in scan_object()
1564 } else if (hlist_empty(&object->area_list) || in scan_object()
1565 object->flags & OBJECT_FULL_SCAN) { in scan_object()
1566 void *start = object->flags & OBJECT_PHYS ? in scan_object()
1567 __va((phys_addr_t)object->pointer) : in scan_object()
1568 (void *)object->pointer; in scan_object()
1569 void *end = start + object->size; in scan_object()
1574 scan_block(start, next, object); in scan_object()
1580 raw_spin_unlock_irqrestore(&object->lock, flags); in scan_object()
1582 raw_spin_lock_irqsave(&object->lock, flags); in scan_object()
1583 } while (object->flags & OBJECT_ALLOCATED); in scan_object()
1585 hlist_for_each_entry(area, &object->area_list, node) in scan_object()
1588 object); in scan_object()
1591 raw_spin_unlock_irqrestore(&object->lock, flags); in scan_object()
1600 struct kmemleak_object *object, *tmp; in scan_gray_list() local
1607 object = list_entry(gray_list.next, typeof(*object), gray_list); in scan_gray_list()
1608 while (&object->gray_list != &gray_list) { in scan_gray_list()
1613 scan_object(object); in scan_gray_list()
1615 tmp = list_entry(object->gray_list.next, typeof(*object), in scan_gray_list()
1618 /* remove the object from the list and release it */ in scan_gray_list()
1619 list_del(&object->gray_list); in scan_gray_list()
1620 put_object(object); in scan_gray_list()
1622 object = tmp; in scan_gray_list()
1628 * Conditionally call resched() in an object iteration loop while making sure
1629 * that the given object won't go away without RCU read lock by performing a
1632 static void kmemleak_cond_resched(struct kmemleak_object *object) in kmemleak_cond_resched() argument
1634 if (!get_object(object)) in kmemleak_cond_resched()
1635 return; /* Try next object */ in kmemleak_cond_resched()
1638 if (object->del_state & DELSTATE_REMOVED) in kmemleak_cond_resched()
1639 goto unlock_put; /* Object removed */ in kmemleak_cond_resched()
1640 object->del_state |= DELSTATE_NO_DELETE; in kmemleak_cond_resched()
1648 if (object->del_state & DELSTATE_REMOVED) in kmemleak_cond_resched()
1649 list_del_rcu(&object->object_list); in kmemleak_cond_resched()
1650 object->del_state &= ~DELSTATE_NO_DELETE; in kmemleak_cond_resched()
1653 put_object(object); in kmemleak_cond_resched()
1663 struct kmemleak_object *object; in kmemleak_scan() local
1672 list_for_each_entry_rcu(object, &object_list, object_list) { in kmemleak_scan()
1673 raw_spin_lock_irq(&object->lock); in kmemleak_scan()
1677 * 1 reference to any object at this point. in kmemleak_scan()
1679 if (atomic_read(&object->use_count) > 1) { in kmemleak_scan()
1680 pr_debug("object->use_count = %d\n", in kmemleak_scan()
1681 atomic_read(&object->use_count)); in kmemleak_scan()
1682 dump_object_info(object); in kmemleak_scan()
1687 if ((object->flags & OBJECT_PHYS) && in kmemleak_scan()
1688 !(object->flags & OBJECT_NO_SCAN)) { in kmemleak_scan()
1689 unsigned long phys = object->pointer; in kmemleak_scan()
1692 PHYS_PFN(phys + object->size) > max_low_pfn) in kmemleak_scan()
1693 __paint_it(object, KMEMLEAK_BLACK); in kmemleak_scan()
1696 /* reset the reference count (whiten the object) */ in kmemleak_scan()
1697 object->count = 0; in kmemleak_scan()
1698 if (color_gray(object) && get_object(object)) in kmemleak_scan()
1699 list_add_tail(&object->gray_list, &gray_list); in kmemleak_scan()
1701 raw_spin_unlock_irq(&object->lock); in kmemleak_scan()
1704 kmemleak_cond_resched(object); in kmemleak_scan()
1772 list_for_each_entry_rcu(object, &object_list, object_list) { in kmemleak_scan()
1774 kmemleak_cond_resched(object); in kmemleak_scan()
1781 if (!color_white(object)) in kmemleak_scan()
1783 raw_spin_lock_irq(&object->lock); in kmemleak_scan()
1784 if (color_white(object) && (object->flags & OBJECT_ALLOCATED) in kmemleak_scan()
1785 && update_checksum(object) && get_object(object)) { in kmemleak_scan()
1787 object->count = object->min_count; in kmemleak_scan()
1788 list_add_tail(&object->gray_list, &gray_list); in kmemleak_scan()
1790 raw_spin_unlock_irq(&object->lock); in kmemleak_scan()
1809 list_for_each_entry_rcu(object, &object_list, object_list) { in kmemleak_scan()
1811 kmemleak_cond_resched(object); in kmemleak_scan()
1818 if (!color_white(object)) in kmemleak_scan()
1820 raw_spin_lock_irq(&object->lock); in kmemleak_scan()
1821 if (unreferenced_object(object) && in kmemleak_scan()
1822 !(object->flags & OBJECT_REPORTED)) { in kmemleak_scan()
1823 object->flags |= OBJECT_REPORTED; in kmemleak_scan()
1826 print_unreferenced(NULL, object); in kmemleak_scan()
1830 raw_spin_unlock_irq(&object->lock); in kmemleak_scan()
1908 * Iterate over the object_list and return the first valid object at or after
1914 struct kmemleak_object *object; in kmemleak_seq_start() local
1923 list_for_each_entry_rcu(object, &object_list, object_list) { in kmemleak_seq_start()
1926 if (get_object(object)) in kmemleak_seq_start()
1929 object = NULL; in kmemleak_seq_start()
1931 return object; in kmemleak_seq_start()
1935 * Return the next object in the object_list. The function decrements the
1936 * use_count of the previous object and increases that of the next one.
1958 * Decrement the use_count of the last object required, if any.
1975 * Print the information for an unreferenced object to the seq file.
1979 struct kmemleak_object *object = v; in kmemleak_seq_show() local
1982 raw_spin_lock_irqsave(&object->lock, flags); in kmemleak_seq_show()
1983 if ((object->flags & OBJECT_REPORTED) && unreferenced_object(object)) in kmemleak_seq_show()
1984 print_unreferenced(seq, object); in kmemleak_seq_show()
1985 raw_spin_unlock_irqrestore(&object->lock, flags); in kmemleak_seq_show()
2004 struct kmemleak_object *object; in dump_str_object_info() local
2009 object = find_and_get_object(addr, 0); in dump_str_object_info()
2010 if (!object) { in dump_str_object_info()
2011 pr_info("Unknown object at 0x%08lx\n", addr); in dump_str_object_info()
2015 raw_spin_lock_irqsave(&object->lock, flags); in dump_str_object_info()
2016 dump_object_info(object); in dump_str_object_info()
2017 raw_spin_unlock_irqrestore(&object->lock, flags); in dump_str_object_info()
2019 put_object(object); in dump_str_object_info()
2031 struct kmemleak_object *object; in kmemleak_clear() local
2034 list_for_each_entry_rcu(object, &object_list, object_list) { in kmemleak_clear()
2035 raw_spin_lock_irq(&object->lock); in kmemleak_clear()
2036 if ((object->flags & OBJECT_REPORTED) && in kmemleak_clear()
2037 unreferenced_object(object)) in kmemleak_clear()
2038 __paint_it(object, KMEMLEAK_GREY); in kmemleak_clear()
2039 raw_spin_unlock_irq(&object->lock); in kmemleak_clear()
2062 * dump=... - dump information about the object found at the given address
2148 struct kmemleak_object *object, *tmp; in __kmemleak_do_cleanup() local
2154 list_for_each_entry_safe(object, tmp, &object_list, object_list) { in __kmemleak_do_cleanup()
2155 __remove_object(object); in __kmemleak_do_cleanup()
2156 __delete_object(object); in __kmemleak_do_cleanup()
2172 * longer track object freeing. Ordering of the scan thread stopping and in kmemleak_do_cleanup()