4fe9852927
Inserts past 75% load call schedule_work(&ht->run_work) to kick an
async resize. If a caller holds a raw spinlock (e.g. an
insecure_elasticity user), schedule_work() under that lock records
caller_lock -> pool->lock -> pi_lock -> rq->__lock
A cycle forms if any of these locks is acquired in the reverse
direction elsewhere. sched_ext, the only current insecure_elasticity
user, hits this: it holds scx_sched_lock across rhashtable inserts of
sub-schedulers, while scx_bypass() takes rq->__lock -> scx_sched_lock.
Exercising the resize path produces:
Chain exists of:
&pool->lock --> &rq->__lock --> scx_sched_lock
Bounce the kick from the insert paths through irq_work so
schedule_work() runs from hard IRQ context with the caller's lock no
longer held. rht_deferred_worker()'s self-rearm on error stays on
schedule_work(&ht->run_work) - the worker runs in process context with
no caller lock held, and keeping the self-requeue on @run_work lets
cancel_work_sync() in rhashtable_free_and_destroy() drain it.
v3: Keep rht_deferred_worker()'s self-rearm on schedule_work(&run_work).
Routing it through irq_work in v2 broke cancel_work_sync()'s
self-requeue handling - an irq_work queued after irq_work_sync()
returned but while cancel_work_sync() was still waiting could fire
post-teardown.
v2: Bounce unconditionally instead of gating on insecure_elasticity,
as suggested by Herbert.
Signed-off-by: Tejun Heo <tj@kernel.org>
Acked-by: Herbert Xu <herbert@gondor.apana.org.au>
147 lines
4 KiB
C
147 lines
4 KiB
C
/* SPDX-License-Identifier: GPL-2.0 */
|
|
/*
|
|
* Resizable, Scalable, Concurrent Hash Table
|
|
*
|
|
* Simple structures that might be needed in include
|
|
* files.
|
|
*/
|
|
|
|
#ifndef _LINUX_RHASHTABLE_TYPES_H
|
|
#define _LINUX_RHASHTABLE_TYPES_H
|
|
|
|
#include <linux/alloc_tag.h>
|
|
#include <linux/atomic.h>
|
|
#include <linux/compiler.h>
|
|
#include <linux/irq_work_types.h>
|
|
#include <linux/mutex.h>
|
|
#include <linux/workqueue_types.h>
|
|
|
|
struct rhash_head {
|
|
struct rhash_head __rcu *next;
|
|
};
|
|
|
|
struct rhlist_head {
|
|
struct rhash_head rhead;
|
|
struct rhlist_head __rcu *next;
|
|
};
|
|
|
|
struct bucket_table;
|
|
|
|
/**
|
|
* struct rhashtable_compare_arg - Key for the function rhashtable_compare
|
|
* @ht: Hash table
|
|
* @key: Key to compare against
|
|
*/
|
|
struct rhashtable_compare_arg {
|
|
struct rhashtable *ht;
|
|
const void *key;
|
|
};
|
|
|
|
typedef u32 (*rht_hashfn_t)(const void *data, u32 len, u32 seed);
|
|
typedef u32 (*rht_obj_hashfn_t)(const void *data, u32 len, u32 seed);
|
|
typedef int (*rht_obj_cmpfn_t)(struct rhashtable_compare_arg *arg,
|
|
const void *obj);
|
|
|
|
/**
|
|
* struct rhashtable_params - Hash table construction parameters
|
|
* @nelem_hint: Hint on number of elements, should be 75% of desired size
|
|
* @key_len: Length of key
|
|
* @key_offset: Offset of key in struct to be hashed
|
|
* @head_offset: Offset of rhash_head in struct to be hashed
|
|
* @max_size: Maximum size while expanding
|
|
* @min_size: Minimum size while shrinking
|
|
* @insecure_elasticity: Set to true to disable chain length checks
|
|
* @automatic_shrinking: Enable automatic shrinking of tables
|
|
* @hashfn: Hash function (default: jhash2 if !(key_len % 4), or jhash)
|
|
* @obj_hashfn: Function to hash object
|
|
* @obj_cmpfn: Function to compare key with object
|
|
*/
|
|
struct rhashtable_params {
|
|
u16 nelem_hint;
|
|
u16 key_len;
|
|
u16 key_offset;
|
|
u16 head_offset;
|
|
unsigned int max_size;
|
|
u16 min_size;
|
|
bool insecure_elasticity;
|
|
bool automatic_shrinking;
|
|
rht_hashfn_t hashfn;
|
|
rht_obj_hashfn_t obj_hashfn;
|
|
rht_obj_cmpfn_t obj_cmpfn;
|
|
};
|
|
|
|
/**
|
|
* struct rhashtable - Hash table handle
|
|
* @tbl: Bucket table
|
|
* @key_len: Key length for hashfn
|
|
* @max_elems: Maximum number of elements in table
|
|
* @p: Configuration parameters
|
|
* @rhlist: True if this is an rhltable
|
|
* @run_work: Deferred worker to expand/shrink asynchronously
|
|
* @run_irq_work: Bounces the @run_work kick through hard IRQ context.
|
|
* @mutex: Mutex to protect current/future table swapping
|
|
* @lock: Spin lock to protect walker list
|
|
* @nelems: Number of elements in table
|
|
*/
|
|
struct rhashtable {
|
|
struct bucket_table __rcu *tbl;
|
|
unsigned int key_len;
|
|
unsigned int max_elems;
|
|
struct rhashtable_params p;
|
|
bool rhlist;
|
|
struct work_struct run_work;
|
|
struct irq_work run_irq_work;
|
|
struct mutex mutex;
|
|
spinlock_t lock;
|
|
atomic_t nelems;
|
|
#ifdef CONFIG_MEM_ALLOC_PROFILING
|
|
struct alloc_tag *alloc_tag;
|
|
#endif
|
|
};
|
|
|
|
/**
|
|
* struct rhltable - Hash table with duplicate objects in a list
|
|
* @ht: Underlying rhtable
|
|
*/
|
|
struct rhltable {
|
|
struct rhashtable ht;
|
|
};
|
|
|
|
/**
|
|
* struct rhashtable_walker - Hash table walker
|
|
* @list: List entry on list of walkers
|
|
* @tbl: The table that we were walking over
|
|
*/
|
|
struct rhashtable_walker {
|
|
struct list_head list;
|
|
struct bucket_table *tbl;
|
|
};
|
|
|
|
/**
|
|
* struct rhashtable_iter - Hash table iterator
|
|
* @ht: Table to iterate through
|
|
* @p: Current pointer
|
|
* @list: Current hash list pointer
|
|
* @walker: Associated rhashtable walker
|
|
* @slot: Current slot
|
|
* @skip: Number of entries to skip in slot
|
|
*/
|
|
struct rhashtable_iter {
|
|
struct rhashtable *ht;
|
|
struct rhash_head *p;
|
|
struct rhlist_head *list;
|
|
struct rhashtable_walker walker;
|
|
unsigned int slot;
|
|
unsigned int skip;
|
|
bool end_of_table;
|
|
};
|
|
|
|
int rhashtable_init_noprof(struct rhashtable *ht,
|
|
const struct rhashtable_params *params);
|
|
#define rhashtable_init(...) alloc_hooks(rhashtable_init_noprof(__VA_ARGS__))
|
|
|
|
int rhltable_init_noprof(struct rhltable *hlt,
|
|
const struct rhashtable_params *params);
|
|
#define rhltable_init(...) alloc_hooks(rhltable_init_noprof(__VA_ARGS__))
|
|
|
|
#endif /* _LINUX_RHASHTABLE_TYPES_H */
|