diff options
| author | Thomas Gleixner <tglx@linutronix.de> | 2011-06-06 12:20:33 +0200 |
|---|---|---|
| committer | Luis Claudio R. Goncalves <lgoncalv@redhat.com> | 2023-02-18 22:05:18 -0300 |
| commit | e3fa2b9a1f19cffa65d139fa7a5a8585b2655632 (patch) | |
| tree | 16a18ceb6622f89a9dd3faf97e22f21904b31082 | |
| parent | 53402862a9c80f2ad7195ad33a77d901ccd11d87 (diff) | |
| download | linux-rt-e3fa2b9a1f19cffa65d139fa7a5a8585b2655632.tar.gz | |
sched: Move mmdrop to RCU on RT
Takes sleeping locks and calls into the memory allocator, so nothing
we want to do in task switch and oder atomic contexts.
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
| -rw-r--r-- | include/linux/mm_types.h | 4 | ||||
| -rw-r--r-- | include/linux/sched/mm.h | 11 | ||||
| -rw-r--r-- | kernel/fork.c | 13 | ||||
| -rw-r--r-- | kernel/sched/core.c | 7 |
4 files changed, 34 insertions, 1 deletions
diff --git a/include/linux/mm_types.h b/include/linux/mm_types.h index 4eb38918da8f..2a83d965b00e 100644 --- a/include/linux/mm_types.h +++ b/include/linux/mm_types.h @@ -12,6 +12,7 @@ #include <linux/completion.h> #include <linux/cpumask.h> #include <linux/uprobes.h> +#include <linux/rcupdate.h> #include <linux/page-flags-layout.h> #include <linux/workqueue.h> #include <linux/seqlock.h> @@ -570,6 +571,9 @@ struct mm_struct { bool tlb_flush_batched; #endif struct uprobes_state uprobes_state; +#ifdef CONFIG_PREEMPT_RT + struct rcu_head delayed_drop; +#endif #ifdef CONFIG_HUGETLB_PAGE atomic_long_t hugetlb_usage; #endif diff --git a/include/linux/sched/mm.h b/include/linux/sched/mm.h index e3e5e149b00e..6d39ad0f55bb 100644 --- a/include/linux/sched/mm.h +++ b/include/linux/sched/mm.h @@ -49,6 +49,17 @@ static inline void mmdrop(struct mm_struct *mm) __mmdrop(mm); } +#ifdef CONFIG_PREEMPT_RT +extern void __mmdrop_delayed(struct rcu_head *rhp); +static inline void mmdrop_delayed(struct mm_struct *mm) +{ + if (atomic_dec_and_test(&mm->mm_count)) + call_rcu(&mm->delayed_drop, __mmdrop_delayed); +} +#else +# define mmdrop_delayed(mm) mmdrop(mm) +#endif + /** * mmget() - Pin the address space associated with a &struct mm_struct. * @mm: The address space to pin. diff --git a/kernel/fork.c b/kernel/fork.c index e9718d560e30..125dd5007004 100644 --- a/kernel/fork.c +++ b/kernel/fork.c @@ -688,6 +688,19 @@ void __mmdrop(struct mm_struct *mm) } EXPORT_SYMBOL_GPL(__mmdrop); +#ifdef CONFIG_PREEMPT_RT +/* + * RCU callback for delayed mm drop. Not strictly rcu, but we don't + * want another facility to make this work. + */ +void __mmdrop_delayed(struct rcu_head *rhp) +{ + struct mm_struct *mm = container_of(rhp, struct mm_struct, delayed_drop); + + __mmdrop(mm); +} +#endif + static void mmdrop_async_fn(struct work_struct *work) { struct mm_struct *mm; diff --git a/kernel/sched/core.c b/kernel/sched/core.c index 47066d5b9dfc..80149c7d4bbd 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -4266,9 +4266,13 @@ static struct rq *finish_task_switch(struct task_struct *prev) * provided by mmdrop(), * - a sync_core for SYNC_CORE. */ + /* + * We use mmdrop_delayed() here so we don't have to do the + * full __mmdrop() when we are the last user. + */ if (mm) { membarrier_mm_sync_core_before_usermode(mm); - mmdrop(mm); + mmdrop_delayed(mm); } if (unlikely(prev_state == TASK_DEAD)) { if (prev->sched_class->task_dead) @@ -7266,6 +7270,7 @@ void sched_setnuma(struct task_struct *p, int nid) #endif /* CONFIG_NUMA_BALANCING */ #ifdef CONFIG_HOTPLUG_CPU + /* * Ensure that the idle task is using init_mm right before its CPU goes * offline. |