mirror of
https://github.com/torvalds/linux.git
synced 2025-04-12 16:47:42 +00:00
494e7fe591
-----BEGIN PGP SIGNATURE-----
iQIzBAABCAAdFiEE+soXsSLHKoYyzcli6rmadz2vbToFAmfcq3kACgkQ6rmadz2v
bToxkw/8DHIqjVnzU2O9hbRM1anYo6yM8e34IxCt0ajHTSEVJ93+C161QDWo/6Dk
+RNlaeGekaBUk+QOLb4u+rzZ2eR/pWSm37xuDRAiBCQ+3MgR60gGRaSljpS3IUem
0FvS6C1HObBCEUXMU2rNv/5cJB5/qrQYa9FEEjRvBTLqgQkdS7yaW/KKuZaNb+Ts
KiEeWvPrPSZXStfRGy8Wr4eS2rYhxPAikUR+xde9CM+HtMWwKTCTSp8qXrqA92Dj
Cz9ix01scznuf78QCRDZp09im3lZys8ZQprmPgMxyEscN+CDL7n68wAhmTJq0uo3
3NqIv7zBQ8wMChj0f0HjwZ0Wrj7BJAveY2Q0RterxdzT4vMKdtNkThX46ISaCoX/
XQAAhZHemK6MvBJk+LKkqqMgrD+3FAzvY7O+SCyUBAMs4FK1myRJQihdLXHGfiBU
DMDZE1jsE8qBaeUbz4LIuCy8fx2LhtVwVNwbNIBUZHdyfjxIXnQT/8Cnrgklwy2i
tnYekhAsHDQY+QDkrvJpc4E1vUtiXwSDI5ErcnWdSzctEOyVeUg7OuuGD4riCd1c
emdJmtASM1z9Ajqa1dytDxVaF6wjKlbhQgnKamuex5JLGCK6makk8ZoB+DBfKYHD
VoWummTu8ldf+Dp4ehBh7AbeF2vn4kLqcF1PLRsBO6ytJs4HIt8=
=5O7h
-----END PGP SIGNATURE-----
Merge tag 'bpf_res_spin_lock' of git://git.kernel.org/pub/scm/linux/kernel/git/bpf/bpf-next
Pull bpf relisient spinlock support from Alexei Starovoitov:
"This patch set introduces Resilient Queued Spin Lock (or rqspinlock
with res_spin_lock() and res_spin_unlock() APIs).
This is a qspinlock variant which recovers the kernel from a stalled
state when the lock acquisition path cannot make forward progress.
This can occur when a lock acquisition attempt enters a deadlock
situation (e.g. AA, or ABBA), or more generally, when the owner of the
lock (which we’re trying to acquire) isn’t making forward progress.
Deadlock detection is the main mechanism used to provide instant
recovery, with the timeout mechanism acting as a final line of
defense. Detection is triggered immediately when beginning the waiting
loop of a lock slow path.
Additionally, BPF programs attached to different parts of the kernel
can introduce new control flow into the kernel, which increases the
likelihood of deadlocks in code not written to handle reentrancy.
There have been multiple syzbot reports surfacing deadlocks in
internal kernel code due to the diverse ways in which BPF programs can
be attached to different parts of the kernel. By switching the BPF
subsystem’s lock usage to rqspinlock, all of these issues are
mitigated at runtime.
This spin lock implementation allows BPF maps to become safer and
remove mechanisms that have fallen short in assuring safety when
nesting programs in arbitrary ways in the same context or across
different contexts.
We run benchmarks that stress locking scalability and perform
comparison against the baseline (qspinlock). For the rqspinlock case,
we replace the default qspinlock with it in the kernel, such that all
spin locks in the kernel use the rqspinlock slow path. As such,
benchmarks that stress kernel spin locks end up exercising rqspinlock.
More details in the cover letter in commit 6ffb9017e932 ("Merge branch
'resilient-queued-spin-lock'")"
* tag 'bpf_res_spin_lock' of git://git.kernel.org/pub/scm/linux/kernel/git/bpf/bpf-next: (24 commits)
selftests/bpf: Add tests for rqspinlock
bpf: Maintain FIFO property for rqspinlock unlock
bpf: Implement verifier support for rqspinlock
bpf: Introduce rqspinlock kfuncs
bpf: Convert lpm_trie.c to rqspinlock
bpf: Convert percpu_freelist.c to rqspinlock
bpf: Convert hashtab.c to rqspinlock
rqspinlock: Add locktorture support
rqspinlock: Add entry to Makefile, MAINTAINERS
rqspinlock: Add macros for rqspinlock usage
rqspinlock: Add basic support for CONFIG_PARAVIRT
rqspinlock: Add a test-and-set fallback
rqspinlock: Add deadlock detection and recovery
rqspinlock: Protect waiters in trylock fallback from stalls
rqspinlock: Protect waiters in queue from stalls
rqspinlock: Protect pending bit owners from stalls
rqspinlock: Hardcode cond_acquire loops for arm64
rqspinlock: Add support for timeouts
rqspinlock: Drop PV and virtualization support
rqspinlock: Add rqspinlock.h header
...
103 lines
4.2 KiB
C
103 lines
4.2 KiB
C
/* SPDX-License-Identifier: GPL-2.0 */
|
|
/*
|
|
* This program is free software; you can redistribute it and/or modify
|
|
* it under the terms of the GNU General Public License as published by
|
|
* the Free Software Foundation; either version 2 of the License, or
|
|
* (at your option) any later version.
|
|
*
|
|
* This program is distributed in the hope that it will be useful,
|
|
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
* GNU General Public License for more details.
|
|
*
|
|
* Authors: Waiman Long <longman@redhat.com>
|
|
*/
|
|
|
|
#ifndef LOCK_EVENT
|
|
#define LOCK_EVENT(name) LOCKEVENT_ ## name,
|
|
#endif
|
|
|
|
#ifdef CONFIG_QUEUED_SPINLOCKS
|
|
#ifdef CONFIG_PARAVIRT_SPINLOCKS
|
|
/*
|
|
* Locking events for PV qspinlock.
|
|
*/
|
|
LOCK_EVENT(pv_hash_hops) /* Average # of hops per hashing operation */
|
|
LOCK_EVENT(pv_kick_unlock) /* # of vCPU kicks issued at unlock time */
|
|
LOCK_EVENT(pv_kick_wake) /* # of vCPU kicks for pv_latency_wake */
|
|
LOCK_EVENT(pv_latency_kick) /* Average latency (ns) of vCPU kick */
|
|
LOCK_EVENT(pv_latency_wake) /* Average latency (ns) of kick-to-wakeup */
|
|
LOCK_EVENT(pv_lock_stealing) /* # of lock stealing operations */
|
|
LOCK_EVENT(pv_spurious_wakeup) /* # of spurious wakeups in non-head vCPUs */
|
|
LOCK_EVENT(pv_wait_again) /* # of wait's after queue head vCPU kick */
|
|
LOCK_EVENT(pv_wait_early) /* # of early vCPU wait's */
|
|
LOCK_EVENT(pv_wait_head) /* # of vCPU wait's at the queue head */
|
|
LOCK_EVENT(pv_wait_node) /* # of vCPU wait's at non-head queue node */
|
|
#endif /* CONFIG_PARAVIRT_SPINLOCKS */
|
|
|
|
/*
|
|
* Locking events for qspinlock
|
|
*
|
|
* Subtracting lock_use_node[234] from lock_slowpath will give you
|
|
* lock_use_node1.
|
|
*/
|
|
LOCK_EVENT(lock_pending) /* # of locking ops via pending code */
|
|
LOCK_EVENT(lock_slowpath) /* # of locking ops via MCS lock queue */
|
|
LOCK_EVENT(lock_use_node2) /* # of locking ops that use 2nd percpu node */
|
|
LOCK_EVENT(lock_use_node3) /* # of locking ops that use 3rd percpu node */
|
|
LOCK_EVENT(lock_use_node4) /* # of locking ops that use 4th percpu node */
|
|
LOCK_EVENT(lock_no_node) /* # of locking ops w/o using percpu node */
|
|
#endif /* CONFIG_QUEUED_SPINLOCKS */
|
|
|
|
/*
|
|
* Locking events for Resilient Queued Spin Lock
|
|
*/
|
|
LOCK_EVENT(rqspinlock_lock_timeout) /* # of locking ops that timeout */
|
|
|
|
/*
|
|
* Locking events for rwsem
|
|
*/
|
|
LOCK_EVENT(rwsem_sleep_reader) /* # of reader sleeps */
|
|
LOCK_EVENT(rwsem_sleep_writer) /* # of writer sleeps */
|
|
LOCK_EVENT(rwsem_wake_reader) /* # of reader wakeups */
|
|
LOCK_EVENT(rwsem_wake_writer) /* # of writer wakeups */
|
|
LOCK_EVENT(rwsem_opt_lock) /* # of opt-acquired write locks */
|
|
LOCK_EVENT(rwsem_opt_fail) /* # of failed optspins */
|
|
LOCK_EVENT(rwsem_opt_nospin) /* # of disabled optspins */
|
|
LOCK_EVENT(rwsem_rlock) /* # of read locks acquired */
|
|
LOCK_EVENT(rwsem_rlock_steal) /* # of read locks by lock stealing */
|
|
LOCK_EVENT(rwsem_rlock_fast) /* # of fast read locks acquired */
|
|
LOCK_EVENT(rwsem_rlock_fail) /* # of failed read lock acquisitions */
|
|
LOCK_EVENT(rwsem_rlock_handoff) /* # of read lock handoffs */
|
|
LOCK_EVENT(rwsem_wlock) /* # of write locks acquired */
|
|
LOCK_EVENT(rwsem_wlock_fail) /* # of failed write lock acquisitions */
|
|
LOCK_EVENT(rwsem_wlock_handoff) /* # of write lock handoffs */
|
|
|
|
/*
|
|
* Locking events for rtlock_slowlock()
|
|
*/
|
|
LOCK_EVENT(rtlock_slowlock) /* # of rtlock_slowlock() calls */
|
|
LOCK_EVENT(rtlock_slow_acq1) /* # of locks acquired after wait_lock */
|
|
LOCK_EVENT(rtlock_slow_acq2) /* # of locks acquired in for loop */
|
|
LOCK_EVENT(rtlock_slow_sleep) /* # of sleeps */
|
|
LOCK_EVENT(rtlock_slow_wake) /* # of wakeup's */
|
|
|
|
/*
|
|
* Locking events for rt_mutex_slowlock()
|
|
*/
|
|
LOCK_EVENT(rtmutex_slowlock) /* # of rt_mutex_slowlock() calls */
|
|
LOCK_EVENT(rtmutex_slow_block) /* # of rt_mutex_slowlock_block() calls */
|
|
LOCK_EVENT(rtmutex_slow_acq1) /* # of locks acquired after wait_lock */
|
|
LOCK_EVENT(rtmutex_slow_acq2) /* # of locks acquired at the end */
|
|
LOCK_EVENT(rtmutex_slow_acq3) /* # of locks acquired in *block() */
|
|
LOCK_EVENT(rtmutex_slow_sleep) /* # of sleeps */
|
|
LOCK_EVENT(rtmutex_slow_wake) /* # of wakeup's */
|
|
LOCK_EVENT(rtmutex_deadlock) /* # of rt_mutex_handle_deadlock()'s */
|
|
|
|
/*
|
|
* Locking events for lockdep
|
|
*/
|
|
LOCK_EVENT(lockdep_acquire)
|
|
LOCK_EVENT(lockdep_lock)
|
|
LOCK_EVENT(lockdep_nocheck)
|