mutex.c 25.3 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12
/*
 * kernel/mutex.c
 *
 * Mutexes: blocking mutual exclusion locks
 *
 * Started by Ingo Molnar:
 *
 *  Copyright (C) 2004, 2005, 2006 Red Hat, Inc., Ingo Molnar <mingo@redhat.com>
 *
 * Many thanks to Arjan van de Ven, Thomas Gleixner, Steven Rostedt and
 * David Howells for suggestions and improvements.
 *
13 14 15 16 17
 *  - Adaptive spinning for mutexes by Peter Zijlstra. (Ported to mainline
 *    from the -rt tree, where it was originally implemented for rtmutexes
 *    by Steven Rostedt, based on work by Gregory Haskins, Peter Morreale
 *    and Sven Dietrich.
 *
18 19 20
 * Also see Documentation/mutex-design.txt.
 */
#include <linux/mutex.h>
21
#include <linux/ww_mutex.h>
22
#include <linux/sched.h>
23
#include <linux/sched/rt.h>
24
#include <linux/export.h>
25 26
#include <linux/spinlock.h>
#include <linux/interrupt.h>
27
#include <linux/debug_locks.h>
28 29 30 31 32 33 34 35 36 37 38 39 40

/*
 * In the DEBUG case we are using the "NULL fastpath" for mutexes,
 * which forces all calls into the slowpath:
 */
#ifdef CONFIG_DEBUG_MUTEXES
# include "mutex-debug.h"
# include <asm-generic/mutex-null.h>
#else
# include "mutex.h"
# include <asm/mutex.h>
#endif

41
/*
42 43
 * A negative mutex count indicates that waiters are sleeping waiting for the
 * mutex.
44 45 46
 */
#define	MUTEX_SHOW_NO_WAITER(mutex)	(atomic_read(&(mutex)->count) >= 0)

47 48
void
__mutex_init(struct mutex *lock, const char *name, struct lock_class_key *key)
49 50 51 52
{
	atomic_set(&lock->count, 1);
	spin_lock_init(&lock->wait_lock);
	INIT_LIST_HEAD(&lock->wait_list);
53
	mutex_clear_owner(lock);
54 55 56
#ifdef CONFIG_MUTEX_SPIN_ON_OWNER
	lock->spin_mlock = NULL;
#endif
57

58
	debug_mutex_init(lock, name, key);
59 60 61 62
}

EXPORT_SYMBOL(__mutex_init);

63
#ifndef CONFIG_DEBUG_LOCK_ALLOC
64 65 66 67 68 69
/*
 * We split the mutex lock/unlock logic into separate fastpath and
 * slowpath functions, to reduce the register pressure on the fastpath.
 * We also put the fastpath first in the kernel image, to make sure the
 * branch is predicted by the CPU as default-untaken.
 */
70
static __used noinline void __sched
71
__mutex_lock_slowpath(atomic_t *lock_count);
72

73
/**
74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93
 * mutex_lock - acquire the mutex
 * @lock: the mutex to be acquired
 *
 * Lock the mutex exclusively for this task. If the mutex is not
 * available right now, it will sleep until it can get it.
 *
 * The mutex must later on be released by the same task that
 * acquired it. Recursive locking is not allowed. The task
 * may not exit without first unlocking the mutex. Also, kernel
 * memory where the mutex resides mutex must not be freed with
 * the mutex still locked. The mutex must first be initialized
 * (or statically defined) before it can be locked. memset()-ing
 * the mutex to 0 is not allowed.
 *
 * ( The CONFIG_DEBUG_MUTEXES .config option turns on debugging
 *   checks that will enforce the restrictions and will also do
 *   deadlock debugging. )
 *
 * This function is similar to (but not equivalent to) down().
 */
94
void __sched mutex_lock(struct mutex *lock)
95
{
96
	might_sleep();
97 98 99 100 101
	/*
	 * The locking fastpath is the 1->0 transition from
	 * 'unlocked' into 'locked' state.
	 */
	__mutex_fastpath_lock(&lock->count, __mutex_lock_slowpath);
102
	mutex_set_owner(lock);
103 104 105
}

EXPORT_SYMBOL(mutex_lock);
106
#endif
107

108
#ifdef CONFIG_MUTEX_SPIN_ON_OWNER
109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162
/*
 * In order to avoid a stampede of mutex spinners from acquiring the mutex
 * more or less simultaneously, the spinners need to acquire a MCS lock
 * first before spinning on the owner field.
 *
 * We don't inline mspin_lock() so that perf can correctly account for the
 * time spent in this lock function.
 */
struct mspin_node {
	struct mspin_node *next ;
	int		  locked;	/* 1 if lock acquired */
};
#define	MLOCK(mutex)	((struct mspin_node **)&((mutex)->spin_mlock))

static noinline
void mspin_lock(struct mspin_node **lock, struct mspin_node *node)
{
	struct mspin_node *prev;

	/* Init node */
	node->locked = 0;
	node->next   = NULL;

	prev = xchg(lock, node);
	if (likely(prev == NULL)) {
		/* Lock acquired */
		node->locked = 1;
		return;
	}
	ACCESS_ONCE(prev->next) = node;
	smp_wmb();
	/* Wait until the lock holder passes the lock down */
	while (!ACCESS_ONCE(node->locked))
		arch_mutex_cpu_relax();
}

static void mspin_unlock(struct mspin_node **lock, struct mspin_node *node)
{
	struct mspin_node *next = ACCESS_ONCE(node->next);

	if (likely(!next)) {
		/*
		 * Release the lock by setting it to NULL
		 */
		if (cmpxchg(lock, node, NULL) == node)
			return;
		/* Wait until the next pointer is set */
		while (!(next = ACCESS_ONCE(node->next)))
			arch_mutex_cpu_relax();
	}
	ACCESS_ONCE(next->locked) = 1;
	smp_wmb();
}

163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205
/*
 * Mutex spinning code migrated from kernel/sched/core.c
 */

static inline bool owner_running(struct mutex *lock, struct task_struct *owner)
{
	if (lock->owner != owner)
		return false;

	/*
	 * Ensure we emit the owner->on_cpu, dereference _after_ checking
	 * lock->owner still matches owner, if that fails, owner might
	 * point to free()d memory, if it still matches, the rcu_read_lock()
	 * ensures the memory stays valid.
	 */
	barrier();

	return owner->on_cpu;
}

/*
 * Look out! "owner" is an entirely speculative pointer
 * access and not reliable.
 */
static noinline
int mutex_spin_on_owner(struct mutex *lock, struct task_struct *owner)
{
	rcu_read_lock();
	while (owner_running(lock, owner)) {
		if (need_resched())
			break;

		arch_mutex_cpu_relax();
	}
	rcu_read_unlock();

	/*
	 * We break out the loop above on need_resched() and when the
	 * owner changed, which is a sign for heavy contention. Return
	 * success only when lock->owner is NULL.
	 */
	return lock->owner == NULL;
}
206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223

/*
 * Initial check for entering the mutex spinning loop
 */
static inline int mutex_can_spin_on_owner(struct mutex *lock)
{
	int retval = 1;

	rcu_read_lock();
	if (lock->owner)
		retval = lock->owner->on_cpu;
	rcu_read_unlock();
	/*
	 * if lock->owner is not set, the mutex owner may have just acquired
	 * it and not set the owner yet or the mutex has been released.
	 */
	return retval;
}
224 225
#endif

226
static __used noinline void __sched __mutex_unlock_slowpath(atomic_t *lock_count);
227

228
/**
229 230 231 232 233 234 235 236 237 238
 * mutex_unlock - release the mutex
 * @lock: the mutex to be released
 *
 * Unlock a mutex that has been locked by this task previously.
 *
 * This function must not be used in interrupt context. Unlocking
 * of a not locked mutex is not allowed.
 *
 * This function is similar to (but not equivalent to) up().
 */
239
void __sched mutex_unlock(struct mutex *lock)
240 241 242 243 244
{
	/*
	 * The unlocking fastpath is the 0->1 transition from 'locked'
	 * into 'unlocked' state:
	 */
245 246 247 248 249 250 251 252
#ifndef CONFIG_DEBUG_MUTEXES
	/*
	 * When debugging is enabled we must not clear the owner before time,
	 * the slow path will always be taken, and that clears the owner field
	 * after verifying that it was indeed current.
	 */
	mutex_clear_owner(lock);
#endif
253 254 255 256
	__mutex_fastpath_unlock(&lock->count, __mutex_unlock_slowpath);
}

EXPORT_SYMBOL(mutex_unlock);
257 258 259 260 261 262 263 264 265 266
/**
 * ww_mutex_unlock - release the w/w mutex
 * @lock: the mutex to be released
 *
 * Unlock a mutex that has been locked by this task previously with any of the
 * ww_mutex_lock* functions (with or without an acquire context). It is
 * forbidden to release the locks after releasing the acquire context. 
 *
 * This function must not be used in interrupt context. Unlocking
 * of a unlocked mutex is not allowed.
267
 */
268
void __sched ww_mutex_unlock(struct ww_mutex *lock)
269
{
270 271 272 273 274 275 276
        /*
         * The unlocking fastpath is the 0->1 transition from 'locked'
         * into 'unlocked' state:
         */
        if (lock->ctx) {
#ifdef CONFIG_DEBUG_MUTEXES
                DEBUG_LOCKS_WARN_ON(!lock->ctx->acquired);
277
#endif
278 279 280 281
                if (lock->ctx->acquired > 0)
                        lock->ctx->acquired--;
                lock->ctx = NULL;
        }
282

283 284 285 286 287 288 289 290 291 292 293
#ifndef CONFIG_DEBUG_MUTEXES
        /*
         * When debugging is enabled we must not clear the owner before time,
         * the slow path will always be taken, and that clears the owner field
         * after verifying that it was indeed current.
         */
        mutex_clear_owner(&lock->base);
#endif
        __mutex_fastpath_unlock(&lock->base.count, __mutex_unlock_slowpath);
}
EXPORT_SYMBOL(ww_mutex_unlock);
294

295 296 297 298 299
static inline int __sched
__mutex_lock_check_stamp(struct mutex *lock, struct ww_acquire_ctx *ctx) 
{
        struct ww_mutex *ww = container_of(lock, struct ww_mutex, base);
        struct ww_acquire_ctx *hold_ctx = ACCESS_ONCE(ww->ctx);
300

301 302
        if (!hold_ctx)
                return 0;
303

304 305
        if (unlikely(ctx == hold_ctx))
                return -EALREADY;
306

307 308 309 310 311 312 313 314 315 316 317
        if (ctx->stamp - hold_ctx->stamp <= LONG_MAX &&
            (ctx->stamp != hold_ctx->stamp || ctx > hold_ctx)) {
#ifdef CONFIG_DEBUG_MUTEXES
                DEBUG_LOCKS_WARN_ON(ctx->contending_lock);
                ctx->contending_lock = ww;
#endif
                return -EDEADLK;
        }
 
        return 0;
}
318

319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357
static __always_inline void ww_mutex_lock_acquired(struct ww_mutex *ww,
                                                   struct ww_acquire_ctx *ww_ctx)
{
#ifdef CONFIG_DEBUG_MUTEXES
        /*
         * If this WARN_ON triggers, you used ww_mutex_lock to acquire,
         * but released with a normal mutex_unlock in this call.
         *
         * This should never happen, always use ww_mutex_unlock.
         */
        DEBUG_LOCKS_WARN_ON(ww->ctx);

        /*
         * Not quite done after calling ww_acquire_done() ?
         */
        DEBUG_LOCKS_WARN_ON(ww_ctx->done_acquire);

        if (ww_ctx->contending_lock) {
                /*
                 * After -EDEADLK you tried to
                 * acquire a different ww_mutex? Bad!
                 */
                DEBUG_LOCKS_WARN_ON(ww_ctx->contending_lock != ww);

                /*
                 * You called ww_mutex_lock after receiving -EDEADLK,
                 * but 'forgot' to unlock everything else first?
                 */
                DEBUG_LOCKS_WARN_ON(ww_ctx->acquired > 0);
                ww_ctx->contending_lock = NULL;
        }

        /*
         * Naughty, using a different class will lead to undefined behavior!
         */
        DEBUG_LOCKS_WARN_ON(ww_ctx->ww_class != ww->ww_class);
#endif
        ww_ctx->acquired++;
}
358

359 360 361 362 363 364 365 366 367 368 369 370
/*
 * Lock a mutex (possibly interruptible), slowpath:
 */
static __always_inline int __sched
__mutex_lock_common(struct mutex *lock, long state, unsigned int subclass,
                    struct lockdep_map *nest_lock, unsigned long ip,
                    struct ww_acquire_ctx *ww_ctx, const bool use_ww_ctx)
{
        struct task_struct *task = current;
        struct mutex_waiter waiter;
        unsigned long flags;
        int ret;
371

372 373
        preempt_disable();
        mutex_acquire_nest(&lock->dep_map, subclass, 0, nest_lock, ip);
374

375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568
#ifdef CONFIG_MUTEX_SPIN_ON_OWNER
        /*
         * Optimistic spinning.
         *
         * We try to spin for acquisition when we find that there are no
         * pending waiters and the lock owner is currently running on a 
         * (different) CPU.
         *
         * The rationale is that if the lock owner is running, it is likely to
         * release the lock soon.
         *
         * Since this needs the lock owner, and this mutex implementation
         * doesn't track the owner atomically in the lock field, we need to
         * track it non-atomically.
         *
         * We can't do this for DEBUG_MUTEXES because that relies on wait_lock
         * to serialize everything.
         *
         * The mutex spinners are queued up using MCS lock so that only one
         * spinner can compete for the mutex. However, if mutex spinning isn't
         * going to happen, there is no point in going through the lock/unlock
         * overhead.
         */
        if (!mutex_can_spin_on_owner(lock))
                goto slowpath;

        if (!osq_lock(&lock->osq))
                goto slowpath;

        for (;;) {
                struct task_struct *owner;

                if (use_ww_ctx && ww_ctx->acquired > 0) {
                        struct ww_mutex *ww;

                        ww = container_of(lock, struct ww_mutex, base);
                        /*
                         * If ww->ctx is set the contents are undefined, only
                         * by acquiring wait_lock there is a guarantee that  
                         * they are not invalid when reading.
                         *
                         * As such, when deadlock detection needs to be
                         * performed the optimistic spinning cannot be done.
                         */
                        if (ACCESS_ONCE(ww->ctx))
                                break;
                }

                /*
                 * If there's an owner, wait for it to either
                 * release the lock or go to sleep.
                 */
                owner = ACCESS_ONCE(lock->owner);
                if (owner && !mutex_spin_on_owner(lock, owner))
                        break;

                if ((atomic_read(&lock->count) == 1) &&
                    (atomic_cmpxchg(&lock->count, 1, 0) == 1)) {
                        lock_acquired(&lock->dep_map, ip); //mdrjr
                        if (use_ww_ctx) {
                                struct ww_mutex *ww;
                                ww = container_of(lock, struct ww_mutex, base);

                                ww_mutex_set_context_fastpath(ww, ww_ctx);
                        }

                        mutex_set_owner(lock);
                        osq_unlock(&lock->osq);
                        preempt_enable();
                        return 0;
                }

                /*
                 * When there's no owner, we might have preempted between the
                 * owner acquiring the lock and setting the owner field. If  
                 * we're an RT task that will live-lock because we won't let 
                 * the owner complete.
                 */
                if (!owner && (need_resched() || rt_task(task)))
                        break;

                /*
                 * The cpu_relax() call is a compiler barrier which forces
                 * everything in this loop to be re-loaded. We don't need 
                 * memory barriers as we'll eventually observe the right  
                 * values at the cost of a few extra spins.
                 */
                arch_mutex_cpu_relax();
        }
        osq_unlock(&lock->osq);
slowpath:
        /*
         * If we fell out of the spin path because of need_resched(),
         * reschedule now, before we try-lock the mutex. This avoids getting
         * scheduled out right after we obtained the mutex.
         */
        if (need_resched())
                schedule_preempt_disabled();
#endif
        spin_lock_mutex(&lock->wait_lock, flags);

        /* once more, can we acquire the lock? */
        if (MUTEX_SHOW_NO_WAITER(lock) && (atomic_xchg(&lock->count, 0) == 1))
                goto skip_wait;

        debug_mutex_lock_common(lock, &waiter);
        debug_mutex_add_waiter(lock, &waiter, task_thread_info(task));

        /* add waiting tasks to the end of the waitqueue (FIFO): */
        list_add_tail(&waiter.list, &lock->wait_list);
        waiter.task = task;

        lock_contended(&lock->dep_map, ip);

        for (;;) {
                /*
                 * Lets try to take the lock again - this is needed even if
                 * we get here for the first time (shortly after failing to
                 * acquire the lock), to make sure that we get a wakeup once
                 * it's unlocked. Later on, if we sleep, this is the
                 * operation that gives us the lock. We xchg it to -1, so
                 * that when we release the lock, we properly wake up the
                 * other waiters:
                 */
                if (MUTEX_SHOW_NO_WAITER(lock) &&
                    (atomic_xchg(&lock->count, -1) == 1))
                        break;

                /*
                 * got a signal? (This code gets eliminated in the
                 * TASK_UNINTERRUPTIBLE case.)
                 */
                if (unlikely(signal_pending_state(state, task))) {
                        ret = -EINTR; //mdrjr
                        goto err;
                }

                if (use_ww_ctx && ww_ctx->acquired > 0) {
                        ret = __mutex_lock_check_stamp(lock, ww_ctx);
                        if (ret)
                                goto err;
                }

                __set_task_state(task, state);

                /* didn't get the lock, go to sleep: */
                spin_unlock_mutex(&lock->wait_lock, flags);
                schedule_preempt_disabled();
                spin_lock_mutex(&lock->wait_lock, flags);
        }
        mutex_remove_waiter(lock, &waiter, current_thread_info());
        /* set it to 0 if there are no waiters left: */
        if (likely(list_empty(&lock->wait_list)))
                atomic_set(&lock->count, 0);
        debug_mutex_free_waiter(&waiter);   

skip_wait:
        /* got the lock - cleanup and rejoice! */
        lock_acquired(&lock->dep_map, ip);
        mutex_set_owner(lock);

        if (use_ww_ctx) {
                struct ww_mutex *ww = container_of(lock, struct ww_mutex, base);
                struct mutex_waiter *cur;

                /*
                 * This branch gets optimized out for the common case,
                 * and is only important for ww_mutex_lock.
                 */
                ww_mutex_lock_acquired(ww, ww_ctx);
                ww->ctx = ww_ctx;

                /*
                 * Give any possible sleeping processes the chance to wake up,
                 * so they can recheck if they have to back off.
                 */
                list_for_each_entry(cur, &lock->wait_list, list) {
                        debug_mutex_wake_waiter(lock, cur);
                        wake_up_process(cur->task);
                }
        }

        spin_unlock_mutex(&lock->wait_lock, flags);
        preempt_enable();
        return 0;

err:
        mutex_remove_waiter(lock, &waiter, task_thread_info(task));
        spin_unlock_mutex(&lock->wait_lock, flags);
        debug_mutex_free_waiter(&waiter);
        mutex_release(&lock->dep_map, 1, ip);
        preempt_enable();
        return ret;
}
569 570 571



572 573 574 575 576
#ifdef CONFIG_DEBUG_LOCK_ALLOC
void __sched
mutex_lock_nested(struct mutex *lock, unsigned int subclass)
{
	might_sleep();
577
	__mutex_lock_common(lock, TASK_UNINTERRUPTIBLE, subclass, NULL, _RET_IP_, NULL, 0);
578 579 580
}

EXPORT_SYMBOL_GPL(mutex_lock_nested);
581

582 583 584 585
void __sched
_mutex_lock_nest_lock(struct mutex *lock, struct lockdep_map *nest)
{
	might_sleep();
586
	__mutex_lock_common(lock, TASK_UNINTERRUPTIBLE, 0, nest, _RET_IP_, NULL, 0);
587 588 589 590
}

EXPORT_SYMBOL_GPL(_mutex_lock_nest_lock);

591 592 593 594
int __sched
mutex_lock_killable_nested(struct mutex *lock, unsigned int subclass)
{
	might_sleep();
595
	return __mutex_lock_common(lock, TASK_KILLABLE, subclass, NULL, _RET_IP_, NULL, 0);
596 597 598
}
EXPORT_SYMBOL_GPL(mutex_lock_killable_nested);

599 600 601 602
int __sched
mutex_lock_interruptible_nested(struct mutex *lock, unsigned int subclass)
{
	might_sleep();
603
	return __mutex_lock_common(lock, TASK_INTERRUPTIBLE,
604
				   subclass, NULL, _RET_IP_, NULL, 0);
605 606 607
}

EXPORT_SYMBOL_GPL(mutex_lock_interruptible_nested);
608 609 610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664
static inline int
ww_mutex_deadlock_injection(struct ww_mutex *lock, struct ww_acquire_ctx *ctx)
{
#ifdef CONFIG_DEBUG_WW_MUTEX_SLOWPATH
        unsigned tmp;

        if (ctx->deadlock_inject_countdown-- == 0) {
                tmp = ctx->deadlock_inject_interval;
                if (tmp > UINT_MAX/4)
                        tmp = UINT_MAX;
                else
                        tmp = tmp*2 + tmp + tmp/2;

                ctx->deadlock_inject_interval = tmp;
                ctx->deadlock_inject_countdown = tmp;
                ctx->contending_lock = lock;

                ww_mutex_unlock(lock);

                return -EDEADLK;
        }
#endif
   
        return 0;
}

int __sched
__ww_mutex_lock(struct ww_mutex *lock, struct ww_acquire_ctx *ctx)
{
        int ret;
   
        might_sleep();
        ret =  __mutex_lock_common(&lock->base, TASK_UNINTERRUPTIBLE,
                                   0, &ctx->dep_map, _RET_IP_, ctx, 1);
        if (!ret && ctx->acquired > 1)
                return ww_mutex_deadlock_injection(lock, ctx);

        return ret;
} 
EXPORT_SYMBOL_GPL(__ww_mutex_lock);

int __sched
__ww_mutex_lock_interruptible(struct ww_mutex *lock, struct ww_acquire_ctx *ctx)
{
        int ret;

        might_sleep();
        ret = __mutex_lock_common(&lock->base, TASK_INTERRUPTIBLE,
                                  0, &ctx->dep_map, _RET_IP_, ctx, 1);

        if (!ret && ctx->acquired > 1)
                return ww_mutex_deadlock_injection(lock, ctx);

        return ret;
}
EXPORT_SYMBOL_GPL(__ww_mutex_lock_interruptible);

665 666
#endif

667 668 669
/*
 * Release the lock, slowpath:
 */
670
static inline void
671
__mutex_unlock_common_slowpath(atomic_t *lock_count, int nested)
672
{
673
	struct mutex *lock = container_of(lock_count, struct mutex, count);
674
	unsigned long flags;
675

676
	spin_lock_mutex(&lock->wait_lock, flags);
677
	mutex_release(&lock->dep_map, nested, _RET_IP_);
678
	debug_mutex_unlock(lock);
679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698

	/*
	 * some architectures leave the lock unlocked in the fastpath failure
	 * case, others need to leave it locked. In the later case we have to
	 * unlock it here
	 */
	if (__mutex_slowpath_needs_to_unlock())
		atomic_set(&lock->count, 1);

	if (!list_empty(&lock->wait_list)) {
		/* get the first entry from the wait-list: */
		struct mutex_waiter *waiter =
				list_entry(lock->wait_list.next,
					   struct mutex_waiter, list);

		debug_mutex_wake_waiter(lock, waiter);

		wake_up_process(waiter->task);
	}

699
	spin_unlock_mutex(&lock->wait_lock, flags);
700 701
}

702 703 704
/*
 * Release the lock, slowpath:
 */
705
static __used noinline void
706 707
__mutex_unlock_slowpath(atomic_t *lock_count)
{
708
	__mutex_unlock_common_slowpath(lock_count, 1);
709 710
}

711
#ifndef CONFIG_DEBUG_LOCK_ALLOC
712 713 714 715
/*
 * Here come the less common (and hence less performance-critical) APIs:
 * mutex_lock_interruptible() and mutex_trylock().
 */
716
static noinline int __sched
717 718
__mutex_lock_killable_slowpath(atomic_t *lock_count);

719
static noinline int __sched
720
__mutex_lock_interruptible_slowpath(atomic_t *lock_count);
721

722 723
/**
 * mutex_lock_interruptible - acquire the mutex, interruptible
724 725 726 727 728 729 730 731 732
 * @lock: the mutex to be acquired
 *
 * Lock the mutex like mutex_lock(), and return 0 if the mutex has
 * been acquired or sleep until the mutex becomes available. If a
 * signal arrives while waiting for the lock then this function
 * returns -EINTR.
 *
 * This function is similar to (but not equivalent to) down_interruptible().
 */
733
int __sched mutex_lock_interruptible(struct mutex *lock)
734
{
735 736
	int ret;

737
	might_sleep();
738
	ret =  __mutex_fastpath_lock_retval
739
			(&lock->count, __mutex_lock_interruptible_slowpath);
740 741 742 743
	if (!ret)
		mutex_set_owner(lock);

	return ret;
744 745 746 747
}

EXPORT_SYMBOL(mutex_lock_interruptible);

748
int __sched mutex_lock_killable(struct mutex *lock)
749
{
750 751
	int ret;

752
	might_sleep();
753
	ret = __mutex_fastpath_lock_retval
754
			(&lock->count, __mutex_lock_killable_slowpath);
755 756 757 758
	if (!ret)
		mutex_set_owner(lock);

	return ret;
759 760 761
}
EXPORT_SYMBOL(mutex_lock_killable);

762
static __used noinline void __sched
763 764 765 766
__mutex_lock_slowpath(atomic_t *lock_count)
{
	struct mutex *lock = container_of(lock_count, struct mutex, count);

767
	__mutex_lock_common(lock, TASK_UNINTERRUPTIBLE, 0, NULL, _RET_IP_);
768 769
}

770
static noinline int __sched
771 772 773 774
__mutex_lock_killable_slowpath(atomic_t *lock_count)
{
	struct mutex *lock = container_of(lock_count, struct mutex, count);

775
	return __mutex_lock_common(lock, TASK_KILLABLE, 0, NULL, _RET_IP_);
776 777
}

778
static noinline int __sched
779
__mutex_lock_interruptible_slowpath(atomic_t *lock_count)
780 781 782
{
	struct mutex *lock = container_of(lock_count, struct mutex, count);

783
	return __mutex_lock_common(lock, TASK_INTERRUPTIBLE, 0, NULL, _RET_IP_);
784
}
785
#endif
786 787 788 789 790 791 792 793

/*
 * Spinlock based trylock, we take the spinlock and check whether we
 * can get the lock:
 */
static inline int __mutex_trylock_slowpath(atomic_t *lock_count)
{
	struct mutex *lock = container_of(lock_count, struct mutex, count);
794
	unsigned long flags;
795 796
	int prev;

797
	spin_lock_mutex(&lock->wait_lock, flags);
798 799

	prev = atomic_xchg(&lock->count, -1);
800
	if (likely(prev == 1)) {
801
		mutex_set_owner(lock);
802 803
		mutex_acquire(&lock->dep_map, 0, 1, _RET_IP_);
	}
804

805 806 807 808
	/* Set it back to 0 if there are no waiters: */
	if (likely(list_empty(&lock->wait_list)))
		atomic_set(&lock->count, 0);

809
	spin_unlock_mutex(&lock->wait_lock, flags);
810 811 812 813

	return prev == 1;
}

814 815
/**
 * mutex_trylock - try to acquire the mutex, without waiting
816 817 818 819 820 821
 * @lock: the mutex to be acquired
 *
 * Try to acquire the mutex atomically. Returns 1 if the mutex
 * has been acquired successfully, and 0 on contention.
 *
 * NOTE: this function follows the spin_trylock() convention, so
822
 * it is negated from the down_trylock() return values! Be careful
823 824 825 826 827
 * about this when converting semaphore users to mutexes.
 *
 * This function must not be used in interrupt context. The
 * mutex must be released by the same task that acquired it.
 */
828
int __sched mutex_trylock(struct mutex *lock)
829
{
830 831 832 833 834 835 836
	int ret;

	ret = __mutex_fastpath_trylock(&lock->count, __mutex_trylock_slowpath);
	if (ret)
		mutex_set_owner(lock);

	return ret;
837 838
}
EXPORT_SYMBOL(mutex_trylock);
839 840 841 842 843 844 845 846 847 848 849 850 851 852 853 854 855 856 857 858 859 860 861 862

/**
 * atomic_dec_and_mutex_lock - return holding mutex if we dec to 0
 * @cnt: the atomic which we are to dec
 * @lock: the mutex to return holding if we dec to 0
 *
 * return true and hold lock if we dec to 0, return false otherwise
 */
int atomic_dec_and_mutex_lock(atomic_t *cnt, struct mutex *lock)
{
	/* dec if we can't possibly hit 0 */
	if (atomic_add_unless(cnt, -1, 1))
		return 0;
	/* we might hit 0, so take the lock */
	mutex_lock(lock);
	if (!atomic_dec_and_test(cnt)) {
		/* when we actually did the dec, we didn't hit 0 */
		mutex_unlock(lock);
		return 0;
	}
	/* we hit 0, and we hold the lock */
	return 1;
}
EXPORT_SYMBOL(atomic_dec_and_mutex_lock);