fs-writeback.c 39.9 KB
Newer Older
Linus Torvalds's avatar
Linus Torvalds committed
1 2 3 4 5 6 7 8 9 10
/*
 * fs/fs-writeback.c
 *
 * Copyright (C) 2002, Linus Torvalds.
 *
 * Contains all the functions related to writing back and waiting
 * upon dirty inodes against superblocks, and writing back dirty
 * pages against inodes.  ie: data writeback.  Writeout of the
 * inode itself is not handled here.
 *
11
 * 10Apr2002	Andrew Morton
Linus Torvalds's avatar
Linus Torvalds committed
12 13 14 15 16
 *		Split out of fs/inode.c
 *		Additions for address_space-based writeback
 */

#include <linux/kernel.h>
17
#include <linux/export.h>
Linus Torvalds's avatar
Linus Torvalds committed
18
#include <linux/spinlock.h>
19
#include <linux/slab.h>
Linus Torvalds's avatar
Linus Torvalds committed
20 21 22
#include <linux/sched.h>
#include <linux/fs.h>
#include <linux/mm.h>
23
#include <linux/pagemap.h>
24
#include <linux/kthread.h>
Linus Torvalds's avatar
Linus Torvalds committed
25 26 27
#include <linux/writeback.h>
#include <linux/blkdev.h>
#include <linux/backing-dev.h>
28
#include <linux/tracepoint.h>
29
#include "internal.h"
Linus Torvalds's avatar
Linus Torvalds committed
30

31 32 33 34 35
/*
 * 4MB minimal write chunk size
 */
#define MIN_WRITEBACK_PAGES	(4096UL >> (PAGE_CACHE_SHIFT - 10))

36 37 38
/*
 * Passed into wb_writeback(), essentially a subset of writeback_control
 */
39
struct wb_writeback_work {
40 41
	long nr_pages;
	struct super_block *sb;
42
	unsigned long *older_than_this;
43
	enum writeback_sync_modes sync_mode;
44
	unsigned int tagged_writepages:1;
45 46 47
	unsigned int for_kupdate:1;
	unsigned int range_cyclic:1;
	unsigned int for_background:1;
48
	enum wb_reason reason;		/* why was writeback initiated? */
49

50
	struct list_head list;		/* pending work list */
51
	struct completion *done;	/* set if the caller waits */
52 53
};

54 55 56 57
/**
 * writeback_in_progress - determine whether there is writeback in progress
 * @bdi: the device's backing_dev_info structure.
 *
58 59
 * Determine whether there is writeback waiting to be handled against a
 * backing device.
60 61 62
 */
int writeback_in_progress(struct backing_dev_info *bdi)
{
63
	return test_bit(BDI_writeback_running, &bdi->state);
64
}
65
EXPORT_SYMBOL(writeback_in_progress);
66

67 68 69 70
static inline struct backing_dev_info *inode_to_bdi(struct inode *inode)
{
	struct super_block *sb = inode->i_sb;

71 72 73 74
	if (strcmp(sb->s_type->name, "bdev") == 0)
		return inode->i_mapping->backing_dev_info;

	return sb->s_bdi;
75 76
}

77 78 79 80 81
static inline struct inode *wb_inode(struct list_head *head)
{
	return list_entry(head, struct inode, i_wb_list);
}

82 83 84 85 86 87 88 89
/*
 * Include the creation of the trace points after defining the
 * wb_writeback_work structure and inline functions so that the definition
 * remains local to this file.
 */
#define CREATE_TRACE_POINTS
#include <trace/events/writeback.h>

90 91 92 93 94 95 96 97
static void bdi_wakeup_thread(struct backing_dev_info *bdi)
{
	spin_lock_bh(&bdi->wb_lock);
	if (test_bit(BDI_registered, &bdi->state))
		mod_delayed_work(bdi_wq, &bdi->wb.dwork, 0);
	spin_unlock_bh(&bdi->wb_lock);
}

98 99 100 101 102 103
static void bdi_queue_work(struct backing_dev_info *bdi,
			   struct wb_writeback_work *work)
{
	trace_writeback_queue(bdi, work);

	spin_lock_bh(&bdi->wb_lock);
104 105 106 107 108
	if (!test_bit(BDI_registered, &bdi->state)) {
		if (work->done)
			complete(work->done);
		goto out_unlock;
	}
109
	list_add_tail(&work->list, &bdi->work_list);
110
	mod_delayed_work(bdi_wq, &bdi->wb.dwork, 0);
111 112
out_unlock:
	spin_unlock_bh(&bdi->wb_lock);
Linus Torvalds's avatar
Linus Torvalds committed
113 114
}

115 116
static void
__bdi_start_writeback(struct backing_dev_info *bdi, long nr_pages,
117
		      bool range_cyclic, enum wb_reason reason)
Linus Torvalds's avatar
Linus Torvalds committed
118
{
119
	struct wb_writeback_work *work;
120

121 122 123 124
	/*
	 * This is WB_SYNC_NONE writeback, so if allocation fails just
	 * wakeup the thread for old dirty data writeback
	 */
125 126
	work = kzalloc(sizeof(*work), GFP_ATOMIC);
	if (!work) {
127
		trace_writeback_nowork(bdi);
128
		bdi_wakeup_thread(bdi);
129
		return;
130
	}
131

132 133 134
	work->sync_mode	= WB_SYNC_NONE;
	work->nr_pages	= nr_pages;
	work->range_cyclic = range_cyclic;
135
	work->reason	= reason;
136

137
	bdi_queue_work(bdi, work);
138 139 140 141 142 143
}

/**
 * bdi_start_writeback - start writeback
 * @bdi: the backing device to write from
 * @nr_pages: the number of pages to write
144
 * @reason: reason why some writeback work was initiated
145 146 147
 *
 * Description:
 *   This does WB_SYNC_NONE opportunistic writeback. The IO is only
148
 *   started when this function returns, we make no guarantees on
149
 *   completion. Caller need not hold sb s_umount semaphore.
150 151
 *
 */
152 153
void bdi_start_writeback(struct backing_dev_info *bdi, long nr_pages,
			enum wb_reason reason)
154
{
155
	__bdi_start_writeback(bdi, nr_pages, true, reason);
156
}
157

158 159 160 161 162
/**
 * bdi_start_background_writeback - start background writeback
 * @bdi: the backing device to write from
 *
 * Description:
163 164 165 166
 *   This makes sure WB_SYNC_NONE background writeback happens. When
 *   this function returns, it is only guaranteed that for given BDI
 *   some IO is happening if we are over background dirty threshold.
 *   Caller need not hold sb s_umount semaphore.
167 168 169
 */
void bdi_start_background_writeback(struct backing_dev_info *bdi)
{
170 171 172 173
	/*
	 * We just wake up the flusher thread. It will perform background
	 * writeback as soon as there is no other work to do.
	 */
174
	trace_writeback_wake_background(bdi);
175
	bdi_wakeup_thread(bdi);
Linus Torvalds's avatar
Linus Torvalds committed
176 177
}

178 179 180 181 182
/*
 * Remove the inode from the writeback list it is on.
 */
void inode_wb_list_del(struct inode *inode)
{
183 184 185
	struct backing_dev_info *bdi = inode_to_bdi(inode);

	spin_lock(&bdi->wb.list_lock);
186
	list_del_init(&inode->i_wb_list);
187
	spin_unlock(&bdi->wb.list_lock);
188 189
}

190 191 192 193 194
/*
 * Redirty an inode: set its when-it-was dirtied timestamp and move it to the
 * furthest end of its superblock's dirty-inode list.
 *
 * Before stamping the inode's ->dirtied_when, we check to see whether it is
195
 * already the most-recently-dirtied inode on the b_dirty list.  If that is
196 197 198
 * the case then the inode must have been redirtied while it was being written
 * out and we don't reset its dirtied_when.
 */
199
static void redirty_tail(struct inode *inode, struct bdi_writeback *wb)
200
{
201
	assert_spin_locked(&wb->list_lock);
202
	if (!list_empty(&wb->b_dirty)) {
203
		struct inode *tail;
204

205
		tail = wb_inode(wb->b_dirty.next);
206
		if (time_before(inode->dirtied_when, tail->dirtied_when))
207 208
			inode->dirtied_when = jiffies;
	}
209
	list_move(&inode->i_wb_list, &wb->b_dirty);
210 211
}

212
/*
213
 * requeue inode for re-scanning after bdi->b_io list is exhausted.
214
 */
215
static void requeue_io(struct inode *inode, struct bdi_writeback *wb)
216
{
217
	assert_spin_locked(&wb->list_lock);
218
	list_move(&inode->i_wb_list, &wb->b_more_io);
219 220
}

Joern Engel's avatar
Joern Engel committed
221 222
static void inode_sync_complete(struct inode *inode)
{
223
	inode->i_state &= ~I_SYNC;
224 225
	/* If inode is clean an unused, put it into LRU now... */
	inode_add_lru(inode);
226
	/* Waiters must see I_SYNC cleared before being woken up */
Joern Engel's avatar
Joern Engel committed
227 228 229 230
	smp_mb();
	wake_up_bit(&inode->i_state, __I_SYNC);
}

231 232 233 234 235 236 237 238
static bool inode_dirtied_after(struct inode *inode, unsigned long t)
{
	bool ret = time_after(inode->dirtied_when, t);
#ifndef CONFIG_64BIT
	/*
	 * For inodes being constantly redirtied, dirtied_when can get stuck.
	 * It _appears_ to be in the future, but is actually in distant past.
	 * This test is necessary to prevent such wrapped-around relative times
239
	 * from permanently stopping the whole bdi writeback.
240 241 242 243 244 245
	 */
	ret = ret && time_before_eq(inode->dirtied_when, jiffies);
#endif
	return ret;
}

246
/*
247
 * Move expired (dirtied before work->older_than_this) dirty inodes from
248
 * @delaying_queue to @dispatch_queue.
249
 */
250
static int move_expired_inodes(struct list_head *delaying_queue,
251
			       struct list_head *dispatch_queue,
252
			       struct wb_writeback_work *work)
253
{
254 255
	LIST_HEAD(tmp);
	struct list_head *pos, *node;
256
	struct super_block *sb = NULL;
257
	struct inode *inode;
258
	int do_sb_sort = 0;
259
	int moved = 0;
260

261
	while (!list_empty(delaying_queue)) {
262
		inode = wb_inode(delaying_queue->prev);
263 264
		if (work->older_than_this &&
		    inode_dirtied_after(inode, *work->older_than_this))
265
			break;
266 267 268
		if (sb && sb != inode->i_sb)
			do_sb_sort = 1;
		sb = inode->i_sb;
269
		list_move(&inode->i_wb_list, &tmp);
270
		moved++;
271 272
	}

273 274 275
	/* just one sb in list, splice to dispatch_queue and we're done */
	if (!do_sb_sort) {
		list_splice(&tmp, dispatch_queue);
276
		goto out;
277 278
	}

279 280
	/* Move inodes from one superblock together */
	while (!list_empty(&tmp)) {
281
		sb = wb_inode(tmp.prev)->i_sb;
282
		list_for_each_prev_safe(pos, node, &tmp) {
283
			inode = wb_inode(pos);
284
			if (inode->i_sb == sb)
285
				list_move(&inode->i_wb_list, dispatch_queue);
286
		}
287
	}
288 289
out:
	return moved;
290 291 292 293
}

/*
 * Queue all expired dirty inodes for io, eldest first.
294 295 296 297 298 299 300 301
 * Before
 *         newly dirtied     b_dirty    b_io    b_more_io
 *         =============>    gf         edc     BA
 * After
 *         newly dirtied     b_dirty    b_io    b_more_io
 *         =============>    g          fBAedc
 *                                           |
 *                                           +--> dequeue for IO
302
 */
303
static void queue_io(struct bdi_writeback *wb, struct wb_writeback_work *work)
304
{
305
	int moved;
306
	assert_spin_locked(&wb->list_lock);
307
	list_splice_init(&wb->b_more_io, &wb->b_io);
308 309
	moved = move_expired_inodes(&wb->b_dirty, &wb->b_io, work);
	trace_writeback_queue_io(wb, work, moved);
310 311
}

312
static int write_inode(struct inode *inode, struct writeback_control *wbc)
313
{
314 315 316 317 318 319 320 321
	int ret;

	if (inode->i_sb->s_op->write_inode && !is_bad_inode(inode)) {
		trace_writeback_write_inode_start(inode, wbc);
		ret = inode->i_sb->s_op->write_inode(inode, wbc);
		trace_writeback_write_inode(inode, wbc);
		return ret;
	}
322
	return 0;
323 324
}

Linus Torvalds's avatar
Linus Torvalds committed
325
/*
326 327
 * Wait for writeback on an inode to complete. Called with i_lock held.
 * Caller must make sure inode cannot go away when we drop i_lock.
328
 */
329 330 331
static void __inode_wait_for_writeback(struct inode *inode)
	__releases(inode->i_lock)
	__acquires(inode->i_lock)
332 333 334 335 336
{
	DEFINE_WAIT_BIT(wq, &inode->i_state, __I_SYNC);
	wait_queue_head_t *wqh;

	wqh = bit_waitqueue(&inode->i_state, __I_SYNC);
337 338
	while (inode->i_state & I_SYNC) {
		spin_unlock(&inode->i_lock);
339
		__wait_on_bit(wqh, &wq, inode_wait, TASK_UNINTERRUPTIBLE);
340
		spin_lock(&inode->i_lock);
341
	}
342 343
}

344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373
/*
 * Wait for writeback on an inode to complete. Caller must have inode pinned.
 */
void inode_wait_for_writeback(struct inode *inode)
{
	spin_lock(&inode->i_lock);
	__inode_wait_for_writeback(inode);
	spin_unlock(&inode->i_lock);
}

/*
 * Sleep until I_SYNC is cleared. This function must be called with i_lock
 * held and drops it. It is aimed for callers not holding any inode reference
 * so once i_lock is dropped, inode can go away.
 */
static void inode_sleep_on_writeback(struct inode *inode)
	__releases(inode->i_lock)
{
	DEFINE_WAIT(wait);
	wait_queue_head_t *wqh = bit_waitqueue(&inode->i_state, __I_SYNC);
	int sleep;

	prepare_to_wait(wqh, &wait, TASK_UNINTERRUPTIBLE);
	sleep = inode->i_state & I_SYNC;
	spin_unlock(&inode->i_lock);
	if (sleep)
		schedule();
	finish_wait(wqh, &wait);
}

374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396
/*
 * Find proper writeback list for the inode depending on its current state and
 * possibly also change of its state while we were doing writeback.  Here we
 * handle things such as livelock prevention or fairness of writeback among
 * inodes. This function can be called only by flusher thread - noone else
 * processes all inodes in writeback lists and requeueing inodes behind flusher
 * thread's back can have unexpected consequences.
 */
static void requeue_inode(struct inode *inode, struct bdi_writeback *wb,
			  struct writeback_control *wbc)
{
	if (inode->i_state & I_FREEING)
		return;

	/*
	 * Sync livelock prevention. Each inode is tagged and synced in one
	 * shot. If still dirty, it will be redirty_tail()'ed below.  Update
	 * the dirty time to prevent enqueue and sync it again.
	 */
	if ((inode->i_state & I_DIRTY) &&
	    (wbc->sync_mode == WB_SYNC_ALL || wbc->tagged_writepages))
		inode->dirtied_when = jiffies;

397 398 399 400 401 402 403 404 405
	if (wbc->pages_skipped) {
		/*
		 * writeback is not making progress due to locked
		 * buffers. Skip this inode for now.
		 */
		redirty_tail(inode, wb);
		return;
	}

406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436
	if (mapping_tagged(inode->i_mapping, PAGECACHE_TAG_DIRTY)) {
		/*
		 * We didn't write back all the pages.  nfs_writepages()
		 * sometimes bales out without doing anything.
		 */
		if (wbc->nr_to_write <= 0) {
			/* Slice used up. Queue for next turn. */
			requeue_io(inode, wb);
		} else {
			/*
			 * Writeback blocked by something other than
			 * congestion. Delay the inode for some time to
			 * avoid spinning on the CPU (100% iowait)
			 * retrying writeback of the dirty page/inode
			 * that cannot be performed immediately.
			 */
			redirty_tail(inode, wb);
		}
	} else if (inode->i_state & I_DIRTY) {
		/*
		 * Filesystems can dirty the inode during writeback operations,
		 * such as delayed allocation during submission or metadata
		 * updates after data IO completion.
		 */
		redirty_tail(inode, wb);
	} else {
		/* The inode is clean. Remove from writeback lists. */
		list_del_init(&inode->i_wb_list);
	}
}

437
/*
438 439 440
 * Write out an inode and its dirty pages. Do not update the writeback list
 * linkage. That is left to the caller. The caller is also responsible for
 * setting I_SYNC flag and calling inode_sync_complete() to clear it.
Linus Torvalds's avatar
Linus Torvalds committed
441 442
 */
static int
443
__writeback_single_inode(struct inode *inode, struct writeback_control *wbc)
Linus Torvalds's avatar
Linus Torvalds committed
444 445
{
	struct address_space *mapping = inode->i_mapping;
446
	long nr_to_write = wbc->nr_to_write;
447
	unsigned dirty;
Linus Torvalds's avatar
Linus Torvalds committed
448 449
	int ret;

450
	WARN_ON(!(inode->i_state & I_SYNC));
Linus Torvalds's avatar
Linus Torvalds committed
451

452 453
	trace_writeback_single_inode_start(inode, wbc, nr_to_write);

Linus Torvalds's avatar
Linus Torvalds committed
454 455
	ret = do_writepages(mapping, wbc);

456 457 458 459 460
	/*
	 * Make sure to wait on the data before writing out the metadata.
	 * This is important for filesystems that modify metadata on data
	 * I/O completion.
	 */
461
	if (wbc->sync_mode == WB_SYNC_ALL) {
462
		int err = filemap_fdatawait(mapping);
Linus Torvalds's avatar
Linus Torvalds committed
463 464 465 466
		if (ret == 0)
			ret = err;
	}

467 468 469 470 471
	/*
	 * Some filesystems may redirty the inode during the writeback
	 * due to delalloc, clear dirty metadata flags right before
	 * write_inode()
	 */
472
	spin_lock(&inode->i_lock);
473

474
	dirty = inode->i_state & I_DIRTY;
475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492
	inode->i_state &= ~I_DIRTY;

	/*
	 * Paired with smp_mb() in __mark_inode_dirty().  This allows
	 * __mark_inode_dirty() to test i_state without grabbing i_lock -
	 * either they see the I_DIRTY bits cleared or we see the dirtied
	 * inode.
	 *
	 * I_DIRTY_PAGES is always cleared together above even if @mapping
	 * still has dirty pages.  The flag is reinstated after smp_mb() if
	 * necessary.  This guarantees that either __mark_inode_dirty()
	 * sees clear I_DIRTY_PAGES or we see PAGECACHE_TAG_DIRTY.
	 */
	smp_mb();

	if (mapping_tagged(mapping, PAGECACHE_TAG_DIRTY))
		inode->i_state |= I_DIRTY_PAGES;

493
	spin_unlock(&inode->i_lock);
494

495 496
	/* Don't write the inode if only I_DIRTY_PAGES was set */
	if (dirty & (I_DIRTY_SYNC | I_DIRTY_DATASYNC)) {
497
		int err = write_inode(inode, wbc);
Linus Torvalds's avatar
Linus Torvalds committed
498 499 500
		if (ret == 0)
			ret = err;
	}
501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528
	trace_writeback_single_inode(inode, wbc, nr_to_write);
	return ret;
}

/*
 * Write out an inode's dirty pages. Either the caller has an active reference
 * on the inode or the inode has I_WILL_FREE set.
 *
 * This function is designed to be called for writing back one inode which
 * we go e.g. from filesystem. Flusher thread uses __writeback_single_inode()
 * and does more profound writeback list handling in writeback_sb_inodes().
 */
static int
writeback_single_inode(struct inode *inode, struct bdi_writeback *wb,
		       struct writeback_control *wbc)
{
	int ret = 0;

	spin_lock(&inode->i_lock);
	if (!atomic_read(&inode->i_count))
		WARN_ON(!(inode->i_state & (I_WILL_FREE|I_FREEING)));
	else
		WARN_ON(inode->i_state & I_WILL_FREE);

	if (inode->i_state & I_SYNC) {
		if (wbc->sync_mode != WB_SYNC_ALL)
			goto out;
		/*
529 530 531
		 * It's a data-integrity sync. We must wait. Since callers hold
		 * inode reference or inode has I_WILL_FREE set, it cannot go
		 * away under us.
532
		 */
533
		__inode_wait_for_writeback(inode);
534 535 536
	}
	WARN_ON(inode->i_state & I_SYNC);
	/*
537 538 539 540 541 542
	 * Skip inode if it is clean and we have no outstanding writeback in
	 * WB_SYNC_ALL mode. We don't want to mess with writeback lists in this
	 * function since flusher thread may be doing for example sync in
	 * parallel and if we move the inode, it could get skipped. So here we
	 * make sure inode is on some writeback list and leave it there unless
	 * we have completely cleaned the inode.
543
	 */
544 545 546
	if (!(inode->i_state & I_DIRTY) &&
	    (wbc->sync_mode != WB_SYNC_ALL ||
	     !mapping_tagged(inode->i_mapping, PAGECACHE_TAG_WRITEBACK)))
547 548 549 550
		goto out;
	inode->i_state |= I_SYNC;
	spin_unlock(&inode->i_lock);

551
	ret = __writeback_single_inode(inode, wbc);
Linus Torvalds's avatar
Linus Torvalds committed
552

553
	spin_lock(&wb->list_lock);
554
	spin_lock(&inode->i_lock);
555 556 557 558 559 560 561
	/*
	 * If inode is clean, remove it from writeback lists. Otherwise don't
	 * touch it. See comment above for explanation.
	 */
	if (!(inode->i_state & I_DIRTY))
		list_del_init(&inode->i_wb_list);
	spin_unlock(&wb->list_lock);
Joern Engel's avatar
Joern Engel committed
562
	inode_sync_complete(inode);
563 564
out:
	spin_unlock(&inode->i_lock);
Linus Torvalds's avatar
Linus Torvalds committed
565 566 567
	return ret;
}

568 569
static long writeback_chunk_size(struct backing_dev_info *bdi,
				 struct wb_writeback_work *work)
570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587
{
	long pages;

	/*
	 * WB_SYNC_ALL mode does livelock avoidance by syncing dirty
	 * inodes/pages in one big loop. Setting wbc.nr_to_write=LONG_MAX
	 * here avoids calling into writeback_inodes_wb() more than once.
	 *
	 * The intended call sequence for WB_SYNC_ALL writeback is:
	 *
	 *      wb_writeback()
	 *          writeback_sb_inodes()       <== called only once
	 *              write_cache_pages()     <== called once for each inode
	 *                   (quickly) tag currently dirty pages
	 *                   (maybe slowly) sync all tagged pages
	 */
	if (work->sync_mode == WB_SYNC_ALL || work->tagged_writepages)
		pages = LONG_MAX;
588 589 590 591 592 593 594
	else {
		pages = min(bdi->avg_write_bandwidth / 2,
			    global_dirty_limit / DIRTY_SCOPE);
		pages = min(pages, work->nr_pages);
		pages = round_down(pages + MIN_WRITEBACK_PAGES,
				   MIN_WRITEBACK_PAGES);
	}
595 596 597 598

	return pages;
}

599 600
/*
 * Write a portion of b_io inodes which belong to @sb.
601
 *
602
 * Return the number of pages and/or inodes written.
603
 */
604 605 606
static long writeback_sb_inodes(struct super_block *sb,
				struct bdi_writeback *wb,
				struct wb_writeback_work *work)
Linus Torvalds's avatar
Linus Torvalds committed
607
{
608 609 610 611 612 613 614 615 616 617 618 619 620
	struct writeback_control wbc = {
		.sync_mode		= work->sync_mode,
		.tagged_writepages	= work->tagged_writepages,
		.for_kupdate		= work->for_kupdate,
		.for_background		= work->for_background,
		.range_cyclic		= work->range_cyclic,
		.range_start		= 0,
		.range_end		= LLONG_MAX,
	};
	unsigned long start_time = jiffies;
	long write_chunk;
	long wrote = 0;  /* count both pages and inodes */

621
	while (!list_empty(&wb->b_io)) {
622
		struct inode *inode = wb_inode(wb->b_io.prev);
623 624

		if (inode->i_sb != sb) {
625
			if (work->sb) {
626 627 628 629 630
				/*
				 * We only want to write back data for this
				 * superblock, move all inodes not belonging
				 * to it back onto the dirty list.
				 */
631
				redirty_tail(inode, wb);
632 633 634 635 636 637 638 639
				continue;
			}

			/*
			 * The inode belongs to a different superblock.
			 * Bounce back to the caller to unpin this and
			 * pin the next superblock.
			 */
640
			break;
641 642
		}

643
		/*
644 645
		 * Don't bother with new inodes or inodes being freed, first
		 * kind does not need periodic writeout yet, and for the latter
646 647
		 * kind writeout is handled by the freer.
		 */
648
		spin_lock(&inode->i_lock);
649
		if (inode->i_state & (I_NEW | I_FREEING | I_WILL_FREE)) {
650
			spin_unlock(&inode->i_lock);
651
			redirty_tail(inode, wb);
652 653
			continue;
		}
654 655 656 657 658 659 660 661 662 663 664 665 666 667 668
		if ((inode->i_state & I_SYNC) && wbc.sync_mode != WB_SYNC_ALL) {
			/*
			 * If this inode is locked for writeback and we are not
			 * doing writeback-for-data-integrity, move it to
			 * b_more_io so that writeback can proceed with the
			 * other inodes on s_io.
			 *
			 * We'll have another go at writing back this inode
			 * when we completed a full scan of b_io.
			 */
			spin_unlock(&inode->i_lock);
			requeue_io(inode, wb);
			trace_writeback_sb_inodes_requeue(inode);
			continue;
		}
669 670
		spin_unlock(&wb->list_lock);

671 672 673 674 675
		/*
		 * We already requeued the inode if it had I_SYNC set and we
		 * are doing WB_SYNC_NONE writeback. So this catches only the
		 * WB_SYNC_ALL case.
		 */
676 677 678 679
		if (inode->i_state & I_SYNC) {
			/* Wait for I_SYNC. This function drops i_lock... */
			inode_sleep_on_writeback(inode);
			/* Inode may be gone, start again */
680
			spin_lock(&wb->list_lock);
681 682
			continue;
		}
683 684
		inode->i_state |= I_SYNC;
		spin_unlock(&inode->i_lock);
685

686
		write_chunk = writeback_chunk_size(wb->bdi, work);
687 688
		wbc.nr_to_write = write_chunk;
		wbc.pages_skipped = 0;
689

690 691 692 693
		/*
		 * We use I_SYNC to pin the inode in memory. While it is set
		 * evict_inode() will wait so the inode cannot be freed.
		 */
694
		__writeback_single_inode(inode, &wbc);
695

696 697
		work->nr_pages -= write_chunk - wbc.nr_to_write;
		wrote += write_chunk - wbc.nr_to_write;
698 699
		spin_lock(&wb->list_lock);
		spin_lock(&inode->i_lock);
700 701
		if (!(inode->i_state & I_DIRTY))
			wrote++;
702 703
		requeue_inode(inode, wb, &wbc);
		inode_sync_complete(inode);
704
		spin_unlock(&inode->i_lock);
705
		cond_resched_lock(&wb->list_lock);
706 707 708 709 710 711 712 713 714
		/*
		 * bail out to wb_writeback() often enough to check
		 * background threshold and other termination conditions.
		 */
		if (wrote) {
			if (time_is_before_jiffies(start_time + HZ / 10UL))
				break;
			if (work->nr_pages <= 0)
				break;
715
		}
Linus Torvalds's avatar
Linus Torvalds committed
716
	}
717
	return wrote;
718 719
}

720 721
static long __writeback_inodes_wb(struct bdi_writeback *wb,
				  struct wb_writeback_work *work)
722
{
723 724
	unsigned long start_time = jiffies;
	long wrote = 0;
Nick Piggin's avatar
Nick Piggin committed
725

726
	while (!list_empty(&wb->b_io)) {
727
		struct inode *inode = wb_inode(wb->b_io.prev);
728
		struct super_block *sb = inode->i_sb;
729

730
		if (!grab_super_passive(sb)) {
731 732 733 734 735 736
			/*
			 * grab_super_passive() may fail consistently due to
			 * s_umount being grabbed by someone else. Don't use
			 * requeue_io() to avoid busy retrying the inode/sb.
			 */
			redirty_tail(inode, wb);
737
			continue;
738
		}
739
		wrote += writeback_sb_inodes(sb, wb, work);
740
		drop_super(sb);
741

742 743 744 745 746 747 748
		/* refer to the same tests at the end of writeback_sb_inodes */
		if (wrote) {
			if (time_is_before_jiffies(start_time + HZ / 10UL))
				break;
			if (work->nr_pages <= 0)
				break;
		}
749
	}
750
	/* Leave any unwritten inodes on b_io */
751
	return wrote;
752 753
}

754 755
long writeback_inodes_wb(struct bdi_writeback *wb, long nr_pages,
				enum wb_reason reason)
756
{
757 758 759 760
	struct wb_writeback_work work = {
		.nr_pages	= nr_pages,
		.sync_mode	= WB_SYNC_NONE,
		.range_cyclic	= 1,
761
		.reason		= reason,
762
	};
763

764
	spin_lock(&wb->list_lock);
765
	if (list_empty(&wb->b_io))
766
		queue_io(wb, &work);
767
	__writeback_inodes_wb(wb, &work);
768
	spin_unlock(&wb->list_lock);
769

770 771
	return nr_pages - work.nr_pages;
}
772

773
static bool over_bground_thresh(struct backing_dev_info *bdi)
774 775 776
{
	unsigned long background_thresh, dirty_thresh;

777
	global_dirty_limits(&background_thresh, &dirty_thresh);
778

779 780 781 782 783 784 785 786 787
	if (global_page_state(NR_FILE_DIRTY) +
	    global_page_state(NR_UNSTABLE_NFS) > background_thresh)
		return true;

	if (bdi_stat(bdi, BDI_RECLAIMABLE) >
				bdi_dirty_limit(bdi, background_thresh))
		return true;

	return false;
788 789
}

790 791 792 793 794 795 796
/*
 * Called under wb->list_lock. If there are multiple wb per bdi,
 * only the flusher working on the first wb should do it.
 */
static void wb_update_bandwidth(struct bdi_writeback *wb,
				unsigned long start_time)
{
797
	__bdi_update_bandwidth(wb->bdi, 0, 0, 0, 0, 0, start_time);
798 799
}

800 801
/*
 * Explicit flushing or periodic writeback of "old" data.
802
 *
803 804 805 806
 * Define "old": the first time one of an inode's pages is dirtied, we mark the
 * dirtying-time in the inode's address_space.  So this periodic writeback code
 * just walks the superblock inode list, writing back any inodes which are
 * older than a specific point in time.
807
 *
808 809 810
 * Try to run once per dirty_writeback_interval.  But if a writeback event
 * takes longer than a dirty_writeback_interval interval, then leave a
 * one-second gap.
811
 *
812 813
 * older_than_this takes precedence over nr_to_write.  So we'll only write back
 * all dirty pages if they are all attached to "old" mappings.
814
 */
815
static long wb_writeback(struct bdi_writeback *wb,
816
			 struct wb_writeback_work *work)
817
{
818
	unsigned long wb_start = jiffies;
819
	long nr_pages = work->nr_pages;
820
	unsigned long oldest_jif;
821
	struct inode *inode;
822
	long progress;
823

824
	oldest_jif = jiffies;
825
	work->older_than_this = &oldest_jif;
Nick Piggin's avatar
Nick Piggin committed
826

827
	spin_lock(&wb->list_lock);
828 829
	for (;;) {
		/*
830
		 * Stop writeback when nr_pages has been consumed
831
		 */
832
		if (work->nr_pages <= 0)
833
			break;
834

835 836 837 838 839 840 841 842 843 844
		/*
		 * Background writeout and kupdate-style writeback may
		 * run forever. Stop them if there is other work to do
		 * so that e.g. sync can proceed. They'll be restarted
		 * after the other works are all done.
		 */
		if ((work->for_background || work->for_kupdate) &&
		    !list_empty(&wb->bdi->work_list))
			break;

Nick Piggin's avatar
Nick Piggin committed
845
		/*
846 847
		 * For background writeout, stop when we are below the
		 * background dirty threshold
Nick Piggin's avatar
Nick Piggin committed
848
		 */
849
		if (work->for_background && !over_bground_thresh(wb->bdi))
850
			break;
Nick Piggin's avatar
Nick Piggin committed
851

852 853 854 855 856 857
		/*
		 * Kupdate and background works are special and we want to
		 * include all inodes that need writing. Livelock avoidance is
		 * handled by these works yielding to any other work so we are
		 * safe.
		 */
858 859 860
		if (work->for_kupdate) {
			oldest_jif = jiffies -
				msecs_to_jiffies(dirty_expire_interval * 10);
861 862
		} else if (work->for_background)
			oldest_jif = jiffies;
863

864
		trace_writeback_start(wb->bdi, work);
865
		if (list_empty(&wb->b_io))
866
			queue_io(wb, work);
867
		if (work->sb)
868
			progress = writeback_sb_inodes(work->sb, wb, work);
869
		else
870 871
			progress = __writeback_inodes_wb(wb, work);
		trace_writeback_written(wb->bdi, work);
872

873
		wb_update_bandwidth(wb, wb_start);
874 875

		/*
876 877 878 879 880 881
		 * Did we write something? Try for more
		 *
		 * Dirty inodes are moved to b_io for writeback in batches.
		 * The completion of the current batch does not necessarily
		 * mean the overall work is done. So we keep looping as long
		 * as made some progress on cleaning pages or inodes.
882
		 */
883
		if (progress)
884 885
			continue;
		/*
886
		 * No more inodes for IO, bail
887
		 */
888
		if (list_empty(&wb->b_more_io))
889
			break;
890 891 892 893 894 895
		/*
		 * Nothing written. Wait for some inode to
		 * become available for writeback. Otherwise
		 * we'll just busyloop.
		 */
		if (!list_empty(&wb->b_more_io))  {
896
			trace_writeback_wait(wb->bdi, work);
897
			inode = wb_inode(wb->b_more_io.prev);
898
			spin_lock(&inode->i_lock);
899
			spin_unlock(&wb->list_lock);
900 901
			/* This function drops i_lock... */
			inode_sleep_on_writeback(inode);
902
			spin_lock(&wb->list_lock);
903 904
		}
	}
905
	spin_unlock(&wb->list_lock);
906

907
	return nr_pages - work->nr_pages;
908 909 910
}

/*
911
 * Return the next wb_writeback_work struct that hasn't been processed yet.
912
 */
913
static struct wb_writeback_work *
914
get_next_work_item(struct backing_dev_info *bdi)
915
{
916
	struct wb_writeback_work *work = NULL;
917

918
	spin_lock_bh(&bdi->wb_lock);
919 920 921 922
	if (!list_empty(&bdi->work_list)) {
		work = list_entry(bdi->work_list.next,
				  struct wb_writeback_work, list);
		list_del_init(&work->list);
923
	}
924
	spin_unlock_bh(&bdi->wb_lock);
925
	return work;
926 927
}

928 929 930 931 932 933 934 935 936 937 938
/*
 * Add in the number of potentially dirty inodes, because each inode
 * write can dirty pagecache in the underlying blockdev.
 */
static unsigned long get_nr_dirty_pages(void)
{
	return global_page_state(NR_FILE_DIRTY) +
		global_page_state(NR_UNSTABLE_NFS) +
		get_nr_dirty_inodes();
}

939 940
static long wb_check_background_flush(struct bdi_writeback *wb)
{
941
	if (over_bground_thresh(wb->bdi)) {
942 943 944 945 946 947

		struct wb_writeback_work work = {
			.nr_pages	= LONG_MAX,
			.sync_mode	= WB_SYNC_NONE,
			.for_background	= 1,
			.range_cyclic	= 1,
948
			.reason		= WB_REASON_BACKGROUND,
949 950 951 952 953 954 955 956
		};

		return wb_writeback(wb, &work);
	}

	return 0;
}

957 958 959 960 961
static long wb_check_old_data_flush(struct bdi_writeback *wb)
{
	unsigned long expired;
	long nr_pages;

962 963 964 965 966 967
	/*
	 * When set to zero, disable periodic writeback
	 */
	if (!dirty_writeback_interval)
		return 0;

968 969 970 971 972 973
	expired = wb->last_old_flush +
			msecs_to_jiffies(dirty_writeback_interval * 10);
	if (time_before(jiffies, expired))
		return 0;

	wb->last_old_flush = jiffies;
974
	nr_pages = get_nr_dirty_pages();
975

976
	if (nr_pages) {
977
		struct wb_writeback_work work = {
978 979 980 981
			.nr_pages	= nr_pages,
			.sync_mode	= WB_SYNC_NONE,
			.for_kupdate	= 1,
			.range_cyclic	= 1,
982
			.reason		= WB_REASON_PERIODIC,
983 984
		};

985
		return wb_writeback(wb, &work);
986
	}
987 988 989 990 991 992 993 994 995 996

	return 0;
}

/*
 * Retrieve work items and do the writeback they describe
 */
long wb_do_writeback(struct bdi_writeback *wb, int force_wait)
{
	struct backing_dev_info *bdi = wb->bdi;
997
	struct wb_writeback_work *work;
998
	long wrote = 0;
999

1000
	set_bit(BDI_writeback_running, &wb->bdi->state);
1001
	while ((work = get_next_work_item(bdi)) != NULL) {
1002 1003
		/*
		 * Override sync mode, in case we must wait for completion
1004
		 * because this thread is exiting now.
1005 1006
		 */
		if (force_wait)
1007
			work->sync_mode = WB_SYNC_ALL;
1008

1009 1010
		trace_writeback_exec(bdi, work);

1011
		wrote += wb_writeback(wb, work);
1012 1013

		/*
1014 1015
		 * Notify the caller of completion if this is a synchronous
		 * work item, otherwise just free it.
1016
		 */
1017 1018 1019 1020
		if (work->done)
			complete(work->done);
		else
			kfree(work);
1021 1022 1023 1024 1025 1026
	}

	/*
	 * Check for periodic writeback, kupdated() style
	 */
	wrote += wb_check_old_data_flush(wb);
1027
	wrote += wb_check_background_flush(wb);
1028
	clear_bit(BDI_writeback_running, &wb->bdi->state);
1029 1030 1031 1032 1033 1034

	return wrote;
}

/*
 * Handle writeback of dirty data for the device backed by this bdi. Also
1035
 * reschedules periodically and does kupdated style flushing.
1036
 */
1037
void bdi_writeback_workfn(struct work_struct *work)
1038
{
1039 1040
	struct bdi_writeback *wb = container_of(to_delayed_work(work),
						struct bdi_writeback, dwork);
1041
	struct backing_dev_info *bdi = wb->bdi;
1042 1043
	long pages_written;

1044
	set_worker_desc("flush-%s", dev_name(bdi->dev));
1045
	current->flags |= PF_SWAPWRITE;
1046

1047
	if (likely(!current_is_workqueue_rescuer() ||
1048
		   !test_bit(BDI_registered, &bdi->state))) {
1049
		/*
1050 1051 1052 1053
		 * The normal path.  Keep writing back @bdi until its
		 * work_list is empty.  Note that this path is also taken
		 * if @bdi is shutting down even when we're running off the
		 * rescuer as work_list needs to be drained.
1054
		 */
1055 1056 1057 1058 1059 1060 1061 1062 1063 1064 1065 1066
		do {
			pages_written = wb_do_writeback(wb, 0);
			trace_writeback_pages_written(pages_written);
		} while (!list_empty(&bdi->work_list));
	} else {
		/*
		 * bdi_wq can't get enough workers and we're running off
		 * the emergency worker.  Don't hog it.  Hopefully, 1024 is
		 * enough for efficient IO.
		 */
		pages_written = writeback_inodes_wb(&bdi->wb, 1024,
						    WB_REASON_FORKER_THREAD);
1067
		trace_writeback_pages_written(pages_written);
1068 1069
	}

1070 1071 1072 1073
	if (!list_empty(&bdi->work_list))
		mod_delayed_work(bdi_wq, &wb->dwork, 0);
	else if (wb_has_dirty_io(wb) && dirty_writeback_interval)
		bdi_wakeup_thread_delayed(bdi);
1074

1075
	current->flags &= ~PF_SWAPWRITE;
1076 1077 1078
}

/*
1079 1080
 * Start writeback of `nr_pages' pages.  If `nr_pages' is zero, write back
 * the whole world.
1081
 */
1082
void wakeup_flusher_threads(long nr_pages, enum wb_reason reason)
1083
{
1084
	struct backing_dev_info *bdi;
1085

1086 1087
	if (!nr_pages) {
		nr_pages = global_page_state(NR_FILE_DIRTY) +
1088 1089
				global_page_state(NR_UNSTABLE_NFS);
	}
1090

1091
	rcu_read_lock();
1092
	list_for_each_entry_rcu(bdi, &bdi_list, bdi_list) {
1093 1094
		if (!bdi_has_dirty_io(bdi))
			continue;
1095
		__bdi_start_writeback(bdi, nr_pages, false, reason);
1096
	}
1097
	rcu_read_unlock();
Linus Torvalds's avatar
Linus Torvalds committed
1098 1099
}

1100 1101 1102 1103 1104 1105 1106 1107 1108 1109 1110 1111 1112 1113 1114 1115 1116 1117 1118 1119 1120 1121 1122 1123 1124 1125 1126 1127
static noinline void block_dump___mark_inode_dirty(struct inode *inode)
{
	if (inode->i_ino || strcmp(inode->i_sb->s_id, "bdev")) {
		struct dentry *dentry;
		const char *name = "?";

		dentry = d_find_alias(inode);
		if (dentry) {
			spin_lock(&dentry->d_lock);
			name = (const char *) dentry->d_name.name;
		}
		printk(KERN_DEBUG
		       "%s(%d): dirtied inode %lu (%s) on %s\n",
		       current->comm, task_pid_nr(current), inode->i_ino,
		       name, inode->i_sb->s_id);
		if (dentry) {
			spin_unlock(&dentry->d_lock);
			dput(dentry);
		}
	}
}

/**
 *	__mark_inode_dirty -	internal function
 *	@inode: inode to mark
 *	@flags: what kind of dirty (i.e. I_DIRTY_SYNC)
 *	Mark an inode as dirty. Callers should use mark_inode_dirty or
 *  	mark_inode_dirty_sync.
Linus Torvalds's avatar
Linus Torvalds committed
1128
 *
1129 1130 1131 1132 1133 1134 1135 1136 1137
 * Put the inode on the super block's dirty list.
 *
 * CAREFUL! We mark it dirty unconditionally, but move it onto the
 * dirty list only if it is hashed or if it refers to a blockdev.
 * If it was not hashed, it will never be added to the dirty list
 * even if it is later hashed, as it will have been marked dirty already.
 *
 * In short, make sure you hash any inodes _before_ you start marking
 * them dirty.
Linus Torvalds's avatar
Linus Torvalds committed
1138
 *
1139 1140 1141 1142 1143 1144
 * Note that for blockdevs, inode->dirtied_when represents the dirtying time of
 * the block-special inode (/dev/hda1) itself.  And the ->dirtied_when field of
 * the kernel-internal blockdev inode represents the dirtying time of the
 * blockdev's pages.  This is why for I_DIRTY_PAGES we always use
 * page->mapping->host, so the page-dirtying time is recorded in the internal
 * blockdev inode.
Linus Torvalds's avatar
Linus Torvalds committed
1145
 */
1146
void __mark_inode_dirty(struct inode *inode, int flags)
Linus Torvalds's avatar
Linus Torvalds committed
1147
{
1148
	struct super_block *sb = inode->i_sb;
1149
	struct backing_dev_info *bdi = NULL;
Linus Torvalds's avatar
Linus Torvalds committed
1150

1151 1152 1153 1154 1155
	/*
	 * Don't do this for I_DIRTY_PAGES - that doesn't actually
	 * dirty the inode itself
	 */
	if (flags & (I_DIRTY_SYNC | I_DIRTY_DATASYNC)) {
1156 1157
		trace_writeback_dirty_inode_start(inode, flags);

1158
		if (sb->s_op->dirty_inode)
1159
			sb->s_op->dirty_inode(inode, flags);
1160 1161

		trace_writeback_dirty_inode(inode, flags);
1162 1163 1164
	}

	/*
1165 1166
	 * Paired with smp_mb() in __writeback_single_inode() for the
	 * following lockless i_state test.  See there for details.
1167 1168 1169 1170 1171 1172
	 */
	smp_mb();

	if ((inode->i_state & flags) == flags)
		return;

Mauro Ribeiro's avatar
Mauro Ribeiro committed
1173
	if (unlikely(block_dump > 1))
1174 1175
		block_dump___mark_inode_dirty(inode);

1176
	spin_lock(&inode->i_lock);
1177 1178 1179 1180 1181 1182 1183 1184 1185 1186 1187
	if ((inode->i_state & flags) != flags) {
		const int was_dirty = inode->i_state & I_DIRTY;

		inode->i_state |= flags;

		/*
		 * If the inode is being synced, just update its dirty state.
		 * The unlocker will place the inode on the appropriate
		 * superblock list, based upon its state.
		 */
		if (inode->i_state & I_SYNC)
1188
			goto out_unlock_inode;
1189 1190 1191 1192 1193 1194

		/*
		 * Only add valid (hashed) inodes to the superblock's
		 * dirty list.  Add blockdev inodes as well.
		 */
		if (!S_ISBLK(inode->i_mode)) {
Al Viro's avatar
Al Viro committed
1195
			if (inode_unhashed(inode))
1196
				goto out_unlock_inode;
1197
		}
1198
		if (inode->i_state & I_FREEING)
1199
			goto out_unlock_inode;
1200 1201 1202 1203 1204 1205

		/*
		 * If the inode was already on b_dirty/b_io/b_more_io, don't
		 * reposition it (that would break b_dirty time-ordering).
		 */
		if (!was_dirty) {
1206
			bool wakeup_bdi = false;
1207 1208 1209 1210 1211 1212 1213 1214 1215 1216 1217 1218 1219 1220
			bdi = inode_to_bdi(inode);

			if (bdi_cap_writeback_dirty(bdi)) {
				WARN(!test_bit(BDI_registered, &bdi->state),
				     "bdi-%s not registered\n", bdi->name);

				/*
				 * If this is the first dirty inode for this
				 * bdi, we have to wake-up the corresponding
				 * bdi thread to make sure background
				 * write-back happens later.
				 */
				if (!wb_has_dirty_io(&bdi->wb))
					wakeup_bdi = true;
1221
			}
1222

1223
			spin_unlock(&inode->i_lock);
1224
			spin_lock(&bdi->wb.list_lock);
1225
			inode->dirtied_when = jiffies;
1226
			list_move(&inode->i_wb_list, &bdi->wb.b_dirty);
1227
			spin_unlock(&bdi->wb.list_lock);
1228 1229 1230 1231

			if (wakeup_bdi)
				bdi_wakeup_thread_delayed(bdi);
			return;
Linus Torvalds's avatar
Linus Torvalds committed
1232 1233
		}
	}
1234 1235
out_unlock_inode:
	spin_unlock(&inode->i_lock);
1236

1237 1238 1239
}
EXPORT_SYMBOL(__mark_inode_dirty);

1240
static void wait_sb_inodes(struct super_block *sb)
1241 1242 1243 1244 1245 1246 1247
{
	struct inode *inode, *old_inode = NULL;

	/*
	 * We need to be protected against the filesystem going from
	 * r/o to r/w or vice versa.
	 */
1248
	WARN_ON(!rwsem_is_locked(&sb->s_umount));
1249

1250
	spin_lock(&inode_sb_list_lock);
1251 1252 1253 1254 1255 1256 1257 1258

	/*
	 * Data integrity sync. Must wait for all pages under writeback,
	 * because there may have been pages dirtied before our sync
	 * call, but which had writeout started before we write it out.
	 * In which case, the inode may not be on the dirty list, but
	 * we still have to wait for that writeout.
	 */
1259
	list_for_each_entry(inode, &sb->s_inodes, i_sb_list) {
1260
		struct address_space *mapping = inode->i_mapping;
1261

1262 1263 1264 1265
		spin_lock(&inode->i_lock);
		if ((inode->i_state & (I_FREEING|I_WILL_FREE|I_NEW)) ||
		    (mapping->nrpages == 0)) {
			spin_unlock(&inode->i_lock);
1266
			continue;
1267
		}
1268
		__iget(inode);
1269
		spin_unlock(&inode->i_lock);
1270 1271
		spin_unlock(&inode_sb_list_lock);

1272
		/*
1273 1274