Linux 软raid - - Barrier

什么是Barriers

在linux软raid中，用来处理正常IO和同步IO的并发问题，可以简单理解为专用于软raid的锁。

软raid在做resync/recovery，或者配置操作时需要raise 屏障，于此同时必须暂停正常IO。
barrier是可以被多次raise的一个计数器，来计算有多少个相关活动事件在发生，其中不包括正常IO。

raise 屏障的条件是没有pending的IO即nr_pending=0。

只有在没有人等待barrier down的情况下，才会选择raise barrier。这意味着，一旦IO请求准备就绪，在IO请求有机会之前，不会启动其他需要屏障的操作。

常规IO调用“wait_barrier”。当返回时，没有后台组IO发生，它必须安排在完成IO后调用allow_barrier。

后台组IO调用必须调用raise_barrier。一旦返回，就没有正常的IO发生。它必须安排在特定后台IO完成时调用lower_barrier。

/* Barriers....* Sometimes we need to suspend IO while we do something else,* either some resync/recovery, or reconfigure the array.* To do this we raise a 'barrier'.* The 'barrier' is a counter that can be raised multiple times* to count how many activities are happening which preclude* normal IO.* We can only raise the barrier if there is no pending IO.* i.e. if nr_pending == 0.* We choose only to raise the barrier if no-one is waiting for the* barrier to go down.  This means that as soon as an IO request* is ready, no other operations which require a barrier will start* until the IO request has had a chance.** So: regular IO calls 'wait_barrier'.  When that returns there*    is no backgroup IO happening,  It must arrange to call*    allow_barrier when it has finished its IO.* backgroup IO calls must call raise_barrier.  Once that returns*    there is no normal IO happeing.  It must arrange to call*    lower_barrier when the particular background IO completes.*/

相关数据结构

用来描述软raid配置相关的所有信息。

struct r1conf {struct mddev		*mddev;struct raid1_info	*mirrors;	/* twice 'raid_disks' to* allow for replacements.*/int			raid_disks;spinlock_t		device_lock;/* list of 'struct r1bio' that need to be processed by raid1d,* whether to retry a read, writeout a resync or recovery* block, or anything else.*/struct list_head	retry_list;/* A separate list of r1bio which just need raid_end_bio_io called.* This mustn't happen for writes which had any errors if the superblock* needs to be written.*/struct list_head	bio_end_io_list;/* queue pending writes to be submitted on unplug */struct bio_list		pending_bio_list;int			pending_count;/* for use when syncing mirrors:* We don't allow both normal IO and resync/recovery IO at* the same time - resync/recovery can only happen when there* is no other IO.  So when either is active, the other has to wait.* See more details description in raid1.c near raise_barrier().*/wait_queue_head_t	wait_barrier;spinlock_t		resync_lock;atomic_t		nr_sync_pending;atomic_t		*nr_pending;atomic_t		*nr_waiting;atomic_t		*nr_queued;atomic_t		*barrier;int			array_frozen;/* Set to 1 if a full sync is needed, (fresh device added).* Cleared when a sync completes.*/int			fullsync;/* When the same as mddev->recovery_disabled we don't allow* recovery to be attempted as we expect a read error.*/int			recovery_disabled;/* poolinfo contains information about the content of the* mempools - it changes when the array grows or shrinks*/struct pool_info	*poolinfo;mempool_t		r1bio_pool;mempool_t		r1buf_pool;struct bio_set		bio_split;/* temporary buffer to synchronous IO when attempting to repair* a read error.*/struct page		*tmppage;/* When taking over an array from a different personality, we store* the new thread here until we fully activate the array.*/struct md_thread	*thread;/* Keep track of cluster resync window to send to other* nodes.*/sector_t		cluster_sync_low;sector_t		cluster_sync_high;};

在当前的例子中，我们需要关注3个成员。

• nr_pending
  正在处理的正常IO
• nr_waitting
  等待同步完成的正常IO
• barrier
  正在处理的同步IO

相关内核函数

raise_barrier

raise_barrier只有在同步IO的场景下raid1_sync_request才会被调用，这就意味着，只有等待正常IO完成之后，才能把屏障加起来。

static sector_t raise_barrier(struct r1conf *conf, sector_t sector_nr)
{int idx = sector_to_idx(sector_nr);	// 获取在bucket中的index。spin_lock_irq(&conf->resync_lock);/* Wait until no block IO is waiting */wait_event_lock_irq(conf->wait_barrier,!atomic_read(&conf->nr_waiting[idx]),conf->resync_lock);/* block any new IO from starting */atomic_inc(&conf->barrier[idx]);/** In raise_barrier() we firstly increase conf->barrier[idx] then* check conf->nr_pending[idx]. In _wait_barrier() we firstly* increase conf->nr_pending[idx] then check conf->barrier[idx].* A memory barrier here to make sure conf->nr_pending[idx] won't* be fetched before conf->barrier[idx] is increased. Otherwise* there will be a race between raise_barrier() and _wait_barrier().*/smp_mb__after_atomic();	// 内存屏障。/* For these conditions we must wait:* A: while the array is in frozen state* B: while conf->nr_pending[idx] is not 0, meaning regular I/O*    existing in corresponding I/O barrier bucket.* C: while conf->barrier[idx] >= RESYNC_DEPTH, meaning reaches*    max resync count which allowed on current I/O barrier bucket.*/wait_event_lock_irq(conf->wait_barrier,(!conf->array_frozen &&!atomic_read(&conf->nr_pending[idx]) &&atomic_read(&conf->barrier[idx]) < RESYNC_DEPTH) ||test_bit(MD_RECOVERY_INTR, &conf->mddev->recovery),conf->resync_lock);if (test_bit(MD_RECOVERY_INTR, &conf->mddev->recovery)) {atomic_dec(&conf->barrier[idx]);spin_unlock_irq(&conf->resync_lock);wake_up(&conf->wait_barrier);return -EINTR;}atomic_inc(&conf->nr_sync_pending);spin_unlock_irq(&conf->resync_lock);return 0;
}

wait_barrier

wait_barrier只有在向下发写请求raid1_write_request时被调用，如果此时对应的磁盘扇区存在barrier，nr_waiting会被添加，表示同一时刻，同一扇区存在同步IO。

static void _wait_barrier(struct r1conf *conf, int idx)
{/** We need to increase conf->nr_pending[idx] very early here,* then raise_barrier() can be blocked when it waits for* conf->nr_pending[idx] to be 0. Then we can avoid holding* conf->resync_lock when there is no barrier raised in same* barrier unit bucket. Also if the array is frozen, I/O* should be blocked until array is unfrozen.*/atomic_inc(&conf->nr_pending[idx]);/** In _wait_barrier() we firstly increase conf->nr_pending[idx], then* check conf->barrier[idx]. In raise_barrier() we firstly increase* conf->barrier[idx], then check conf->nr_pending[idx]. A memory* barrier is necessary here to make sure conf->barrier[idx] won't be* fetched before conf->nr_pending[idx] is increased. Otherwise there* will be a race between _wait_barrier() and raise_barrier().*/smp_mb__after_atomic();/** Don't worry about checking two atomic_t variables at same time* here. If during we check conf->barrier[idx], the array is* frozen (conf->array_frozen is 1), and chonf->barrier[idx] is* 0, it is safe to return and make the I/O continue. Because the* array is frozen, all I/O returned here will eventually complete* or be queued, no race will happen. See code comment in* frozen_array().*/if (!READ_ONCE(conf->array_frozen) &&!atomic_read(&conf->barrier[idx]))return;/** After holding conf->resync_lock, conf->nr_pending[idx]* should be decreased before waiting for barrier to drop.* Otherwise, we may encounter a race condition because* raise_barrer() might be waiting for conf->nr_pending[idx]* to be 0 at same time.*/spin_lock_irq(&conf->resync_lock);atomic_inc(&conf->nr_waiting[idx]);atomic_dec(&conf->nr_pending[idx]);/** In case freeze_array() is waiting for* get_unqueued_pending() == extra*/wake_up(&conf->wait_barrier);/* Wait for the barrier in same barrier unit bucket to drop. */wait_event_lock_irq(conf->wait_barrier,!conf->array_frozen &&!atomic_read(&conf->barrier[idx]),conf->resync_lock);atomic_inc(&conf->nr_pending[idx]);atomic_dec(&conf->nr_waiting[idx]);spin_unlock_irq(&conf->resync_lock);
}static void wait_barrier(struct r1conf *conf, sector_t sector_nr)
{int idx = sector_to_idx(sector_nr);_wait_barrier(conf, idx);
}

wait_read_barrier

wait_read_barrier只有在下发IO读请求时被调用raid1_write_request，读请求入口将对应的bio状态置为pending状态，如果raid处于非frozen状态，直接返回。

static void wait_read_barrier(struct r1conf *conf, sector_t sector_nr)
{int idx = sector_to_idx(sector_nr);/** Very similar to _wait_barrier(). The difference is, for read* I/O we don't need wait for sync I/O, but if the whole array* is frozen, the read I/O still has to wait until the array is* unfrozen. Since there is no ordering requirement with* conf->barrier[idx] here, memory barrier is unnecessary as well.*/atomic_inc(&conf->nr_pending[idx]);if (!READ_ONCE(conf->array_frozen))return;spin_lock_irq(&conf->resync_lock);atomic_inc(&conf->nr_waiting[idx]);atomic_dec(&conf->nr_pending[idx]);/** In case freeze_array() is waiting for* get_unqueued_pending() == extra*/wake_up(&conf->wait_barrier);/* Wait for array to be unfrozen */wait_event_lock_irq(conf->wait_barrier,!conf->array_frozen,conf->resync_lock);atomic_inc(&conf->nr_pending[idx]);atomic_dec(&conf->nr_waiting[idx]);spin_unlock_irq(&conf->resync_lock);
}

lower_barrier

static void lower_barrier(struct r1conf *conf, sector_t sector_nr)
{int idx = sector_to_idx(sector_nr);BUG_ON(atomic_read(&conf->barrier[idx]) <= 0);atomic_dec(&conf->barrier[idx]);atomic_dec(&conf->nr_sync_pending);wake_up(&conf->wait_barrier);
}