From e254de6bcf3f5b6e78a92ac95fb91acef8adfe1a Mon Sep 17 00:00:00 2001 From: Shaohua Li Date: Wed, 29 Aug 2018 11:05:42 -0700 Subject: md/raid5-cache: disable reshape completely We don't support reshape yet if an array supports log device. Previously we determine the fact by checking ->log. However, ->log could be NULL after a log device is removed, but the array is still marked to support log device. Don't allow reshape in this case too. User can disable log device support by setting 'consistency_policy' to 'resync' then do reshape. Reported-by: Xiao Ni Tested-by: Xiao Ni Signed-off-by: Shaohua Li --- drivers/md/raid5-log.h | 5 +++++ drivers/md/raid5.c | 6 +++--- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/drivers/md/raid5-log.h b/drivers/md/raid5-log.h index a001808a2b77..bfb811407061 100644 --- a/drivers/md/raid5-log.h +++ b/drivers/md/raid5-log.h @@ -46,6 +46,11 @@ extern int ppl_modify_log(struct r5conf *conf, struct md_rdev *rdev, bool add); extern void ppl_quiesce(struct r5conf *conf, int quiesce); extern int ppl_handle_flush_request(struct r5l_log *log, struct bio *bio); +static inline bool raid5_has_log(struct r5conf *conf) +{ + return test_bit(MD_HAS_JOURNAL, &conf->mddev->flags); +} + static inline bool raid5_has_ppl(struct r5conf *conf) { return test_bit(MD_HAS_PPL, &conf->mddev->flags); diff --git a/drivers/md/raid5.c b/drivers/md/raid5.c index 4ce0d7502fad..e4e98f47865d 100644 --- a/drivers/md/raid5.c +++ b/drivers/md/raid5.c @@ -733,7 +733,7 @@ static bool stripe_can_batch(struct stripe_head *sh) { struct r5conf *conf = sh->raid_conf; - if (conf->log || raid5_has_ppl(conf)) + if (raid5_has_log(conf) || raid5_has_ppl(conf)) return false; return test_bit(STRIPE_BATCH_READY, &sh->state) && !test_bit(STRIPE_BITMAP_PENDING, &sh->state) && @@ -7737,7 +7737,7 @@ static int raid5_resize(struct mddev *mddev, sector_t sectors) sector_t newsize; struct r5conf *conf = mddev->private; - if (conf->log || raid5_has_ppl(conf)) + if (raid5_has_log(conf) || raid5_has_ppl(conf)) return -EINVAL; sectors &= ~((sector_t)conf->chunk_sectors - 1); newsize = raid5_size(mddev, sectors, mddev->raid_disks); @@ -7788,7 +7788,7 @@ static int check_reshape(struct mddev *mddev) { struct r5conf *conf = mddev->private; - if (conf->log || raid5_has_ppl(conf)) + if (raid5_has_log(conf) || raid5_has_ppl(conf)) return -EINVAL; if (mddev->delta_disks == 0 && mddev->new_layout == mddev->layout && -- cgit v1.2.3 From 1d0ffd264204eba1861865560f1f7f7a92919384 Mon Sep 17 00:00:00 2001 From: Xiao Ni Date: Thu, 30 Aug 2018 15:57:09 +0800 Subject: RAID10 BUG_ON in raise_barrier when force is true and conf->barrier is 0 In raid10 reshape_request it gets max_sectors in read_balance. If the underlayer disks have bad blocks, the max_sectors is less than last. It will call goto read_more many times. It calls raise_barrier(conf, sectors_done != 0) every time. In this condition sectors_done is not 0. So the value passed to the argument force of raise_barrier is true. In raise_barrier it checks conf->barrier when force is true. If force is true and conf->barrier is 0, it panic. In this case reshape_request submits bio to under layer disks. And in the callback function of the bio it calls lower_barrier. If the bio finishes before calling raise_barrier again, it can trigger the BUG_ON. Add one pair of raise_barrier/lower_barrier to fix this bug. Signed-off-by: Xiao Ni Suggested-by: Neil Brown Signed-off-by: Shaohua Li --- drivers/md/raid10.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/drivers/md/raid10.c b/drivers/md/raid10.c index 981898049491..d6f7978b4449 100644 --- a/drivers/md/raid10.c +++ b/drivers/md/raid10.c @@ -4529,11 +4529,12 @@ static sector_t reshape_request(struct mddev *mddev, sector_t sector_nr, allow_barrier(conf); } + raise_barrier(conf, 0); read_more: /* Now schedule reads for blocks from sector_nr to last */ r10_bio = raid10_alloc_init_r10buf(conf); r10_bio->state = 0; - raise_barrier(conf, sectors_done != 0); + raise_barrier(conf, 1); atomic_set(&r10_bio->remaining, 0); r10_bio->mddev = mddev; r10_bio->sector = sector_nr; @@ -4629,6 +4630,8 @@ read_more: if (sector_nr <= last) goto read_more; + lower_barrier(conf); + /* Now that we have done the whole section we can * update reshape_progress */ -- cgit v1.2.3 From 41a95041126522a921fb73df22cbdd520dfdebad Mon Sep 17 00:00:00 2001 From: Guoqing Jiang Date: Fri, 31 Aug 2018 10:05:57 +0800 Subject: md-cluster: release RESYNC lock after the last resync message All the RESYNC messages are sent with resync lock held, the only exception is resync_finish which releases resync_lockres before send the last resync message, this should be changed as well. Otherwise, we can see deadlock issue as follows: clustermd2-gqjiang2:~ # cat /proc/mdstat Personalities : [raid10] [raid1] md0 : active raid1 sdg[0] sdf[1] 134144 blocks super 1.2 [2/2] [UU] [===================>.] resync = 99.6% (134144/134144) finish=0.0min speed=26K/sec bitmap: 1/1 pages [4KB], 65536KB chunk unused devices: clustermd2-gqjiang2:~ # ps aux|grep md|grep D root 20497 0.0 0.0 0 0 ? D 16:00 0:00 [md0_raid1] clustermd2-gqjiang2:~ # cat /proc/20497/stack [] dlm_lock_sync+0x8e/0xc0 [md_cluster] [] __sendmsg+0x98/0x130 [md_cluster] [] sendmsg+0x20/0x30 [md_cluster] [] resync_info_update+0xb5/0xc0 [md_cluster] [] md_reap_sync_thread+0x134/0x170 [md_mod] [] md_check_recovery+0x28c/0x510 [md_mod] [] raid1d+0x42/0x800 [raid1] [] md_thread+0x121/0x150 [md_mod] [] kthread+0xff/0x140 [] ret_from_fork+0x35/0x40 [] 0xffffffffffffffff clustermd-gqjiang1:~ # ps aux|grep md|grep D root 20531 0.0 0.0 0 0 ? D 16:00 0:00 [md0_raid1] root 20537 0.0 0.0 0 0 ? D 16:00 0:00 [md0_cluster_rec] root 20676 0.0 0.0 0 0 ? D 16:01 0:00 [md0_resync] clustermd-gqjiang1:~ # cat /proc/mdstat Personalities : [raid10] [raid1] md0 : active raid1 sdf[1] sdg[0] 134144 blocks super 1.2 [2/2] [UU] [===================>.] resync = 97.3% (131072/134144) finish=8076.8min speed=0K/sec bitmap: 1/1 pages [4KB], 65536KB chunk unused devices: clustermd-gqjiang1:~ # cat /proc/20531/stack [] metadata_update_start+0xcd/0xd0 [md_cluster] [] md_update_sb.part.61+0x97/0x820 [md_mod] [] md_check_recovery+0x29b/0x510 [md_mod] [] raid1d+0x42/0x800 [raid1] [] md_thread+0x121/0x150 [md_mod] [] kthread+0xff/0x140 [] ret_from_fork+0x35/0x40 [] 0xffffffffffffffff clustermd-gqjiang1:~ # cat /proc/20537/stack [] freeze_array+0xf2/0x140 [raid1] [] recv_daemon+0x41e/0x580 [md_cluster] [] md_thread+0x121/0x150 [md_mod] [] kthread+0xff/0x140 [] ret_from_fork+0x35/0x40 [] 0xffffffffffffffff clustermd-gqjiang1:~ # cat /proc/20676/stack [] dlm_lock_sync+0x8e/0xc0 [md_cluster] [] lock_token+0x2f/0xa0 [md_cluster] [] lock_comm+0x32/0x90 [md_cluster] [] sendmsg+0x15/0x30 [md_cluster] [] resync_info_update+0x8a/0xc0 [md_cluster] [] raid1_sync_request+0xa9a/0xb10 [raid1] [] md_do_sync+0xbaa/0xf90 [md_mod] [] md_thread+0x121/0x150 [md_mod] [] kthread+0xff/0x140 [] ret_from_fork+0x35/0x40 [] 0xffffffffffffffff Reviewed-by: NeilBrown Signed-off-by: Guoqing Jiang Signed-off-by: Shaohua Li --- drivers/md/md-cluster.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/drivers/md/md-cluster.c b/drivers/md/md-cluster.c index 94329e03001e..0b2af6e74fc3 100644 --- a/drivers/md/md-cluster.c +++ b/drivers/md/md-cluster.c @@ -1276,18 +1276,18 @@ static int resync_info_update(struct mddev *mddev, sector_t lo, sector_t hi) static int resync_finish(struct mddev *mddev) { struct md_cluster_info *cinfo = mddev->cluster_info; + int ret = 0; clear_bit(MD_RESYNCING_REMOTE, &mddev->recovery); - dlm_unlock_sync(cinfo->resync_lockres); /* * If resync thread is interrupted so we can't say resync is finished, * another node will launch resync thread to continue. */ - if (test_bit(MD_CLOSING, &mddev->flags)) - return 0; - else - return resync_info_update(mddev, 0, 0); + if (!test_bit(MD_CLOSING, &mddev->flags)) + ret = resync_info_update(mddev, 0, 0); + dlm_unlock_sync(cinfo->resync_lockres); + return ret; } static int area_resyncing(struct mddev *mddev, int direction, -- cgit v1.2.3