diff options
Diffstat (limited to 'drivers/md')
| -rw-r--r-- | drivers/md/dm-core.h | 22 | ||||
| -rw-r--r-- | drivers/md/dm.c | 30 |
2 files changed, 38 insertions, 14 deletions
diff --git a/drivers/md/dm-core.h b/drivers/md/dm-core.h index 8d3d11887343..e127cbcaf33d 100644 --- a/drivers/md/dm-core.h +++ b/drivers/md/dm-core.h @@ -232,18 +232,36 @@ struct dm_io { struct mapped_device *md; struct bio *orig_bio; blk_status_t status; - bool start_io_acct:1; - int was_accounted; + unsigned short flags; unsigned long start_time; void *data; struct hlist_node node; struct task_struct *map_task; + spinlock_t startio_lock; spinlock_t endio_lock; struct dm_stats_aux stats_aux; /* last member of dm_target_io is 'struct bio' */ struct dm_target_io tio; }; +/* + * dm_io flags + */ +enum { + DM_IO_START_ACCT, + DM_IO_ACCOUNTED +}; + +static inline bool dm_io_flagged(struct dm_io *io, unsigned int bit) +{ + return (io->flags & (1U << bit)) != 0; +} + +static inline void dm_io_set_flag(struct dm_io *io, unsigned int bit) +{ + io->flags |= (1U << bit); +} + static inline void dm_io_inc_pending(struct dm_io *io) { atomic_inc(&io->io_count); diff --git a/drivers/md/dm.c b/drivers/md/dm.c index df7664f3028c..83328f03bcb2 100644 --- a/drivers/md/dm.c +++ b/drivers/md/dm.c @@ -541,11 +541,18 @@ static void dm_start_io_acct(struct dm_io *io, struct bio *clone) * Expect no possibility for race unless is_duplicate_bio. */ if (!clone || likely(!clone_to_tio(clone)->is_duplicate_bio)) { - if (WARN_ON_ONCE(io->was_accounted)) + if (WARN_ON_ONCE(dm_io_flagged(io, DM_IO_ACCOUNTED))) return; - io->was_accounted = 1; - } else if (xchg(&io->was_accounted, 1) == 1) - return; + dm_io_set_flag(io, DM_IO_ACCOUNTED); + } else { + unsigned long flags; + if (dm_io_flagged(io, DM_IO_ACCOUNTED)) + return; + /* Can afford locking given is_duplicate_bio */ + spin_lock_irqsave(&io->startio_lock, flags); + dm_io_set_flag(io, DM_IO_ACCOUNTED); + spin_unlock_irqrestore(&io->startio_lock, flags); + } __dm_start_io_acct(io, bio); } @@ -575,11 +582,10 @@ static struct dm_io *alloc_io(struct mapped_device *md, struct bio *bio) io->orig_bio = NULL; io->md = md; io->map_task = current; + spin_lock_init(&io->startio_lock); spin_lock_init(&io->endio_lock); - io->start_time = jiffies; - io->start_io_acct = false; - io->was_accounted = 0; + io->flags = 0; dm_stats_record_start(&md->stats, &io->stats_aux); @@ -868,7 +874,7 @@ static void dm_io_complete(struct dm_io *io) } io_error = io->status; - if (io->was_accounted) + if (dm_io_flagged(io, DM_IO_ACCOUNTED)) dm_end_io_acct(io, bio); else if (!io_error) { /* @@ -1218,7 +1224,7 @@ void dm_submit_bio_remap(struct bio *clone, struct bio *tgt_clone) */ if (io->map_task == current) { /* Still in target's map function */ - io->start_io_acct = true; + dm_io_set_flag(io, DM_IO_START_ACCT); } else { /* * Called by another thread, managed by DM target, @@ -1288,7 +1294,7 @@ static void __map_bio(struct bio *clone) case DM_MAPIO_SUBMITTED: /* target has assumed ownership of this io */ if (!ti->accounts_remapped_io) - io->start_io_acct = true; + dm_io_set_flag(io, DM_IO_START_ACCT); break; case DM_MAPIO_REMAPPED: /* @@ -1297,7 +1303,7 @@ static void __map_bio(struct bio *clone) */ __dm_submit_bio_remap(clone, disk_devt(io->md->disk), tio->old_sector); - io->start_io_acct = true; + dm_io_set_flag(io, DM_IO_START_ACCT); break; case DM_MAPIO_KILL: case DM_MAPIO_REQUEUE: @@ -1591,7 +1597,7 @@ out: if (!orig_bio) orig_bio = bio; smp_store_release(&ci.io->orig_bio, orig_bio); - if (ci.io->start_io_acct) + if (dm_io_flagged(ci.io, DM_IO_START_ACCT)) dm_start_io_acct(ci.io, NULL); /* |