summaryrefslogtreecommitdiffstats
path: root/drivers/md
diff options
context:
space:
mode:
Diffstat (limited to 'drivers/md')
-rw-r--r--drivers/md/dm-core.h22
-rw-r--r--drivers/md/dm.c30
2 files changed, 38 insertions, 14 deletions
diff --git a/drivers/md/dm-core.h b/drivers/md/dm-core.h
index 8d3d11887343..e127cbcaf33d 100644
--- a/drivers/md/dm-core.h
+++ b/drivers/md/dm-core.h
@@ -232,18 +232,36 @@ struct dm_io {
struct mapped_device *md;
struct bio *orig_bio;
blk_status_t status;
- bool start_io_acct:1;
- int was_accounted;
+ unsigned short flags;
unsigned long start_time;
void *data;
struct hlist_node node;
struct task_struct *map_task;
+ spinlock_t startio_lock;
spinlock_t endio_lock;
struct dm_stats_aux stats_aux;
/* last member of dm_target_io is 'struct bio' */
struct dm_target_io tio;
};
+/*
+ * dm_io flags
+ */
+enum {
+ DM_IO_START_ACCT,
+ DM_IO_ACCOUNTED
+};
+
+static inline bool dm_io_flagged(struct dm_io *io, unsigned int bit)
+{
+ return (io->flags & (1U << bit)) != 0;
+}
+
+static inline void dm_io_set_flag(struct dm_io *io, unsigned int bit)
+{
+ io->flags |= (1U << bit);
+}
+
static inline void dm_io_inc_pending(struct dm_io *io)
{
atomic_inc(&io->io_count);
diff --git a/drivers/md/dm.c b/drivers/md/dm.c
index df7664f3028c..83328f03bcb2 100644
--- a/drivers/md/dm.c
+++ b/drivers/md/dm.c
@@ -541,11 +541,18 @@ static void dm_start_io_acct(struct dm_io *io, struct bio *clone)
* Expect no possibility for race unless is_duplicate_bio.
*/
if (!clone || likely(!clone_to_tio(clone)->is_duplicate_bio)) {
- if (WARN_ON_ONCE(io->was_accounted))
+ if (WARN_ON_ONCE(dm_io_flagged(io, DM_IO_ACCOUNTED)))
return;
- io->was_accounted = 1;
- } else if (xchg(&io->was_accounted, 1) == 1)
- return;
+ dm_io_set_flag(io, DM_IO_ACCOUNTED);
+ } else {
+ unsigned long flags;
+ if (dm_io_flagged(io, DM_IO_ACCOUNTED))
+ return;
+ /* Can afford locking given is_duplicate_bio */
+ spin_lock_irqsave(&io->startio_lock, flags);
+ dm_io_set_flag(io, DM_IO_ACCOUNTED);
+ spin_unlock_irqrestore(&io->startio_lock, flags);
+ }
__dm_start_io_acct(io, bio);
}
@@ -575,11 +582,10 @@ static struct dm_io *alloc_io(struct mapped_device *md, struct bio *bio)
io->orig_bio = NULL;
io->md = md;
io->map_task = current;
+ spin_lock_init(&io->startio_lock);
spin_lock_init(&io->endio_lock);
-
io->start_time = jiffies;
- io->start_io_acct = false;
- io->was_accounted = 0;
+ io->flags = 0;
dm_stats_record_start(&md->stats, &io->stats_aux);
@@ -868,7 +874,7 @@ static void dm_io_complete(struct dm_io *io)
}
io_error = io->status;
- if (io->was_accounted)
+ if (dm_io_flagged(io, DM_IO_ACCOUNTED))
dm_end_io_acct(io, bio);
else if (!io_error) {
/*
@@ -1218,7 +1224,7 @@ void dm_submit_bio_remap(struct bio *clone, struct bio *tgt_clone)
*/
if (io->map_task == current) {
/* Still in target's map function */
- io->start_io_acct = true;
+ dm_io_set_flag(io, DM_IO_START_ACCT);
} else {
/*
* Called by another thread, managed by DM target,
@@ -1288,7 +1294,7 @@ static void __map_bio(struct bio *clone)
case DM_MAPIO_SUBMITTED:
/* target has assumed ownership of this io */
if (!ti->accounts_remapped_io)
- io->start_io_acct = true;
+ dm_io_set_flag(io, DM_IO_START_ACCT);
break;
case DM_MAPIO_REMAPPED:
/*
@@ -1297,7 +1303,7 @@ static void __map_bio(struct bio *clone)
*/
__dm_submit_bio_remap(clone, disk_devt(io->md->disk),
tio->old_sector);
- io->start_io_acct = true;
+ dm_io_set_flag(io, DM_IO_START_ACCT);
break;
case DM_MAPIO_KILL:
case DM_MAPIO_REQUEUE:
@@ -1591,7 +1597,7 @@ out:
if (!orig_bio)
orig_bio = bio;
smp_store_release(&ci.io->orig_bio, orig_bio);
- if (ci.io->start_io_acct)
+ if (dm_io_flagged(ci.io, DM_IO_START_ACCT))
dm_start_io_acct(ci.io, NULL);
/*