summaryrefslogtreecommitdiffstats
path: root/mm/percpu.c
diff options
context:
space:
mode:
authorDennis Zhou (Facebook) <dennisszhou@gmail.com>2017-07-24 19:02:13 -0400
committerTejun Heo <tj@kernel.org>2017-07-26 17:41:05 -0400
commit86b442fbce74d6cd0805410ef228776cbd0338d7 (patch)
tree1d1fb3282d54d73cc47c0ba7f8a09a9050db23ae /mm/percpu.c
parentca460b3c96274d79f84b31a3fea23a6eed479917 (diff)
downloadlinux-86b442fbce74d6cd0805410ef228776cbd0338d7.tar.bz2
percpu: add first_bit to keep track of the first free in the bitmap
This patch adds first_bit to keep track of the first free bit in the bitmap. This hint helps prevent scanning of fully allocated blocks. Signed-off-by: Dennis Zhou <dennisszhou@gmail.com> Reviewed-by: Josef Bacik <jbacik@fb.com> Signed-off-by: Tejun Heo <tj@kernel.org>
Diffstat (limited to 'mm/percpu.c')
-rw-r--r--mm/percpu.c17
1 files changed, 15 insertions, 2 deletions
diff --git a/mm/percpu.c b/mm/percpu.c
index 708c6de237b9..83abb190ca5a 100644
--- a/mm/percpu.c
+++ b/mm/percpu.c
@@ -427,7 +427,7 @@ static void pcpu_chunk_refresh_hint(struct pcpu_chunk *chunk)
chunk->contig_bits = 0;
bits = nr_empty_pop_pages = 0;
- pcpu_for_each_unpop_region(chunk->alloc_map, rs, re, 0,
+ pcpu_for_each_unpop_region(chunk->alloc_map, rs, re, chunk->first_bit,
pcpu_chunk_map_bits(chunk)) {
bits = re - rs;
@@ -646,7 +646,8 @@ static int pcpu_find_block_fit(struct pcpu_chunk *chunk, int alloc_bits,
int bit_off, bits;
int re; /* region end */
- pcpu_for_each_unpop_region(chunk->alloc_map, bit_off, re, 0,
+ pcpu_for_each_unpop_region(chunk->alloc_map, bit_off, re,
+ chunk->first_bit,
pcpu_chunk_map_bits(chunk)) {
bits = re - bit_off;
@@ -715,6 +716,13 @@ static int pcpu_alloc_area(struct pcpu_chunk *chunk, int alloc_bits,
chunk->free_bytes -= alloc_bits * PCPU_MIN_ALLOC_SIZE;
+ /* update first free bit */
+ if (bit_off == chunk->first_bit)
+ chunk->first_bit = find_next_zero_bit(
+ chunk->alloc_map,
+ pcpu_chunk_map_bits(chunk),
+ bit_off + alloc_bits);
+
pcpu_block_update_hint_alloc(chunk, bit_off, alloc_bits);
pcpu_chunk_relocate(chunk, oslot);
@@ -750,6 +758,9 @@ static void pcpu_free_area(struct pcpu_chunk *chunk, int off)
/* update metadata */
chunk->free_bytes += bits * PCPU_MIN_ALLOC_SIZE;
+ /* update first free bit */
+ chunk->first_bit = min(chunk->first_bit, bit_off);
+
pcpu_block_update_hint_free(chunk, bit_off, bits);
pcpu_chunk_relocate(chunk, oslot);
@@ -841,6 +852,8 @@ static struct pcpu_chunk * __init pcpu_alloc_first_chunk(unsigned long tmp_addr,
set_bit(0, chunk->bound_map);
set_bit(offset_bits, chunk->bound_map);
+ chunk->first_bit = offset_bits;
+
pcpu_block_update_hint_alloc(chunk, 0, offset_bits);
}