summaryrefslogtreecommitdiffstats
path: root/mm/migrate.c
diff options
context:
space:
mode:
Diffstat (limited to 'mm/migrate.c')
-rw-r--r--mm/migrate.c57
1 files changed, 38 insertions, 19 deletions
diff --git a/mm/migrate.c b/mm/migrate.c
index e581253ef330..618aeb5e9cde 100644
--- a/mm/migrate.c
+++ b/mm/migrate.c
@@ -36,6 +36,7 @@
#include <linux/hugetlb.h>
#include <linux/hugetlb_cgroup.h>
#include <linux/gfp.h>
+#include <linux/pfn_t.h>
#include <linux/memremap.h>
#include <linux/userfaultfd_k.h>
#include <linux/balloon_compaction.h>
@@ -239,10 +240,14 @@ static bool remove_migration_pte(struct page *page, struct vm_area_struct *vma,
if (is_write_migration_entry(entry))
pte = maybe_mkwrite(pte, vma);
- if (unlikely(is_zone_device_page(new)) &&
- is_device_private_page(new)) {
- entry = make_device_private_entry(new, pte_write(pte));
- pte = swp_entry_to_pte(entry);
+ if (unlikely(is_zone_device_page(new))) {
+ if (is_device_private_page(new)) {
+ entry = make_device_private_entry(new, pte_write(pte));
+ pte = swp_entry_to_pte(entry);
+ } else if (is_device_public_page(new)) {
+ pte = pte_mkdevmap(pte);
+ flush_dcache_page(new);
+ }
} else
flush_dcache_page(new);
@@ -437,12 +442,11 @@ int migrate_page_move_mapping(struct address_space *mapping,
void **pslot;
/*
- * ZONE_DEVICE pages have 1 refcount always held by their device
- *
- * Note that DAX memory will never reach that point as it does not have
- * the MEMORY_DEVICE_ALLOW_MIGRATE flag set (see memory_hotplug.h).
+ * Device public or private pages have an extra refcount as they are
+ * ZONE_DEVICE pages.
*/
- expected_count += is_zone_device_page(page);
+ expected_count += is_device_private_page(page);
+ expected_count += is_device_public_page(page);
if (!mapping) {
/* Anonymous page without mapping */
@@ -2123,7 +2127,6 @@ out_unlock:
#endif /* CONFIG_NUMA */
-
struct migrate_vma {
struct vm_area_struct *vma;
unsigned long *dst;
@@ -2263,7 +2266,7 @@ again:
pfn = 0;
goto next;
}
- page = vm_normal_page(migrate->vma, addr, pte);
+ page = _vm_normal_page(migrate->vma, addr, pte, true);
mpfn = migrate_pfn(pfn) | MIGRATE_PFN_MIGRATE;
mpfn |= pte_write(pte) ? MIGRATE_PFN_WRITE : 0;
}
@@ -2406,10 +2409,19 @@ static bool migrate_vma_check_page(struct page *page)
if (is_device_private_page(page))
return true;
- /* Other ZONE_DEVICE memory type are not supported */
- return false;
+ /*
+ * Only allow device public page to be migrated and account for
+ * the extra reference count imply by ZONE_DEVICE pages.
+ */
+ if (!is_device_public_page(page))
+ return false;
+ extra++;
}
+ /* For file back page */
+ if (page_mapping(page))
+ extra += 1 + page_has_private(page);
+
if ((page_count(page) - extra) > page_mapcount(page))
return false;
@@ -2647,11 +2659,18 @@ static void migrate_vma_insert_page(struct migrate_vma *migrate,
*/
__SetPageUptodate(page);
- if (is_zone_device_page(page) && is_device_private_page(page)) {
- swp_entry_t swp_entry;
-
- swp_entry = make_device_private_entry(page, vma->vm_flags & VM_WRITE);
- entry = swp_entry_to_pte(swp_entry);
+ if (is_zone_device_page(page)) {
+ if (is_device_private_page(page)) {
+ swp_entry_t swp_entry;
+
+ swp_entry = make_device_private_entry(page, vma->vm_flags & VM_WRITE);
+ entry = swp_entry_to_pte(swp_entry);
+ } else if (is_device_public_page(page)) {
+ entry = pte_mkold(mk_pte(page, READ_ONCE(vma->vm_page_prot)));
+ if (vma->vm_flags & VM_WRITE)
+ entry = pte_mkwrite(pte_mkdirty(entry));
+ entry = pte_mkdevmap(entry);
+ }
} else {
entry = mk_pte(page, vma->vm_page_prot);
if (vma->vm_flags & VM_WRITE)
@@ -2768,7 +2787,7 @@ static void migrate_vma_pages(struct migrate_vma *migrate)
migrate->src[i] &= ~MIGRATE_PFN_MIGRATE;
continue;
}
- } else {
+ } else if (!is_device_public_page(newpage)) {
/*
* Other types of ZONE_DEVICE page are not
* supported.