1 files changed, 47 insertions, 21 deletions

diff --git a/arch/powerpc/kvm/book3s_64_mmu_radix.c b/arch/powerpc/kvm/book3s_64_mmu_radix.c
index b0ba3628adc2..176f911ee983 100644
--- a/arch/powerpc/kvm/book3s_64_mmu_radix.c
+++ b/arch/powerpc/kvm/book3s_64_mmu_radix.c
@@ -156,7 +156,7 @@ static void kvmppc_radix_flush_pwc(struct kvm *kvm)
 	radix__flush_pwc_lpid(kvm->arch.lpid);
 }
 
-unsigned long kvmppc_radix_update_pte(struct kvm *kvm, pte_t *ptep,
+static unsigned long kvmppc_radix_update_pte(struct kvm *kvm, pte_t *ptep,
 				      unsigned long clr, unsigned long set,
 				      unsigned long addr, unsigned int shift)
 {
@@ -360,6 +360,15 @@ static void kvmppc_unmap_free_pud_entry_table(struct kvm *kvm, pud_t *pud,
 	kvmppc_unmap_free_pmd(kvm, pmd, false);
 }
 
+/*
+ * There are a number of bits which may differ between different faults to
+ * the same partition scope entry. RC bits, in the course of cleaning and
+ * aging. And the write bit can change, either the access could have been
+ * upgraded, or a read fault could happen concurrently with a write fault
+ * that sets those bits first.
+ */
+#define PTE_BITS_MUST_MATCH (~(_PAGE_WRITE | _PAGE_DIRTY | _PAGE_ACCESSED))
+
 static int kvmppc_create_pte(struct kvm *kvm, pte_t pte, unsigned long gpa,
 			     unsigned int level, unsigned long mmu_seq)
 {
@@ -404,19 +413,28 @@ static int kvmppc_create_pte(struct kvm *kvm, pte_t pte, unsigned long gpa,
 	if (pud_huge(*pud)) {
 		unsigned long hgpa = gpa & PUD_MASK;
 
+		/* Check if we raced and someone else has set the same thing */
+		if (level == 2) {
+			if (pud_raw(*pud) == pte_raw(pte)) {
+				ret = 0;
+				goto out_unlock;
+			}
+			/* Valid 1GB page here already, add our extra bits */
+			WARN_ON_ONCE((pud_val(*pud) ^ pte_val(pte)) &
+							PTE_BITS_MUST_MATCH);
+			kvmppc_radix_update_pte(kvm, (pte_t *)pud,
+					      0, pte_val(pte), hgpa, PUD_SHIFT);
+			ret = 0;
+			goto out_unlock;
+		}
 		/*
 		 * If we raced with another CPU which has just put
 		 * a 1GB pte in after we saw a pmd page, try again.
 		 */
-		if (level <= 1 && !new_pmd) {
+		if (!new_pmd) {
 			ret = -EAGAIN;
 			goto out_unlock;
 		}
-		/* Check if we raced and someone else has set the same thing */
-		if (level == 2 && pud_raw(*pud) == pte_raw(pte)) {
-			ret = 0;
-			goto out_unlock;
-		}
 		/* Valid 1GB page here already, remove it */
 		kvmppc_unmap_pte(kvm, (pte_t *)pud, hgpa, PUD_SHIFT);
 	}
@@ -443,19 +461,29 @@ static int kvmppc_create_pte(struct kvm *kvm, pte_t pte, unsigned long gpa,
 	if (pmd_is_leaf(*pmd)) {
 		unsigned long lgpa = gpa & PMD_MASK;
 
+		/* Check if we raced and someone else has set the same thing */
+		if (level == 1) {
+			if (pmd_raw(*pmd) == pte_raw(pte)) {
+				ret = 0;
+				goto out_unlock;
+			}
+			/* Valid 2MB page here already, add our extra bits */
+			WARN_ON_ONCE((pmd_val(*pmd) ^ pte_val(pte)) &
+							PTE_BITS_MUST_MATCH);
+			kvmppc_radix_update_pte(kvm, pmdp_ptep(pmd),
+					      0, pte_val(pte), lgpa, PMD_SHIFT);
+			ret = 0;
+			goto out_unlock;
+		}
+
 		/*
 		 * If we raced with another CPU which has just put
 		 * a 2MB pte in after we saw a pte page, try again.
 		 */
-		if (level == 0 && !new_ptep) {
+		if (!new_ptep) {
 			ret = -EAGAIN;
 			goto out_unlock;
 		}
-		/* Check if we raced and someone else has set the same thing */
-		if (level == 1 && pmd_raw(*pmd) == pte_raw(pte)) {
-			ret = 0;
-			goto out_unlock;
-		}
 		/* Valid 2MB page here already, remove it */
 		kvmppc_unmap_pte(kvm, pmdp_ptep(pmd), lgpa, PMD_SHIFT);
 	}
@@ -480,19 +508,17 @@ static int kvmppc_create_pte(struct kvm *kvm, pte_t pte, unsigned long gpa,
 	}
 	ptep = pte_offset_kernel(pmd, gpa);
 	if (pte_present(*ptep)) {
-		unsigned long old;
-
 		/* Check if someone else set the same thing */
 		if (pte_raw(*ptep) == pte_raw(pte)) {
 			ret = 0;
 			goto out_unlock;
 		}
-		/* PTE was previously valid, so invalidate it */
-		old = kvmppc_radix_update_pte(kvm, ptep, _PAGE_PRESENT,
-					      0, gpa, 0);
-		kvmppc_radix_tlbie_page(kvm, gpa, 0);
-		if (old & _PAGE_DIRTY)
-			mark_page_dirty(kvm, gpa >> PAGE_SHIFT);
+		/* Valid page here already, add our extra bits */
+		WARN_ON_ONCE((pte_val(*ptep) ^ pte_val(pte)) &
+							PTE_BITS_MUST_MATCH);
+		kvmppc_radix_update_pte(kvm, ptep, 0, pte_val(pte), gpa, 0);
+		ret = 0;
+		goto out_unlock;
 	}
 	kvmppc_radix_set_pte_at(kvm, gpa, ptep, pte);
 	ret = 0;