Home Home > GIT Browse
summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorMichal Suchanek <msuchanek@suse.de>2019-10-02 14:42:47 +0200
committerMichal Suchanek <msuchanek@suse.de>2019-10-02 21:10:15 +0200
commit90cdc0916cbe514ddf3650c9dca50015112847f0 (patch)
treea694e43ca066fcaae4c0b634a6821e63c8e547d2
parent80c5a9ebd6bcb83d0d084bafa6227630ba397966 (diff)
powerpc/64s/radix: Improve TLB flushing for page table freeing (bsc#1152161 ltc#181664).
- Refresh patches.suse/powerpc-mm-Fix-typo-in-comments.patch. - Refresh patches.suse/powerpc-mm-Fixup-tlbie-vs-store-ordering-issue-on-PO.patch. - Refresh patches.suse/powerpc-mm-Workaround-Nest-MMU-bug-with-TLB-invalida.patch. - Refresh patches.suse/powerpc-mm-radix-Move-the-functions-that-does-the-ac.patch.
-rw-r--r--patches.suse/powerpc-64s-radix-Improve-TLB-flushing-for-page-tabl.patch217
-rw-r--r--patches.suse/powerpc-mm-Fix-typo-in-comments.patch18
-rw-r--r--patches.suse/powerpc-mm-Fixup-tlbie-vs-store-ordering-issue-on-PO.patch53
-rw-r--r--patches.suse/powerpc-mm-Workaround-Nest-MMU-bug-with-TLB-invalida.patch66
-rw-r--r--patches.suse/powerpc-mm-radix-Move-the-functions-that-does-the-ac.patch15
-rw-r--r--series.conf1
6 files changed, 297 insertions, 73 deletions
diff --git a/patches.suse/powerpc-64s-radix-Improve-TLB-flushing-for-page-tabl.patch b/patches.suse/powerpc-64s-radix-Improve-TLB-flushing-for-page-tabl.patch
new file mode 100644
index 0000000000..e21795ce3e
--- /dev/null
+++ b/patches.suse/powerpc-64s-radix-Improve-TLB-flushing-for-page-tabl.patch
@@ -0,0 +1,217 @@
+From 0b2f5a8a792755c88bd786f89712a9fac9967b2b Mon Sep 17 00:00:00 2001
+From: Nicholas Piggin <npiggin@gmail.com>
+Date: Tue, 7 Nov 2017 18:53:09 +1100
+Subject: [PATCH] powerpc/64s/radix: Improve TLB flushing for page table
+ freeing
+
+References: bsc#1152161 ltc#181664
+Patch-mainline: v4.15-rc1
+Git-commit: 0b2f5a8a792755c88bd786f89712a9fac9967b2b
+
+Unmaps that free page tables always flush the entire PID, which is
+sub-optimal. Provide TLB range flushing with an additional PWC flush
+that can be use for va range invalidations with PWC flush.
+
+ Time to munmap N pages of memory including last level page table
+ teardown (after mmap, touch), local invalidate:
+ N 1 2 4 8 16 32 64
+ vanilla 3.2us 3.3us 3.4us 3.6us 4.1us 5.2us 7.2us
+ patched 1.4us 1.5us 1.7us 1.9us 2.6us 3.7us 6.2us
+
+ Global invalidate:
+ N 1 2 4 8 16 32 64
+ vanilla 2.2us 2.3us 2.4us 2.6us 3.2us 4.1us 6.2us
+ patched 2.1us 2.5us 3.4us 5.2us 8.7us 15.7us 6.2us
+
+Local invalidates get much better across the board. Global ones have
+the same issue where multiple tlbies for va flush do get slower than
+the single tlbie to invalidate the PID. None of this test captures
+the TLB benefits of avoiding killing everything.
+
+Global gets worse, but it is brought in to line with global invalidate
+for munmap()s that do not free page tables.
+
+Signed-off-by: Nicholas Piggin <npiggin@gmail.com>
+Signed-off-by: Michael Ellerman <mpe@ellerman.id.au>
+Acked-by: Michal Suchanek <msuchanek@suse.de>
+---
+ arch/powerpc/mm/tlb-radix.c | 90 +++++++++++++++++++++++++------------
+ 1 file changed, 61 insertions(+), 29 deletions(-)
+
+diff --git a/arch/powerpc/mm/tlb-radix.c b/arch/powerpc/mm/tlb-radix.c
+index cfa08da534a7..884f4b705b57 100644
+--- a/arch/powerpc/mm/tlb-radix.c
++++ b/arch/powerpc/mm/tlb-radix.c
+@@ -39,6 +39,20 @@ static inline void __tlbiel_pid(unsigned long pid, int set,
+ trace_tlbie(0, 1, rb, rs, ric, prs, r);
+ }
+
++static inline void __tlbie_pid(unsigned long pid, unsigned long ric)
++{
++ unsigned long rb,rs,prs,r;
++
++ rb = PPC_BIT(53); /* IS = 1 */
++ rs = pid << PPC_BITLSHIFT(31);
++ prs = 1; /* process scoped */
++ r = 1; /* raidx format */
++
++ asm volatile(PPC_TLBIE_5(%0, %4, %3, %2, %1)
++ : : "r"(rb), "i"(r), "i"(prs), "i"(ric), "r"(rs) : "memory");
++ trace_tlbie(0, 0, rb, rs, ric, prs, r);
++}
++
+ /*
+ * We use 128 set in radix mode and 256 set in hpt mode.
+ */
+@@ -70,18 +84,9 @@ static inline void _tlbiel_pid(unsigned long pid, unsigned long ric)
+
+ static inline void _tlbie_pid(unsigned long pid, unsigned long ric)
+ {
+- unsigned long rb,rs,prs,r;
+-
+- rb = PPC_BIT(53); /* IS = 1 */
+- rs = pid << PPC_BITLSHIFT(31);
+- prs = 1; /* process scoped */
+- r = 1; /* raidx format */
+-
+ asm volatile("ptesync": : :"memory");
+- asm volatile(PPC_TLBIE_5(%0, %4, %3, %2, %1)
+- : : "r"(rb), "i"(r), "i"(prs), "i"(ric), "r"(rs) : "memory");
++ __tlbie_pid(pid, ric);
+ asm volatile("eieio; tlbsync; ptesync": : :"memory");
+- trace_tlbie(0, 0, rb, rs, ric, prs, r);
+ }
+
+ static inline void __tlbiel_va(unsigned long va, unsigned long pid,
+@@ -123,9 +128,11 @@ static inline void _tlbiel_va(unsigned long va, unsigned long pid,
+
+ static inline void _tlbiel_va_range(unsigned long start, unsigned long end,
+ unsigned long pid, unsigned long page_size,
+- unsigned long psize)
++ unsigned long psize, bool also_pwc)
+ {
+ asm volatile("ptesync": : :"memory");
++ if (also_pwc)
++ __tlbiel_pid(pid, 0, RIC_FLUSH_PWC);
+ __tlbiel_va_range(start, end, pid, page_size, psize);
+ asm volatile("ptesync": : :"memory");
+ }
+@@ -169,9 +176,11 @@ static inline void _tlbie_va(unsigned long va, unsigned long pid,
+
+ static inline void _tlbie_va_range(unsigned long start, unsigned long end,
+ unsigned long pid, unsigned long page_size,
+- unsigned long psize)
++ unsigned long psize, bool also_pwc)
+ {
+ asm volatile("ptesync": : :"memory");
++ if (also_pwc)
++ __tlbie_pid(pid, RIC_FLUSH_PWC);
+ __tlbie_va_range(start, end, pid, page_size, psize);
+ asm volatile("eieio; tlbsync; ptesync": : :"memory");
+ }
+@@ -412,13 +421,15 @@ static int radix_get_mmu_psize(int page_size)
+ return psize;
+ }
+
++static void radix__flush_tlb_pwc_range_psize(struct mm_struct *mm, unsigned long start,
++ unsigned long end, int psize);
++
+ void radix__tlb_flush(struct mmu_gather *tlb)
+ {
+ int psize = 0;
+ struct mm_struct *mm = tlb->mm;
+ int page_size = tlb->page_size;
+
+- psize = radix_get_mmu_psize(page_size);
+ /*
+ * if page size is not something we understand, do a full mm flush
+ *
+@@ -426,17 +437,28 @@ void radix__tlb_flush(struct mmu_gather *tlb)
+ * that flushes the process table entry cache upon process teardown.
+ * See the comment for radix in arch_exit_mmap().
+ */
+- if (psize != -1 && !tlb->fullmm && !tlb->need_flush_all)
+- radix__flush_tlb_range_psize(mm, tlb->start, tlb->end, psize);
+- else if (tlb->fullmm || tlb->need_flush_all) {
+- tlb->need_flush_all = 0;
++ if (tlb->fullmm) {
+ radix__flush_all_mm(mm);
+- } else
+- radix__flush_tlb_mm(mm);
++ } else if ( (psize = radix_get_mmu_psize(page_size)) == -1) {
++ if (!tlb->need_flush_all)
++ radix__flush_tlb_mm(mm);
++ else
++ radix__flush_all_mm(mm);
++ } else {
++ unsigned long start = tlb->start;
++ unsigned long end = tlb->end;
++
++ if (!tlb->need_flush_all)
++ radix__flush_tlb_range_psize(mm, start, end, psize);
++ else
++ radix__flush_tlb_pwc_range_psize(mm, start, end, psize);
++ }
++ tlb->need_flush_all = 0;
+ }
+
+-void radix__flush_tlb_range_psize(struct mm_struct *mm, unsigned long start,
+- unsigned long end, int psize)
++static inline void __radix__flush_tlb_range_psize(struct mm_struct *mm,
++ unsigned long start, unsigned long end,
++ int psize, bool also_pwc)
+ {
+ unsigned long pid;
+ unsigned int page_shift = mmu_psize_defs[psize].shift;
+@@ -461,18 +483,30 @@ void radix__flush_tlb_range_psize(struct mm_struct *mm, unsigned long start,
+
+ if (full) {
+ if (local)
+- _tlbiel_pid(pid, RIC_FLUSH_TLB);
++ _tlbiel_pid(pid, also_pwc ? RIC_FLUSH_ALL : RIC_FLUSH_TLB);
+ else
+- _tlbie_pid(pid, RIC_FLUSH_TLB);
++ _tlbie_pid(pid, also_pwc ? RIC_FLUSH_ALL: RIC_FLUSH_TLB);
+ } else {
+ if (local)
+- _tlbiel_va_range(start, end, pid, page_size, psize);
++ _tlbiel_va_range(start, end, pid, page_size, psize, also_pwc);
+ else
+- _tlbie_va_range(start, end, pid, page_size, psize);
++ _tlbie_va_range(start, end, pid, page_size, psize, also_pwc);
+ }
+ preempt_enable();
+ }
+
++void radix__flush_tlb_range_psize(struct mm_struct *mm, unsigned long start,
++ unsigned long end, int psize)
++{
++ return __radix__flush_tlb_range_psize(mm, start, end, psize, false);
++}
++
++static void radix__flush_tlb_pwc_range_psize(struct mm_struct *mm, unsigned long start,
++ unsigned long end, int psize)
++{
++ __radix__flush_tlb_range_psize(mm, start, end, psize, true);
++}
++
+ #ifdef CONFIG_TRANSPARENT_HUGEPAGE
+ void radix__flush_tlb_collapsed_pmd(struct mm_struct *mm, unsigned long addr)
+ {
+@@ -494,11 +528,9 @@ void radix__flush_tlb_collapsed_pmd(struct mm_struct *mm, unsigned long addr)
+ preempt_disable();
+
+ if (mm_is_thread_local(mm)) {
+- _tlbiel_pid(pid, RIC_FLUSH_PWC);
+- _tlbiel_va_range(addr, end, pid, PAGE_SIZE, mmu_virtual_psize);
++ _tlbiel_va_range(addr, end, pid, PAGE_SIZE, mmu_virtual_psize, true);
+ } else {
+- _tlbie_pid(pid, RIC_FLUSH_PWC);
+- _tlbie_va_range(addr, end, pid, PAGE_SIZE, mmu_virtual_psize);
++ _tlbie_va_range(addr, end, pid, PAGE_SIZE, mmu_virtual_psize, true);
+ }
+
+ preempt_enable();
+--
+2.23.0
+
diff --git a/patches.suse/powerpc-mm-Fix-typo-in-comments.patch b/patches.suse/powerpc-mm-Fix-typo-in-comments.patch
index fe1148aec4..c490a9db28 100644
--- a/patches.suse/powerpc-mm-Fix-typo-in-comments.patch
+++ b/patches.suse/powerpc-mm-Fix-typo-in-comments.patch
@@ -26,34 +26,34 @@ Acked-by: Michal Suchanek <msuchanek@suse.de>
asm volatile(PPC_TLBIEL(%0, %4, %3, %2, %1)
: : "r"(rb), "i"(r), "i"(prs), "i"(ric), "r"(rs) : "memory");
-@@ -114,7 +114,7 @@ static inline void __tlbiel_va(unsigned
- rb |= ap << PPC_BITLSHIFT(58);
+@@ -112,7 +112,7 @@ static inline void __tlbie_pid(unsigned
+ rb = PPC_BIT(53); /* IS = 1 */
rs = pid << PPC_BITLSHIFT(31);
prs = 1; /* process scoped */
- r = 1; /* raidx format */
+ r = 1; /* radix format */
- asm volatile(PPC_TLBIEL(%0, %4, %3, %2, %1)
+ asm volatile(PPC_TLBIE_5(%0, %4, %3, %2, %1)
: : "r"(rb), "i"(r), "i"(prs), "i"(ric), "r"(rs) : "memory");
-@@ -130,7 +130,7 @@ static inline void __tlbie_va(unsigned l
+@@ -128,7 +128,7 @@ static inline void __tlbiel_va(unsigned
rb |= ap << PPC_BITLSHIFT(58);
rs = pid << PPC_BITLSHIFT(31);
prs = 1; /* process scoped */
- r = 1; /* raidx format */
+ r = 1; /* radix format */
- asm volatile(PPC_TLBIE_5(%0, %4, %3, %2, %1)
+ asm volatile(PPC_TLBIEL(%0, %4, %3, %2, %1)
: : "r"(rb), "i"(r), "i"(prs), "i"(ric), "r"(rs) : "memory");
-@@ -184,7 +184,7 @@ static inline void _tlbie_pid(unsigned l
- rb = PPC_BIT(53); /* IS = 1 */
+@@ -144,7 +144,7 @@ static inline void __tlbie_va(unsigned l
+ rb |= ap << PPC_BITLSHIFT(58);
rs = pid << PPC_BITLSHIFT(31);
prs = 1; /* process scoped */
- r = 1; /* raidx format */
+ r = 1; /* radix format */
- asm volatile("ptesync": : :"memory");
asm volatile(PPC_TLBIE_5(%0, %4, %3, %2, %1)
-@@ -528,7 +528,7 @@ void radix__flush_tlb_all(void)
+ : : "r"(rb), "i"(r), "i"(prs), "i"(ric), "r"(rs) : "memory");
+@@ -653,7 +653,7 @@ void radix__flush_tlb_all(void)
rb = 0x3 << PPC_BITLSHIFT(53); /* IS = 3 */
prs = 0; /* partition scoped */
diff --git a/patches.suse/powerpc-mm-Fixup-tlbie-vs-store-ordering-issue-on-PO.patch b/patches.suse/powerpc-mm-Fixup-tlbie-vs-store-ordering-issue-on-PO.patch
index 3c71c5e9b6..75966561d2 100644
--- a/patches.suse/powerpc-mm-Fixup-tlbie-vs-store-ordering-issue-on-PO.patch
+++ b/patches.suse/powerpc-mm-Fixup-tlbie-vs-store-ordering-issue-on-PO.patch
@@ -20,20 +20,6 @@ Signed-off-by: Aneesh Kumar K.V <aneesh.kumar@linux.vnet.ibm.com>
[mpe: Enable the feature in the DT CPU features code for all Power9,
rename the feature to CPU_FTR_P9_TLBIE_BUG per benh.]
Signed-off-by: Michael Ellerman <mpe@ellerman.id.au>
-[mauricfo: backport: tlb-radix.c :: cover '__tlbie_va()'
- - hunk 2: update context lines (includes the inline assembly still in
- _tlbie_pid() in SLES15 as __tlbie_pie() does not exist yet.
- equivalent logic: the fixup is still between inline assembly
- statements PPC_TLBIE_5 and the syncs)
- - hunk 3: update context line 1
- - hunk 4: move fixup_tlbie() into radix__flush_tlb_range_psize():
- SLES15 does not have __tlbie_va_range(), in upstream its
- callers are __radix__flush_tlb_range_psize() (which here
- calls __tlbie_va() directly, so add the fixup after that)
- and radix__flush_tlb_collapsed_pmd() (in SLES15 it calls
- the wrapper directly, so it's already fixed up).]
-Signed-off-by: Mauricio Faria de Oliveira <mauricfo@linux.vnet.ibm.com>
-
Acked-by: Michal Suchanek <msuchanek@suse.de>
---
arch/powerpc/include/asm/cputable.h | 3 ++-
@@ -46,7 +32,7 @@ Acked-by: Michal Suchanek <msuchanek@suse.de>
7 files changed, 50 insertions(+), 2 deletions(-)
diff --git a/arch/powerpc/include/asm/cputable.h b/arch/powerpc/include/asm/cputable.h
-index a2c5c95..2e2bacb 100644
+index a2c5c95882cf..2e2bacbdf6ed 100644
--- a/arch/powerpc/include/asm/cputable.h
+++ b/arch/powerpc/include/asm/cputable.h
@@ -203,6 +203,7 @@ static inline void cpu_feature_keys_init(void) { }
@@ -67,7 +53,7 @@ index a2c5c95..2e2bacb 100644
(~CPU_FTR_SAO))
#define CPU_FTRS_POWER9_DD2_0 CPU_FTRS_POWER9
diff --git a/arch/powerpc/kernel/dt_cpu_ftrs.c b/arch/powerpc/kernel/dt_cpu_ftrs.c
-index 0bcfb0f..8ca5d5b7 100644
+index 0bcfb0f256e1..8ca5d5b74618 100644
--- a/arch/powerpc/kernel/dt_cpu_ftrs.c
+++ b/arch/powerpc/kernel/dt_cpu_ftrs.c
@@ -709,6 +709,9 @@ static __init void cpufeatures_cpu_quirks(void)
@@ -81,7 +67,7 @@ index 0bcfb0f..8ca5d5b7 100644
static void __init cpufeatures_setup_finished(void)
diff --git a/arch/powerpc/kvm/book3s_64_mmu_radix.c b/arch/powerpc/kvm/book3s_64_mmu_radix.c
-index 0c85481..0837b97 100644
+index 0c854816e653..0837b9738d76 100644
--- a/arch/powerpc/kvm/book3s_64_mmu_radix.c
+++ b/arch/powerpc/kvm/book3s_64_mmu_radix.c
@@ -157,6 +157,9 @@ static void kvmppc_radix_tlbie_page(struct kvm *kvm, unsigned long addr,
@@ -95,7 +81,7 @@ index 0c85481..0837b97 100644
}
diff --git a/arch/powerpc/kvm/book3s_hv_rm_mmu.c b/arch/powerpc/kvm/book3s_hv_rm_mmu.c
-index 8888e62..e1c083f 100644
+index 8888e625a999..e1c083fbe434 100644
--- a/arch/powerpc/kvm/book3s_hv_rm_mmu.c
+++ b/arch/powerpc/kvm/book3s_hv_rm_mmu.c
@@ -473,6 +473,17 @@ static void do_tlbies(struct kvm *kvm, unsigned long *rbvalues,
@@ -117,7 +103,7 @@ index 8888e62..e1c083f 100644
kvm->arch.tlbie_lock = 0;
} else {
diff --git a/arch/powerpc/mm/hash_native_64.c b/arch/powerpc/mm/hash_native_64.c
-index a0675e9..656933c 100644
+index a0675e91ad7d..656933c85925 100644
--- a/arch/powerpc/mm/hash_native_64.c
+++ b/arch/powerpc/mm/hash_native_64.c
@@ -201,6 +201,15 @@ static inline unsigned long ___tlbie(unsigned long vpn, int psize,
@@ -165,7 +151,7 @@ index a0675e9..656933c 100644
if (lock_tlbie)
diff --git a/arch/powerpc/mm/pgtable_64.c b/arch/powerpc/mm/pgtable_64.c
-index 28c980e..adf469f 100644
+index 28c980eb4422..adf469f312f2 100644
--- a/arch/powerpc/mm/pgtable_64.c
+++ b/arch/powerpc/mm/pgtable_64.c
@@ -481,6 +481,7 @@ void mmu_partition_table_set_entry(unsigned int lpid, unsigned long dw0,
@@ -176,9 +162,11 @@ index 28c980e..adf469f 100644
asm volatile("eieio; tlbsync; ptesync" : : : "memory");
}
EXPORT_SYMBOL_GPL(mmu_partition_table_set_entry);
+diff --git a/arch/powerpc/mm/tlb-radix.c b/arch/powerpc/mm/tlb-radix.c
+index 74354c26d316..a07f5372a4bf 100644
--- a/arch/powerpc/mm/tlb-radix.c
+++ b/arch/powerpc/mm/tlb-radix.c
-@@ -137,6 +137,17 @@ static inline void __tlbie_va(unsigned l
+@@ -151,6 +151,17 @@ static inline void __tlbie_va(unsigned long va, unsigned long pid,
trace_tlbie(0, 0, rb, rs, ric, prs, r);
}
@@ -196,15 +184,15 @@ index 28c980e..adf469f 100644
/*
* We use 128 set in radix mode and 256 set in hpt mode.
*/
-@@ -178,6 +189,7 @@ static inline void _tlbie_pid(unsigned l
- asm volatile("ptesync": : :"memory");
- asm volatile(PPC_TLBIE_5(%0, %4, %3, %2, %1)
- : : "r"(rb), "i"(r), "i"(prs), "i"(ric), "r"(rs) : "memory");
+@@ -200,6 +211,7 @@ static inline void _tlbie_pid(unsigned long pid, unsigned long ric)
+ default:
+ __tlbie_pid(pid, RIC_FLUSH_ALL);
+ }
+ fixup_tlbie();
asm volatile("eieio; tlbsync; ptesync": : :"memory");
- trace_tlbie(0, 0, rb, rs, ric, prs, r);
}
-@@ -212,6 +224,7 @@ static inline void _tlbie_va(unsigned lo
+
+@@ -253,6 +265,7 @@ static inline void _tlbie_va(unsigned long va, unsigned long pid,
asm volatile("ptesync": : :"memory");
__tlbie_va(va, pid, ap, ric);
@@ -212,15 +200,15 @@ index 28c980e..adf469f 100644
asm volatile("eieio; tlbsync; ptesync": : :"memory");
}
-@@ -239,6 +252,7 @@ static inline void _tlbie_va_range(unsig
- {
- asm volatile("ptesync": : :"memory");
+@@ -264,6 +277,7 @@ static inline void _tlbie_va_range(unsigned long start, unsigned long end,
+ if (also_pwc)
+ __tlbie_pid(pid, RIC_FLUSH_PWC);
__tlbie_va_range(start, end, pid, page_size, psize);
+ fixup_tlbie();
asm volatile("eieio; tlbsync; ptesync": : :"memory");
}
-@@ -465,6 +479,7 @@ void radix__flush_tlb_range(struct vm_ar
+@@ -498,6 +512,7 @@ void radix__flush_tlb_range(struct vm_area_struct *vma, unsigned long start,
if (hflush)
__tlbie_va_range(hstart, hend, pid,
HPAGE_PMD_SIZE, MMU_PAGE_2M);
@@ -228,3 +216,6 @@ index 28c980e..adf469f 100644
asm volatile("eieio; tlbsync; ptesync": : :"memory");
}
}
+--
+2.23.0
+
diff --git a/patches.suse/powerpc-mm-Workaround-Nest-MMU-bug-with-TLB-invalida.patch b/patches.suse/powerpc-mm-Workaround-Nest-MMU-bug-with-TLB-invalida.patch
index e0bafd6523..27f469371e 100644
--- a/patches.suse/powerpc-mm-Workaround-Nest-MMU-bug-with-TLB-invalida.patch
+++ b/patches.suse/powerpc-mm-Workaround-Nest-MMU-bug-with-TLB-invalida.patch
@@ -22,27 +22,39 @@ Signed-off-by: Balbir Singh <bsingharora@gmail.com>
[balbirs: fixed spelling and coding style to quiesce checkpatch.pl]
Tested-by: Balbir Singh <bsingharora@gmail.com>
Signed-off-by: Michael Ellerman <mpe@ellerman.id.au>
-[mauricfo: backport: essentially, cover '_tlbie_pid(pid, RIC_FLUSH_TLB)'
- - hunk 1: removed -- not required since there's no __tlbie_pid()
- in SLES 15, so _tlbie_pid() is called directly using
- the compile-time constraint.
- - hunk 3: update context lines.
- - hunk 4: update context lines (to match a different function;
- in SLES 15, original target radix__flush_tlb_range()
- just calls radix__flush_tlb_mm() which is already
- patched; but there is another instance of _tlbie_pid(
- (pid, RIC_FLUSH_TLB) in radix__flush_tlb_range_psize(),
- so cover that one).]
-Signed-off-by: Mauricio Faria de Oliveira <mauricfo@linux.vnet.ibm.com>
-
Acked-by: Michal Suchanek <msuchanek@suse.de>
---
- arch/powerpc/mm/tlb-radix.c | 29 +++++++++++++++++++++++------
- 1 file changed, 23 insertions(+), 6 deletions(-)
+ arch/powerpc/mm/tlb-radix.c | 50 +++++++++++++++++++++++++++++++------
+ 1 file changed, 43 insertions(+), 7 deletions(-)
--- a/arch/powerpc/mm/tlb-radix.c
+++ b/arch/powerpc/mm/tlb-radix.c
-@@ -302,6 +302,16 @@ void radix__local_flush_tlb_page(struct
+@@ -151,7 +151,23 @@ static inline void _tlbiel_pid(unsigned
+ static inline void _tlbie_pid(unsigned long pid, unsigned long ric)
+ {
+ asm volatile("ptesync": : :"memory");
+- __tlbie_pid(pid, ric);
++
++ /*
++ * Workaround the fact that the "ric" argument to __tlbie_pid
++ * must be a compile-time contraint to match the "i" constraint
++ * in the asm statement.
++ */
++ switch (ric) {
++ case RIC_FLUSH_TLB:
++ __tlbie_pid(pid, RIC_FLUSH_TLB);
++ break;
++ case RIC_FLUSH_PWC:
++ __tlbie_pid(pid, RIC_FLUSH_PWC);
++ break;
++ case RIC_FLUSH_ALL:
++ default:
++ __tlbie_pid(pid, RIC_FLUSH_ALL);
++ }
+ asm volatile("eieio; tlbsync; ptesync": : :"memory");
+ }
+
+@@ -311,6 +327,16 @@ void radix__local_flush_tlb_page(struct
}
EXPORT_SYMBOL(radix__local_flush_tlb_page);
@@ -59,7 +71,7 @@ Acked-by: Michal Suchanek <msuchanek@suse.de>
#ifdef CONFIG_SMP
void radix__flush_tlb_mm(struct mm_struct *mm)
{
-@@ -312,9 +322,12 @@ void radix__flush_tlb_mm(struct mm_struc
+@@ -321,9 +347,12 @@ void radix__flush_tlb_mm(struct mm_struc
return;
preempt_disable();
@@ -75,7 +87,7 @@ Acked-by: Michal Suchanek <msuchanek@suse.de>
_tlbiel_pid(pid, RIC_FLUSH_TLB);
preempt_enable();
}
-@@ -418,10 +431,14 @@ void radix__flush_tlb_range(struct vm_ar
+@@ -427,10 +456,14 @@ void radix__flush_tlb_range(struct vm_ar
full = (end == TLB_FLUSH_ALL || nr_pages > tlb_single_page_flush_ceiling);
if (full) {
@@ -93,15 +105,13 @@ Acked-by: Michal Suchanek <msuchanek@suse.de>
} else {
bool hflush = false;
unsigned long hstart, hend;
-@@ -514,7 +531,10 @@ void radix__flush_tlb_range_psize(struct
+@@ -533,6 +566,9 @@ static inline void __radix__flush_tlb_ra
+ full = (end == TLB_FLUSH_ALL || nr_pages > tlb_single_page_flush_ceiling);
+
+ if (full) {
++ if (!local && mm_needs_flush_escalation(mm))
++ also_pwc = true;
++
if (local)
- _tlbiel_pid(pid, RIC_FLUSH_TLB);
+ _tlbiel_pid(pid, also_pwc ? RIC_FLUSH_ALL : RIC_FLUSH_TLB);
else
-- _tlbie_pid(pid, RIC_FLUSH_TLB);
-+ if (mm_needs_flush_escalation(mm))
-+ _tlbie_pid(pid, RIC_FLUSH_ALL);
-+ else
-+ _tlbie_pid(pid, RIC_FLUSH_TLB);
- } else {
- if (local)
- _tlbiel_va_range(start, end, pid, page_size, psize);
diff --git a/patches.suse/powerpc-mm-radix-Move-the-functions-that-does-the-ac.patch b/patches.suse/powerpc-mm-radix-Move-the-functions-that-does-the-ac.patch
index 466007a689..f9d22d2256 100644
--- a/patches.suse/powerpc-mm-radix-Move-the-functions-that-does-the-ac.patch
+++ b/patches.suse/powerpc-mm-radix-Move-the-functions-that-does-the-ac.patch
@@ -17,10 +17,12 @@ Acked-by: Michal Suchanek <msuchanek@suse.de>
arch/powerpc/mm/tlb-radix.c | 64 ++++++++++++++++++-------------------
1 file changed, 32 insertions(+), 32 deletions(-)
+diff --git a/arch/powerpc/mm/tlb-radix.c b/arch/powerpc/mm/tlb-radix.c
+index e47ee8c867c5..74354c26d316 100644
--- a/arch/powerpc/mm/tlb-radix.c
+++ b/arch/powerpc/mm/tlb-radix.c
-@@ -105,6 +105,38 @@ static inline void __tlbiel_pid(unsigned
- trace_tlbie(0, 1, rb, rs, ric, prs, r);
+@@ -119,6 +119,38 @@ static inline void __tlbie_pid(unsigned long pid, unsigned long ric)
+ trace_tlbie(0, 0, rb, rs, ric, prs, r);
}
+static inline void __tlbiel_va(unsigned long va, unsigned long pid,
@@ -58,8 +60,8 @@ Acked-by: Michal Suchanek <msuchanek@suse.de>
/*
* We use 128 set in radix mode and 256 set in hpt mode.
*/
-@@ -150,22 +182,6 @@ static inline void _tlbie_pid(unsigned l
- trace_tlbie(0, 0, rb, rs, ric, prs, r);
+@@ -171,22 +203,6 @@ static inline void _tlbie_pid(unsigned long pid, unsigned long ric)
+ asm volatile("eieio; tlbsync; ptesync": : :"memory");
}
-static inline void __tlbiel_va(unsigned long va, unsigned long pid,
@@ -81,7 +83,7 @@ Acked-by: Michal Suchanek <msuchanek@suse.de>
static inline void __tlbiel_va_range(unsigned long start, unsigned long end,
unsigned long pid, unsigned long page_size,
unsigned long psize)
-@@ -196,22 +212,6 @@ static inline void _tlbiel_va_range(unsi
+@@ -219,22 +235,6 @@ static inline void _tlbiel_va_range(unsigned long start, unsigned long end,
asm volatile("ptesync": : :"memory");
}
@@ -104,3 +106,6 @@ Acked-by: Michal Suchanek <msuchanek@suse.de>
static inline void __tlbie_va_range(unsigned long start, unsigned long end,
unsigned long pid, unsigned long page_size,
unsigned long psize)
+--
+2.23.0
+
diff --git a/series.conf b/series.conf
index db2ccfab2d..7e049d894d 100644
--- a/series.conf
+++ b/series.conf
@@ -10195,6 +10195,7 @@
patches.suse/powerpc-64s-radix-Optimize-TLB-range-flush-barriers.patch
patches.suse/powerpc-64s-radix-Implement-_tlbie-l-_va_range-flush.patch
patches.suse/powerpc-64s-radix-Optimize-flush_tlb_range.patch
+ patches.suse/powerpc-64s-radix-Improve-TLB-flushing-for-page-tabl.patch
patches.suse/powerpc-mm-radix-Fix-crashes-on-Power9-DD1-with-radix.patch
patches.suse/powerpc-kprobes-Disable-preemption-before-invoking-p.patch
patches.suse/powerpc-powernv-idle-Round-up-latency-and-residency-.patch