Home Home > GIT Browse
summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorMichal Suchanek <msuchanek@suse.de>2019-07-25 11:35:37 +0200
committerMichal Suchanek <msuchanek@suse.de>2019-10-02 21:10:16 +0200
commit9a6b5c8544705010d6153422ac93d11cbc51b4ac (patch)
tree360274c2f2a5477e116b38305a4956e79a87e9a5
parent90cdc0916cbe514ddf3650c9dca50015112847f0 (diff)
powerpc/64s/radix: Introduce local single page ceiling for TLB range
flush (bsc#1055117 bsc#1152161 ltc#181664). - Refresh patches.suse/powerpc-mm-Workaround-Nest-MMU-bug-with-TLB-invalida.patch.
-rw-r--r--patches.suse/powerpc-64s-radix-Introduce-local-single-page-ceilin.patch99
-rw-r--r--patches.suse/powerpc-mm-Workaround-Nest-MMU-bug-with-TLB-invalida.patch19
-rw-r--r--series.conf1
3 files changed, 112 insertions, 7 deletions
diff --git a/patches.suse/powerpc-64s-radix-Introduce-local-single-page-ceilin.patch b/patches.suse/powerpc-64s-radix-Introduce-local-single-page-ceilin.patch
new file mode 100644
index 0000000000..9689b166d7
--- /dev/null
+++ b/patches.suse/powerpc-64s-radix-Introduce-local-single-page-ceilin.patch
@@ -0,0 +1,99 @@
+From f6f27951fdf84a6edca3ea14077268ad629b57ac Mon Sep 17 00:00:00 2001
+From: Nicholas Piggin <npiggin@gmail.com>
+Date: Tue, 7 Nov 2017 18:53:08 +1100
+Subject: [PATCH] powerpc/64s/radix: Introduce local single page ceiling for
+ TLB range flush
+
+References: bsc#1055117 bsc#1152161 ltc#181664
+Patch-mainline: v4.15-rc1
+Git-commit: f6f27951fdf84a6edca3ea14077268ad629b57ac
+
+The single page flush ceiling is the cut-off point at which we switch
+from invalidating individual pages, to invalidating the entire process
+address space in response to a range flush.
+
+Introduce a local variant of this heuristic because local and global
+tlbie have significantly different properties:
+- Local tlbiel requires 128 instructions to invalidate a PID, global
+ tlbie only 1 instruction.
+- Global tlbie instructions are expensive broadcast operations.
+
+The local ceiling has been made much higher, 2x the number of
+instructions required to invalidate the entire PID (i.e., 256 pages).
+
+ Time to mprotect N pages of memory (after mmap, touch), local invalidate:
+ N 32 34 64 128 256 512
+ vanilla 7.4us 9.0us 14.6us 26.4us 50.2us 98.3us
+ patched 7.4us 7.8us 13.8us 26.4us 51.9us 98.3us
+
+The behaviour of both is identical at N=32 and N=512. Between there,
+the vanilla kernel does a PID invalidate and the patched kernel does
+a va range invalidate.
+
+At N=128, these require the same number of tlbiel instructions, so
+the patched version can be sen to be cheaper when < 128, and more
+expensive when > 128. However this does not well capture the cost
+of invalidated TLB.
+
+The additional cost at 256 pages does not seem prohibitive. It may
+be the case that increasing the limit further would continue to be
+beneficial to avoid invalidating all of the process's TLB entries.
+
+Signed-off-by: Nicholas Piggin <npiggin@gmail.com>
+Signed-off-by: Michael Ellerman <mpe@ellerman.id.au>
+Acked-by: Michal Suchanek <msuchanek@suse.de>
+---
+ arch/powerpc/mm/tlb-radix.c | 23 +++++++++++++++++++----
+ 1 file changed, 19 insertions(+), 4 deletions(-)
+
+diff --git a/arch/powerpc/mm/tlb-radix.c b/arch/powerpc/mm/tlb-radix.c
+index b4b49de551a9..cfa08da534a7 100644
+--- a/arch/powerpc/mm/tlb-radix.c
++++ b/arch/powerpc/mm/tlb-radix.c
+@@ -326,6 +326,7 @@ EXPORT_SYMBOL(radix__flush_tlb_kernel_range);
+ * individual page flushes to full-pid flushes.
+ */
+ static unsigned long tlb_single_page_flush_ceiling __read_mostly = 33;
++static unsigned long tlb_local_single_page_flush_ceiling __read_mostly = POWER9_TLB_SETS_RADIX * 2;
+
+ void radix__flush_tlb_range(struct vm_area_struct *vma, unsigned long start,
+ unsigned long end)
+@@ -348,8 +349,15 @@ void radix__flush_tlb_range(struct vm_area_struct *vma, unsigned long start,
+ return;
+
+ preempt_disable();
+- local = mm_is_thread_local(mm);
+- full = (end == TLB_FLUSH_ALL || nr_pages > tlb_single_page_flush_ceiling);
++ if (mm_is_thread_local(mm)) {
++ local = true;
++ full = (end == TLB_FLUSH_ALL ||
++ nr_pages > tlb_local_single_page_flush_ceiling);
++ } else {
++ local = false;
++ full = (end == TLB_FLUSH_ALL ||
++ nr_pages > tlb_single_page_flush_ceiling);
++ }
+
+ if (full) {
+ if (local)
+@@ -441,8 +449,15 @@ void radix__flush_tlb_range_psize(struct mm_struct *mm, unsigned long start,
+ return;
+
+ preempt_disable();
+- local = mm_is_thread_local(mm);
+- full = (end == TLB_FLUSH_ALL || nr_pages > tlb_single_page_flush_ceiling);
++ if (mm_is_thread_local(mm)) {
++ local = true;
++ full = (end == TLB_FLUSH_ALL ||
++ nr_pages > tlb_local_single_page_flush_ceiling);
++ } else {
++ local = false;
++ full = (end == TLB_FLUSH_ALL ||
++ nr_pages > tlb_single_page_flush_ceiling);
++ }
+
+ if (full) {
+ if (local)
+--
+2.21.0
+
diff --git a/patches.suse/powerpc-mm-Workaround-Nest-MMU-bug-with-TLB-invalida.patch b/patches.suse/powerpc-mm-Workaround-Nest-MMU-bug-with-TLB-invalida.patch
index 27f469371e..fbba0209c3 100644
--- a/patches.suse/powerpc-mm-Workaround-Nest-MMU-bug-with-TLB-invalida.patch
+++ b/patches.suse/powerpc-mm-Workaround-Nest-MMU-bug-with-TLB-invalida.patch
@@ -27,9 +27,11 @@ Acked-by: Michal Suchanek <msuchanek@suse.de>
arch/powerpc/mm/tlb-radix.c | 50 +++++++++++++++++++++++++++++++------
1 file changed, 43 insertions(+), 7 deletions(-)
+diff --git a/arch/powerpc/mm/tlb-radix.c b/arch/powerpc/mm/tlb-radix.c
+index 71d1b19ad1c0..8e4c6cb4a808 100644
--- a/arch/powerpc/mm/tlb-radix.c
+++ b/arch/powerpc/mm/tlb-radix.c
-@@ -151,7 +151,23 @@ static inline void _tlbiel_pid(unsigned
+@@ -151,7 +151,23 @@ static inline void _tlbiel_pid(unsigned long pid, unsigned long ric)
static inline void _tlbie_pid(unsigned long pid, unsigned long ric)
{
asm volatile("ptesync": : :"memory");
@@ -54,7 +56,7 @@ Acked-by: Michal Suchanek <msuchanek@suse.de>
asm volatile("eieio; tlbsync; ptesync": : :"memory");
}
-@@ -311,6 +327,16 @@ void radix__local_flush_tlb_page(struct
+@@ -311,6 +327,16 @@ void radix__local_flush_tlb_page(struct vm_area_struct *vma, unsigned long vmadd
}
EXPORT_SYMBOL(radix__local_flush_tlb_page);
@@ -71,7 +73,7 @@ Acked-by: Michal Suchanek <msuchanek@suse.de>
#ifdef CONFIG_SMP
void radix__flush_tlb_mm(struct mm_struct *mm)
{
-@@ -321,9 +347,12 @@ void radix__flush_tlb_mm(struct mm_struc
+@@ -321,9 +347,12 @@ void radix__flush_tlb_mm(struct mm_struct *mm)
return;
preempt_disable();
@@ -87,8 +89,8 @@ Acked-by: Michal Suchanek <msuchanek@suse.de>
_tlbiel_pid(pid, RIC_FLUSH_TLB);
preempt_enable();
}
-@@ -427,10 +456,14 @@ void radix__flush_tlb_range(struct vm_ar
- full = (end == TLB_FLUSH_ALL || nr_pages > tlb_single_page_flush_ceiling);
+@@ -435,10 +464,14 @@ void radix__flush_tlb_range(struct vm_area_struct *vma, unsigned long start,
+ }
if (full) {
- if (local)
@@ -105,8 +107,8 @@ Acked-by: Michal Suchanek <msuchanek@suse.de>
} else {
bool hflush = false;
unsigned long hstart, hend;
-@@ -533,6 +566,9 @@ static inline void __radix__flush_tlb_ra
- full = (end == TLB_FLUSH_ALL || nr_pages > tlb_single_page_flush_ceiling);
+@@ -548,6 +581,9 @@ static inline void __radix__flush_tlb_range_psize(struct mm_struct *mm,
+ }
if (full) {
+ if (!local && mm_needs_flush_escalation(mm))
@@ -115,3 +117,6 @@ Acked-by: Michal Suchanek <msuchanek@suse.de>
if (local)
_tlbiel_pid(pid, also_pwc ? RIC_FLUSH_ALL : RIC_FLUSH_TLB);
else
+--
+2.23.0
+
diff --git a/series.conf b/series.conf
index 7e049d894d..45d948f36d 100644
--- a/series.conf
+++ b/series.conf
@@ -10195,6 +10195,7 @@
patches.suse/powerpc-64s-radix-Optimize-TLB-range-flush-barriers.patch
patches.suse/powerpc-64s-radix-Implement-_tlbie-l-_va_range-flush.patch
patches.suse/powerpc-64s-radix-Optimize-flush_tlb_range.patch
+ patches.suse/powerpc-64s-radix-Introduce-local-single-page-ceilin.patch
patches.suse/powerpc-64s-radix-Improve-TLB-flushing-for-page-tabl.patch
patches.suse/powerpc-mm-radix-Fix-crashes-on-Power9-DD1-with-radix.patch
patches.suse/powerpc-kprobes-Disable-preemption-before-invoking-p.patch