Home Home > GIT Browse
summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorVlastimil Babka <vbabka@suse.cz>2019-07-29 14:12:36 +0200
committerVlastimil Babka <vbabka@suse.cz>2019-07-30 11:15:53 +0200
commit5d887703afce78058c5fff4eb37f8a228153459c (patch)
treeae18bb7007ccc3156281b777fbc27ba12046d5ea
parent841c4f04e610142aec773bb7986cd1eed283526d (diff)
x86, mm: fix fast GUP with hyper-based TLB flushing (VM
Functionality, bsc#1140903). suse-commit: 0d4f36ea2abf95120559eb7574bf8af5a10abe2d
-rw-r--r--arch/x86/mm/gup.c27
1 files changed, 25 insertions, 2 deletions
diff --git a/arch/x86/mm/gup.c b/arch/x86/mm/gup.c
index 7eeffee21ace..d658dbd6113f 100644
--- a/arch/x86/mm/gup.c
+++ b/arch/x86/mm/gup.c
@@ -98,6 +98,20 @@ static inline int pte_allows_gup(unsigned long pteval, int write)
}
/*
+ * Return the compund head page with ref appropriately incremented,
+ * or NULL if that failed.
+ */
+static inline struct page *try_get_compound_head(struct page *page, int refs)
+{
+ struct page *head = compound_head(page);
+ if (WARN_ON_ONCE(page_ref_count(head) < 0))
+ return NULL;
+ if (unlikely(!page_cache_add_speculative(head, refs)))
+ return NULL;
+ return head;
+}
+
+/*
* The performance critical leaf functions are made noinline otherwise gcc
* inlines everything into a single function which results in too much
* register pressure.
@@ -117,7 +131,7 @@ static noinline int gup_pte_range(pmd_t pmd, unsigned long addr,
ptem = ptep = pte_offset_map(&pmd, addr);
do {
pte_t pte = gup_get_pte(ptep);
- struct page *page;
+ struct page *head, *page;
/* Similar to the PMD case, NUMA hinting must take slow path */
if (pte_protnone(pte))
@@ -137,10 +151,19 @@ static noinline int gup_pte_range(pmd_t pmd, unsigned long addr,
VM_BUG_ON(!pfn_valid(pte_pfn(pte)));
page = pte_page(pte);
- if (unlikely(!try_get_page(page))) {
+
+ head = try_get_compound_head(page, 1);
+ if (!head) {
+ put_dev_pagemap(pgmap);
+ break;
+ }
+
+ if (unlikely(pte_val(pte) != pte_val(*ptep))) {
+ put_page(head);
put_dev_pagemap(pgmap);
break;
}
+
put_dev_pagemap(pgmap);
SetPageReferenced(page);
pages[*nr] = page;