Home Home > GIT Browse
summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorTakashi Iwai <tiwai@suse.de>2019-07-30 12:51:48 +0200
committerTakashi Iwai <tiwai@suse.de>2019-07-30 12:51:48 +0200
commit8125d45016433d7b444d40d0a33627088e4cd6c1 (patch)
tree5ba40d5ce4abf1b5a3b3408a6d2135ba7290c822
parent99a791b675d990975c696b05e8e47dcd859bdf0b (diff)
parent66c4424fc6a0b068bd729506a07f46d82ca38049 (diff)
Merge branch 'SLE15' into users/tiwai/SLE15/bsc1139358
-rw-r--r--arch/x86/mm/gup.c27
1 files changed, 25 insertions, 2 deletions
diff --git a/arch/x86/mm/gup.c b/arch/x86/mm/gup.c
index 7eeffee21ace..d658dbd6113f 100644
--- a/arch/x86/mm/gup.c
+++ b/arch/x86/mm/gup.c
@@ -98,6 +98,20 @@ static inline int pte_allows_gup(unsigned long pteval, int write)
}
/*
+ * Return the compund head page with ref appropriately incremented,
+ * or NULL if that failed.
+ */
+static inline struct page *try_get_compound_head(struct page *page, int refs)
+{
+ struct page *head = compound_head(page);
+ if (WARN_ON_ONCE(page_ref_count(head) < 0))
+ return NULL;
+ if (unlikely(!page_cache_add_speculative(head, refs)))
+ return NULL;
+ return head;
+}
+
+/*
* The performance critical leaf functions are made noinline otherwise gcc
* inlines everything into a single function which results in too much
* register pressure.
@@ -117,7 +131,7 @@ static noinline int gup_pte_range(pmd_t pmd, unsigned long addr,
ptem = ptep = pte_offset_map(&pmd, addr);
do {
pte_t pte = gup_get_pte(ptep);
- struct page *page;
+ struct page *head, *page;
/* Similar to the PMD case, NUMA hinting must take slow path */
if (pte_protnone(pte))
@@ -137,10 +151,19 @@ static noinline int gup_pte_range(pmd_t pmd, unsigned long addr,
VM_BUG_ON(!pfn_valid(pte_pfn(pte)));
page = pte_page(pte);
- if (unlikely(!try_get_page(page))) {
+
+ head = try_get_compound_head(page, 1);
+ if (!head) {
+ put_dev_pagemap(pgmap);
+ break;
+ }
+
+ if (unlikely(pte_val(pte) != pte_val(*ptep))) {
+ put_page(head);
put_dev_pagemap(pgmap);
break;
}
+
put_dev_pagemap(pgmap);
SetPageReferenced(page);
pages[*nr] = page;