Skip to content

Commit c33c794

Browse files
Ryan Robertsakpm00
Ryan Roberts
authored and
akpm00
committed
mm: ptep_get() conversion
Convert all instances of direct pte_t* dereferencing to instead use ptep_get() helper. This means that by default, the accesses change from a C dereference to a READ_ONCE(). This is technically the correct thing to do since where pgtables are modified by HW (for access/dirty) they are volatile and therefore we should always ensure READ_ONCE() semantics. But more importantly, by always using the helper, it can be overridden by the architecture to fully encapsulate the contents of the pte. Arch code is deliberately not converted, as the arch code knows best. It is intended that arch code (arm64) will override the default with its own implementation that can (e.g.) hide certain bits from the core code, or determine young/dirty status by mixing in state from another source. Conversion was done using Coccinelle: ---- // $ make coccicheck \ // COCCI=ptepget.cocci \ // SPFLAGS="--include-headers" \ // MODE=patch virtual patch @ depends on patch @ pte_t *v; @@ - *v + ptep_get(v) ---- Then reviewed and hand-edited to avoid multiple unnecessary calls to ptep_get(), instead opting to store the result of a single call in a variable, where it is correct to do so. This aims to negate any cost of READ_ONCE() and will benefit arch-overrides that may be more complex. Included is a fix for an issue in an earlier version of this patch that was pointed out by kernel test robot. The issue arose because config MMU=n elides definition of the ptep helper functions, including ptep_get(). HUGETLB_PAGE=n configs still define a simple huge_ptep_clear_flush() for linking purposes, which dereferences the ptep. So when both configs are disabled, this caused a build error because ptep_get() is not defined. Fix by continuing to do a direct dereference when MMU=n. This is safe because for this config the arch code cannot be trying to virtualize the ptes because none of the ptep helpers are defined. Link: https://lkml.kernel.org/r/[email protected] Reported-by: kernel test robot <[email protected]> Link: https://lore.kernel.org/oe-kbuild-all/[email protected]/ Signed-off-by: Ryan Roberts <[email protected]> Cc: Adrian Hunter <[email protected]> Cc: Alexander Potapenko <[email protected]> Cc: Alexander Shishkin <[email protected]> Cc: Alex Williamson <[email protected]> Cc: Al Viro <[email protected]> Cc: Andrey Konovalov <[email protected]> Cc: Andrey Ryabinin <[email protected]> Cc: Christian Brauner <[email protected]> Cc: Christoph Hellwig <[email protected]> Cc: Daniel Vetter <[email protected]> Cc: Dave Airlie <[email protected]> Cc: Dimitri Sivanich <[email protected]> Cc: Dmitry Vyukov <[email protected]> Cc: Ian Rogers <[email protected]> Cc: Jason Gunthorpe <[email protected]> Cc: Jérôme Glisse <[email protected]> Cc: Jiri Olsa <[email protected]> Cc: Johannes Weiner <[email protected]> Cc: Kirill A. Shutemov <[email protected]> Cc: Lorenzo Stoakes <[email protected]> Cc: Mark Rutland <[email protected]> Cc: Matthew Wilcox <[email protected]> Cc: Miaohe Lin <[email protected]> Cc: Michal Hocko <[email protected]> Cc: Mike Kravetz <[email protected]> Cc: Mike Rapoport (IBM) <[email protected]> Cc: Muchun Song <[email protected]> Cc: Namhyung Kim <[email protected]> Cc: Naoya Horiguchi <[email protected]> Cc: Oleksandr Tyshchenko <[email protected]> Cc: Pavel Tatashin <[email protected]> Cc: Roman Gushchin <[email protected]> Cc: SeongJae Park <[email protected]> Cc: Shakeel Butt <[email protected]> Cc: Uladzislau Rezki (Sony) <[email protected]> Cc: Vincenzo Frascino <[email protected]> Cc: Yu Zhao <[email protected]> Signed-off-by: Andrew Morton <[email protected]>
1 parent 6c1d2a0 commit c33c794

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

47 files changed

+301
-228
lines changed

‎drivers/gpu/drm/i915/gem/selftests/i915_gem_mman.c

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1681,7 +1681,9 @@ static int igt_mmap_gpu(void *arg)
16811681

16821682
static int check_present_pte(pte_t *pte, unsigned long addr, void *data)
16831683
{
1684-
if (!pte_present(*pte) || pte_none(*pte)) {
1684+
pte_t ptent = ptep_get(pte);
1685+
1686+
if (!pte_present(ptent) || pte_none(ptent)) {
16851687
pr_err("missing PTE:%lx\n",
16861688
(addr - (unsigned long)data) >> PAGE_SHIFT);
16871689
return -EINVAL;
@@ -1692,7 +1694,9 @@ static int check_present_pte(pte_t *pte, unsigned long addr, void *data)
16921694

16931695
static int check_absent_pte(pte_t *pte, unsigned long addr, void *data)
16941696
{
1695-
if (pte_present(*pte) && !pte_none(*pte)) {
1697+
pte_t ptent = ptep_get(pte);
1698+
1699+
if (pte_present(ptent) && !pte_none(ptent)) {
16961700
pr_err("present PTE:%lx; expected to be revoked\n",
16971701
(addr - (unsigned long)data) >> PAGE_SHIFT);
16981702
return -EINVAL;

‎drivers/misc/sgi-gru/grufault.c

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -228,7 +228,7 @@ static int atomic_pte_lookup(struct vm_area_struct *vma, unsigned long vaddr,
228228
goto err;
229229
#ifdef CONFIG_X86_64
230230
if (unlikely(pmd_large(*pmdp)))
231-
pte = *(pte_t *) pmdp;
231+
pte = ptep_get((pte_t *)pmdp);
232232
else
233233
#endif
234234
pte = *pte_offset_kernel(pmdp, vaddr);

‎drivers/vfio/vfio_iommu_type1.c

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -514,6 +514,7 @@ static int follow_fault_pfn(struct vm_area_struct *vma, struct mm_struct *mm,
514514
bool write_fault)
515515
{
516516
pte_t *ptep;
517+
pte_t pte;
517518
spinlock_t *ptl;
518519
int ret;
519520

@@ -536,10 +537,12 @@ static int follow_fault_pfn(struct vm_area_struct *vma, struct mm_struct *mm,
536537
return ret;
537538
}
538539

539-
if (write_fault && !pte_write(*ptep))
540+
pte = ptep_get(ptep);
541+
542+
if (write_fault && !pte_write(pte))
540543
ret = -EFAULT;
541544
else
542-
*pfn = pte_pfn(*ptep);
545+
*pfn = pte_pfn(pte);
543546

544547
pte_unmap_unlock(ptep, ptl);
545548
return ret;

‎drivers/xen/privcmd.c

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -949,7 +949,7 @@ static int privcmd_mmap(struct file *file, struct vm_area_struct *vma)
949949
*/
950950
static int is_mapped_fn(pte_t *pte, unsigned long addr, void *data)
951951
{
952-
return pte_none(*pte) ? 0 : -EBUSY;
952+
return pte_none(ptep_get(pte)) ? 0 : -EBUSY;
953953
}
954954

955955
static int privcmd_vma_range_is_mapped(

‎fs/proc/task_mmu.c

Lines changed: 18 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -538,13 +538,14 @@ static void smaps_pte_entry(pte_t *pte, unsigned long addr,
538538
bool locked = !!(vma->vm_flags & VM_LOCKED);
539539
struct page *page = NULL;
540540
bool migration = false, young = false, dirty = false;
541+
pte_t ptent = ptep_get(pte);
541542

542-
if (pte_present(*pte)) {
543-
page = vm_normal_page(vma, addr, *pte);
544-
young = pte_young(*pte);
545-
dirty = pte_dirty(*pte);
546-
} else if (is_swap_pte(*pte)) {
547-
swp_entry_t swpent = pte_to_swp_entry(*pte);
543+
if (pte_present(ptent)) {
544+
page = vm_normal_page(vma, addr, ptent);
545+
young = pte_young(ptent);
546+
dirty = pte_dirty(ptent);
547+
} else if (is_swap_pte(ptent)) {
548+
swp_entry_t swpent = pte_to_swp_entry(ptent);
548549

549550
if (!non_swap_entry(swpent)) {
550551
int mapcount;
@@ -732,11 +733,12 @@ static int smaps_hugetlb_range(pte_t *pte, unsigned long hmask,
732733
struct mem_size_stats *mss = walk->private;
733734
struct vm_area_struct *vma = walk->vma;
734735
struct page *page = NULL;
736+
pte_t ptent = ptep_get(pte);
735737

736-
if (pte_present(*pte)) {
737-
page = vm_normal_page(vma, addr, *pte);
738-
} else if (is_swap_pte(*pte)) {
739-
swp_entry_t swpent = pte_to_swp_entry(*pte);
738+
if (pte_present(ptent)) {
739+
page = vm_normal_page(vma, addr, ptent);
740+
} else if (is_swap_pte(ptent)) {
741+
swp_entry_t swpent = pte_to_swp_entry(ptent);
740742

741743
if (is_pfn_swap_entry(swpent))
742744
page = pfn_swap_entry_to_page(swpent);
@@ -1105,7 +1107,7 @@ static inline void clear_soft_dirty(struct vm_area_struct *vma,
11051107
* Documentation/admin-guide/mm/soft-dirty.rst for full description
11061108
* of how soft-dirty works.
11071109
*/
1108-
pte_t ptent = *pte;
1110+
pte_t ptent = ptep_get(pte);
11091111

11101112
if (pte_present(ptent)) {
11111113
pte_t old_pte;
@@ -1194,7 +1196,7 @@ static int clear_refs_pte_range(pmd_t *pmd, unsigned long addr,
11941196
return 0;
11951197
}
11961198
for (; addr != end; pte++, addr += PAGE_SIZE) {
1197-
ptent = *pte;
1199+
ptent = ptep_get(pte);
11981200

11991201
if (cp->type == CLEAR_REFS_SOFT_DIRTY) {
12001202
clear_soft_dirty(vma, addr, pte);
@@ -1550,7 +1552,7 @@ static int pagemap_pmd_range(pmd_t *pmdp, unsigned long addr, unsigned long end,
15501552
for (; addr < end; pte++, addr += PAGE_SIZE) {
15511553
pagemap_entry_t pme;
15521554

1553-
pme = pte_to_pagemap_entry(pm, vma, addr, *pte);
1555+
pme = pte_to_pagemap_entry(pm, vma, addr, ptep_get(pte));
15541556
err = add_to_pagemap(addr, &pme, pm);
15551557
if (err)
15561558
break;
@@ -1893,10 +1895,11 @@ static int gather_pte_stats(pmd_t *pmd, unsigned long addr,
18931895
return 0;
18941896
}
18951897
do {
1896-
struct page *page = can_gather_numa_stats(*pte, vma, addr);
1898+
pte_t ptent = ptep_get(pte);
1899+
struct page *page = can_gather_numa_stats(ptent, vma, addr);
18971900
if (!page)
18981901
continue;
1899-
gather_stats(page, md, pte_dirty(*pte), 1);
1902+
gather_stats(page, md, pte_dirty(ptent), 1);
19001903

19011904
} while (pte++, addr += PAGE_SIZE, addr != end);
19021905
pte_unmap_unlock(orig_pte, ptl);

‎fs/userfaultfd.c

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -335,6 +335,7 @@ static inline bool userfaultfd_must_wait(struct userfaultfd_ctx *ctx,
335335
pud_t *pud;
336336
pmd_t *pmd, _pmd;
337337
pte_t *pte;
338+
pte_t ptent;
338339
bool ret = true;
339340

340341
mmap_assert_locked(mm);
@@ -374,9 +375,10 @@ static inline bool userfaultfd_must_wait(struct userfaultfd_ctx *ctx,
374375
* changes under us. PTE markers should be handled the same as none
375376
* ptes here.
376377
*/
377-
if (pte_none_mostly(*pte))
378+
ptent = ptep_get(pte);
379+
if (pte_none_mostly(ptent))
378380
ret = true;
379-
if (!pte_write(*pte) && (reason & VM_UFFD_WP))
381+
if (!pte_write(ptent) && (reason & VM_UFFD_WP))
380382
ret = true;
381383
pte_unmap(pte);
382384

‎include/linux/hugetlb.h

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1185,7 +1185,11 @@ static inline void hugetlb_count_sub(long l, struct mm_struct *mm)
11851185
static inline pte_t huge_ptep_clear_flush(struct vm_area_struct *vma,
11861186
unsigned long addr, pte_t *ptep)
11871187
{
1188+
#ifdef CONFIG_MMU
1189+
return ptep_get(ptep);
1190+
#else
11881191
return *ptep;
1192+
#endif
11891193
}
11901194

11911195
static inline void set_huge_pte_at(struct mm_struct *mm, unsigned long addr,

‎include/linux/mm_inline.h

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -555,7 +555,7 @@ pte_install_uffd_wp_if_needed(struct vm_area_struct *vma, unsigned long addr,
555555
bool arm_uffd_pte = false;
556556

557557
/* The current status of the pte should be "cleared" before calling */
558-
WARN_ON_ONCE(!pte_none(*pte));
558+
WARN_ON_ONCE(!pte_none(ptep_get(pte)));
559559

560560
/*
561561
* NOTE: userfaultfd_wp_unpopulated() doesn't need this whole

‎include/linux/pgtable.h

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -231,7 +231,7 @@ static inline int ptep_test_and_clear_young(struct vm_area_struct *vma,
231231
unsigned long address,
232232
pte_t *ptep)
233233
{
234-
pte_t pte = *ptep;
234+
pte_t pte = ptep_get(ptep);
235235
int r = 1;
236236
if (!pte_young(pte))
237237
r = 0;
@@ -318,7 +318,7 @@ static inline pte_t ptep_get_and_clear(struct mm_struct *mm,
318318
unsigned long address,
319319
pte_t *ptep)
320320
{
321-
pte_t pte = *ptep;
321+
pte_t pte = ptep_get(ptep);
322322
pte_clear(mm, address, ptep);
323323
page_table_check_pte_clear(mm, address, pte);
324324
return pte;
@@ -519,7 +519,7 @@ extern pud_t pudp_huge_clear_flush(struct vm_area_struct *vma,
519519
struct mm_struct;
520520
static inline void ptep_set_wrprotect(struct mm_struct *mm, unsigned long address, pte_t *ptep)
521521
{
522-
pte_t old_pte = *ptep;
522+
pte_t old_pte = ptep_get(ptep);
523523
set_pte_at(mm, address, ptep, pte_wrprotect(old_pte));
524524
}
525525
#endif

‎kernel/events/uprobes.c

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -192,7 +192,7 @@ static int __replace_page(struct vm_area_struct *vma, unsigned long addr,
192192
inc_mm_counter(mm, MM_ANONPAGES);
193193
}
194194

195-
flush_cache_page(vma, addr, pte_pfn(*pvmw.pte));
195+
flush_cache_page(vma, addr, pte_pfn(ptep_get(pvmw.pte)));
196196
ptep_clear_flush_notify(vma, addr, pvmw.pte);
197197
if (new_page)
198198
set_pte_at_notify(mm, addr, pvmw.pte,

‎mm/damon/ops-common.c

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -39,7 +39,7 @@ struct folio *damon_get_folio(unsigned long pfn)
3939

4040
void damon_ptep_mkold(pte_t *pte, struct vm_area_struct *vma, unsigned long addr)
4141
{
42-
struct folio *folio = damon_get_folio(pte_pfn(*pte));
42+
struct folio *folio = damon_get_folio(pte_pfn(ptep_get(pte)));
4343

4444
if (!folio)
4545
return;

‎mm/damon/paddr.c

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -89,7 +89,7 @@ static bool __damon_pa_young(struct folio *folio, struct vm_area_struct *vma,
8989
while (page_vma_mapped_walk(&pvmw)) {
9090
addr = pvmw.address;
9191
if (pvmw.pte) {
92-
*accessed = pte_young(*pvmw.pte) ||
92+
*accessed = pte_young(ptep_get(pvmw.pte)) ||
9393
!folio_test_idle(folio) ||
9494
mmu_notifier_test_young(vma->vm_mm, addr);
9595
} else {

‎mm/damon/vaddr.c

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -323,7 +323,7 @@ static int damon_mkold_pmd_entry(pmd_t *pmd, unsigned long addr,
323323
walk->action = ACTION_AGAIN;
324324
return 0;
325325
}
326-
if (!pte_present(*pte))
326+
if (!pte_present(ptep_get(pte)))
327327
goto out;
328328
damon_ptep_mkold(pte, walk->vma, addr);
329329
out:
@@ -433,6 +433,7 @@ static int damon_young_pmd_entry(pmd_t *pmd, unsigned long addr,
433433
unsigned long next, struct mm_walk *walk)
434434
{
435435
pte_t *pte;
436+
pte_t ptent;
436437
spinlock_t *ptl;
437438
struct folio *folio;
438439
struct damon_young_walk_private *priv = walk->private;
@@ -471,12 +472,13 @@ static int damon_young_pmd_entry(pmd_t *pmd, unsigned long addr,
471472
walk->action = ACTION_AGAIN;
472473
return 0;
473474
}
474-
if (!pte_present(*pte))
475+
ptent = ptep_get(pte);
476+
if (!pte_present(ptent))
475477
goto out;
476-
folio = damon_get_folio(pte_pfn(*pte));
478+
folio = damon_get_folio(pte_pfn(ptent));
477479
if (!folio)
478480
goto out;
479-
if (pte_young(*pte) || !folio_test_idle(folio) ||
481+
if (pte_young(ptent) || !folio_test_idle(folio) ||
480482
mmu_notifier_test_young(walk->mm, addr))
481483
priv->young = true;
482484
*priv->folio_sz = folio_size(folio);

‎mm/filemap.c

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3523,7 +3523,7 @@ vm_fault_t filemap_map_pages(struct vm_fault *vmf,
35233523
* handled in the specific fault path, and it'll prohibit the
35243524
* fault-around logic.
35253525
*/
3526-
if (!pte_none(*vmf->pte))
3526+
if (!pte_none(ptep_get(vmf->pte)))
35273527
goto unlock;
35283528

35293529
/* We're about to handle the fault */

‎mm/gup.c

Lines changed: 12 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -477,13 +477,14 @@ static int follow_pfn_pte(struct vm_area_struct *vma, unsigned long address,
477477
pte_t *pte, unsigned int flags)
478478
{
479479
if (flags & FOLL_TOUCH) {
480-
pte_t entry = *pte;
480+
pte_t orig_entry = ptep_get(pte);
481+
pte_t entry = orig_entry;
481482

482483
if (flags & FOLL_WRITE)
483484
entry = pte_mkdirty(entry);
484485
entry = pte_mkyoung(entry);
485486

486-
if (!pte_same(*pte, entry)) {
487+
if (!pte_same(orig_entry, entry)) {
487488
set_pte_at(vma->vm_mm, address, pte, entry);
488489
update_mmu_cache(vma, address, pte);
489490
}
@@ -549,7 +550,7 @@ static struct page *follow_page_pte(struct vm_area_struct *vma,
549550
ptep = pte_offset_map_lock(mm, pmd, address, &ptl);
550551
if (!ptep)
551552
return no_page_table(vma, flags);
552-
pte = *ptep;
553+
pte = ptep_get(ptep);
553554
if (!pte_present(pte))
554555
goto no_page;
555556
if (pte_protnone(pte) && !gup_can_follow_protnone(flags))
@@ -821,6 +822,7 @@ static int get_gate_page(struct mm_struct *mm, unsigned long address,
821822
pud_t *pud;
822823
pmd_t *pmd;
823824
pte_t *pte;
825+
pte_t entry;
824826
int ret = -EFAULT;
825827

826828
/* user gate pages are read-only */
@@ -844,16 +846,17 @@ static int get_gate_page(struct mm_struct *mm, unsigned long address,
844846
pte = pte_offset_map(pmd, address);
845847
if (!pte)
846848
return -EFAULT;
847-
if (pte_none(*pte))
849+
entry = ptep_get(pte);
850+
if (pte_none(entry))
848851
goto unmap;
849852
*vma = get_gate_vma(mm);
850853
if (!page)
851854
goto out;
852-
*page = vm_normal_page(*vma, address, *pte);
855+
*page = vm_normal_page(*vma, address, entry);
853856
if (!*page) {
854-
if ((gup_flags & FOLL_DUMP) || !is_zero_pfn(pte_pfn(*pte)))
857+
if ((gup_flags & FOLL_DUMP) || !is_zero_pfn(pte_pfn(entry)))
855858
goto unmap;
856-
*page = pte_page(*pte);
859+
*page = pte_page(entry);
857860
}
858861
ret = try_grab_page(*page, gup_flags);
859862
if (unlikely(ret))
@@ -2496,7 +2499,7 @@ static int gup_pte_range(pmd_t pmd, pmd_t *pmdp, unsigned long addr,
24962499
}
24972500

24982501
if (unlikely(pmd_val(pmd) != pmd_val(*pmdp)) ||
2499-
unlikely(pte_val(pte) != pte_val(*ptep))) {
2502+
unlikely(pte_val(pte) != pte_val(ptep_get(ptep)))) {
25002503
gup_put_folio(folio, 1, flags);
25012504
goto pte_unmap;
25022505
}
@@ -2693,7 +2696,7 @@ static int gup_hugepte(pte_t *ptep, unsigned long sz, unsigned long addr,
26932696
if (!folio)
26942697
return 0;
26952698

2696-
if (unlikely(pte_val(pte) != pte_val(*ptep))) {
2699+
if (unlikely(pte_val(pte) != pte_val(ptep_get(ptep)))) {
26972700
gup_put_folio(folio, refs, flags);
26982701
return 0;
26992702
}

0 commit comments

Comments
 (0)