Skip to content

Commit ff31e10

Browse files
kvaneeshmpe
kvaneesh
authored and
mpe
committed
powerpc/mm/hash64: Store the slot information at the right offset for hugetlb
The hugetlb pte entries are at the PMD and PUD level, so we can't use PTRS_PER_PTE to find the second half of the page table. Use the right offset for PUD/PMD to get to the second half of the table. Fixes: bf9a95f ("powerpc: Free up four 64K PTE bits in 64K backed HPTE pages") Signed-off-by: Aneesh Kumar K.V <[email protected]> Reviewed-by: Ram Pai <[email protected]> Signed-off-by: Michael Ellerman <[email protected]>
1 parent 4a7aa4f commit ff31e10

File tree

7 files changed

+28
-17
lines changed

7 files changed

+28
-17
lines changed

‎arch/powerpc/include/asm/book3s/64/hash-4k.h‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -63,7 +63,8 @@ static inline int hash__hugepd_ok(hugepd_t hpd)
6363
* keeping the prototype consistent across the two formats.
6464
*/
6565
static inline unsigned long pte_set_hidx(pte_t *ptep, real_pte_t rpte,
66-
unsigned int subpg_index, unsigned long hidx)
66+
unsigned int subpg_index, unsigned long hidx,
67+
int offset)
6768
{
6869
return (hidx << H_PAGE_F_GIX_SHIFT) &
6970
(H_PAGE_F_SECOND | H_PAGE_F_GIX);

‎arch/powerpc/include/asm/book3s/64/hash-64k.h‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -45,7 +45,7 @@
4545
* generic accessors and iterators here
4646
*/
4747
#define __real_pte __real_pte
48-
static inline real_pte_t __real_pte(pte_t pte, pte_t *ptep)
48+
static inline real_pte_t __real_pte(pte_t pte, pte_t *ptep, int offset)
4949
{
5050
real_pte_t rpte;
5151
unsigned long *hidxp;
@@ -59,7 +59,7 @@ static inline real_pte_t __real_pte(pte_t pte, pte_t *ptep)
5959
*/
6060
smp_rmb();
6161

62-
hidxp = (unsigned long *)(ptep + PTRS_PER_PTE);
62+
hidxp = (unsigned long *)(ptep + offset);
6363
rpte.hidx = *hidxp;
6464
return rpte;
6565
}
@@ -86,9 +86,10 @@ static inline unsigned long __rpte_to_hidx(real_pte_t rpte, unsigned long index)
8686
* expected to modify the PTE bits accordingly and commit the PTE to memory.
8787
*/
8888
static inline unsigned long pte_set_hidx(pte_t *ptep, real_pte_t rpte,
89-
unsigned int subpg_index, unsigned long hidx)
89+
unsigned int subpg_index,
90+
unsigned long hidx, int offset)
9091
{
91-
unsigned long *hidxp = (unsigned long *)(ptep + PTRS_PER_PTE);
92+
unsigned long *hidxp = (unsigned long *)(ptep + offset);
9293

9394
rpte.hidx &= ~HIDX_BITS(0xfUL, subpg_index);
9495
*hidxp = rpte.hidx | HIDX_BITS(HIDX_SHIFT_BY_ONE(hidx), subpg_index);

‎arch/powerpc/include/asm/book3s/64/pgtable.h‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -350,7 +350,7 @@ extern unsigned long pci_io_base;
350350
*/
351351
#ifndef __real_pte
352352

353-
#define __real_pte(e,p) ((real_pte_t){(e)})
353+
#define __real_pte(e, p, o) ((real_pte_t){(e)})
354354
#define __rpte_to_pte(r) ((r).pte)
355355
#define __rpte_to_hidx(r,index) (pte_val(__rpte_to_pte(r)) >> H_PAGE_F_GIX_SHIFT)
356356

‎arch/powerpc/mm/hash64_4k.c‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -55,7 +55,7 @@ int __hash_page_4K(unsigned long ea, unsigned long access, unsigned long vsid,
5555
* need to add in 0x1 if it's a read-only user page
5656
*/
5757
rflags = htab_convert_pte_flags(new_pte);
58-
rpte = __real_pte(__pte(old_pte), ptep);
58+
rpte = __real_pte(__pte(old_pte), ptep, PTRS_PER_PTE);
5959

6060
if (cpu_has_feature(CPU_FTR_NOEXECUTE) &&
6161
!cpu_has_feature(CPU_FTR_COHERENT_ICACHE))
@@ -117,7 +117,7 @@ int __hash_page_4K(unsigned long ea, unsigned long access, unsigned long vsid,
117117
return -1;
118118
}
119119
new_pte = (new_pte & ~_PAGE_HPTEFLAGS) | H_PAGE_HASHPTE;
120-
new_pte |= pte_set_hidx(ptep, rpte, 0, slot);
120+
new_pte |= pte_set_hidx(ptep, rpte, 0, slot, PTRS_PER_PTE);
121121
}
122122
*ptep = __pte(new_pte & ~H_PAGE_BUSY);
123123
return 0;

‎arch/powerpc/mm/hash64_64k.c‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -86,7 +86,7 @@ int __hash_page_4K(unsigned long ea, unsigned long access, unsigned long vsid,
8686

8787
subpg_index = (ea & (PAGE_SIZE - 1)) >> shift;
8888
vpn = hpt_vpn(ea, vsid, ssize);
89-
rpte = __real_pte(__pte(old_pte), ptep);
89+
rpte = __real_pte(__pte(old_pte), ptep, PTRS_PER_PTE);
9090
/*
9191
*None of the sub 4k page is hashed
9292
*/
@@ -214,7 +214,7 @@ int __hash_page_4K(unsigned long ea, unsigned long access, unsigned long vsid,
214214
return -1;
215215
}
216216

217-
new_pte |= pte_set_hidx(ptep, rpte, subpg_index, slot);
217+
new_pte |= pte_set_hidx(ptep, rpte, subpg_index, slot, PTRS_PER_PTE);
218218
new_pte |= H_PAGE_HASHPTE;
219219

220220
*ptep = __pte(new_pte & ~H_PAGE_BUSY);
@@ -262,7 +262,7 @@ int __hash_page_64K(unsigned long ea, unsigned long access,
262262
} while (!pte_xchg(ptep, __pte(old_pte), __pte(new_pte)));
263263

264264
rflags = htab_convert_pte_flags(new_pte);
265-
rpte = __real_pte(__pte(old_pte), ptep);
265+
rpte = __real_pte(__pte(old_pte), ptep, PTRS_PER_PTE);
266266

267267
if (cpu_has_feature(CPU_FTR_NOEXECUTE) &&
268268
!cpu_has_feature(CPU_FTR_COHERENT_ICACHE))
@@ -327,7 +327,7 @@ int __hash_page_64K(unsigned long ea, unsigned long access,
327327
}
328328

329329
new_pte = (new_pte & ~_PAGE_HPTEFLAGS) | H_PAGE_HASHPTE;
330-
new_pte |= pte_set_hidx(ptep, rpte, 0, slot);
330+
new_pte |= pte_set_hidx(ptep, rpte, 0, slot, PTRS_PER_PTE);
331331
}
332332
*ptep = __pte(new_pte & ~H_PAGE_BUSY);
333333
return 0;

‎arch/powerpc/mm/hugetlbpage-hash64.c‎

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,7 @@ int __hash_page_huge(unsigned long ea, unsigned long access, unsigned long vsid,
2727
unsigned long vpn;
2828
unsigned long old_pte, new_pte;
2929
unsigned long rflags, pa, sz;
30-
long slot;
30+
long slot, offset;
3131

3232
BUG_ON(shift != mmu_psize_defs[mmu_psize].shift);
3333

@@ -63,7 +63,11 @@ int __hash_page_huge(unsigned long ea, unsigned long access, unsigned long vsid,
6363
} while(!pte_xchg(ptep, __pte(old_pte), __pte(new_pte)));
6464

6565
rflags = htab_convert_pte_flags(new_pte);
66-
rpte = __real_pte(__pte(old_pte), ptep);
66+
if (unlikely(mmu_psize == MMU_PAGE_16G))
67+
offset = PTRS_PER_PUD;
68+
else
69+
offset = PTRS_PER_PMD;
70+
rpte = __real_pte(__pte(old_pte), ptep, offset);
6771

6872
sz = ((1UL) << shift);
6973
if (!cpu_has_feature(CPU_FTR_COHERENT_ICACHE))
@@ -104,7 +108,7 @@ int __hash_page_huge(unsigned long ea, unsigned long access, unsigned long vsid,
104108
return -1;
105109
}
106110

107-
new_pte |= pte_set_hidx(ptep, rpte, 0, slot);
111+
new_pte |= pte_set_hidx(ptep, rpte, 0, slot, offset);
108112
}
109113

110114
/*

‎arch/powerpc/mm/tlb_hash64.c‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -51,7 +51,7 @@ void hpte_need_flush(struct mm_struct *mm, unsigned long addr,
5151
unsigned int psize;
5252
int ssize;
5353
real_pte_t rpte;
54-
int i;
54+
int i, offset;
5555

5656
i = batch->index;
5757

@@ -67,6 +67,10 @@ void hpte_need_flush(struct mm_struct *mm, unsigned long addr,
6767
psize = get_slice_psize(mm, addr);
6868
/* Mask the address for the correct page size */
6969
addr &= ~((1UL << mmu_psize_defs[psize].shift) - 1);
70+
if (unlikely(psize == MMU_PAGE_16G))
71+
offset = PTRS_PER_PUD;
72+
else
73+
offset = PTRS_PER_PMD;
7074
#else
7175
BUG();
7276
psize = pte_pagesize_index(mm, addr, pte); /* shutup gcc */
@@ -78,6 +82,7 @@ void hpte_need_flush(struct mm_struct *mm, unsigned long addr,
7882
* support 64k pages, this might be different from the
7983
* hardware page size encoded in the slice table. */
8084
addr &= PAGE_MASK;
85+
offset = PTRS_PER_PTE;
8186
}
8287

8388

@@ -91,7 +96,7 @@ void hpte_need_flush(struct mm_struct *mm, unsigned long addr,
9196
}
9297
WARN_ON(vsid == 0);
9398
vpn = hpt_vpn(addr, vsid, ssize);
94-
rpte = __real_pte(__pte(pte), ptep);
99+
rpte = __real_pte(__pte(pte), ptep, offset);
95100

96101
/*
97102
* Check if we have an active batch on this CPU. If not, just

0 commit comments

Comments
 (0)