Skip to content

Commit

Permalink
mm: hugetlb: replace hugetlb_free_vmemmap_enabled with a static_key
Browse files Browse the repository at this point in the history
The page_fixed_fake_head() is used throughout memory management and the
conditional check requires checking a global variable, although the
overhead of this check may be small, it increases when the memory cache
comes under pressure.  Also, the global variable will not be modified
after system boot, so it is very appropriate to use static key machanism.

Link: https://lkml.kernel.org/r/[email protected]
Signed-off-by: Muchun Song <[email protected]>
Reviewed-by: Barry Song <[email protected]>
Cc: Bodeddula Balasubramaniam <[email protected]>
Cc: Chen Huang <[email protected]>
Cc: David Hildenbrand <[email protected]>
Cc: Fam Zheng <[email protected]>
Cc: Jonathan Corbet <[email protected]>
Cc: Matthew Wilcox <[email protected]>
Cc: Michal Hocko <[email protected]>
Cc: Mike Kravetz <[email protected]>
Cc: Oscar Salvador <[email protected]>
Cc: Qi Zheng <[email protected]>
Cc: Xiongchun Duan <[email protected]>
Signed-off-by: Andrew Morton <[email protected]>
Signed-off-by: Linus Torvalds <[email protected]>
  • Loading branch information
Muchun Song authored and torvalds committed Mar 22, 2022
1 parent e7d3248 commit a6b4085
Show file tree
Hide file tree
Showing 4 changed files with 21 additions and 15 deletions.
6 changes: 0 additions & 6 deletions include/linux/hugetlb.h
Original file line number Diff line number Diff line change
Expand Up @@ -1075,12 +1075,6 @@ static inline void set_huge_swap_pte_at(struct mm_struct *mm, unsigned long addr
}
#endif /* CONFIG_HUGETLB_PAGE */

#ifdef CONFIG_HUGETLB_PAGE_FREE_VMEMMAP
extern bool hugetlb_free_vmemmap_enabled;
#else
#define hugetlb_free_vmemmap_enabled false
#endif

static inline spinlock_t *huge_pte_lock(struct hstate *h,
struct mm_struct *mm, pte_t *pte)
{
Expand Down
16 changes: 14 additions & 2 deletions include/linux/page-flags.h
Original file line number Diff line number Diff line change
Expand Up @@ -191,7 +191,14 @@ enum pageflags {
#ifndef __GENERATING_BOUNDS_H

#ifdef CONFIG_HUGETLB_PAGE_FREE_VMEMMAP
extern bool hugetlb_free_vmemmap_enabled;
DECLARE_STATIC_KEY_MAYBE(CONFIG_HUGETLB_PAGE_FREE_VMEMMAP_DEFAULT_ON,
hugetlb_free_vmemmap_enabled_key);

static __always_inline bool hugetlb_free_vmemmap_enabled(void)
{
return static_branch_maybe(CONFIG_HUGETLB_PAGE_FREE_VMEMMAP_DEFAULT_ON,
&hugetlb_free_vmemmap_enabled_key);
}

/*
* If the feature of freeing some vmemmap pages associated with each HugeTLB
Expand All @@ -211,7 +218,7 @@ extern bool hugetlb_free_vmemmap_enabled;
*/
static __always_inline const struct page *page_fixed_fake_head(const struct page *page)
{
if (!hugetlb_free_vmemmap_enabled)
if (!hugetlb_free_vmemmap_enabled())
return page;

/*
Expand Down Expand Up @@ -239,6 +246,11 @@ static inline const struct page *page_fixed_fake_head(const struct page *page)
{
return page;
}

static inline bool hugetlb_free_vmemmap_enabled(void)
{
return false;
}
#endif

static __always_inline int page_is_fake_head(struct page *page)
Expand Down
12 changes: 6 additions & 6 deletions mm/hugetlb_vmemmap.c
Original file line number Diff line number Diff line change
Expand Up @@ -188,9 +188,9 @@
#define RESERVE_VMEMMAP_NR 1U
#define RESERVE_VMEMMAP_SIZE (RESERVE_VMEMMAP_NR << PAGE_SHIFT)

bool hugetlb_free_vmemmap_enabled __read_mostly =
IS_ENABLED(CONFIG_HUGETLB_PAGE_FREE_VMEMMAP_DEFAULT_ON);
EXPORT_SYMBOL(hugetlb_free_vmemmap_enabled);
DEFINE_STATIC_KEY_MAYBE(CONFIG_HUGETLB_PAGE_FREE_VMEMMAP_DEFAULT_ON,
hugetlb_free_vmemmap_enabled_key);
EXPORT_SYMBOL(hugetlb_free_vmemmap_enabled_key);

static int __init early_hugetlb_free_vmemmap_param(char *buf)
{
Expand All @@ -204,9 +204,9 @@ static int __init early_hugetlb_free_vmemmap_param(char *buf)
return -EINVAL;

if (!strcmp(buf, "on"))
hugetlb_free_vmemmap_enabled = true;
static_branch_enable(&hugetlb_free_vmemmap_enabled_key);
else if (!strcmp(buf, "off"))
hugetlb_free_vmemmap_enabled = false;
static_branch_disable(&hugetlb_free_vmemmap_enabled_key);
else
return -EINVAL;

Expand Down Expand Up @@ -284,7 +284,7 @@ void __init hugetlb_vmemmap_init(struct hstate *h)
BUILD_BUG_ON(__NR_USED_SUBPAGE >=
RESERVE_VMEMMAP_SIZE / sizeof(struct page));

if (!hugetlb_free_vmemmap_enabled)
if (!hugetlb_free_vmemmap_enabled())
return;

vmemmap_pages = (nr_pages * sizeof(struct page)) >> PAGE_SHIFT;
Expand Down
2 changes: 1 addition & 1 deletion mm/memory_hotplug.c
Original file line number Diff line number Diff line change
Expand Up @@ -1327,7 +1327,7 @@ bool mhp_supports_memmap_on_memory(unsigned long size)
* populate a single PMD.
*/
return memmap_on_memory &&
!hugetlb_free_vmemmap_enabled &&
!hugetlb_free_vmemmap_enabled() &&
IS_ENABLED(CONFIG_MHP_MEMMAP_ON_MEMORY) &&
size == memory_block_size_bytes() &&
IS_ALIGNED(vmemmap_size, PMD_SIZE) &&
Expand Down

0 comments on commit a6b4085

Please sign in to comment.