Skip to content
Snippets Groups Projects
page_alloc.c 174 KiB
Newer Older
  • Learn to ignore specific revisions
  • 		.zone = page_zone(pfn_to_page(start)),
    		.sync = true,
    		.ignore_skip_hint = true,
    	};
    	INIT_LIST_HEAD(&cc.migratepages);
    
    
    	/*
    	 * What we do here is we mark all pageblocks in range as
    	 * MIGRATE_ISOLATE.  Because pageblock and max order pages may
    	 * have different sizes, and due to the way page allocator
    	 * work, we align the range to biggest of the two pages so
    	 * that page allocator won't try to merge buddies from
    	 * different pageblocks and change MIGRATE_ISOLATE to some
    	 * other migration type.
    	 *
    	 * Once the pageblocks are marked as MIGRATE_ISOLATE, we
    	 * migrate the pages from an unaligned range (ie. pages that
    	 * we are interested in).  This will put all the pages in
    	 * range back to page allocator as MIGRATE_ISOLATE.
    	 *
    	 * When this is done, we take the pages in range from page
    	 * allocator removing them from the buddy system.  This way
    	 * page allocator will never consider using them.
    	 *
    	 * This lets us mark the pageblocks back as
    	 * MIGRATE_CMA/MIGRATE_MOVABLE so that free pages in the
    	 * aligned range but not in the unaligned, original range are
    	 * put back to page allocator so that buddy can use them.
    	 */
    
    	ret = start_isolate_page_range(pfn_max_align_down(start),
    
    				       pfn_max_align_up(end), migratetype,
    				       false);
    
    	ret = __alloc_contig_migrate_range(&cc, start, end);
    
    	if (ret)
    		goto done;
    
    	/*
    	 * Pages from [start, end) are within a MAX_ORDER_NR_PAGES
    	 * aligned blocks that are marked as MIGRATE_ISOLATE.  What's
    	 * more, all pages in [start, end) are free in page allocator.
    	 * What we are going to do is to allocate all pages from
    	 * [start, end) (that is remove them from page allocator).
    	 *
    	 * The only problem is that pages at the beginning and at the
    	 * end of interesting range may be not aligned with pages that
    	 * page allocator holds, ie. they can be part of higher order
    	 * pages.  Because of this, we reserve the bigger range and
    	 * once this is done free the pages we are not interested in.
    	 *
    	 * We don't have to hold zone->lock here because the pages are
    	 * isolated thus they won't get removed from buddy.
    	 */
    
    	lru_add_drain_all();
    	drain_all_pages();
    
    	order = 0;
    	outer_start = start;
    	while (!PageBuddy(pfn_to_page(outer_start))) {
    		if (++order >= MAX_ORDER) {
    			ret = -EBUSY;
    			goto done;
    		}
    		outer_start &= ~0UL << order;
    	}
    
    	/* Make sure the range is really isolated. */
    
    	if (test_pages_isolated(outer_start, end, false)) {
    
    		pr_warn("alloc_contig_range test_pages_isolated(%lx, %lx) failed\n",
    		       outer_start, end);
    		ret = -EBUSY;
    		goto done;
    	}
    
    
    	outer_end = isolate_freepages_range(&cc, outer_start, end);
    
    	if (!outer_end) {
    		ret = -EBUSY;
    		goto done;
    	}
    
    	/* Free head and tail (if any) */
    	if (start != outer_start)
    		free_contig_range(outer_start, start - outer_start);
    	if (end != outer_end)
    		free_contig_range(end, outer_end - end);
    
    done:
    	undo_isolate_page_range(pfn_max_align_down(start),
    
    				pfn_max_align_up(end), migratetype);
    
    	return ret;
    }
    
    void free_contig_range(unsigned long pfn, unsigned nr_pages)
    {
    
    	unsigned int count = 0;
    
    	for (; nr_pages--; pfn++) {
    		struct page *page = pfn_to_page(pfn);
    
    		count += page_count(page) != 1;
    		__free_page(page);
    	}
    	WARN(count != 0, "%d pages are still in use!\n", count);
    
    #ifdef CONFIG_MEMORY_HOTPLUG
    
    /*
     * The zone indicated has a new number of managed_pages; batch sizes and percpu
     * page high values need to be recalulated.
     */
    
    void __meminit zone_pcp_update(struct zone *zone)
    {
    
    	mutex_lock(&pcp_batch_high_lock);
    
    		pageset_set_high_and_batch(zone,
    				per_cpu_ptr(zone->pageset, cpu));
    
    	mutex_unlock(&pcp_batch_high_lock);
    
    void zone_pcp_reset(struct zone *zone)
    {
    	unsigned long flags;
    
    	int cpu;
    	struct per_cpu_pageset *pset;
    
    
    	/* avoid races with drain_pages()  */
    	local_irq_save(flags);
    	if (zone->pageset != &boot_pageset) {
    
    		for_each_online_cpu(cpu) {
    			pset = per_cpu_ptr(zone->pageset, cpu);
    			drain_zonestat(zone, pset);
    		}
    
    		free_percpu(zone->pageset);
    		zone->pageset = &boot_pageset;
    	}
    	local_irq_restore(flags);
    }
    
    
    #ifdef CONFIG_MEMORY_HOTREMOVE
    
    /*
     * All pages in the range must be isolated before calling this.
     */
    void
    __offline_isolated_pages(unsigned long start_pfn, unsigned long end_pfn)
    {
    	struct page *page;
    	struct zone *zone;
    	int order, i;
    	unsigned long pfn;
    	unsigned long flags;
    	/* find the first valid pfn */
    	for (pfn = start_pfn; pfn < end_pfn; pfn++)
    		if (pfn_valid(pfn))
    			break;
    	if (pfn == end_pfn)
    		return;
    	zone = page_zone(pfn_to_page(pfn));
    	spin_lock_irqsave(&zone->lock, flags);
    	pfn = start_pfn;
    	while (pfn < end_pfn) {
    		if (!pfn_valid(pfn)) {
    			pfn++;
    			continue;
    		}
    		page = pfn_to_page(pfn);
    
    		/*
    		 * The HWPoisoned page may be not in buddy system, and
    		 * page_count() is not 0.
    		 */
    		if (unlikely(!PageBuddy(page) && PageHWPoison(page))) {
    			pfn++;
    			SetPageReserved(page);
    			continue;
    		}
    
    
    		BUG_ON(page_count(page));
    		BUG_ON(!PageBuddy(page));
    		order = page_order(page);
    #ifdef CONFIG_DEBUG_VM
    		printk(KERN_INFO "remove from free list %lx %d %lx\n",
    		       pfn, 1 << order, end_pfn);
    #endif
    		list_del(&page->lru);
    		rmv_page_order(page);
    		zone->free_area[order].nr_free--;
    
    #ifdef CONFIG_HIGHMEM
    		if (PageHighMem(page))
    			totalhigh_pages -= 1 << order;
    #endif
    
    		for (i = 0; i < (1 << order); i++)
    			SetPageReserved((page+i));
    		pfn += (1 << order);
    	}
    	spin_unlock_irqrestore(&zone->lock, flags);
    }
    #endif
    
    
    #ifdef CONFIG_MEMORY_FAILURE
    bool is_free_buddy_page(struct page *page)
    {
    	struct zone *zone = page_zone(page);
    	unsigned long pfn = page_to_pfn(page);
    	unsigned long flags;
    	int order;
    
    	spin_lock_irqsave(&zone->lock, flags);
    	for (order = 0; order < MAX_ORDER; order++) {
    		struct page *page_head = page - (pfn & ((1 << order) - 1));
    
    		if (PageBuddy(page_head) && page_order(page_head) >= order)
    			break;
    	}
    	spin_unlock_irqrestore(&zone->lock, flags);
    
    	return order < MAX_ORDER;
    }
    #endif
    
    Andrew Morton's avatar
    Andrew Morton committed
    static const struct trace_print_flags pageflag_names[] = {
    
    	{1UL << PG_locked,		"locked"	},
    	{1UL << PG_error,		"error"		},
    	{1UL << PG_referenced,		"referenced"	},
    	{1UL << PG_uptodate,		"uptodate"	},
    	{1UL << PG_dirty,		"dirty"		},
    	{1UL << PG_lru,			"lru"		},
    	{1UL << PG_active,		"active"	},
    	{1UL << PG_slab,		"slab"		},
    	{1UL << PG_owner_priv_1,	"owner_priv_1"	},
    	{1UL << PG_arch_1,		"arch_1"	},
    	{1UL << PG_reserved,		"reserved"	},
    	{1UL << PG_private,		"private"	},
    	{1UL << PG_private_2,		"private_2"	},
    	{1UL << PG_writeback,		"writeback"	},
    #ifdef CONFIG_PAGEFLAGS_EXTENDED
    	{1UL << PG_head,		"head"		},
    	{1UL << PG_tail,		"tail"		},
    #else
    	{1UL << PG_compound,		"compound"	},
    #endif
    	{1UL << PG_swapcache,		"swapcache"	},
    	{1UL << PG_mappedtodisk,	"mappedtodisk"	},
    	{1UL << PG_reclaim,		"reclaim"	},
    	{1UL << PG_swapbacked,		"swapbacked"	},
    	{1UL << PG_unevictable,		"unevictable"	},
    #ifdef CONFIG_MMU
    	{1UL << PG_mlocked,		"mlocked"	},
    #endif
    #ifdef CONFIG_ARCH_USES_PG_UNCACHED
    	{1UL << PG_uncached,		"uncached"	},
    #endif
    #ifdef CONFIG_MEMORY_FAILURE
    	{1UL << PG_hwpoison,		"hwpoison"	},
    
    #endif
    #ifdef CONFIG_TRANSPARENT_HUGEPAGE
    	{1UL << PG_compound_lock,	"compound_lock"	},
    
    #endif
    };
    
    static void dump_page_flags(unsigned long flags)
    {
    	const char *delim = "";
    	unsigned long mask;
    	int i;
    
    
    Andrew Morton's avatar
    Andrew Morton committed
    	BUILD_BUG_ON(ARRAY_SIZE(pageflag_names) != __NR_PAGEFLAGS);
    
    	printk(KERN_ALERT "page flags: %#lx(", flags);
    
    	/* remove zone id */
    	flags &= (1UL << NR_PAGEFLAGS) - 1;
    
    
    Andrew Morton's avatar
    Andrew Morton committed
    	for (i = 0; i < ARRAY_SIZE(pageflag_names) && flags; i++) {
    
    
    		mask = pageflag_names[i].mask;
    		if ((flags & mask) != mask)
    			continue;
    
    		flags &= ~mask;
    		printk("%s%s", delim, pageflag_names[i].name);
    		delim = "|";
    	}
    
    	/* check for left over flags */
    	if (flags)
    		printk("%s%#lx", delim, flags);
    
    	printk(")\n");
    }
    
    void dump_page(struct page *page)
    {
    	printk(KERN_ALERT
    	       "page:%p count:%d mapcount:%d mapping:%p index:%#lx\n",
    
    		page, atomic_read(&page->_count), page_mapcount(page),
    
    		page->mapping, page->index);
    	dump_page_flags(page->flags);
    
    	mem_cgroup_print_bad_page(page);