memory/SlicePool: add option populate

Author: MaxKellermann

src/memory/SliceArea.hxx

@@ -55,6 +55,8 @@ public:
 	}
 
 	void ForkCow(bool inherit) noexcept;
+	void Populate() noexcept;
+	void CollapseHugePages() noexcept;
 
 	bool IsEmpty() const noexcept {
 		return allocated_count == 0;

src/memory/SlicePool.cxx

@@ -273,11 +273,33 @@ SlicePool::ForkCow(bool inherit) noexcept
 		area.ForkCow(fork_cow);
 }
 
+inline void
+SliceArea::Populate() noexcept
+{
+	PagesPopulateWrite(this, pool.area_size);
+}
+
+inline void
+SliceArea::CollapseHugePages() noexcept
+{
+	::CollapseHugePages(this, pool.area_size);
+}
+
+void
+SlicePool::Populate() noexcept
+{
+	populate = true;
+
+	if (areas.empty() && empty_areas.empty() && full_areas.empty())
+		CreateArea().CollapseHugePages();
+}
+
 void
 SlicePool::Compress() noexcept
 {
-	for (auto &area : areas)
-		area.Compress();
+	if (!populate)
+		for (auto &area : areas)
+			area.Compress();
 
 	empty_areas.clear_and_dispose(SliceArea::Disposer());
 
@@ -302,6 +324,8 @@ SlicePool::CreateArea() noexcept
 {
 	auto *area = SliceArea::New(*this);
 	area->ForkCow(fork_cow);
+	if (populate)
+		area->Populate();
 	empty_areas.push_front(*area);
 	return *area;
 }

src/memory/SlicePool.hxx

@@ -61,6 +61,8 @@ class SlicePool {
 
 	bool fork_cow = true;
 
+	bool populate = false;
+
 public:
 	SlicePool(std::size_t _slice_size, unsigned _slices_per_area,
 		  const char *_vma_name) noexcept;
@@ -76,6 +78,13 @@ public:
 	 */
 	void ForkCow(bool inherit) noexcept;
 
+	/**
+	 * Always been at least one area completely populated (using
+	 * MADV_POPULATE_WRITE).  This reduces waits for Linux kernel
+	 * VM compaction/migration.
+	 */
+	void Populate() noexcept;
+
 	void AddStats(AllocatorStats &stats, const AreaList &list) const noexcept;
 
 	[[gnu::pure]]