diff options
Diffstat (limited to 'target/linux/generic/backport-5.10/630-v5.15-page_pool_frag_support.patch')
-rw-r--r-- | target/linux/generic/backport-5.10/630-v5.15-page_pool_frag_support.patch | 798 |
1 files changed, 0 insertions, 798 deletions
diff --git a/target/linux/generic/backport-5.10/630-v5.15-page_pool_frag_support.patch b/target/linux/generic/backport-5.10/630-v5.15-page_pool_frag_support.patch deleted file mode 100644 index dad4803848..0000000000 --- a/target/linux/generic/backport-5.10/630-v5.15-page_pool_frag_support.patch +++ /dev/null @@ -1,798 +0,0 @@ ---- a/include/net/page_pool.h -+++ b/include/net/page_pool.h -@@ -45,7 +45,10 @@ - * Please note DMA-sync-for-CPU is still - * device driver responsibility - */ --#define PP_FLAG_ALL (PP_FLAG_DMA_MAP | PP_FLAG_DMA_SYNC_DEV) -+#define PP_FLAG_PAGE_FRAG BIT(2) /* for page frag feature */ -+#define PP_FLAG_ALL (PP_FLAG_DMA_MAP |\ -+ PP_FLAG_DMA_SYNC_DEV |\ -+ PP_FLAG_PAGE_FRAG) - - /* - * Fast allocation side cache array/stack -@@ -65,7 +68,7 @@ - #define PP_ALLOC_CACHE_REFILL 64 - struct pp_alloc_cache { - u32 count; -- void *cache[PP_ALLOC_CACHE_SIZE]; -+ struct page *cache[PP_ALLOC_CACHE_SIZE]; - }; - - struct page_pool_params { -@@ -79,6 +82,22 @@ struct page_pool_params { - unsigned int offset; /* DMA addr offset */ - }; - -+ -+static inline int page_pool_ethtool_stats_get_count(void) -+{ -+ return 0; -+} -+ -+static inline u8 *page_pool_ethtool_stats_get_strings(u8 *data) -+{ -+ return data; -+} -+ -+static inline u64 *page_pool_ethtool_stats_get(u64 *data, void *stats) -+{ -+ return data; -+} -+ - struct page_pool { - struct page_pool_params p; - -@@ -88,6 +107,9 @@ struct page_pool { - unsigned long defer_warn; - - u32 pages_state_hold_cnt; -+ unsigned int frag_offset; -+ struct page *frag_page; -+ long frag_users; - - /* - * Data structure for allocation side -@@ -137,6 +159,18 @@ static inline struct page *page_pool_dev - return page_pool_alloc_pages(pool, gfp); - } - -+struct page *page_pool_alloc_frag(struct page_pool *pool, unsigned int *offset, -+ unsigned int size, gfp_t gfp); -+ -+static inline struct page *page_pool_dev_alloc_frag(struct page_pool *pool, -+ unsigned int *offset, -+ unsigned int size) -+{ -+ gfp_t gfp = (GFP_ATOMIC | __GFP_NOWARN); -+ -+ return page_pool_alloc_frag(pool, offset, size, gfp); -+} -+ - /* get the stored dma direction. A driver might decide to treat this locally and - * avoid the extra cache line from page_pool to determine the direction - */ -@@ -146,6 +180,8 @@ inline enum dma_data_direction page_pool - return pool->p.dma_dir; - } - -+bool page_pool_return_skb_page(struct page *page); -+ - struct page_pool *page_pool_create(const struct page_pool_params *params); - - #ifdef CONFIG_PAGE_POOL -@@ -165,6 +201,7 @@ static inline void page_pool_release_pag - struct page *page) - { - } -+ - #endif - - void page_pool_put_page(struct page_pool *pool, struct page *page, -@@ -189,19 +226,48 @@ static inline void page_pool_recycle_dir - page_pool_put_full_page(pool, page, true); - } - -+#define PAGE_POOL_DMA_USE_PP_FRAG_COUNT \ -+ (sizeof(dma_addr_t) > sizeof(unsigned long)) -+ - static inline dma_addr_t page_pool_get_dma_addr(struct page *page) - { -- dma_addr_t ret = page->dma_addr[0]; -- if (sizeof(dma_addr_t) > sizeof(unsigned long)) -- ret |= (dma_addr_t)page->dma_addr[1] << 16 << 16; -+ dma_addr_t ret = page->dma_addr; -+ -+ if (PAGE_POOL_DMA_USE_PP_FRAG_COUNT) -+ ret |= (dma_addr_t)page->dma_addr_upper << 16 << 16; -+ - return ret; - } - - static inline void page_pool_set_dma_addr(struct page *page, dma_addr_t addr) - { -- page->dma_addr[0] = addr; -- if (sizeof(dma_addr_t) > sizeof(unsigned long)) -- page->dma_addr[1] = upper_32_bits(addr); -+ page->dma_addr = addr; -+ if (PAGE_POOL_DMA_USE_PP_FRAG_COUNT) -+ page->dma_addr_upper = upper_32_bits(addr); -+} -+ -+static inline void page_pool_set_frag_count(struct page *page, long nr) -+{ -+ atomic_long_set(&page->pp_frag_count, nr); -+} -+ -+static inline long page_pool_atomic_sub_frag_count_return(struct page *page, -+ long nr) -+{ -+ long ret; -+ -+ /* As suggested by Alexander, atomic_long_read() may cover up the -+ * reference count errors, so avoid calling atomic_long_read() in -+ * the cases of freeing or draining the page_frags, where we would -+ * not expect it to match or that are slowpath anyway. -+ */ -+ if (__builtin_constant_p(nr) && -+ atomic_long_read(&page->pp_frag_count) == nr) -+ return 0; -+ -+ ret = atomic_long_sub_return(nr, &page->pp_frag_count); -+ WARN_ON(ret < 0); -+ return ret; - } - - static inline bool is_page_pool_compiled_in(void) -@@ -225,4 +291,23 @@ static inline void page_pool_nid_changed - if (unlikely(pool->p.nid != new_nid)) - page_pool_update_nid(pool, new_nid); - } -+ -+static inline void page_pool_ring_lock(struct page_pool *pool) -+ __acquires(&pool->ring.producer_lock) -+{ -+ if (in_serving_softirq()) -+ spin_lock(&pool->ring.producer_lock); -+ else -+ spin_lock_bh(&pool->ring.producer_lock); -+} -+ -+static inline void page_pool_ring_unlock(struct page_pool *pool) -+ __releases(&pool->ring.producer_lock) -+{ -+ if (in_serving_softirq()) -+ spin_unlock(&pool->ring.producer_lock); -+ else -+ spin_unlock_bh(&pool->ring.producer_lock); -+} -+ - #endif /* _NET_PAGE_POOL_H */ ---- a/net/core/page_pool.c -+++ b/net/core/page_pool.c -@@ -11,16 +11,22 @@ - #include <linux/device.h> - - #include <net/page_pool.h> -+#include <net/xdp.h> -+ - #include <linux/dma-direction.h> - #include <linux/dma-mapping.h> - #include <linux/page-flags.h> - #include <linux/mm.h> /* for __put_page() */ -+#include <linux/poison.h> -+#include <linux/ethtool.h> - - #include <trace/events/page_pool.h> - - #define DEFER_TIME (msecs_to_jiffies(1000)) - #define DEFER_WARN_INTERVAL (60 * HZ) - -+#define BIAS_MAX LONG_MAX -+ - static int page_pool_init(struct page_pool *pool, - const struct page_pool_params *params) - { -@@ -64,6 +70,10 @@ static int page_pool_init(struct page_po - */ - } - -+ if (PAGE_POOL_DMA_USE_PP_FRAG_COUNT && -+ pool->p.flags & PP_FLAG_PAGE_FRAG) -+ return -EINVAL; -+ - if (ptr_ring_init(&pool->ring, ring_qsize, GFP_KERNEL) < 0) - return -ENOMEM; - -@@ -180,40 +190,10 @@ static void page_pool_dma_sync_for_devic - pool->p.dma_dir); - } - --/* slow path */ --noinline --static struct page *__page_pool_alloc_pages_slow(struct page_pool *pool, -- gfp_t _gfp) -+static bool page_pool_dma_map(struct page_pool *pool, struct page *page) - { -- struct page *page; -- gfp_t gfp = _gfp; - dma_addr_t dma; - -- /* We could always set __GFP_COMP, and avoid this branch, as -- * prep_new_page() can handle order-0 with __GFP_COMP. -- */ -- if (pool->p.order) -- gfp |= __GFP_COMP; -- -- /* FUTURE development: -- * -- * Current slow-path essentially falls back to single page -- * allocations, which doesn't improve performance. This code -- * need bulk allocation support from the page allocator code. -- */ -- -- /* Cache was empty, do real allocation */ --#ifdef CONFIG_NUMA -- page = alloc_pages_node(pool->p.nid, gfp, pool->p.order); --#else -- page = alloc_pages(gfp, pool->p.order); --#endif -- if (!page) -- return NULL; -- -- if (!(pool->p.flags & PP_FLAG_DMA_MAP)) -- goto skip_dma_map; -- - /* Setup DMA mapping: use 'struct page' area for storing DMA-addr - * since dma_addr_t can be either 32 or 64 bits and does not always fit - * into page private data (i.e 32bit cpu with 64bit DMA caps) -@@ -222,22 +202,53 @@ static struct page *__page_pool_alloc_pa - dma = dma_map_page_attrs(pool->p.dev, page, 0, - (PAGE_SIZE << pool->p.order), - pool->p.dma_dir, DMA_ATTR_SKIP_CPU_SYNC); -- if (dma_mapping_error(pool->p.dev, dma)) { -- put_page(page); -- return NULL; -- } -+ if (dma_mapping_error(pool->p.dev, dma)) -+ return false; -+ - page_pool_set_dma_addr(page, dma); - - if (pool->p.flags & PP_FLAG_DMA_SYNC_DEV) - page_pool_dma_sync_for_device(pool, page, pool->p.max_len); - --skip_dma_map: -+ return true; -+} -+ -+static void page_pool_set_pp_info(struct page_pool *pool, -+ struct page *page) -+{ -+ page->pp = pool; -+ page->pp_magic |= PP_SIGNATURE; -+} -+ -+static void page_pool_clear_pp_info(struct page *page) -+{ -+ page->pp_magic = 0; -+ page->pp = NULL; -+} -+ -+/* slow path */ -+noinline -+static struct page *__page_pool_alloc_pages_slow(struct page_pool *pool, -+ gfp_t gfp) -+{ -+ struct page *page; -+ -+ gfp |= __GFP_COMP; -+ page = alloc_pages_node(pool->p.nid, gfp, pool->p.order); -+ if (unlikely(!page)) -+ return NULL; -+ -+ if ((pool->p.flags & PP_FLAG_DMA_MAP) && -+ unlikely(!page_pool_dma_map(pool, page))) { -+ put_page(page); -+ return NULL; -+ } -+ -+ page_pool_set_pp_info(pool, page); -+ - /* Track how many pages are held 'in-flight' */ - pool->pages_state_hold_cnt++; -- - trace_page_pool_state_hold(pool, page, pool->pages_state_hold_cnt); -- -- /* When page just alloc'ed is should/must have refcnt 1. */ - return page; - } - -@@ -302,10 +313,12 @@ void page_pool_release_page(struct page_ - DMA_ATTR_SKIP_CPU_SYNC); - page_pool_set_dma_addr(page, 0); - skip_dma_unmap: -+ page_pool_clear_pp_info(page); -+ - /* This may be the last page returned, releasing the pool, so - * it is not safe to reference pool afterwards. - */ -- count = atomic_inc_return(&pool->pages_state_release_cnt); -+ count = atomic_inc_return_relaxed(&pool->pages_state_release_cnt); - trace_page_pool_state_release(pool, page, count); - } - EXPORT_SYMBOL(page_pool_release_page); -@@ -331,7 +344,10 @@ static bool page_pool_recycle_in_ring(st - else - ret = ptr_ring_produce_bh(&pool->ring, page); - -- return (ret == 0) ? true : false; -+ if (!ret) -+ return true; -+ -+ return false; - } - - /* Only allow direct recycling in special circumstances, into the -@@ -350,46 +366,43 @@ static bool page_pool_recycle_in_cache(s - return true; - } - --/* page is NOT reusable when: -- * 1) allocated when system is under some pressure. (page_is_pfmemalloc) -- */ --static bool pool_page_reusable(struct page_pool *pool, struct page *page) --{ -- return !page_is_pfmemalloc(page); --} -- - /* If the page refcnt == 1, this will try to recycle the page. - * if PP_FLAG_DMA_SYNC_DEV is set, we'll try to sync the DMA area for - * the configured size min(dma_sync_size, pool->max_len). - * If the page refcnt != 1, then the page will be returned to memory - * subsystem. - */ --void page_pool_put_page(struct page_pool *pool, struct page *page, -- unsigned int dma_sync_size, bool allow_direct) --{ -+static __always_inline struct page * -+__page_pool_put_page(struct page_pool *pool, struct page *page, -+ unsigned int dma_sync_size, bool allow_direct) -+{ -+ /* It is not the last user for the page frag case */ -+ if (pool->p.flags & PP_FLAG_PAGE_FRAG && -+ page_pool_atomic_sub_frag_count_return(page, 1)) -+ return NULL; -+ - /* This allocator is optimized for the XDP mode that uses - * one-frame-per-page, but have fallbacks that act like the - * regular page allocator APIs. - * - * refcnt == 1 means page_pool owns page, and can recycle it. -+ * -+ * page is NOT reusable when allocated when system is under -+ * some pressure. (page_is_pfmemalloc) - */ -- if (likely(page_ref_count(page) == 1 && -- pool_page_reusable(pool, page))) { -+ if (likely(page_ref_count(page) == 1 && !page_is_pfmemalloc(page))) { - /* Read barrier done in page_ref_count / READ_ONCE */ - - if (pool->p.flags & PP_FLAG_DMA_SYNC_DEV) - page_pool_dma_sync_for_device(pool, page, - dma_sync_size); - -- if (allow_direct && in_serving_softirq()) -- if (page_pool_recycle_in_cache(page, pool)) -- return; -+ if (allow_direct && in_serving_softirq() && -+ page_pool_recycle_in_cache(page, pool)) -+ return NULL; - -- if (!page_pool_recycle_in_ring(pool, page)) { -- /* Cache full, fallback to free pages */ -- page_pool_return_page(pool, page); -- } -- return; -+ /* Page found as candidate for recycling */ -+ return page; - } - /* Fallback/non-XDP mode: API user have elevated refcnt. - * -@@ -407,9 +420,98 @@ void page_pool_put_page(struct page_pool - /* Do not replace this with page_pool_return_page() */ - page_pool_release_page(pool, page); - put_page(page); -+ -+ return NULL; -+} -+ -+void page_pool_put_page(struct page_pool *pool, struct page *page, -+ unsigned int dma_sync_size, bool allow_direct) -+{ -+ page = __page_pool_put_page(pool, page, dma_sync_size, allow_direct); -+ if (page && !page_pool_recycle_in_ring(pool, page)) -+ /* Cache full, fallback to free pages */ -+ page_pool_return_page(pool, page); - } - EXPORT_SYMBOL(page_pool_put_page); - -+static struct page *page_pool_drain_frag(struct page_pool *pool, -+ struct page *page) -+{ -+ long drain_count = BIAS_MAX - pool->frag_users; -+ -+ /* Some user is still using the page frag */ -+ if (likely(page_pool_atomic_sub_frag_count_return(page, -+ drain_count))) -+ return NULL; -+ -+ if (page_ref_count(page) == 1 && !page_is_pfmemalloc(page)) { -+ if (pool->p.flags & PP_FLAG_DMA_SYNC_DEV) -+ page_pool_dma_sync_for_device(pool, page, -1); -+ -+ return page; -+ } -+ -+ page_pool_return_page(pool, page); -+ return NULL; -+} -+ -+static void page_pool_free_frag(struct page_pool *pool) -+{ -+ long drain_count = BIAS_MAX - pool->frag_users; -+ struct page *page = pool->frag_page; -+ -+ pool->frag_page = NULL; -+ -+ if (!page || -+ page_pool_atomic_sub_frag_count_return(page, drain_count)) -+ return; -+ -+ page_pool_return_page(pool, page); -+} -+ -+struct page *page_pool_alloc_frag(struct page_pool *pool, -+ unsigned int *offset, -+ unsigned int size, gfp_t gfp) -+{ -+ unsigned int max_size = PAGE_SIZE << pool->p.order; -+ struct page *page = pool->frag_page; -+ -+ if (WARN_ON(!(pool->p.flags & PP_FLAG_PAGE_FRAG) || -+ size > max_size)) -+ return NULL; -+ -+ size = ALIGN(size, dma_get_cache_alignment()); -+ *offset = pool->frag_offset; -+ -+ if (page && *offset + size > max_size) { -+ page = page_pool_drain_frag(pool, page); -+ if (page) -+ goto frag_reset; -+ } -+ -+ if (!page) { -+ page = page_pool_alloc_pages(pool, gfp); -+ if (unlikely(!page)) { -+ pool->frag_page = NULL; -+ return NULL; -+ } -+ -+ pool->frag_page = page; -+ -+frag_reset: -+ pool->frag_users = 1; -+ *offset = 0; -+ pool->frag_offset = size; -+ page_pool_set_frag_count(page, BIAS_MAX); -+ return page; -+ } -+ -+ pool->frag_users++; -+ pool->frag_offset = *offset + size; -+ return page; -+} -+EXPORT_SYMBOL(page_pool_alloc_frag); -+ - static void page_pool_empty_ring(struct page_pool *pool) - { - struct page *page; -@@ -515,6 +617,8 @@ void page_pool_destroy(struct page_pool - if (!page_pool_put(pool)) - return; - -+ page_pool_free_frag(pool); -+ - if (!page_pool_release(pool)) - return; - -@@ -541,3 +645,32 @@ void page_pool_update_nid(struct page_po - } - } - EXPORT_SYMBOL(page_pool_update_nid); -+ -+bool page_pool_return_skb_page(struct page *page) -+{ -+ struct page_pool *pp; -+ -+ page = compound_head(page); -+ -+ /* page->pp_magic is OR'ed with PP_SIGNATURE after the allocation -+ * in order to preserve any existing bits, such as bit 0 for the -+ * head page of compound page and bit 1 for pfmemalloc page, so -+ * mask those bits for freeing side when doing below checking, -+ * and page_is_pfmemalloc() is checked in __page_pool_put_page() -+ * to avoid recycling the pfmemalloc page. -+ */ -+ if (unlikely((page->pp_magic & ~0x3UL) != PP_SIGNATURE)) -+ return false; -+ -+ pp = page->pp; -+ -+ /* Driver set this to memory recycling info. Reset it on recycle. -+ * This will *not* work for NIC using a split-page memory model. -+ * The page will be returned to the pool here regardless of the -+ * 'flipped' fragment being in use or not. -+ */ -+ page_pool_put_full_page(pp, page, false); -+ -+ return true; -+} -+EXPORT_SYMBOL(page_pool_return_skb_page); ---- a/include/linux/mm_types.h -+++ b/include/linux/mm_types.h -@@ -97,10 +97,25 @@ struct page { - }; - struct { /* page_pool used by netstack */ - /** -- * @dma_addr: might require a 64-bit value on -- * 32-bit architectures. -+ * @pp_magic: magic value to avoid recycling non -+ * page_pool allocated pages. - */ -- unsigned long dma_addr[2]; -+ unsigned long pp_magic; -+ struct page_pool *pp; -+ unsigned long _pp_mapping_pad; -+ unsigned long dma_addr; -+ union { -+ /** -+ * dma_addr_upper: might require a 64-bit -+ * value on 32-bit architectures. -+ */ -+ unsigned long dma_addr_upper; -+ /** -+ * For frag page support, not supported in -+ * 32-bit architectures with 64-bit DMA. -+ */ -+ atomic_long_t pp_frag_count; -+ }; - }; - struct { /* slab, slob and slub */ - union { ---- a/net/core/skbuff.c -+++ b/net/core/skbuff.c -@@ -594,13 +594,22 @@ static void skb_clone_fraglist(struct sk - skb_get(list); - } - -+static bool skb_pp_recycle(struct sk_buff *skb, void *data) -+{ -+ if (!IS_ENABLED(CONFIG_PAGE_POOL) || !skb->pp_recycle) -+ return false; -+ return page_pool_return_skb_page(virt_to_page(data)); -+} -+ - static void skb_free_head(struct sk_buff *skb) - { - unsigned char *head = skb->head; - -- if (skb->head_frag) -+ if (skb->head_frag) { -+ if (skb_pp_recycle(skb, head)) -+ return; - skb_free_frag(head); -- else -+ } else - kfree(head); - } - -@@ -612,16 +621,27 @@ static void skb_release_data(struct sk_b - if (skb->cloned && - atomic_sub_return(skb->nohdr ? (1 << SKB_DATAREF_SHIFT) + 1 : 1, - &shinfo->dataref)) -- return; -+ goto exit; - - for (i = 0; i < shinfo->nr_frags; i++) -- __skb_frag_unref(&shinfo->frags[i]); -+ __skb_frag_unref(&shinfo->frags[i], skb->pp_recycle); - - if (shinfo->frag_list) - kfree_skb_list(shinfo->frag_list); - - skb_zcopy_clear(skb, true); - skb_free_head(skb); -+exit: -+ /* When we clone an SKB we copy the reycling bit. The pp_recycle -+ * bit is only set on the head though, so in order to avoid races -+ * while trying to recycle fragments on __skb_frag_unref() we need -+ * to make one SKB responsible for triggering the recycle path. -+ * So disable the recycling bit if an SKB is cloned and we have -+ * additional references to to the fragmented part of the SKB. -+ * Eventually the last SKB will have the recycling bit set and it's -+ * dataref set to 0, which will trigger the recycling -+ */ -+ skb->pp_recycle = 0; - } - - /* -@@ -1002,6 +1022,7 @@ static struct sk_buff *__skb_clone(struc - n->nohdr = 0; - n->peeked = 0; - C(pfmemalloc); -+ C(pp_recycle); - n->destructor = NULL; - C(tail); - C(end); -@@ -3420,7 +3441,7 @@ int skb_shift(struct sk_buff *tgt, struc - fragto = &skb_shinfo(tgt)->frags[merge]; - - skb_frag_size_add(fragto, skb_frag_size(fragfrom)); -- __skb_frag_unref(fragfrom); -+ __skb_frag_unref(fragfrom, skb->pp_recycle); - } - - /* Reposition in the original skb */ -@@ -5187,6 +5208,20 @@ bool skb_try_coalesce(struct sk_buff *to - if (skb_cloned(to)) - return false; - -+ /* In general, avoid mixing slab allocated and page_pool allocated -+ * pages within the same SKB. However when @to is not pp_recycle and -+ * @from is cloned, we can transition frag pages from page_pool to -+ * reference counted. -+ * -+ * On the other hand, don't allow coalescing two pp_recycle SKBs if -+ * @from is cloned, in case the SKB is using page_pool fragment -+ * references (PP_FLAG_PAGE_FRAG). Since we only take full page -+ * references for cloned SKBs at the moment that would result in -+ * inconsistent reference counts. -+ */ -+ if (to->pp_recycle != (from->pp_recycle && !skb_cloned(from))) -+ return false; -+ - if (len <= skb_tailroom(to)) { - if (len) - BUG_ON(skb_copy_bits(from, 0, skb_put(to, len), len)); ---- a/include/linux/skbuff.h -+++ b/include/linux/skbuff.h -@@ -37,6 +37,7 @@ - #include <linux/in6.h> - #include <linux/if_packet.h> - #include <net/flow.h> -+#include <net/page_pool.h> - #if IS_ENABLED(CONFIG_NF_CONNTRACK) - #include <linux/netfilter/nf_conntrack_common.h> - #endif -@@ -787,7 +788,8 @@ struct sk_buff { - fclone:2, - peeked:1, - head_frag:1, -- pfmemalloc:1; -+ pfmemalloc:1, -+ pp_recycle:1; /* page_pool recycle indicator */ - #ifdef CONFIG_SKB_EXTENSIONS - __u8 active_extensions; - #endif -@@ -3030,9 +3032,15 @@ static inline void skb_frag_ref(struct s - * - * Releases a reference on the paged fragment @frag. - */ --static inline void __skb_frag_unref(skb_frag_t *frag) -+static inline void __skb_frag_unref(skb_frag_t *frag, bool recycle) - { -- put_page(skb_frag_page(frag)); -+ struct page *page = skb_frag_page(frag); -+ -+#ifdef CONFIG_PAGE_POOL -+ if (recycle && page_pool_return_skb_page(page)) -+ return; -+#endif -+ put_page(page); - } - - /** -@@ -3044,7 +3052,7 @@ static inline void __skb_frag_unref(skb_ - */ - static inline void skb_frag_unref(struct sk_buff *skb, int f) - { -- __skb_frag_unref(&skb_shinfo(skb)->frags[f]); -+ __skb_frag_unref(&skb_shinfo(skb)->frags[f], skb->pp_recycle); - } - - /** -@@ -4643,5 +4651,12 @@ static inline u64 skb_get_kcov_handle(st - #endif - } - -+#ifdef CONFIG_PAGE_POOL -+static inline void skb_mark_for_recycle(struct sk_buff *skb) -+{ -+ skb->pp_recycle = 1; -+} -+#endif -+ - #endif /* __KERNEL__ */ - #endif /* _LINUX_SKBUFF_H */ ---- a/drivers/net/ethernet/marvell/sky2.c -+++ b/drivers/net/ethernet/marvell/sky2.c -@@ -2501,7 +2501,7 @@ static void skb_put_frags(struct sk_buff - - if (length == 0) { - /* don't need this page */ -- __skb_frag_unref(frag); -+ __skb_frag_unref(frag, false); - --skb_shinfo(skb)->nr_frags; - } else { - size = min(length, (unsigned) PAGE_SIZE); ---- a/drivers/net/ethernet/mellanox/mlx4/en_rx.c -+++ b/drivers/net/ethernet/mellanox/mlx4/en_rx.c -@@ -526,7 +526,7 @@ static int mlx4_en_complete_rx_desc(stru - fail: - while (nr > 0) { - nr--; -- __skb_frag_unref(skb_shinfo(skb)->frags + nr); -+ __skb_frag_unref(skb_shinfo(skb)->frags + nr, false); - } - return 0; - } ---- a/net/tls/tls_device.c -+++ b/net/tls/tls_device.c -@@ -131,7 +131,7 @@ static void destroy_record(struct tls_re - int i; - - for (i = 0; i < record->num_frags; i++) -- __skb_frag_unref(&record->frags[i]); -+ __skb_frag_unref(&record->frags[i], false); - kfree(record); - } - ---- a/include/linux/poison.h -+++ b/include/linux/poison.h -@@ -82,4 +82,7 @@ - /********** security/ **********/ - #define KEY_DESTROY 0xbd - -+/********** net/core/page_pool.c **********/ -+#define PP_SIGNATURE (0x40 + POISON_POINTER_DELTA) -+ - #endif ---- a/include/linux/mm.h -+++ b/include/linux/mm.h -@@ -1602,7 +1602,7 @@ static inline bool page_is_pfmemalloc(st - * Page index cannot be this large so this must be - * a pfmemalloc page. - */ -- return page->index == -1UL; -+ return (uintptr_t)page->lru.next & BIT(1); - } - - /* -@@ -1611,12 +1611,12 @@ static inline bool page_is_pfmemalloc(st - */ - static inline void set_page_pfmemalloc(struct page *page) - { -- page->index = -1UL; -+ page->lru.next = (void *)BIT(1); - } - - static inline void clear_page_pfmemalloc(struct page *page) - { -- page->index = 0; -+ page->lru.next = NULL; - } - - /* |