#define kmalloc(size, flags) alloc_hooks(kmalloc_noprof(size, flags)) // linux/slab.h 1056 line

// alloc_tag.h
#define DEFINE_ALLOC_TAG(_alloc_tag)

#define alloc_hooks_tag(_tag, _do_alloc)                \
({                                  \
    typeof(_do_alloc) _res;                     \
    if (mem_alloc_profiling_enabled()) {                \
        struct alloc_tag * __maybe_unused _old;         \
        _old = alloc_tag_save(_tag);                \
        _res = _do_alloc;                   \
        alloc_tag_restore(_tag, _old);              \
    } else                              \
        _res = _do_alloc;                   \
    _res;                               \
})

#define alloc_hooks(_do_alloc)                      \
({                                  \
    DEFINE_ALLOC_TAG(_alloc_tag);                   \
    alloc_hooks_tag(&_alloc_tag, _do_alloc);            \
})

// linux/slab.h 997 line
#define kmalloc_noprof(...)         _kmalloc_noprof(__VA_ARGS__, __kmalloc_token(__VA_ARGS__))

#define __kmalloc_token(...) ((kmalloc_token_t){}) /* no-op */ // 533

// 982
static __always_inline __alloc_size(1) void *_kmalloc_noprof(size_t size, gfp_t flags, kmalloc_token_t token)
{
    if (__builtin_constant_p(size) && size) {
        unsigned int index;

        if (size > KMALLOC_MAX_CACHE_SIZE)
            return __kmalloc_large_noprof(size, flags);

        index = kmalloc_index(size);
        return __kmalloc_cache_noprof(
                kmalloc_caches[kmalloc_type(flags, token)][index],
                flags, size);
    }
    return __kmalloc_noprof(PASS_TOKEN_PARAMS(size, token), flags);
}

/*
 * Whenever changing this, take care of that kmalloc_type() and
 * create_kmalloc_caches() still work as intended.
 *
 * KMALLOC_NORMAL can contain only unaccounted objects whereas KMALLOC_CGROUP
 * is for accounted but unreclaimable and non-dma objects. All the other
 * kmem caches can have both accounted and unaccounted objects.
 */
enum kmalloc_cache_type {
    KMALLOC_NORMAL = 0,
#ifndef CONFIG_ZONE_DMA
    KMALLOC_DMA = KMALLOC_NORMAL,
#endif
#ifndef CONFIG_MEMCG
    KMALLOC_CGROUP = KMALLOC_NORMAL,
#endif
#ifndef CONFIG_SLAB_OBJ_EXT
    KMALLOC_NO_OBJ_EXT = KMALLOC_NORMAL,
#endif
    KMALLOC_PARTITION_START = KMALLOC_NORMAL,
    KMALLOC_PARTITION_END = KMALLOC_PARTITION_START + KMALLOC_PARTITION_CACHES_NR,
#ifdef CONFIG_SLUB_TINY
    KMALLOC_RECLAIM = KMALLOC_NORMAL,
#else
    KMALLOC_RECLAIM,
#endif
#ifdef CONFIG_ZONE_DMA
    KMALLOC_DMA,
#endif
#ifdef CONFIG_MEMCG
    KMALLOC_CGROUP,
#endif
#ifdef CONFIG_SLAB_OBJ_EXT
    KMALLOC_NO_OBJ_EXT,
#endif
    NR_KMALLOC_TYPES
};

static __always_inline enum kmalloc_cache_type kmalloc_type(gfp_t flags, kmalloc_token_t token)
{
    /*
     * The most common case is KMALLOC_NORMAL, so test for it
     * with a single branch for all the relevant flags.
     */
    if (likely((flags & KMALLOC_NOT_NORMAL_BITS) == 0))
#ifdef CONFIG_KMALLOC_PARTITION_RANDOM
        /* KMALLOC_PARTITION_CACHES_NR (=15) copies + the KMALLOC_NORMAL */
        return KMALLOC_PARTITION_START + hash_64(token.v ^ random_kmalloc_seed,
                             ilog2(KMALLOC_PARTITION_CACHES_NR + 1));
#elif defined(CONFIG_KMALLOC_PARTITION_TYPED)
        return KMALLOC_PARTITION_START + token.v;
#else
        return KMALLOC_NORMAL;
#endif

    /*
     * At least one of the flags has to be set. Their priorities in
     * decreasing order are:
     *  1) __GFP_DMA
     *  2) __GFP_RECLAIMABLE
     *  3) __GFP_ACCOUNT
     */
    if (IS_ENABLED(CONFIG_ZONE_DMA) && (flags & __GFP_DMA))
        return KMALLOC_DMA;
    if (!IS_ENABLED(CONFIG_MEMCG) || (flags & __GFP_RECLAIMABLE))
        return KMALLOC_RECLAIM;
    else
        return KMALLOC_CGROUP;
}

typedef struct kmem_cache * kmem_buckets[KMALLOC_SHIFT_HIGH + 1];

kmem_buckets kmalloc_caches[NR_KMALLOC_TYPES] __ro_after_init =
{ /* initialization for https://llvm.org/pr42570 */ };

#define kmalloc_index(s) __kmalloc_index(s, true)
/*
 * Figure out which kmalloc slab an allocation of a certain size
 * belongs to.
 * 0 = zero alloc
 * 1 =  65 .. 96 bytes
 * 2 = 129 .. 192 bytes
 * n = 2^(n-1)+1 .. 2^n
 *
 * Note: __kmalloc_index() is compile-time optimized, and not runtime optimized;
 * typical usage is via kmalloc_index() and therefore evaluated at compile-time.
 * Callers where !size_is_constant should only be test modules, where runtime
 * overheads of __kmalloc_index() can be tolerated.  Also see kmalloc_slab().
 */
static __always_inline unsigned int __kmalloc_index(size_t size,
                            bool size_is_constant)
{
    if (!size)
        return 0;

    if (size <= KMALLOC_MIN_SIZE)
        return KMALLOC_SHIFT_LOW;

    if (KMALLOC_MIN_SIZE <= 32 && size > 64 && size <= 96)
        return 1;
    if (KMALLOC_MIN_SIZE <= 64 && size > 128 && size <= 192)
        return 2;
    if (size <=          8) return 3;
    if (size <=         16) return 4;
    if (size <=         32) return 5;
    if (size <=         64) return 6;
    if (size <=        128) return 7;
    if (size <=        256) return 8;
    if (size <=        512) return 9;
    if (size <=       1024) return 10;
    if (size <=   2 * 1024) return 11;
    if (size <=   4 * 1024) return 12;
    if (size <=   8 * 1024) return 13;
    if (size <=  16 * 1024) return 14;
    if (size <=  32 * 1024) return 15;
    if (size <=  64 * 1024) return 16;
    if (size <= 128 * 1024) return 17;
    if (size <= 256 * 1024) return 18;
    if (size <= 512 * 1024) return 19;
    if (size <= 1024 * 1024) return 20;
    if (size <=  2 * 1024 * 1024) return 21;

    if (!IS_ENABLED(CONFIG_PROFILE_ALL_BRANCHES) && size_is_constant)
        BUILD_BUG_ON_MSG(1, "unexpected size in kmalloc_index()");
    else
        BUG();

    /* Will never be reached. Needed because the compiler may complain */
    return -1;
}

void *__kmalloc_cache_noprof(struct kmem_cache *s, gfp_t gfpflags, size_t size)
{
    void *ret;
    const struct slab_alloc_context ac = {
        .caller_addr = _RET_IP_,
        .orig_size = size,
        .alloc_flags = SLAB_ALLOC_DEFAULT,
    };

    ret = slab_alloc_node(s, gfpflags, NUMA_NO_NODE, &ac);

    trace_kmalloc(_RET_IP_, ret, size, s->size, gfpflags, NUMA_NO_NODE);

    ret = kasan_kmalloc(s, ret, size, gfpflags);
    return ret;
}

void *__kmalloc_noprof(DECL_TOKEN_PARAMS(size, token), gfp_t flags)
{
    const struct slab_alloc_context ac = {
        .caller_addr = _RET_IP_,
        .orig_size = size,
        .alloc_flags = SLAB_ALLOC_DEFAULT,
    };

    return __do_kmalloc_node(NULL, flags,  NUMA_NO_NODE,
                 PASS_TOKEN_PARAM(token), &ac);
}

static __always_inline
void *__do_kmalloc_node(kmem_buckets *b, gfp_t flags, int node,
            kmalloc_token_t token, const struct slab_alloc_context *ac)
{
    const size_t size = ac->orig_size;
    struct kmem_cache *s;
    void *ret;

    if (unlikely(size > KMALLOC_MAX_CACHE_SIZE)) {
        ret = __kmalloc_large_node_noprof(size, flags, node);
        trace_kmalloc(ac->caller_addr, ret, size,
                  PAGE_SIZE << get_order(size), flags, node);
        return ret;
    }

    if (unlikely(!size))
        return ZERO_SIZE_PTR;

    s = kmalloc_slab(size, b, flags, token, ac->alloc_flags);

    ret = slab_alloc_node(s, flags, node, ac);
    ret = kasan_kmalloc(s, ret, size, flags);
    trace_kmalloc(ac->caller_addr, ret, size, s->size, flags, node);
    return ret;
}

/*
 * Find the kmem_cache structure that serves a given size of
 * allocation
 *
 * This assumes size is larger than zero and not larger than
 * KMALLOC_MAX_CACHE_SIZE and the caller must check that.
 */
static inline struct kmem_cache *
kmalloc_slab(size_t size, kmem_buckets *b, gfp_t flags, kmalloc_token_t token,
         unsigned int alloc_flags)
{
    unsigned int index;
    enum kmalloc_cache_type type = kmalloc_type(flags, token);

    if (alloc_flags & SLAB_ALLOC_NO_OBJ_EXT)
        type = KMALLOC_NO_OBJ_EXT;

    if (!b)
        b = &kmalloc_caches[type];
    if (size <= 192)
        index = kmalloc_size_index[size_index_elem(size)];
    else
        index = fls(size - 1);

    return (*b)[index];
}

static inline unsigned int size_index_elem(unsigned int bytes)
{
    return (bytes - 1) / 8;
}

/*
 * Conversion table for small slabs sizes / 8 to the index in the
 * kmalloc array. This is necessary for slabs < 192 since we have non power
 * of two cache sizes there. The size of larger slabs can be determined using
 * fls.
 */
u8 kmalloc_size_index[24] __ro_after_init = {
    3,  /* 8 */
    4,  /* 16 */
    5,  /* 24 */
    5,  /* 32 */
    6,  /* 40 */
    6,  /* 48 */
    6,  /* 56 */
    6,  /* 64 */
    1,  /* 72 */
    1,  /* 80 */
    1,  /* 88 */
    1,  /* 96 */
    7,  /* 104 */
    7,  /* 112 */
    7,  /* 120 */
    7,  /* 128 */
    2,  /* 136 */
    2,  /* 144 */
    2,  /* 152 */
    2,  /* 160 */
    2,  /* 168 */
    2,  /* 176 */
    2,  /* 184 */
    2   /* 192 */
};

// __do_kmalloc_node()
ret = slab_alloc_node(s, flags, node, ac);

/*
 * Inlined fastpath so that allocation functions (kmalloc, kmem_cache_alloc)
 * have the fastpath folded into their functions. So no function call
 * overhead for requests that can be satisfied on the fastpath.
 *
 * The fastpath works by first checking if the lockless freelist can be used.
 * If not then __slab_alloc is called for slow processing.
 *
 * Otherwise we can simply pick the next object from the lockless free list.
 */
static __fastpath_inline void *slab_alloc_node(struct kmem_cache *s,
        gfp_t gfpflags, int node, const struct slab_alloc_context *ac)
{
    void *object;

    s = slab_pre_alloc_hook(s, gfpflags);
    if (unlikely(!s))
        return NULL;

    object = kfence_alloc(s, ac->orig_size, gfpflags);
    if (unlikely(object))
        goto out;

    node = apply_strict_numa_policy(node);

    object = alloc_from_pcs(s, gfpflags, ac->alloc_flags, node);

    if (unlikely(!object))
        object = ___slab_alloc(s, gfpflags, node, ac);

    maybe_wipe_obj_freeptr(s, object);

out:
    /*
     * In case this fails due to memcg_slab_post_alloc_hook(),
     * object is set to NULL
     */
    slab_post_alloc_hook(s, gfpflags, 1, &object, ac);

    return object;
}

    if (unlikely(!object))
        object = ___slab_alloc(s, gfpflags, node, ac);

/**
 * might_alloc - Mark possible allocation sites
 * @gfp_mask: gfp_t flags that would be used to allocate
 *
 * Similar to might_sleep() and other annotations, this can be used in functions
 * that might allocate, but often don't. Compiles to nothing without
 * CONFIG_LOCKDEP. Includes a conditional might_sleep() if @gfp allows blocking.
 */
static inline void might_alloc(gfp_t gfp_mask)
{
    fs_reclaim_acquire(gfp_mask);
    fs_reclaim_release(gfp_mask);

    if (current->flags & PF_MEMALLOC)
        return;

    might_sleep_if(gfpflags_allow_blocking(gfp_mask));
}

static __fastpath_inline
struct kmem_cache *slab_pre_alloc_hook(struct kmem_cache *s, gfp_t flags)
{
    flags &= gfp_allowed_mask;

    might_alloc(flags);

    if (unlikely(should_failslab(s, flags)))
        return NULL;

    return s;
}

// gfp_allowed_mask注释翻译
在系统启动早期，gfp_allowed_mask 会被设置为 GFP_BOOT_MASK，以限制此时可以使用的 GFP 标志，因为中断尚未启用。
中断启用后，系统正常运行期间，它会被设置为 __GFP_BITS_MASK，允许使用完整的 GFP 标志集合。
在系统休眠期间，电源管理（PM）子系统也会使用该掩码：当设备已经被挂起时，它通过限制内存分配所使用的 GFP 标志，避免分配过程触发 I/O 操作。

    fs_reclaim_acquire(gfp_mask);
    fs_reclaim_release(gfp_mask);

static inline bool gfpflags_allow_blocking(const gfp_t gfp_flags)
{
    return !!(gfp_flags & __GFP_DIRECT_RECLAIM);
}

    if (current->flags & PF_MEMALLOC)
        return;

if (unlikely(should_failslab(s, flags)))
    return NULL;

// 内核文档翻译
Kernel Electric-Fence（KFENCE）是一种基于采样技术的内存安全错误检测器，其运行成本较低。KFENCE 能够检测到堆内存越界访问、释放后使用错误以及无效释放等内存安全漏洞。

KFENCE 旨在能够在生产环境中被启用，其性能开销几乎为零。与 KASAN 相比，KFENCE 在性能与精度之间做出了权衡。设计 KFENCE 的主要目的是为了在总运行时间足够长的情况下，能够检测到那些非生产测试负载无法发现的代码中的错误。当该工具被部署在大量机器上时，就能快速实现足够长的总运行时间。

/**
 * kfence_alloc() - 以较低概率分配一个 KFENCE 对象
 * @s:     指定对象要求的 struct kmem_cache
 * @size:  要分配对象的确切大小（可以小于 @s->size，
 *         例如使用 kmalloc 缓存时）
 * @flags: GFP 标志
 *
 * 返回值：
 * * NULL     - 必须继续按常规方式分配内存，
 * * 非 NULL  - 指向 KFENCE 对象的指针。
 *
 * 应将 kfence_alloc() 插入堆内存分配的快速路径中，
 * 通过静态分支，以较低概率透明地返回由 KFENCE 分配的对象。
 * 该概率由启动参数 kfence.sample_interval 控制。
 */

node = apply_strict_numa_policy(node);

非统一内存访问架构 Non-Uniform Memory Access(NUMA) 是一种为多处理器的电脑设计的内存架构，内存访问时间取决于内存相对于处理器的位置。在NUMA下，处理器访问它自己的本地内存的速度比非本地内存（内存位于另一个处理器，或者是处理器之间共享的内存）快一些。
非统一内存访问架构的特点是：被共享的内存物理上是分布式的，所有这些内存的集合就是全局地址空间。所以处理器访问这些内存的时间是不一样的，显然访问本地内存的速度要比访问全局共享内存或远程访问外地内存要快些。另外，NUMA中内存可能是分层的：本地内存，群内共享内存，全局共享内存。
—— 维基百科

static __always_inline int apply_strict_numa_policy(int node)
{
#ifdef CONFIG_NUMA
    if (static_branch_unlikely(&strict_numa) &&
            node == NUMA_NO_NODE) {

        struct mempolicy *mpol = current->mempolicy;

        if (mpol) {
            /*
             * Special BIND rule support. If the local node
             * is in permitted set then do not redirect
             * to a particular node.
             * Otherwise we apply the memory policy to get
             * the node we need to allocate on.
             */
            if (mpol->mode != MPOL_BIND ||
                    !node_isset(numa_mem_id(), mpol->nodes))
                node = mempolicy_slab_node();
        }
    }
#endif
    return node;
}

// mempolicy注释翻译
/*
 * 描述内存策略。
 *
 * mempolicy 可以关联到进程，也可以关联到 VMA（虚拟内存区域）。
 * 对于与 VMA 相关的内存分配，优先使用 VMA 的策略；
 * 其他情况下则使用进程的策略。中断处理会忽略当前进程的内存策略。
 */
struct mempolicy {
    atomic_t refcnt;
    unsigned short mode;    /* See MPOL_* above */
    unsigned short flags;   /* See set_mempolicy() MPOL_F_* above */
    nodemask_t nodes;   /* interleave/bind/preferred/etc */
    int home_node;      /* Home node to use for MPOL_BIND and MPOL_PREFERRED_MANY */

    union {
        nodemask_t cpuset_mems_allowed; /* relative to these nodes */
        nodemask_t user_nodemask;   /* nodemask passed by user */
    } w;
    struct rcu_head rcu;
};

mpol->mode       // 策略类型
mpol->nodes      // 策略指定的节点集合
numa_mem_id()    // 当前 CPU 对应的本地内存节点

if (策略是 MPOL_BIND && 本地内存节点在允许集合中)
    不修改 node;
else
    node = mempolicy_slab_node();

/*
 * Depending on the memory policy provide a node from which to allocate the
 * next slab entry.
 */
unsigned int mempolicy_slab_node(void)
{
    struct mempolicy *policy;
    int node = numa_mem_id();

    if (!in_task())
        return node;

    policy = current->mempolicy;
    if (!policy)
        return node;

    switch (policy->mode) {
    case MPOL_PREFERRED:
        return first_node(policy->nodes);

    case MPOL_INTERLEAVE:
        return interleave_nodes(policy);

    case MPOL_WEIGHTED_INTERLEAVE:
        return weighted_interleave_nodes(policy);

    case MPOL_BIND:
    case MPOL_PREFERRED_MANY:
    {
        struct zoneref *z;

        /*
         * Follow bind policy behavior and start allocation at the
         * first node.
         */
        struct zonelist *zonelist;
        enum zone_type highest_zoneidx = gfp_zone(GFP_KERNEL);
        zonelist = &NODE_DATA(node)->node_zonelists[ZONELIST_FALLBACK];
        z = first_zones_zonelist(zonelist, highest_zoneidx,
                            &policy->nodes);
        return zonelist_zone(z) ? zonelist_node_idx(z) : node;
    }
    case MPOL_LOCAL:
        return node;

    default:
        BUG();
    }
}

struct zone
    ├── struct page
    ├── struct page
    ├── struct page
    ├── ...
    └── struct page

struct pglist_data {
    struct zone node_zones[MAX_NR_ZONES];
    ...
};

pg_data_t（NUMA 节点）
    └── node_zones[ZONE_DMA]
    └── node_zones[ZONE_DMA32]
    └── node_zones[ZONE_NORMAL]
    └── node_zones[ZONE_MOVABLE]

Node 3 的 ZONE_NORMAL

NUMA node id = 3
zone_idx     = ZONE_NORMAL

struct zonelist {
    struct zoneref _zonerefs[MAX_ZONES_PER_ZONELIST + 1];
};

/*
 * This struct contains information about a zone in a zonelist. It is stored
 * here to avoid dereferences into large structures and lookups of tables
 */
struct zoneref {
    struct zone *zone;  /* Pointer to actual zone */
    int zone_idx;       /* zone_idx(zoneref->zone) */
};

/*
 * 这个结构体保存 zonelist 中某个 zone 的信息。
 * 将这些信息直接保存在这里，是为了避免对大型结构体进行
 * 多次指针解引用，以及避免额外的查表操作。
 */

object = alloc_from_pcs(s, gfpflags, ac->alloc_flags, node);

static __fastpath_inline
void *alloc_from_pcs(struct kmem_cache *s, gfp_t gfp, unsigned int alloc_flags, int node)
{
    struct slub_percpu_sheaves *pcs;
    bool node_requested;
    void *object;

    node_requested = IS_ENABLED(CONFIG_NUMA) && node != NUMA_NO_NODE;

    /*
     * We assume the percpu sheaves contain only local objects although it's
     * not completely guaranteed, so we verify later.
     */
    if (unlikely(node_requested && node != numa_mem_id())) {
        stat(s, ALLOC_NODE_MISMATCH);
        return NULL;
    }

    if (!local_trylock(&s->cpu_sheaves->lock))
        return NULL;

    pcs = this_cpu_ptr(s->cpu_sheaves);

    if (unlikely(pcs->main->size == 0)) {
        pcs = __pcs_replace_empty_main(s, pcs, gfp, alloc_flags);
        if (unlikely(!pcs))
            return NULL;
    }

    object = pcs->main->objects[pcs->main->size - 1];

    if (unlikely(node_requested)) {
        /*
         * Verify that the object was from the node we want. This could
         * be false because of cpu migration during an unlocked part of
         * the current allocation or previous freeing process.
         */
        if (page_to_nid(virt_to_page(object)) != node) {
            local_unlock(&s->cpu_sheaves->lock);
            stat(s, ALLOC_NODE_MISMATCH);
            return NULL;
        }
    }

    pcs->main->size--;

    local_unlock(&s->cpu_sheaves->lock);

    stat(s, ALLOC_FASTPATH);

    return object;
}

node_requested = IS_ENABLED(CONFIG_NUMA) && node != NUMA_NO_NODE;

if (unlikely(node_requested && node != numa_mem_id())) {
        stat(s, ALLOC_NODE_MISMATCH);
        return NULL;
}

if (!local_trylock(&s->cpu_sheaves->lock))
        return NULL;

CPU 0                         CPU 1
持有锁                        等待锁
修改数据                      自旋……
释放锁                        获得锁

CPU 0 → CPU 0 的 sheaves 和锁
CPU 1 → CPU 1 的 sheaves 和锁

CPU 0 上的普通代码
    │
    ├─ 获得当前 CPU 的 sheaves 锁
    │
    ├─ 正在访问 pcs
    │
    └─ 此时发生中断
           │
           └─ 中断处理程序也申请同一个 kmem_cache 的对象
                  │
                  └─ 进入 alloc_from_pcs()
                         │
                         └─ 尝试获取同一把本地锁

中断处理程序：等原来的代码释放锁
原来的代码：等中断处理结束，才能继续执行并释放锁

struct slub_percpu_sheaves {
    local_trylock_t lock;
    struct slab_sheaf *main; /* never NULL when unlocked */
    struct slab_sheaf *spare; /* empty or full, may be NULL */
    struct slab_sheaf *rcu_free; /* for batching kfree_rcu() */
};

struct node_barn {
    spinlock_t lock;
    struct list_head sheaves_full;
    struct list_head sheaves_empty;
    unsigned int nr_full;
    unsigned int nr_empty;
};

struct slab_sheaf {
    union {
        struct rcu_head rcu_head;
        struct list_head barn_list;
        /* only used to defer call_rcu() in unknown context */
        struct llist_node llnode;
        /* only used for prefilled sheafs */
        struct {
            unsigned int capacity;
            bool pfmemalloc;
        };
    };
    struct kmem_cache *cache;
    unsigned int size;
    int node; /* only used for rcu_sheaf */
    void *objects[];
};

// slub.c 4714
/*
 * Replace the empty main sheaf with a (at least partially) full sheaf.
 *
 * Must be called with the cpu_sheaves local lock locked. If successful, returns
 * the pcs pointer and the local lock locked (possibly on a different cpu than
 * initially called). If not successful, returns NULL and the local lock
 * unlocked.
 */
static struct slub_percpu_sheaves *
__pcs_replace_empty_main(struct kmem_cache *s, struct slub_percpu_sheaves *pcs,
             gfp_t gfp, unsigned int alloc_flags)
{
    struct slab_sheaf *empty = NULL;
    struct slab_sheaf *full;
    struct node_barn *barn;
    bool allow_spin;

    slab_lockdep_assert_held(this_cpu_ptr(&s->cpu_sheaves->lock));

    /* Bootstrap or debug cache, back off */
    if (unlikely(!cache_has_sheaves(s))) {
        local_unlock(&s->cpu_sheaves->lock);
        return NULL;
    }

    if (pcs->spare && pcs->spare->size > 0) {
        swap(pcs->main, pcs->spare);
        return pcs;
    }

    barn = get_barn(s);
    if (!barn) {
        local_unlock(&s->cpu_sheaves->lock);
        return NULL;
    }

    allow_spin = alloc_flags_allow_spinning(alloc_flags);

    full = barn_replace_empty_sheaf(barn, pcs->main, allow_spin);

    if (full) {
        stat(s, BARN_GET);
        pcs->main = full;
        return pcs;
    }

    stat(s, BARN_GET_FAIL);

    if (allow_spin) {
        if (pcs->spare) {
            empty = pcs->spare;
            pcs->spare = NULL;
        } else {
            empty = barn_get_empty_sheaf(barn, true);
        }
    }

    local_unlock(&s->cpu_sheaves->lock);
    pcs = NULL;

    if (!allow_spin)
        return NULL;

    if (!empty) {
        empty = alloc_empty_sheaf(s, gfp, alloc_flags);
        if (!empty)
            return NULL;
    }

    if (refill_sheaf(s, empty, gfp | __GFP_NOMEMALLOC | __GFP_NOWARN)) {
        /*
         * we must be very low on memory so don't bother
         * with the barn
         */
        sheaf_flush_unused(s, empty);
        free_empty_sheaf(s, empty);

        return NULL;
    }

    full = empty;
    empty = NULL;

    if (!local_trylock(&s->cpu_sheaves->lock))
        goto barn_put;
    pcs = this_cpu_ptr(s->cpu_sheaves);

    /*
     * If we put any empty or full sheaf to the barn below, it's due to
     * racing or being migrated to a different cpu. Breaching the barn's
     * sheaf limits should be thus rare enough so just ignore them to
     * simplify the recovery.
     */

    if (pcs->main->size == 0) {
        if (!pcs->spare)
            pcs->spare = pcs->main;
        else
            barn_put_empty_sheaf(barn, pcs->main);
        pcs->main = full;
        return pcs;
    }

    if (!pcs->spare) {
        pcs->spare = full;
        return pcs;
    }

    if (pcs->spare->size == 0) {
        barn_put_empty_sheaf(barn, pcs->spare);
        pcs->spare = full;
        return pcs;
    }

barn_put:
    barn_put_full_sheaf(barn, full);
    stat(s, BARN_PUT);

    return pcs;
}

// slub.c 3229
/*
 * If a full sheaf is available, return it and put the supplied empty one to
 * barn. We ignore the limit on empty sheaves as the number of sheaves doesn't
 * change.
 */
static struct slab_sheaf *
barn_replace_empty_sheaf(struct node_barn *barn, struct slab_sheaf *empty,
             bool allow_spin)
{
    struct slab_sheaf *full = NULL;
    unsigned long flags;

    if (!data_race(barn->nr_full))
        return NULL;

    if (likely(allow_spin))
        spin_lock_irqsave(&barn->lock, flags);
    else if (!spin_trylock_irqsave(&barn->lock, flags))
        return NULL;

    if (likely(barn->nr_full)) {
        full = list_first_entry(&barn->sheaves_full, struct slab_sheaf,
                    barn_list);
        list_del(&full->barn_list);
        list_add(&empty->barn_list, &barn->sheaves_empty);
        barn->nr_full--;
        barn->nr_empty++;
    }

    spin_unlock_irqrestore(&barn->lock, flags);

    return full;
}

static struct slab_sheaf *__alloc_empty_sheaf(struct kmem_cache *s, gfp_t gfp,
                unsigned int alloc_flags, unsigned int capacity)
{
    struct slab_sheaf *sheaf;
    size_t sheaf_size;

    /*
     * Prevent recursion to the same cache, or a deep stack of kmallocs of
     * varying sizes (sheaf capacity might differ for each kmalloc size
     * bucket)
     */
    if (s->flags & SLAB_KMALLOC)
        alloc_flags |= SLAB_ALLOC_NO_RECURSE;

    sheaf_size = struct_size(sheaf, objects, capacity);
    sheaf = kmalloc_flags(sheaf_size, gfp | __GFP_ZERO, alloc_flags, NUMA_NO_NODE);

    if (unlikely(!sheaf))
        return NULL;

    sheaf->cache = s;

    stat(s, SHEAF_ALLOC);

    return sheaf;
}

static inline struct slab_sheaf *alloc_empty_sheaf(struct kmem_cache *s,
                gfp_t gfp, unsigned int alloc_flags)
{
    if (alloc_flags & SLAB_ALLOC_NO_RECURSE)
        return NULL;

    gfp &= ~OBJCGS_CLEAR_MASK;

    return __alloc_empty_sheaf(s, gfp, alloc_flags, s->sheaf_capacity);
}

/*
 * The only version of kmalloc_node() that takes alloc_flags and thus can
 * determine on its own whether to handle the allocation via kmalloc_nolock() or
 * normally
 */
void *__kmalloc_flags_noprof(DECL_TOKEN_PARAMS(size, token), gfp_t flags,
                 unsigned int alloc_flags, int node)
{
    const struct slab_alloc_context ac = {
        .caller_addr = _RET_IP_,
        .orig_size = size,
        .alloc_flags = alloc_flags,
    };

    if (alloc_flags_allow_spinning(alloc_flags)) {
        return __do_kmalloc_node(NULL, flags, node,
                PASS_TOKEN_PARAM(token), &ac);
    } else {
        return __kmalloc_nolock_noprof(PASS_TOKEN_PARAMS(size, token),
                           flags, node, &ac);
    }
}

// alloc_empty_sheaf
if (alloc_flags & SLAB_ALLOC_NO_RECURSE)
    return NULL;

// __alloc_empty_sheaf
/*
 * Prevent recursion to the same cache, or a deep stack of kmallocs of
 * varying sizes (sheaf capacity might differ for each kmalloc size
 * bucket)
 */
if (s->flags & SLAB_KMALLOC)
    alloc_flags |= SLAB_ALLOC_NO_RECURSE;

// slub.c 7396
static unsigned int
refill_objects(struct kmem_cache *s, void **p, gfp_t gfp, unsigned int min,
           unsigned int max)
{
    int local_node = numa_mem_id();
    unsigned int refilled;
    struct slab *slab;

    refilled = __refill_objects_node(s, p, gfp, min, max,
                     get_node(s, local_node),
                     /* allow_spin = */ true);
    if (refilled >= min)
        return refilled;

    refilled += __refill_objects_any(s, p + refilled, gfp, min - refilled,
                     max - refilled);
    if (refilled >= min)
        return refilled;

new_slab:

    slab = new_slab(s, gfp, SLAB_ALLOC_DEFAULT, local_node);
    if (!slab)
        goto out;

    stat(s, ALLOC_SLAB);

    refilled += alloc_from_new_slab(s, slab, p + refilled, max - refilled,
                    /* allow_spin = */ true);

    if (refilled < min)
        goto new_slab;

out:
    return refilled;
}

// slub.c 2876
static int refill_sheaf(struct kmem_cache *s, struct slab_sheaf *sheaf,
             gfp_t gfp)
{
    int to_fill = s->sheaf_capacity - sheaf->size;
    int filled;

    if (!to_fill)
        return 0;

    filled = refill_objects(s, &sheaf->objects[sheaf->size], gfp, to_fill,
                to_fill);

    sheaf->size += filled;

    stat_add(s, SHEAF_REFILL, filled);

    if (filled < to_fill)
        return -ENOMEM;

    return 0;
}

local_unlock(&s->cpu_sheaves->lock);
pcs = NULL;