From: Heinz Mauelshagen <mauelshagen@redhat.com>
Signed-off-by: Heinz Mauelshagen <mauelshagen@redhat.com>
---
drivers/md/Kconfig | 10
drivers/md/Makefile | 2
drivers/md/dm-cache-policy-basic.c | 1985 +++++++++++++++++++++++++++++++++++++
3 files changed, 1997 insertions(+)
Index: linux/drivers/md/Kconfig
===================================================================
--- linux.orig/drivers/md/Kconfig
+++ linux/drivers/md/Kconfig
@@ -291,6 +291,16 @@ config DM_CACHE_CLEANER
---help---
Under development
+config DM_CACHE_BASIC
+ tristate "Basic Cache Policies - a list of replacement policies (EXPERIMENTAL)"
+ depends on DM_CACHE
+ select BTREE
+ default y
+ ---help---
+ Under development
+ A cache replacement policy module providing
+ FIFO, FILO, LRU, MRU, LFU, MFU, LFU_WS, MFU_WS, MULTIQUEUE, MULTIQUEUE_WS, Q2, TWOQUEUE and RANDOM policies
+
config DM_MIRROR
tristate "Mirror target"
depends on BLK_DEV_DM
Index: linux/drivers/md/Makefile
===================================================================
--- linux.orig/drivers/md/Makefile
+++ linux/drivers/md/Makefile
@@ -14,6 +14,7 @@ dm-thin-pool-y += dm-thin.o dm-thin-meta
dm-cache-y += dm-cache-target.o dm-cache-metadata.o dm-cache-policy.o
dm-cache-mq-y += dm-cache-policy-mq.o
dm-cache-cleaner-y += dm-cache-policy-cleaner.o
+dm-cache-basic-y += dm-cache-policy-basic.o
md-mod-y += md.o bitmap.o
raid456-y += raid5.o
@@ -50,6 +51,7 @@ obj-$(CONFIG_DM_VERITY) += dm-verity.o
obj-$(CONFIG_DM_CACHE) += dm-cache.o
obj-$(CONFIG_DM_CACHE_MQ) += dm-cache-mq.o
obj-$(CONFIG_DM_CACHE_CLEANER) += dm-cache-cleaner.o
+obj-$(CONFIG_DM_CACHE_BASIC) += dm-cache-basic.o
obj-$(CONFIG_DM_SWITCH) += dm-switch.o
ifeq ($(CONFIG_DM_UEVENT),y)
Index: linux/drivers/md/dm-cache-policy-basic.c
===================================================================
--- /dev/null
+++ linux/drivers/md/dm-cache-policy-basic.c
@@ -0,0 +1,1985 @@
+/*
+ * Copyright (C) 2012 Red Hat. All rights reserved.
+ *
+ * This file is released under the GPL.
+ *
+ * A selection of cache replacement policies for the dm-cache target:
+ * basic
+ * dumb
+ * fifo
+ * filo
+ * lfu
+ * lfu_ws
+ * lru
+ * mfu
+ * mfu_ws
+ * mru
+ * multiqueue
+ * multiqueue_ws
+ * noop
+ * random
+ * q2
+ * twoqueue
+ */
+
+#include "dm-cache-policy.h"
+#include "dm.h"
+
+#include <linux/btree.h>
+#include <linux/hash.h>
+#include <linux/list.h>
+#include <linux/module.h>
+#include <linux/random.h>
+#include <linux/slab.h>
+
+/* Cache input queue defines. */
+#define READ_PROMOTE_THRESHOLD 1U /* Minimum read cache in queue promote per element threshold. */
+#define WRITE_PROMOTE_THRESHOLD 4U /* Minimum write cache in queue promote per element threshold. */
+
+/* Default "multiqueue" queue timeout. */
+#define MQ_QUEUE_TMO_DEFAULT (5UL * HZ) /* Default seconds queue maximum lifetime per entry. FIXME: dynamic? */
+
+/*----------------------------------------------------------------------------*/
+/*
+ * Large, sequential ios are probably better left on the origin device since
+ * spindles tend to have good bandwidth.
+ *
+ * The io_tracker tries to spot when the io is in
+ * one of these sequential modes.
+ *
+ * Two thresholds to switch between random and sequential io mode are defaulting
+ * as follows and can be adjusted via the constructor and message interfaces.
+ */
+#define RANDOM_THRESHOLD_DEFAULT 4
+#define SEQUENTIAL_THRESHOLD_DEFAULT 512
+
+static struct kmem_cache *basic_entry_cache;
+static struct kmem_cache *track_entry_cache;
+
+enum io_pattern {
+ PATTERN_SEQUENTIAL,
+ PATTERN_RANDOM
+};
+
+struct io_tracker {
+ sector_t next_start_osector, nr_seq_sectors;
+
+ unsigned nr_rand_samples;
+ enum io_pattern pattern;
+
+ unsigned long thresholds[2];
+};
+
+static void iot_init(struct io_tracker *t, int sequential_threshold, int random_threshold)
+{
+ t->pattern = PATTERN_RANDOM;
+ t->nr_seq_sectors = t->nr_rand_samples = t->next_start_osector = 0;
+ t->thresholds[PATTERN_SEQUENTIAL] = sequential_threshold < 0 ? SEQUENTIAL_THRESHOLD_DEFAULT : sequential_threshold;
+ t->thresholds[PATTERN_RANDOM] = random_threshold < 0 ? RANDOM_THRESHOLD_DEFAULT : random_threshold;
+}
+
+static bool iot_sequential_pattern(struct io_tracker *t)
+{
+ return t->pattern == PATTERN_SEQUENTIAL;
+}
+
+static void iot_update_stats(struct io_tracker *t, struct bio *bio)
+{
+ sector_t sectors = bio_sectors(bio);
+
+ if (bio->bi_sector == t->next_start_osector) {
+ t->nr_seq_sectors += sectors;
+
+ } else {
+ /*
+ * Just one non-sequential IO is
+ * enough to reset the counters.
+ */
+ if (t->nr_seq_sectors)
+ t->nr_seq_sectors = t->nr_rand_samples = 0;
+
+ t->nr_rand_samples++;
+ }
+
+ t->next_start_osector = bio->bi_sector + sectors;
+}
+
+static void iot_check_for_pattern_switch(struct io_tracker *t,
+ sector_t block_size)
+{
+ bool reset = iot_sequential_pattern(t) ? (t->nr_rand_samples >= t->thresholds[PATTERN_RANDOM]) :
+ (t->nr_seq_sectors >= t->thresholds[PATTERN_SEQUENTIAL] * block_size);
+ if (reset)
+ t->nr_seq_sectors = t->nr_rand_samples = 0;
+}
+
+/*----------------------------------------------------------------------------*/
+
+/*----------------------------------------------------------------*/
+
+/* The common cache entry part for all policies. */
+struct common_entry {
+ struct hlist_node hlist;
+ struct list_head list;
+ dm_oblock_t oblock;
+ unsigned count[2][2];
+};
+
+/* Cache entry struct. */
+struct basic_cache_entry {
+ struct common_entry ce;
+ struct list_head walk;
+
+ dm_cblock_t cblock;
+ unsigned long access, expire;
+ unsigned saved;
+};
+
+/* Pre and post cache queue entry. */
+struct track_queue_entry {
+ struct common_entry ce;
+};
+
+enum policy_type {
+ p_dumb,
+ p_fifo,
+ p_filo,
+ p_lru,
+ p_mru,
+ p_lfu,
+ p_lfu_ws,
+ p_mfu,
+ p_mfu_ws,
+ p_multiqueue,
+ p_multiqueue_ws,
+ p_noop,
+ p_random,
+ p_q2,
+ p_twoqueue,
+ p_basic /* The default selecting one of the above. */
+};
+
+struct policy;
+typedef void (*queue_add_fn)(struct policy *, struct list_head *);
+typedef void (*queue_del_fn)(struct policy *, struct list_head *);
+typedef struct list_head * (*queue_evict_fn)(struct policy *);
+
+struct queue_fns {
+ queue_add_fn add;
+ queue_del_fn del;
+ queue_evict_fn evict;
+};
+
+static struct list_head *queue_evict_multiqueue(struct policy *);
+static void queue_add_noop(struct policy *, struct list_head *);
+
+#define IS_FILO_MRU(p) (p->queues.fns->add == &queue_add_filo_mru)
+#define IS_LFU(p) (p->queues.fns->add == &queue_add_lfu)
+#define IS_MULTIQUEUE(p) (p->queues.fns->evict == &queue_evict_multiqueue)
+#define IS_Q2(p) (p->queues.fns->add == &queue_add_q2)
+#define IS_TWOQUEUE(p) (p->queues.fns->add == &queue_add_twoqueue)
+#define IS_DUMB(p) (p->queues.fns->add == &queue_add_dumb)
+#define IS_NOOP(p) (p->queues.fns->add == &queue_add_noop)
+
+#define IS_FIFO_FILO(p) (p->queues.fns->del == &queue_del_fifo_filo)
+#define IS_Q2_TWOQUEUE(p) (p->queues.fns->evict == &queue_evict_q2_twoqueue)
+#define IS_MULTIQUEUE_Q2_TWOQUEUE(p) (p->queues.fns->del == &queue_del_multiqueue)
+#define IS_LFU_MFU_WS(p) (p->queues.fns->del == &queue_del_lfu_mfu)
+
+static unsigned next_power(unsigned n, unsigned min)
+{
+ return roundup_pow_of_two(max(n, min));
+}
+
+struct hash {
+ struct hlist_head *table;
+ dm_block_t hash_bits;
+ unsigned nr_buckets;
+};
+
+enum count_type {
+ T_HITS,
+ T_SECTORS
+};
+struct track_queue {
+ struct hash hash;
+ struct track_queue_entry *elts;
+ struct list_head used, free;
+ unsigned count[2][2], size, nr_elts;
+};
+
+struct policy {
+ struct dm_cache_policy policy;
+ struct mutex lock;
+
+ struct io_tracker tracker;
+
+ sector_t origin_size, block_size;
+ unsigned block_shift, calc_threshold_hits, promote_threshold[2], hits;
+
+ struct {
+ /* add/del/evict entry abstractions. */
+ struct queue_fns *fns;
+
+ /* Multiqueue policies. */
+ struct list_head *mq;
+ unsigned long mq_tmo;
+
+ /* Pre- and post-cache queues. */
+ struct track_queue pre, post;
+ enum count_type ctype;
+
+ /*
+ * FIXME:
+ * mempool based kernel lib btree used for lfu,mfu,lfu_ws and mfu_ws
+ *
+ * Now preallocating all objects on creation in order to avoid OOM deadlock.
+ *
+ * Replace with priority heap.
+ */
+ struct btree_head32 fu_head;
+ mempool_t *fu_pool;
+
+ unsigned nr_mqueues, twoqueue_q0_size, twoqueue_q0_max_elts;
+ struct list_head free; /* Free cache entry list */
+ struct list_head used; /* Used cache entry list */
+ struct list_head walk; /* walk_mappings uses this list */
+ } queues;
+
+ /* MINORME: allocate only for multiqueue? */
+ unsigned long jiffies;
+
+ /*
+ * We know exactly how many cblocks will be needed, so we can
+ * allocate them up front.
+ */
+ /* FIXME: unify with track_queue? */
+ dm_cblock_t cache_size;
+ unsigned find_free_nr_words;
+ unsigned find_free_last_word;
+ struct hash chash;
+ unsigned cache_count[2][2];
+
+ /* Cache entry allocation bitset. */
+ unsigned long *allocation_bitset;
+ dm_cblock_t nr_cblocks_allocated;
+
+ struct basic_cache_entry **tmp_entries;
+
+ int threshold_args[2];
+ int mq_tmo_arg, ctype_arg;
+};
+
+/*----------------------------------------------------------------------------*/
+/* Low-level functions. */
+static struct policy *to_policy(struct dm_cache_policy *p)
+{
+ return container_of(p, struct policy, policy);
+}
+
+static int to_rw(struct bio *bio)
+{
+ return (bio_data_dir(bio) == WRITE) ? 1 : 0;
+}
+
+/*----------------------------------------------------------------------------*/
+/* Low-level queue functions. */
+static void queue_init(struct list_head *q)
+{
+ INIT_LIST_HEAD(q);
+}
+
+static bool queue_empty(struct list_head *q)
+{
+ return list_empty(q);
+}
+
+static void queue_add(struct list_head *q, struct list_head *elt)
+{
+ list_add(elt, q);
+}
+
+static void queue_add_tail(struct list_head *q, struct list_head *elt)
+{
+ list_add_tail(elt, q);
+}
+
+static void queue_del(struct list_head *elt)
+{
+ list_del(elt);
+}
+
+static struct list_head *queue_pop(struct list_head *q)
+{
+ struct list_head *r = q->next;
+
+ BUG_ON(!r);
+ list_del(r);
+
+ return r;
+}
+
+static void queue_move_tail(struct list_head *q, struct list_head *elt)
+{
+ list_move_tail(elt, q);
+}
+
+/*----------------------------------------------------------------------------*/
+
+/* Allocate/free various resources. */
+static int alloc_hash(struct hash *hash, unsigned elts)
+{
+ hash->nr_buckets = next_power(elts >> 4, 16);
+ hash->hash_bits = ffs(hash->nr_buckets) - 1;
+ hash->table = vzalloc(sizeof(*hash->table) * hash->nr_buckets);
+
+ return hash->table ? 0 : -ENOMEM;
+}
+
+static void free_hash(struct hash *hash)
+{
+ if (hash->table)
+ vfree(hash->table);
+}
+
+/* Free/alloc basic cache entry structures. */
+static void free_cache_entries(struct policy *p)
+{
+ struct basic_cache_entry *e, *tmp;
+
+ list_for_each_entry_safe(e, tmp, &p->queues.free, ce.list)
+ kmem_cache_free(basic_entry_cache, e);
+
+ list_for_each_entry_safe(e, tmp, &p->queues.walk, walk)
+ kmem_cache_free(basic_entry_cache, e);
+}
+
+static int alloc_cache_blocks_with_hash(struct policy *p, unsigned cache_size)
+{
+ int r = -ENOMEM;
+ unsigned u = cache_size;
+
+ p->nr_cblocks_allocated = to_cblock(0);
+
+ while (u--) {
+ struct basic_cache_entry *e = kmem_cache_zalloc(basic_entry_cache, GFP_KERNEL);
+
+ if (!e)
+ goto bad_cache_alloc;
+
+ queue_add(&p->queues.free, &e->ce.list);
+ }
+
+ /* Cache entries hash. */
+ r = alloc_hash(&p->chash, cache_size);
+ if (!r)
+ return 0;
+
+bad_cache_alloc:
+ free_cache_entries(p);
+
+ return r;
+}
+
+static void free_cache_blocks_and_hash(struct policy *p)
+{
+ free_hash(&p->chash);
+ free_cache_entries(p);
+}
+
+static void free_track_queue(struct track_queue *q)
+{
+ struct track_queue_entry *tqe, *tmp;
+
+ free_hash(&q->hash);
+
+ list_splice(&q->used, &q->free);
+ list_for_each_entry_safe(tqe, tmp, &q->free, ce.list)
+ kmem_cache_free(track_entry_cache, tqe);
+}
+
+static int alloc_track_queue_with_hash(struct track_queue *q, unsigned elts)
+{
+ int r = -ENOMEM;
+ unsigned u = elts;
+
+ while (u--) {
+ struct track_queue_entry *tqe = kmem_cache_zalloc(track_entry_cache, GFP_KERNEL);
+
+ if (!tqe)
+ goto bad_tq_alloc;
+
+ queue_add(&q->free, &tqe->ce.list);
+ }
+
+
+ r = alloc_hash(&q->hash, elts);
+ if (!r)
+ return 0;
+
+bad_tq_alloc:
+ free_track_queue(q);
+
+ return r;
+}
+
+static int alloc_multiqueues(struct policy *p, unsigned mqueues)
+{
+ /* Multiqueue heads. */
+ p->queues.nr_mqueues = mqueues;
+ p->queues.mq = vzalloc(sizeof(*p->queues.mq) * mqueues);
+ if (!p->queues.mq)
+ return -ENOMEM;
+
+ while (mqueues--)
+ queue_init(&p->queues.mq[mqueues]);
+
+ return 0;
+}
+
+static void free_multiqueues(struct policy *p)
+{
+ vfree(p->queues.mq);
+}
+
+static struct basic_cache_entry *alloc_cache_entry(struct policy *p)
+{
+ struct basic_cache_entry *e;
+
+ BUG_ON(from_cblock(p->nr_cblocks_allocated) >= from_cblock(p->cache_size));
+
+ e = list_entry(queue_pop(&p->queues.free), struct basic_cache_entry, ce.list);
+ p->nr_cblocks_allocated = to_cblock(from_cblock(p->nr_cblocks_allocated) + 1);
+
+ return e;
+}
+
+static void alloc_cblock(struct policy *p, dm_cblock_t cblock)
+{
+ BUG_ON(from_cblock(cblock) >= from_cblock(p->cache_size));
+ BUG_ON(test_bit(from_cblock(cblock), p->allocation_bitset));
+ set_bit(from_cblock(cblock), p->allocation_bitset);
+}
+
+static void free_cblock(struct policy *p, dm_cblock_t cblock)
+{
+ BUG_ON(from_cblock(cblock) >= from_cblock(p->cache_size));
+ BUG_ON(!test_bit(from_cblock(cblock), p->allocation_bitset));
+ clear_bit(from_cblock(cblock), p->allocation_bitset);
+}
+
+static void queue_add_twoqueue(struct policy *p, struct list_head *elt);
+static bool any_free_cblocks(struct policy *p)
+{
+ if (IS_TWOQUEUE(p)) {
+ /*
+ * Only allow a certain amount of the total cache size in queue 0
+ * (cblocks with hit count 1).
+ */
+ if (p->queues.twoqueue_q0_size == p->queues.twoqueue_q0_max_elts)
+ return false;
+ }
+
+ return !queue_empty(&p->queues.free);
+}
+
+/*----------------------------------------------------------------*/
+
+static unsigned bit_set_nr_words(unsigned nr_cblocks)
+{
+ return dm_div_up(nr_cblocks, BITS_PER_LONG);
+}
+
+static unsigned long *alloc_bitset(unsigned nr_cblocks)
+{
+ return vzalloc(sizeof(unsigned long) * bit_set_nr_words(nr_cblocks));
+}
+
+static void free_bitset(unsigned long *bits)
+{
+ if (bits)
+ vfree(bits);
+}
+/*----------------------------------------------------------------------------*/
+
+/* Hash functions (lookup, insert, remove). */
+static struct common_entry *__lookup_common_entry(struct hash *hash, dm_oblock_t oblock)
+{
+ unsigned h = hash_64(from_oblock(oblock), hash->hash_bits);
+ struct common_entry *cur;
+ struct hlist_node *tmp;
+ struct hlist_head *bucket = &hash->table[h];
+
+ hlist_for_each_entry(cur, tmp, bucket, hlist) {
+ if (cur->oblock == oblock) {
+ /* Move upfront bucket for faster access. */
+ hlist_del(&cur->hlist);
+ hlist_add_head(&cur->hlist, bucket);
+ return cur;
+ }
+ }
+
+ return NULL;
+}
+
+static struct basic_cache_entry *lookup_cache_entry(struct policy *p,
+ dm_oblock_t oblock)
+{
+ struct common_entry *ce = IS_NOOP(p) ? NULL :
+ __lookup_common_entry(&p->chash, oblock);
+
+ return ce ? container_of(ce, struct basic_cache_entry, ce) : NULL;
+}
+
+static void insert_cache_hash_entry(struct policy *p, struct basic_cache_entry *e)
+{
+ unsigned h = hash_64(from_oblock(e->ce.oblock), p->chash.hash_bits);
+
+ hlist_add_head(&e->ce.hlist, &p->chash.table[h]);
+}
+
+static void remove_cache_hash_entry(struct policy *p, struct basic_cache_entry *e)
+{
+ hlist_del(&e->ce.hlist);
+}
+
+/* Cache track queue. */
+static struct track_queue_entry *lookup_track_queue_entry(struct track_queue *q,
+ dm_oblock_t oblock)
+{
+ struct common_entry *ce = __lookup_common_entry(&q->hash, oblock);
+
+ return ce ? container_of(ce, struct track_queue_entry, ce) : NULL;
+}
+
+static void insert_track_queue_hash_entry(struct track_queue *q,
+ struct track_queue_entry *tqe)
+{
+ unsigned h = hash_64(from_oblock(tqe->ce.oblock), q->hash.hash_bits);
+
+ hlist_add_head(&tqe->ce.hlist, &q->hash.table[h]);
+}
+
+static void remove_track_queue_hash_entry(struct track_queue_entry *tqe)
+{
+ hlist_del(&tqe->ce.hlist);
+}
+/*----------------------------------------------------------------------------*/
+
+/* Out of cache queue support functions. */
+static struct track_queue_entry *pop_track_queue(struct track_queue *q)
+{
+ struct track_queue_entry *r;
+
+ if (queue_empty(&q->free)) {
+ unsigned t, u, end = ARRAY_SIZE(r->ce.count[T_HITS]);
+
+ BUG_ON(queue_empty(&q->used));
+ r = list_entry(queue_pop(&q->used), struct track_queue_entry, ce.list);
+ remove_track_queue_hash_entry(r);
+ q->size--;
+
+ for (t = 0; t < end; t++)
+ for (u = 0; u < end; u++)
+ q->count[t][u] -= q->count[t][u];
+
+ memset(r, 0, sizeof(*r));
+
+ } else
+ r = list_entry(queue_pop(&q->free), struct track_queue_entry, ce.list);
+
+ return r;
+}
+
+/* Retrieve track entry from free list _or_ evict one from track queue. */
+static struct track_queue_entry *
+pop_add_and_insert_track_queue_entry(struct track_queue *q, dm_oblock_t oblock)
+{
+ struct track_queue_entry *r = pop_track_queue(q);
+
+ r->ce.oblock = oblock;
+ queue_add_tail(&q->used, &r->ce.list);
+ insert_track_queue_hash_entry(q, r);
+ q->size++;
+
+ return r;
+}
+
+static unsigned ctype_threshold(struct policy *p, unsigned th)
+{
+ return th << (p->queues.ctype == T_HITS ? 0 : p->block_shift);
+}
+
+static void init_promote_threshold(struct policy *p, bool cache_full)
+{
+ p->promote_threshold[0] = ctype_threshold(p, READ_PROMOTE_THRESHOLD);
+ p->promote_threshold[1] = ctype_threshold(p, WRITE_PROMOTE_THRESHOLD);
+
+ if (cache_full) {
+ p->promote_threshold[0] += ((p->cache_count[p->queues.ctype][0] * READ_PROMOTE_THRESHOLD) << 5) / from_cblock(p->cache_size);
+ p->promote_threshold[1] += ((p->cache_count[p->queues.ctype][1] * WRITE_PROMOTE_THRESHOLD) << 6) / from_cblock(p->cache_size);
+ }
+}
+
+static void calc_rw_threshold(struct policy *p)
+{
+ if (++p->hits > p->calc_threshold_hits && !any_free_cblocks(p)) {
+ p->hits = 0;
+ init_promote_threshold(p, true);
+
+ pr_alert("promote thresholds = %u/%u queue stats = %u/%u\n",
+ p->promote_threshold[0], p->promote_threshold[1], p->queues.pre.size, p->queues.post.size);
+ }
+}
+
+/* Add or update track queue entry. */
+static struct track_queue_entry *
+update_track_queue(struct policy *p, struct track_queue *q, dm_oblock_t oblock,
+ int rw, unsigned hits, sector_t sectors)
+{
+ struct track_queue_entry *r = lookup_track_queue_entry(q, oblock);
+
+ if (r)
+ queue_move_tail(&q->used, &r->ce.list);
+
+ else {
+ r = pop_add_and_insert_track_queue_entry(q, oblock);
+ BUG_ON(!r);
+ }
+
+ r->ce.count[T_HITS][rw] += hits;
+ r->ce.count[T_SECTORS][rw] += sectors;
+ q->count[T_HITS][rw] += hits;
+ q->count[T_SECTORS][rw] += sectors;
+
+ return r;
+}
+
+/* Get hit/sector counts from track queue entry if exists and delete the entry. */
+static void get_any_counts_from_track_queue(struct track_queue *q,
+ struct basic_cache_entry *e,
+ dm_oblock_t oblock)
+{
+ struct track_queue_entry *tqe = lookup_track_queue_entry(q, oblock);
+
+ if (tqe) {
+ /*
+ * On track queue -> retrieve memorized hit count and sectors
+ * in order to sort into appropriate queue on add_cache_entry().
+ */
+ unsigned t, u, end = ARRAY_SIZE(e->ce.count[T_HITS]);
+
+ remove_track_queue_hash_entry(tqe);
+
+ for (t = 0; t < end; t++)
+ for (u = 0; u < end; u++) {
+ e->ce.count[t][u] += tqe->ce.count[t][u];
+ q->count[t][u] -= tqe->ce.count[t][u];
+ }
+
+ memset(&tqe->ce.count, 0, sizeof(tqe->ce.count));
+ queue_move_tail(&q->free, &tqe->ce.list);
+ q->size--;
+ }
+}
+
+static unsigned sum_count(struct policy *p, struct common_entry *ce, enum count_type t)
+{
+ return (ce->count[t][0] + ce->count[t][1]) >> (t == T_HITS ? 0 : p->block_shift);
+}
+
+/*----------------------------------------------------------------------------*/
+
+/* queue_add_.*() functions. */
+static void __queue_add_default(struct policy *p, struct list_head *elt,
+ bool to_head)
+{
+ struct list_head *q = &p->queues.used;
+ struct basic_cache_entry *e = list_entry(elt, struct basic_cache_entry, ce.list);
+
+ to_head ? queue_add(q, elt) : queue_add_tail(q, elt);
+ queue_add_tail(&p->queues.walk, &e->walk);
+}
+
+static void queue_add_default(struct policy *p, struct list_head *elt)
+{
+ __queue_add_default(p, elt, true);
+}
+
+static void queue_add_default_tail(struct policy *p, struct list_head *elt)
+{
+ __queue_add_default(p, elt, false);
+}
+
+static void queue_add_filo_mru(struct policy *p, struct list_head *elt)
+{
+ queue_add_default(p, elt);
+}
+
+static u32 __make_key(u32 k, bool is_lfu)
+{
+ /*
+ * Invert key in case of lfu to allow btree_last() to
+ * retrieve the minimum used list.
+ */
+ return is_lfu ? ~k : k;
+}
+
+static void __queue_add_lfu_mfu(struct policy *p, struct list_head *elt,
+ bool is_lfu, enum count_type ctype)
+{
+ struct list_head *head;
+ struct basic_cache_entry *e = list_entry(elt, struct basic_cache_entry, ce.list);
+ u32 key = __make_key(sum_count(p, &e->ce, ctype), is_lfu);
+
+ /*
+ * Memorize key for deletion (e->ce.count[T_HITS]/e->ce.count[T_SECTORS]
+ * will have changed before)
+ */
+ e->saved = key;
+
+ /*
+ * Key is e->ce.count[T_HITS]/e->ce.count[T_SECTORS] for mfu or
+ * ~e->ce.count[T_HITS]/~e->ce.count[T_SECTORS] for lfu in order to
+ * allow for btree_last() to be able to retrieve the appropriate node.
+ *
+ * A list of cblocks sharing the same hit/sector count is hanging off that node.
+ *
+ * FIXME: replace with priority heap.
+ */
+ head = btree_lookup32(&p->queues.fu_head, key);
+ if (head) {
+ /* Always add to the end where we'll pop cblocks off */
+ list_add_tail(elt, head);
+
+ if (is_lfu) {
+ /*
+ * For lfu, point to added new head, so that
+ * the older entry will get popped first.
+ */
+ int r = btree_update32(&p->queues.fu_head, key, (void *) elt);
+
+ BUG_ON(r);
+ }
+
+ } else {
+ /* New key, insert into tree. */
+ int r = btree_insert32(&p->queues.fu_head, key, (void *) elt, GFP_KERNEL);
+
+ BUG_ON(r);
+ INIT_LIST_HEAD(elt);
+ }
+
+ queue_add_tail(&p->queues.walk, &e->walk);
+}
+
+static void queue_add_lfu(struct policy *p, struct list_head *elt)
+{
+ __queue_add_lfu_mfu(p, elt, true, T_HITS);
+}
+
+static void queue_add_mfu(struct policy *p, struct list_head *elt)
+{
+ __queue_add_lfu_mfu(p, elt, false, T_HITS);
+}
+
+static void queue_add_lfu_ws(struct policy *p, struct list_head *elt)
+{
+ __queue_add_lfu_mfu(p, elt, true, T_SECTORS);
+}
+
+static void queue_add_mfu_ws(struct policy *p, struct list_head *elt)
+{
+ __queue_add_lfu_mfu(p, elt, false, T_SECTORS);
+}
+
+static unsigned __select_multiqueue(struct policy *p, struct basic_cache_entry *e,
+ enum count_type ctype)
+{
+ return min((unsigned) ilog2(sum_count(p, &e->ce, ctype)), p->queues.nr_mqueues - 1U);
+}
+
+static unsigned __get_twoqueue(struct policy *p, struct basic_cache_entry *e)
+{
+ return sum_count(p, &e->ce, T_HITS) > 1 ? 1 : 0;
+}
+
+static unsigned long __queue_tmo_multiqueue(struct policy *p)
+{
+ return p->jiffies + p->queues.mq_tmo;
+}
+
+static void demote_multiqueues(struct policy *p)
+{
+ struct basic_cache_entry *e;
+ struct list_head *cur = p->queues.mq, *end;
+
+ if (!queue_empty(cur))
+ return;
+
+ /*
+ * Start with 2nd queue, because we conditionally move
+ * from queue to queue - 1
+ */
+ end = cur + p->queues.nr_mqueues;
+ while (++cur < end) {
+ while (!queue_empty(cur)) {
+ /* Reference head element. */
+ e = list_first_entry(cur, struct basic_cache_entry, ce.list);
+
+ /*
+ * If expired, move entry from head of higher prio
+ * queue to tail of lower prio one.
+ */
+ if (time_after_eq(p->jiffies, e->expire)) {
+ queue_move_tail(cur - 1, &e->ce.list);
+ e->expire = __queue_tmo_multiqueue(p);
+
+ } else
+ break;
+ }
+ }
+}
+
+static void __queue_add_multiqueue(struct policy *p, struct list_head *elt,
+ enum count_type ctype)
+{
+ struct basic_cache_entry *e = list_entry(elt, struct basic_cache_entry, ce.list);
+ unsigned queue = __select_multiqueue(p, e, ctype);
+
+ e->expire = __queue_tmo_multiqueue(p);
+ queue_add_tail(&p->queues.mq[queue], &e->ce.list);
+ queue_add_tail(&p->queues.walk, &e->walk);
+}
+
+static void queue_add_multiqueue(struct policy *p, struct list_head *elt)
+{
+ __queue_add_multiqueue(p, elt, T_HITS);
+}
+
+static void queue_add_multiqueue_ws(struct policy *p, struct list_head *elt)
+{
+ __queue_add_multiqueue(p, elt, T_SECTORS);
+}
+
+static void queue_add_q2(struct policy *p, struct list_head *elt)
+{
+ struct basic_cache_entry *e = list_entry(elt, struct basic_cache_entry, ce.list);
+
+ queue_add_tail(&p->queues.mq[0], &e->ce.list);
+ queue_add_tail(&p->queues.walk, &e->walk);
+}
+
+static void queue_add_twoqueue(struct policy *p, struct list_head *elt)
+{
+ unsigned queue;
+ struct basic_cache_entry *e = list_entry(elt, struct basic_cache_entry, ce.list);
+
+ queue = e->saved = __get_twoqueue(p, e);
+ if (!queue)
+ p->queues.twoqueue_q0_size++;
+
+ queue_add_tail(&p->queues.mq[queue], &e->ce.list);
+ queue_add_tail(&p->queues.walk, &e->walk);
+}
+
+static void queue_add_dumb(struct policy *p, struct list_head *elt)
+{
+ queue_add_default_tail(p, elt);
+}
+
+static void queue_add_noop(struct policy *p, struct list_head *elt)
+{
+ queue_add_default_tail(p, elt); /* Never called. */
+}
+/*----------------------------------------------------------------------------*/
+
+/* queue_del_.*() functions. */
+static void queue_del_default(struct policy *p, struct list_head *elt)
+{
+ struct basic_cache_entry *e = list_entry(elt, struct basic_cache_entry, ce.list);
+
+ queue_del(&e->ce.list);
+ queue_del(&e->walk);
+}
+
+static void queue_del_fifo_filo(struct policy *p, struct list_head *elt)
+{
+ queue_del_default(p, elt);
+}
+
+static void queue_del_lfu_mfu(struct policy *p, struct list_head *elt)
+{
+ struct list_head *head;
+ struct basic_cache_entry *e = list_entry(elt, struct basic_cache_entry, ce.list);
+ /* Retrieve saved key which has been saved by queue_add_lfu_mfu(). */
+ u32 key = e->saved;
+
+ head = btree_lookup32(&p->queues.fu_head, key);
+ BUG_ON(!head);
+ if (head == elt) {
+ /* Need to remove head, because it's the only element. */
+ if (list_empty(head)) {
+ struct list_head *h = btree_remove32(&p->queues.fu_head, key);
+
+ BUG_ON(!h);
+
+ } else {
+ int r;
+
+ /* Update node to point to next entry as new head. */
+ head = head->next;
+ list_del(elt);
+ r = btree_update32(&p->queues.fu_head, key, (void *) head);
+ BUG_ON(r);
+ }
+
+ } else
+ /* If not head, we can simply remove the element from the list. */
+ list_del(elt);
+
+ queue_del(&e->walk);
+}
+
+static void queue_del_multiqueue(struct policy *p, struct list_head *elt)
+{
+ struct basic_cache_entry *e = list_entry(elt, struct basic_cache_entry, ce.list);
+
+ if (IS_TWOQUEUE(p)) {
+ unsigned queue = e->saved;
+
+ if (!queue)
+ p->queues.twoqueue_q0_size--;
+ }
+
+ queue_del(&e->ce.list);
+ queue_del(&e->walk);
+}
+/*----------------------------------------------------------------------------*/
+
+/* queue_evict_.*() functions. */
+static struct list_head *queue_evict_default(struct policy *p)
+{
+ struct list_head *r = queue_pop(&p->queues.used);
+ struct basic_cache_entry *e = list_entry(r, struct basic_cache_entry, ce.list);
+
+ queue_del(&e->walk);
+
+ return r;
+}
+
+static struct list_head *queue_evict_lfu_mfu(struct policy *p)
+{
+ u32 k;
+ struct list_head *r;
+ struct basic_cache_entry *e;
+
+ /* This'll retrieve lfu/mfu entry because of __make_key(). */
+ r = btree_last32(&p->queues.fu_head, &k);
+ BUG_ON(!r);
+
+ if (list_empty(r))
+ r = btree_remove32(&p->queues.fu_head, k);
+
+ else {
+ /* Retrieve last element in order to minimize btree updates. */
+ r = r->prev;
+ BUG_ON(!r);
+ list_del(r);
+ }
+
+ e = list_entry(r, struct basic_cache_entry, ce.list);
+ e->saved = 0;
+ queue_del(&e->walk);
+
+ return r;
+}
+
+static struct list_head *queue_evict_random(struct policy *p)
+{
+ struct list_head *r = p->queues.used.next;
+ struct basic_cache_entry *e;
+ dm_block_t off = random32();
+
+ BUG_ON(!r);
+
+ /* FIXME: cblock_t is 32 bit for the time being. */
+ /* Be prepared for large caches ;-) */
+ if (from_cblock(p->cache_size) >= UINT_MAX)
+ off |= ((dm_block_t) random32() << 32);
+
+ /* FIXME: overhead walking list. */
+ off = do_div(off, from_cblock(p->cache_size));
+ while (off--)
+ r = r->next;
+
+ e = list_entry(r, struct basic_cache_entry, ce.list);
+ queue_del(r);
+ queue_del(&e->walk);
+
+ return r;
+}
+
+static struct list_head *queue_evict_multiqueue(struct policy *p)
+{
+ struct list_head *cur = p->queues.mq - 1, /* -1 because of ++cur below. */
+ *end = p->queues.mq + p->queues.nr_mqueues;
+
+ while (++cur < end) {
+ if (!queue_empty(cur)) {
+ struct basic_cache_entry *e;
+ struct list_head *r;
+
+ if (IS_TWOQUEUE(p) && cur == p->queues.mq)
+ p->queues.twoqueue_q0_size--;
+
+ r = queue_pop(cur);
+ e = list_entry(r, struct basic_cache_entry, ce.list);
+ queue_del(&e->walk);
+
+ return r;
+ }
+
+ if (IS_MULTIQUEUE(p))
+ break;
+ }
+
+ return NULL;
+}
+
+static struct list_head *queue_evict_q2_twoqueue(struct policy *p)
+{
+ return queue_evict_multiqueue(p);
+}
+
+/*----------------------------------------------------------------------------*/
+
+/*
+ * This doesn't allocate the block.
+ */
+static int __find_free_cblock(struct policy *p, unsigned begin, unsigned end,
+ dm_cblock_t *result, unsigned *last_word)
+{
+ int r = -ENOSPC;
+ unsigned w;
+
+ for (w = begin; w < end; w++) {
+ /*
+ * ffz is undefined if no zero exists
+ */
+ if (p->allocation_bitset[w] != ULONG_MAX) {
+ *last_word = w;
+ *result = to_cblock((w * BITS_PER_LONG) + ffz(p->allocation_bitset[w]));
+ if (from_cblock(*result) < from_cblock(p->cache_size))
+ r = 0;
+
+ break;
+ }
+ }
+
+ return r;
+}
+
+static int find_free_cblock(struct policy *p, dm_cblock_t *result)
+{
+ int r = __find_free_cblock(p, p->find_free_last_word, p->find_free_nr_words, result, &p->find_free_last_word);
+
+ if (r == -ENOSPC && p->find_free_last_word)
+ r = __find_free_cblock(p, 0, p->find_free_last_word, result, &p->find_free_last_word);
+
+ return r;
+}
+
+static void alloc_cblock_insert_cache_and_count_entry(struct policy *p, struct basic_cache_entry *e)
+{
+ unsigned t, u, end = ARRAY_SIZE(e->ce.count[T_HITS]);
+
+ alloc_cblock(p, e->cblock);
+ insert_cache_hash_entry(p, e);
+
+ if (IS_DUMB(p) || IS_NOOP(p))
+ return;
+
+ for (t = 0; t < end; t++)
+ for (u = 0; u < end; u++)
+ p->cache_count[t][u] += e->ce.count[t][u];
+}
+
+static void add_cache_entry(struct policy *p, struct basic_cache_entry *e)
+{
+ p->queues.fns->add(p, &e->ce.list);
+ alloc_cblock_insert_cache_and_count_entry(p, e);
+}
+
+static void remove_cache_entry(struct policy *p, struct basic_cache_entry *e)
+{
+ unsigned t, u, end = ARRAY_SIZE(e->ce.count[T_HITS]);
+
+ remove_cache_hash_entry(p, e);
+ free_cblock(p, e->cblock);
+
+ if (IS_DUMB(p) || IS_NOOP(p))
+ return;
+
+ for (t = 0; t < end; t++)
+ for (u = 0; u < end; u++)
+ p->cache_count[t][u] -= e->ce.count[t][u];
+}
+
+static struct basic_cache_entry *evict_cache_entry(struct policy *p)
+{
+ struct basic_cache_entry *r;
+ struct list_head *elt = p->queues.fns->evict(p);
+
+ if (elt) {
+ r = list_entry(elt, struct basic_cache_entry, ce.list);
+ remove_cache_entry(p, r);
+ } else
+ r = NULL;
+
+ return r;
+}
+
+static void update_cache_entry(struct policy *p, struct basic_cache_entry *e,
+ struct bio *bio, struct policy_result *result)
+{
+ int rw;
+
+ result->op = POLICY_HIT;
+ result->cblock = e->cblock;
+
+ if (IS_DUMB(p) || IS_NOOP(p))
+ return;
+
+ rw = to_rw(bio);
+
+ e->ce.count[T_HITS][rw]++;
+ e->ce.count[T_SECTORS][rw] += bio_sectors(bio);
+
+ /*
+ * No queue deletion and reinsertion needed with fifo/filo; ie.
+ * avoid queue reordering for those.
+ */
+ if (!IS_FIFO_FILO(p)) {
+ p->queues.fns->del(p, &e->ce.list);
+ p->queues.fns->add(p, &e->ce.list);
+ }
+}
+
+static void get_cache_block(struct policy *p, dm_oblock_t oblock, struct bio *bio,
+ struct policy_result *result)
+{
+ int rw = to_rw(bio);
+ struct basic_cache_entry *e;
+
+ if (queue_empty(&p->queues.free)) {
+ if (IS_MULTIQUEUE(p))
+ demote_multiqueues(p);
+
+ e = evict_cache_entry(p);
+ if (!e)
+ return;
+
+ /* Memorize hits and sectors of just evicted entry on out queue. */
+ if (!IS_DUMB(p)) {
+ /* Reads. */
+ update_track_queue(p, &p->queues.post, e->ce.oblock, 0,
+ e->ce.count[T_HITS][0],
+ e->ce.count[T_SECTORS][0]);
+ /* Writes. */
+ update_track_queue(p, &p->queues.post, e->ce.oblock, 1,
+ e->ce.count[T_HITS][1],
+ e->ce.count[T_SECTORS][1]);
+ }
+
+ result->old_oblock = e->ce.oblock;
+ result->op = POLICY_REPLACE;
+
+ } else {
+ int r;
+
+ e = alloc_cache_entry(p);
+ r = find_free_cblock(p, &e->cblock);
+ BUG_ON(r);
+
+ result->op = POLICY_NEW;
+ }
+
+ /*
+ * If an entry for oblock exists on track queues ->
+ * retrieve hit counts and sectors from track queues and delete
+ * the respective tracking entries.
+ */
+ if (!IS_DUMB(p)) {
+ memset(&e->ce.count, 0, sizeof(e->ce.count));
+ e->ce.count[T_HITS][rw] = 1;
+ e->ce.count[T_SECTORS][rw] = bio_sectors(bio);
+ get_any_counts_from_track_queue(&p->queues.pre, e, oblock);
+ get_any_counts_from_track_queue(&p->queues.post, e, oblock);
+ }
+
+ result->cblock = e->cblock;
+ e->ce.oblock = oblock;
+ add_cache_entry(p, e);
+}
+
+static bool in_cache(struct policy *p, dm_oblock_t oblock, struct bio *bio, struct policy_result *result)
+{
+ struct basic_cache_entry *e = lookup_cache_entry(p, oblock);
+
+ if (e) {
+ /* Cache hit: update entry on queues, increment its hit count */
+ update_cache_entry(p, e, bio, result);
+ return true;
+ }
+
+ return false;
+}
+
+static bool should_promote(struct policy *p, struct track_queue_entry *tqe,
+ dm_oblock_t oblock, int rw, bool discarded_oblock,
+ struct policy_result *result)
+{
+ BUG_ON(!tqe);
+ calc_rw_threshold(p);
+
+ if (discarded_oblock && any_free_cblocks(p))
+ /*
+ * We don't need to do any copying at all, so give this a
+ * very low threshold. In practice this only triggers
+ * during initial population after a format.
+ */
+ return true;
+
+ return tqe->ce.count[p->queues.ctype][rw] >= p->promote_threshold[rw];
+}
+
+static void map_prerequisites(struct policy *p, struct bio *bio)
+{
+ /* Update io tracker. */
+ iot_update_stats(&p->tracker, bio);
+ iot_check_for_pattern_switch(&p->tracker, p->block_size);
+
+ /* Get start jiffies needed for time based queue demotion. */
+ if (IS_MULTIQUEUE(p))
+ p->jiffies = get_jiffies_64();
+}
+
+static int map(struct policy *p, dm_oblock_t oblock,
+ bool can_block, bool can_migrate, bool discarded_oblock,
+ struct bio *bio, struct policy_result *result)
+{
+ int rw = to_rw(bio);
+ struct track_queue_entry *tqe;
+
+ if (IS_NOOP(p))
+ return 0;
+
+ if (in_cache(p, oblock, bio, result))
+ return 0;
+
+ if (!IS_DUMB(p))
+ /* Record hits on pre cache track queue. */
+ tqe = update_track_queue(p, &p->queues.pre, oblock, rw, 1, bio_sectors(bio));
+
+ if (!can_migrate)
+ return -EWOULDBLOCK;
+
+ else if (!IS_DUMB(p) && iot_sequential_pattern(&p->tracker))
+ ;
+
+ else if (IS_DUMB(p) || should_promote(p, tqe, oblock, rw, discarded_oblock, result))
+ get_cache_block(p, oblock, bio, result);
+
+ return 0;
+}
+
+/* Public interface (see dm-cache-policy.h */
+static int basic_map(struct dm_cache_policy *pe, dm_oblock_t oblock,
+ bool can_block, bool can_migrate, bool discarded_oblock,
+ struct bio *bio, struct policy_result *result)
+{
+ int r;
+ struct policy *p = to_policy(pe);
+
+ result->op = POLICY_MISS;
+
+ if (can_block)
+ mutex_lock(&p->lock);
+
+ else if (!mutex_trylock(&p->lock))
+ return -EWOULDBLOCK;
+
+ if (!IS_DUMB(p) && !IS_NOOP(p))
+ map_prerequisites(p, bio);
+
+ r = map(p, oblock, can_block, can_migrate, discarded_oblock, bio, result);
+
+ mutex_unlock(&p->lock);
+
+ return r;
+}
+
+static int basic_lookup(struct dm_cache_policy *pe, dm_oblock_t oblock, dm_cblock_t *cblock)
+{
+ int r;
+ struct policy *p = to_policy(pe);
+ struct basic_cache_entry *e;
+
+ if (!mutex_trylock(&p->lock))
+ return -EWOULDBLOCK;
+
+ e = lookup_cache_entry(p, oblock);
+ if (e) {
+ *cblock = e->cblock;
+ r = 0;
+
+ } else
+ r = -ENOENT;
+
+ mutex_unlock(&p->lock);
+
+ return r;
+}
+
+static void basic_destroy(struct dm_cache_policy *pe)
+{
+ struct policy *p = to_policy(pe);
+
+ if (IS_LFU_MFU_WS(p))
+ btree_destroy32(&p->queues.fu_head);
+
+ else if (IS_MULTIQUEUE_Q2_TWOQUEUE(p))
+ free_multiqueues(p);
+
+ free_track_queue(&p->queues.post);
+ free_track_queue(&p->queues.pre);
+ free_bitset(p->allocation_bitset);
+ free_cache_blocks_and_hash(p);
+ kfree(p);
+}
+
+/* FIXME: converters can disappear in case of larger hint cast in metadata. */
+static const uint16_t high_flag = 0x8000;
+static const uint32_t hint_lmask = 0xFFFF;
+static const uint32_t hint_hmask = 0xFFFF0000;
+static uint16_t count_to_hint(unsigned val)
+{
+ uint16_t vh, vl;
+
+ vl = val & hint_lmask;
+ vh = (val & hint_hmask) >> 16;
+
+ if (vh)
+ return vh | high_flag;
+ else
+ return vl & ~high_flag;
+}
+
+static uint32_t counts_to_hint(unsigned read, unsigned write)
+{
+ return count_to_hint(read) & (count_to_hint(write) << 16);
+}
+
+static unsigned check_high(uint16_t v)
+{
+ unsigned r = v;
+
+ if (r & high_flag)
+ r = (r & ~high_flag) << 16;
+
+ return r;
+}
+
+static void hint_to_counts(uint32_t val, unsigned *read, unsigned *write)
+{
+ *read = check_high(val & hint_lmask);
+ *write = check_high((val & hint_hmask) >> 16);
+
+}
+
+static void sort_in_cache_entry(struct policy *p, struct basic_cache_entry *e)
+{
+ struct list_head *elt;
+ struct basic_cache_entry *cur;
+
+ list_for_each(elt, &p->queues.used) {
+ cur = list_entry(elt, struct basic_cache_entry, ce.list);
+ if (e->ce.count[T_HITS][0] > cur->ce.count[T_HITS][0])
+ break;
+ }
+
+ if (elt == &p->queues.used)
+ list_add_tail(&e->ce.list, elt);
+ else
+ list_add(&e->ce.list, elt);
+
+ queue_add_tail(&p->queues.walk, &e->walk);
+}
+
+static int basic_load_mapping(struct dm_cache_policy *pe,
+ dm_oblock_t oblock, dm_cblock_t cblock,
+ uint32_t hint, bool hint_valid)
+{
+ struct policy *p = to_policy(pe);
+ struct basic_cache_entry *e;
+
+ e = alloc_cache_entry(p);
+ if (!e)
+ return -ENOMEM;
+
+ e->cblock = cblock;
+ e->ce.oblock = oblock;
+
+ if (hint_valid) {
+ unsigned reads, writes;
+
+ hint_to_counts(hint, &reads, &writes);
+ e->ce.count[T_HITS][0] = reads;
+ e->ce.count[T_HITS][1] = writes;
+
+ if (IS_MULTIQUEUE(p) || IS_TWOQUEUE(p) || IS_LFU_MFU_WS(p)) {
+ /* FIXME: store also in larger hints rather than making up. */
+ e->ce.count[T_SECTORS][0] = reads << p->block_shift;
+ e->ce.count[T_SECTORS][1] = writes << p->block_shift;
+ }
+ }
+
+ if (IS_MULTIQUEUE(p) || IS_TWOQUEUE(p) || IS_LFU_MFU_WS(p))
+ add_cache_entry(p, e);
+ else {
+ sort_in_cache_entry(p, e);
+ alloc_cblock_insert_cache_and_count_entry(p, e);
+ }
+
+ return 0;
+}
+
+/* Walk mappings */
+static int basic_walk_mappings(struct dm_cache_policy *pe, policy_walk_fn fn,
+ void *context)
+{
+ int r = 0;
+ unsigned nr = 0;
+ struct policy *p = to_policy(pe);
+ struct basic_cache_entry *e;
+
+ mutex_lock(&p->lock);
+
+ list_for_each_entry(e, &p->queues.walk, walk) {
+ unsigned reads, writes;
+
+ if (IS_MULTIQUEUE_Q2_TWOQUEUE(p) || IS_LFU_MFU_WS(p)) {
+ reads = e->ce.count[T_HITS][0];
+ writes = e->ce.count[T_HITS][1];
+
+ } else {
+ reads = nr++;
+
+ if (IS_FILO_MRU(p))
+ reads = from_cblock(p->cache_size) - reads - 1;
+
+ writes = 0;
+ }
+
+ r = fn(context, e->cblock, e->ce.oblock,
+ counts_to_hint(reads, writes));
+ if (r)
+ break;
+ }
+
+ mutex_unlock(&p->lock);
+ return r;
+}
+
+static struct basic_cache_entry *__basic_force_remove_mapping(struct policy *p,
+ dm_oblock_t oblock)
+{
+ struct basic_cache_entry *r = lookup_cache_entry(p, oblock);
+
+ BUG_ON(!r);
+
+ p->queues.fns->del(p, &r->ce.list);
+ remove_cache_entry(p, r);
+
+ return r;
+}
+
+static void basic_remove_mapping(struct dm_cache_policy *pe, dm_oblock_t oblock)
+{
+ struct policy *p = to_policy(pe);
+ struct basic_cache_entry *e;
+
+ mutex_lock(&p->lock);
+ e = __basic_force_remove_mapping(p, oblock);
+ memset(&e->ce.count, 0, sizeof(e->ce.count));
+ queue_add_tail(&p->queues.free, &e->ce.list);
+
+ BUG_ON(!from_cblock(p->nr_cblocks_allocated));
+ p->nr_cblocks_allocated = to_cblock(from_cblock(p->nr_cblocks_allocated) - 1);
+ mutex_unlock(&p->lock);
+}
+
+static void basic_force_mapping(struct dm_cache_policy *pe,
+ dm_oblock_t current_oblock, dm_oblock_t oblock)
+{
+ struct policy *p = to_policy(pe);
+ struct basic_cache_entry *e;
+
+ mutex_lock(&p->lock);
+ e = __basic_force_remove_mapping(p, current_oblock);
+ e->ce.oblock = oblock;
+ add_cache_entry(p, e);
+ mutex_unlock(&p->lock);
+}
+
+static dm_cblock_t basic_residency(struct dm_cache_policy *pe)
+{
+ /* FIXME: lock mutex, not sure we can block here. */
+ return to_policy(pe)->nr_cblocks_allocated;
+}
+
+/* ctr/message optional argument parsing. */
+static int process_threshold_option(struct policy *p, char **argv,
+ enum io_pattern pattern, bool set_ctr_arg)
+{
+ unsigned long tmp;
+
+ if (kstrtoul(argv[1], 10, &tmp))
+ return -EINVAL;
+
+ if (set_ctr_arg) {
+ if (p->threshold_args[pattern] > -1)
+ return -EINVAL;
+
+ p->threshold_args[pattern] = tmp;
+ }
+
+ p->tracker.thresholds[pattern] = tmp;
+
+ return 0;
+}
+
+static int process_multiqueue_timeout_option(struct policy *p, char **argv, bool set_ctr_arg)
+{
+ unsigned long tmp;
+
+ /* multiqueue timeout in milliseconds. */
+ if (kstrtoul(argv[1], 10, &tmp) ||
+ tmp < 1 || tmp > 24*3600*1000) /* 1 day max :) */
+ return -EINVAL;
+
+ if (IS_MULTIQUEUE(p)) {
+ unsigned long ticks = tmp * HZ / 1000;
+
+ if (set_ctr_arg) {
+ if (p->mq_tmo_arg > -1)
+ return -EINVAL;
+
+ p->mq_tmo_arg = tmp;
+ }
+
+ /* Ensure one tick timeout minimum. */
+ p->queues.mq_tmo = ticks ? ticks : 1;
+
+ return 0;
+ }
+
+ return -EINVAL;
+}
+
+static int process_hits_option(struct policy *p, char **argv, bool set_ctr_arg)
+{
+ unsigned long tmp;
+
+ /* Only allow as ctr argument. */
+ if (!set_ctr_arg)
+ return -EINVAL;
+
+ if (kstrtoul(argv[1], 10, &tmp) || tmp > 1)
+ return -EINVAL;
+
+ if (p->ctype_arg > -1)
+ return -EINVAL;
+
+ p->ctype_arg = tmp;
+ p->queues.ctype = tmp ? T_HITS : T_SECTORS;
+
+ return 0;
+}
+
+static int process_config_option(struct policy *p, char **argv, bool set_ctr_arg)
+{
+ if (!strcasecmp(argv[0], "sequential_threshold"))
+ return process_threshold_option(p, argv, PATTERN_SEQUENTIAL, set_ctr_arg);
+
+ else if (!strcasecmp(argv[0], "random_threshold"))
+ return process_threshold_option(p, argv, PATTERN_RANDOM, set_ctr_arg);
+
+ else if (!strcasecmp(argv[0], "multiqueue_timeout"))
+ return process_multiqueue_timeout_option(p, argv, set_ctr_arg);
+
+ else if (!strcasecmp(argv[0], "hits"))
+ return process_hits_option(p, argv, set_ctr_arg);
+
+ return -EINVAL;
+}
+
+static int basic_message(struct dm_cache_policy *pe, unsigned argc, char **argv)
+{
+ struct policy *p = to_policy(pe);
+
+ if (argc != 3)
+ return -EINVAL;
+
+ if (!strcasecmp(argv[0], "set_config"))
+ return process_config_option(p, argv + 1, false);
+
+ return -EINVAL;
+}
+
+static int basic_status(struct dm_cache_policy *pe, status_type_t type,
+ unsigned status_flags, char *result, unsigned maxlen)
+{
+ ssize_t sz = 0;
+ struct policy *p = to_policy(pe);
+
+ switch (type) {
+ case STATUSTYPE_INFO:
+ DMEMIT(" %lu %lu %lu %u",
+ p->tracker.thresholds[PATTERN_SEQUENTIAL],
+ p->tracker.thresholds[PATTERN_RANDOM],
+ p->queues.mq_tmo * 1000 / HZ,
+ p->queues.ctype);
+ break;
+
+ case STATUSTYPE_TABLE:
+ if (p->threshold_args[PATTERN_SEQUENTIAL] > -1)
+ DMEMIT(" sequential_threshold %u", p->threshold_args[PATTERN_SEQUENTIAL]);
+
+ if (p->threshold_args[PATTERN_RANDOM] > -1)
+ DMEMIT(" random_threshold %u", p->threshold_args[PATTERN_RANDOM]);
+
+ if (p->mq_tmo_arg > -1)
+ DMEMIT(" multiqueue_timeout %d", p->mq_tmo_arg);
+
+ if (p->ctype_arg > -1)
+ DMEMIT(" hits %d", p->ctype_arg);
+ }
+
+ return 0;
+}
+
+static int process_policy_args(struct policy *p, int argc, char **argv)
+{
+ int r;
+ unsigned u;
+
+ p->threshold_args[0] = p->threshold_args[1] = p->mq_tmo_arg = p->ctype_arg = -1;
+
+ if (!argc)
+ return 0;
+
+ if (argc != 2 && argc != 4 && argc != 6 && argc != 8)
+ return -EINVAL;
+
+ for (r = u = 0; u < argc && !r; u += 2)
+ r = process_config_option(p, argv + u, true);
+
+ return r;
+}
+
+/* Init the policy plugin interface function pointers. */
+static void init_policy_functions(struct policy *p)
+{
+ p->policy.destroy = basic_destroy;
+ p->policy.map = basic_map;
+ p->policy.lookup = basic_lookup;
+ p->policy.load_mapping = basic_load_mapping;
+ p->policy.walk_mappings = basic_walk_mappings;
+ p->policy.remove_mapping = basic_remove_mapping;
+ p->policy.writeback_work = NULL;
+ p->policy.force_mapping = basic_force_mapping;
+ p->policy.residency = basic_residency;
+ p->policy.tick = NULL;
+ p->policy.status = basic_status;
+ p->policy.message = basic_message;
+}
+
+static struct dm_cache_policy *basic_policy_create(dm_cblock_t cache_size,
+ sector_t origin_size,
+ sector_t block_size,
+ int argc, char **argv,
+ enum policy_type type)
+{
+ int r;
+ unsigned mqueues = 0;
+ static struct queue_fns queue_fns[] = {
+ /* These have to be in 'enum policy_type' order! */
+ { &queue_add_dumb, &queue_del_default, &queue_evict_default }, /* p_dumb */
+ { &queue_add_default_tail, &queue_del_fifo_filo, &queue_evict_default }, /* p_fifo */
+ { &queue_add_filo_mru, &queue_del_fifo_filo, &queue_evict_default }, /* p_filo */
+ { &queue_add_default_tail, &queue_del_default, &queue_evict_default }, /* p_lru */
+ { &queue_add_filo_mru, &queue_del_default, &queue_evict_default }, /* p_mru */
+ { &queue_add_lfu, &queue_del_lfu_mfu, &queue_evict_lfu_mfu }, /* p_lfu */
+ { &queue_add_lfu_ws, &queue_del_lfu_mfu, &queue_evict_lfu_mfu }, /* p_lfu_ws */
+ { &queue_add_mfu, &queue_del_lfu_mfu, &queue_evict_lfu_mfu }, /* p_mfu */
+ { &queue_add_mfu_ws, &queue_del_lfu_mfu, &queue_evict_lfu_mfu }, /* p_mfu_ws */
+ { &queue_add_multiqueue, &queue_del_multiqueue, &queue_evict_multiqueue }, /* p_multiqueue */
+ { &queue_add_multiqueue_ws, &queue_del_multiqueue, &queue_evict_multiqueue }, /* p_multiqueue_ws */
+ { &queue_add_noop, NULL, NULL }, /* p_noop */
+ { &queue_add_default_tail, &queue_del_default, &queue_evict_random }, /* p_random */
+ { &queue_add_q2, &queue_del_multiqueue, &queue_evict_q2_twoqueue }, /* p_q2 */
+ { &queue_add_twoqueue, &queue_del_multiqueue, &queue_evict_q2_twoqueue }, /* p_twoqueue */
+ };
+ struct policy *p = kzalloc(sizeof(*p), GFP_KERNEL);
+
+ if (!p)
+ return NULL;
+
+ /* Set default (aka basic) policy (doesn't need a queue_fns entry above). */
+ if (type == p_basic)
+ type = p_multiqueue_ws;
+
+ /* Distinguish policies */
+ p->queues.fns = queue_fns + type;
+
+ init_policy_functions(p);
+
+ /* Need to do that before iot_init(). */
+ r = process_policy_args(p, argc, argv);
+ if (r)
+ goto bad_free_policy;
+
+ iot_init(&p->tracker, p->threshold_args[PATTERN_SEQUENTIAL], p->threshold_args[PATTERN_RANDOM]);
+
+ p->cache_size = cache_size;
+ p->find_free_nr_words = bit_set_nr_words(from_cblock(cache_size));
+ p->find_free_last_word = 0;
+ p->block_size = block_size;
+ p->block_shift = ffs(block_size);
+ p->origin_size = origin_size;
+ p->calc_threshold_hits = max(from_cblock(cache_size) >> 2, 128U);
+ p->queues.ctype = p->ctype_arg < 0 ? T_HITS : p->queues.ctype;
+ init_promote_threshold(p, false);
+ mutex_init(&p->lock);
+ queue_init(&p->queues.free);
+ queue_init(&p->queues.used);
+ queue_init(&p->queues.walk);
+ queue_init(&p->queues.pre.free);
+ queue_init(&p->queues.pre.used);
+ queue_init(&p->queues.post.free);
+ queue_init(&p->queues.post.used);
+
+ if (IS_NOOP(p))
+ goto out;
+
+ /* Allocate cache entry structs and add them to free list. */
+ r = alloc_cache_blocks_with_hash(p, from_cblock(cache_size));
+ if (r)
+ goto bad_free_policy;
+
+ /* Cache allocation bitset. */
+ p->allocation_bitset = alloc_bitset(from_cblock(cache_size));
+ if (!p->allocation_bitset)
+ goto bad_free_cache_blocks_and_hash;
+
+ if (IS_DUMB(p))
+ goto out;
+
+ /*
+ * Create in queue to track entries waiting for the
+ * cache in order to stear their promotion.
+ */
+ r = alloc_track_queue_with_hash(&p->queues.pre, max(from_cblock(cache_size), 128U));
+ if (r)
+ goto bad_free_allocation_bitset;
+
+ /* Create cache_size queue to track evicted cache entries. */
+ r = alloc_track_queue_with_hash(&p->queues.post, max(from_cblock(cache_size) >> 1, 128U));
+ if (r)
+ goto bad_free_track_queue_pre;
+
+ if (IS_LFU_MFU_WS(p)) {
+ /* FIXME: replace with priority heap. */
+ p->queues.fu_pool = mempool_create(from_cblock(cache_size), btree_alloc, btree_free, NULL);
+ if (!p->queues.fu_pool)
+ goto bad_free_track_queue_post;
+
+ btree_init_mempool32(&p->queues.fu_head, p->queues.fu_pool);
+
+ } else if (IS_Q2(p))
+ mqueues = 1; /* Not really multiple queues but code can be shared */
+
+ else if (IS_TWOQUEUE(p)) {
+ /*
+ * Just 2 prio queues.
+ *
+ * Only allow 25% of the total cache size maximum in queue 0 (hit count 1).
+ * Ie. 75% minimum is reserved for cblocks with multiple hits.
+ */
+ mqueues = 2;
+ p->queues.twoqueue_q0_max_elts =
+ min(max(from_cblock(cache_size) >> 2, 16U), from_cblock(cache_size));
+
+ } else if (IS_MULTIQUEUE(p)) {
+ /* Multiple queues. */
+ mqueues = min(max((unsigned) ilog2(block_size << 13), 8U), (unsigned) from_cblock(cache_size));
+ p->jiffies = get_jiffies_64();
+ p->queues.mq_tmo = p->mq_tmo_arg < 0 ? MQ_QUEUE_TMO_DEFAULT : p->queues.mq_tmo;
+ }
+
+
+ if (mqueues) {
+ r = alloc_multiqueues(p, mqueues);
+ if (r)
+ goto bad_free_track_queue_post;
+
+ }
+
+out:
+ return &p->policy;
+
+bad_free_track_queue_post:
+ free_track_queue(&p->queues.post);
+bad_free_track_queue_pre:
+ free_track_queue(&p->queues.pre);
+bad_free_allocation_bitset:
+ free_bitset(p->allocation_bitset);
+bad_free_cache_blocks_and_hash:
+ free_cache_blocks_and_hash(p);
+bad_free_policy:
+ kfree(p);
+
+ return NULL;
+}
+/*----------------------------------------------------------------------------*/
+
+/* Policy type creation magic. */
+#define __CREATE_POLICY(policy) \
+static struct dm_cache_policy *policy ## _create(dm_cblock_t cache_size, sector_t origin_size, \
+ sector_t block_size, int argc, char **argv) \
+{ \
+ return basic_policy_create(cache_size, origin_size, block_size, argc, argv, p_ ## policy); \
+}
+
+#define __POLICY_TYPE(policy) \
+static struct dm_cache_policy_type policy ## _policy_type = { \
+ .name = #policy, \
+ .hint_size = 0, \
+ .owner = THIS_MODULE, \
+ .create = policy ## _create \
+};
+
+#define __CREATE_POLICY_TYPE(policy) \
+ __CREATE_POLICY(policy); \
+ __POLICY_TYPE(policy);
+
+/*
+ * Create all fifo_create,filo_create,lru_create,... functions and
+ * declare and initialize all fifo_policy_type,filo_policy_type,... structures.
+ */
+__CREATE_POLICY_TYPE(basic);
+__CREATE_POLICY_TYPE(dumb);
+__CREATE_POLICY_TYPE(fifo);
+__CREATE_POLICY_TYPE(filo);
+__CREATE_POLICY_TYPE(lfu);
+__CREATE_POLICY_TYPE(lfu_ws);
+__CREATE_POLICY_TYPE(lru);
+__CREATE_POLICY_TYPE(mfu);
+__CREATE_POLICY_TYPE(mfu_ws);
+__CREATE_POLICY_TYPE(mru);
+__CREATE_POLICY_TYPE(multiqueue);
+__CREATE_POLICY_TYPE(multiqueue_ws);
+__CREATE_POLICY_TYPE(noop);
+__CREATE_POLICY_TYPE(random);
+__CREATE_POLICY_TYPE(q2);
+__CREATE_POLICY_TYPE(twoqueue);
+
+static struct dm_cache_policy_type *policy_types[] = {
+ &basic_policy_type,
+ &dumb_policy_type,
+ &fifo_policy_type,
+ &filo_policy_type,
+ &lfu_policy_type,
+ &lfu_ws_policy_type,
+ &lru_policy_type,
+ &mfu_policy_type,
+ &mfu_ws_policy_type,
+ &mru_policy_type,
+ &multiqueue_policy_type,
+ &multiqueue_ws_policy_type,
+ &noop_policy_type,
+ &random_policy_type,
+ &q2_policy_type,
+ &twoqueue_policy_type
+};
+
+static int __init basic_init(void)
+{
+ int i = ARRAY_SIZE(policy_types), r;
+
+ basic_entry_cache = kmem_cache_create("dm_cache_basic_policy",
+ sizeof(struct basic_cache_entry),
+ __alignof__(struct basic_cache_entry),
+ 0, NULL);
+ if (!basic_entry_cache)
+ goto bad_basic_entry_cache;
+
+ track_entry_cache = kmem_cache_create("dm_cache_basic_policy_tq",
+ sizeof(struct track_queue_entry),
+ __alignof__(struct track_queue_entry),
+ 0, NULL);
+ if (!track_entry_cache)
+ goto bad_track_entry_cache;
+
+ while (i--) {
+ r = dm_cache_policy_register(policy_types[i]);
+ if (r)
+ goto bad_policy;
+ }
+
+ return 0;
+
+bad_policy:
+ kmem_cache_destroy(track_entry_cache);
+bad_track_entry_cache:
+ kmem_cache_destroy(basic_entry_cache);
+bad_basic_entry_cache:
+ return -ENOMEM;
+}
+
+static void __exit basic_exit(void)
+{
+ int i = ARRAY_SIZE(policy_types);
+
+ while (i--)
+ dm_cache_policy_unregister(policy_types[i]);
+
+ kmem_cache_destroy(track_entry_cache);
+ kmem_cache_destroy(basic_entry_cache);
+}
+
+module_init(basic_init);
+module_exit(basic_exit);
+
+MODULE_AUTHOR("Joe Thornber/Heinz Mauelshagen <dm-devel@redhat.com>");
+MODULE_LICENSE("GPL");
+MODULE_DESCRIPTION("basic cache policies (fifo, lru, etc)");
+
+MODULE_ALIAS("dm-cache-basic"); /* basic_policy_create() maps "basic" to one of the following: */
+MODULE_ALIAS("dm-cache-dumb");
+MODULE_ALIAS("dm-cache-fifo");
+MODULE_ALIAS("dm-cache-filo");
+MODULE_ALIAS("dm-cache-lfu");
+MODULE_ALIAS("dm-cache-lfu_ws");
+MODULE_ALIAS("dm-cache-lru");
+MODULE_ALIAS("dm-cache-mfu");
+MODULE_ALIAS("dm-cache-mfu_ws");
+MODULE_ALIAS("dm-cache-mru");
+MODULE_ALIAS("dm-cache-multiqueue");
+MODULE_ALIAS("dm-cache-multiqueue_ws");
+MODULE_ALIAS("dm-cache-noop");
+MODULE_ALIAS("dm-cache-random");
+MODULE_ALIAS("dm-cache-q2");
+MODULE_ALIAS("dm-cache-twoqueue");