exynos9810: schedutil/KAIR: Sync schedutil to latest patches from Samsung

Taken from Note 10 5G & S10 [Exynos 9825/9820] and fully merged with KAIR feature as they quote "AI based Resource Control"
This commit is contained in:
THEBOSS619 2020-09-18 16:40:46 +03:00 committed by Mustafa Gökmen
parent 76f96b92f5
commit 366405377c
No known key found for this signature in database
GPG key ID: 3204D8100CFF21ED
9 changed files with 1922 additions and 6 deletions

View file

@ -6185,4 +6185,6 @@ CONFIG_SBITMAP=y
CONFIG_WIREGUARD=y
# CONFIG_WIREGUARD_DEBUG is not set
# CONFIG_KSU is not set
# CONFIG_KSU_DEBUG is not set
# CONFIG_KSU_DEBUG is not set
CONFIG_KAIR=y
CONFIG_SCHED_KAIR_GLUE=y

419
include/linux/kair.h Normal file
View file

@ -0,0 +1,419 @@
/**
* Classfication and realization of KAIR concept using ECAVE - Elastic Capability
* Vessel - dynamic resource control abstraction theory
**/
#include <linux/sched.h>
#include <linux/bug.h>
#include <linux/list.h>
#include <linux/tfifo.h>
#include <linux/kobject.h>
#include <linux/sysfs.h>
/**
*
**/
#define KAIR_QUANT_STEP (17)
#define KAIR_QUANT_SHIFT (4)
#define KAIR_VARIANCE_SPAN (64)
#define KAIR_VARIANCE_SPAN_SHIFT (6)
#define KAIR_UNDETERMINED (0xFFFFFFFF)
#define KAIR_DIVERGING (-1)
#define KAIR_CONVERGING (0)
#define KAIR_TPDF_CUMSUM (512)
#define KAIR_TPDF_MINVAL (1)
#define KAIR_TPDF_CASCADE_LEVEL (1)
#define KAIR_ALIAS_LEN (8)
#define KAIR_INF_TPDF_WEIGHT (1)
#define KAIR_FIN_TPDF_WEIGHT (2)
#if KAIR_TPDF_CASCADE_LEVEL > 1
#define KAIR_INST_TPDF_WEIGHT (4)
#else
#define KAIR_INST_TPDF_WEIGHT (8)
#endif
/**
* Calculating Energy-Performance(EP) weight ratio, which is described
* in theory as beta / alpha * upscaler.
* where, beta is energy consuming panelty weight, alpha is bottleneck
* panelty weight, and upscaler is 1024.
**/
#define KAIR_BALANCED_EP_RATIO (1024)
/**
*
**/
#define KAIR_WINDOW_INF (0x8000000000000000UL)
#define KAIR_WINDOW_INF_MASK (0x8000000000000000UL)
#define KAIR_WINDOW_NEED_RESCALE (0x4000000000000000UL)
#define KAIR_WINDOW_SIZE_MASK (0x0FFFFFFF00000000UL)
#define KAIR_WINDOW_SIZE_SHIFT (32)
#define KAIR_WINDOW_CNT_MASK (0x000000000FFFFFFFUL)
/**
*
**/
#define KAIR_DEF_RAND_NEUTRAL (2)
#define KAIR_TRIGGER_THROTTLE (3)
#define KAIR_FASTEN_THROTTLE (-100)
/**
* 1-dimensional discrete indexing degree of randomness which correspondingly
* spans upto 64 skewness * 64 flatness long.
* randomness bitfield : | <-- skewness --> | <-- flatness(variance) --> |
**/
#define RAND_SKEW_SHIFT (8)
typedef int randomness;
/**
* Random integer variable for stochastic process
**/
struct rand_var {
int nval;
unsigned int ubound;
unsigned int lbound;
};
/**
* Random integer variable coordinators
**/
#define __RV_INITIALIZER() \
{ \
.nval = 0, \
.ubound = 0, \
.lbound = 0 \
}
#define RV_DECLARE(name) struct rand_var name = __RV_INITIALIZER()
/**
* @rv : random variable structure
* @num : signed number
* @ulimit : positive upper bound
* @llimit : positive lower bound
**/
#define RV_SET(rv, num, ulimit, llimit) \
({ \
rv.nval = (int)(num); \
rv.ubound = (unsigned int)(ulimit); \
rv.lbound = (unsigned int)(llimit); \
})
/* Stretching nval to fit given tpdf->qlvl and returning the proper index
* within >= 0 and < tpdf->qlvl.
*/
#define RV_TAILORED(pdf, rv) \
({ \
typeof((pdf) + 1) __pdf = (pdf); \
typeof((rv) + 1) __rv = (rv); \
int __nval; \
int __hscale = (int)__pdf->qlvl >> 1; \
int __ubound = (int)__rv->ubound; \
int __lbound = (int)__rv->lbound; \
__nval = (__rv->nval >= 0) ? \
(__ubound ? \
((__rv->nval * __hscale / __ubound) + __hscale) : \
__hscale) : \
(__lbound ? \
((__rv->nval * __hscale / __lbound) + __hscale) : \
__hscale); \
__nval; \
})
#define RV_SPAN(rv) \
({ \
typeof((rv) + 1) __rv = (rv); \
unsigned int __span = __rv->ubound + __rv->lbound; \
__span; \
})
/**
* TPDF(Temporal Probability Density Function) container class
**/
struct tpdf {
struct list_head pos;
unsigned int qlvl;
u64 win_size;
struct __tfifo cache;
unsigned int *qtbl;
unsigned int pc_cnt;
/**
* """ TPDF container methods list """
* @tabling : preparing qtbl
* @untabling : removing qtbl
* @rv_register : registration of corresponding rv
* @rv_unregister : moving rv
* @interpolator : calculating moderate value
* @equilibrator : rebalancing qtbl to maintain total PDF integral
* @rescaler : rescaling the whole PDF to have
**/
int (*tabling)(struct tpdf *self);
void (*untabling)(struct tpdf *self);
void (*rv_register)(struct tpdf *self, struct rand_var *rv);
void (*rv_unregister)(struct tpdf *self);
unsigned int (*interpolator)(struct tpdf *self, unsigned int tpos,
unsigned int rpos);
void (*equilibrator)(struct tpdf *self);
void (*rescaler)(struct tpdf *self);
randomness irand;
char alias[KAIR_ALIAS_LEN];
struct rand_var *rv;
unsigned int weight;
};
/**
* """ KAIR statistics """
* @choke_cnt : counting how much the event so-called bottleneck or
* momentary sluggish in ECAVE theory happened.
* @save_total : accumulating amount how much redundant capacity is saved.
* @rand_neutral : an argument used for running betting algorithm.
* @throttling : +- accumulation of continuous choke/relief events.
* 0 is neutral.
**/
struct kair_stats {
unsigned int choke_cnt;
long long save_total;
unsigned int rand_neutral;
int throttling;
};
/**
* KAIR instance kobject created per each kair_obj_creator() call by
* any client and will be used to register to sysfs.
**/
struct krinst_obj {
struct kobject obj;
struct kair_class *inst;
};
#define to_krinst_obj(x) container_of(x, struct krinst_obj, obj)
/**
* KAIR attribute customized for krinst_obj.
**/
struct krinst_attribute {
struct attribute attr;
ssize_t (*show)(struct krinst_obj *inst, struct krinst_attribute *attr, char *buf);
ssize_t (*store)(struct krinst_obj *inst, struct krinst_attribute *attr, const char *buf, size_t count);
};
#define to_krinst_attr(x) container_of(x, struct krinst_attribute, attr)
#define KRINST_ATTR_RO(__name) \
static struct krinst_attribute __name##_attr = __ATTR_RO(__name)
#define KRINST_ATTR_RW(__name) \
static struct krinst_attribute __name##_attr = __ATTR_RW(__name)
/**
* TODO: more studies may be required to apply an external training.
* Kairistics is experimentally determined, which describes the pair of integer
* ratios in consideration of the fixed scaling factor heuristically used in each
* client's legacy mechanism.
**/
struct kairistics {
unsigned int gamma_numer:8;
unsigned int gamma_denom:8;
unsigned int theta_numer:8;
unsigned int theta_denom:8;
};
#define DECLARE_KAIRISTICS(alias, gnumer, gdenum, tnumer, tdenum) \
static struct kairistics kairistic_##alias = { \
.gamma_numer = gnumer, \
.gamma_denom = gdenum, \
.theta_numer = tnumer, \
.theta_denom = tdenum, \
}
/**
* "" KAIR class description """
* Which is virtually integrating Elastic Capacity Vessel abstraction model
* applicable on arbitrary dynamic self resource control systems.
* All real numbered properties must be handled with integers through adequate
* conversion and quantization processes.
**/
struct kair_class {
struct list_head tpdf_cascade;
/**
* bit-shift representation of KAIR capacity denominator
* which is considered as the summation of each level's TPDF weight
* multiplied by maximum randomness.
**/
unsigned int capa_denom;
/**
* """ KAIR methods list """:
* @initializer : self-initializer
* @stopper : stopping to learn tpdf, cleaning up.
* @finalizer : returning all resources.
* @job_learner : learner of capacity-probability-density which is
* actually conducting exclusive on-device learning
* algorithm on the given capacity-random variable.
* @job_inferer : returns randomness index of the TPDF which is most
* resembling given TPDF.
* @cap_bettor : returns betting capacity estimated.
**/
int (*initializer)(struct kair_class *self);
void (*stopper)(struct kair_class *self);
void (*finalizer)(struct kair_class *self);
void (*job_learner)(struct kair_class *self, struct rand_var *v);
int (*job_inferer)(struct kair_class *self);
unsigned int (*cap_bettor)(struct kair_class *self, struct rand_var *v,
unsigned int cap_legecy);
struct kair_stats stats;
unsigned int df_velocity;
unsigned int resilience;
unsigned int ep_ratio;
unsigned int max_capa;
unsigned int min_capa;
unsigned int epsilon;
struct kairistics kairistic;
char alias[KAIR_ALIAS_LEN];
struct krinst_obj *extif;
};
/**
*
*
**/
extern int kair_tpdf_tabling(struct tpdf *self);
extern void kair_tpdf_untabling(struct tpdf *self);
extern void kair_tpdf_rv_register(struct tpdf *self, struct rand_var *rv);
extern void kair_tpdf_rv_unregister(struct tpdf *self);
extern unsigned int linear_interpolator(struct tpdf *self, unsigned int top_pos,
unsigned int raw_pos);
extern void kair_tpdf_equilibrator(struct tpdf *self);
extern void kair_tpdf_rescaler(struct tpdf *self);
#define tpdf_infinite_init(pdf, __tlevel) \
({ \
int __err = 0; \
typeof((pdf) + 1) __pdf = (pdf); \
sprintf(__pdf->alias, "%s_tpdf", #__tlevel); \
*__pdf = (struct tpdf) { \
.pos = LIST_HEAD_INIT(__pdf->pos), \
.qlvl = KAIR_QUANT_STEP, \
.win_size = KAIR_WINDOW_INF, \
.pc_cnt = 0, \
.irand = KAIR_DIVERGING, \
.weight = KAIR_INF_TPDF_WEIGHT, \
.cache = tfifo_init(&__pdf->cache), \
.tabling = kair_tpdf_tabling, \
.untabling = kair_tpdf_untabling, \
.rv_register = kair_tpdf_rv_register, \
.rv_unregister = kair_tpdf_rv_unregister, \
.interpolator = linear_interpolator, \
.equilibrator = kair_tpdf_equilibrator, \
.rescaler = kair_tpdf_rescaler, \
}; \
__err = __pdf->tabling(__pdf); \
__err; \
})
#define tpdf_finite_init(pdf, __tlevel, period) \
({ \
int __err = 0; \
typeof((pdf) + 1) __pdf = (pdf); \
sprintf(__pdf->alias, "%s_tpdf", #__tlevel); \
*__pdf = (struct tpdf) { \
.pos = LIST_HEAD_INIT(__pdf->pos), \
.qlvl = KAIR_QUANT_STEP, \
.win_size = ((u64)period << KAIR_WINDOW_SIZE_SHIFT) | \
KAIR_WINDOW_NEED_RESCALE, \
.pc_cnt = 0, \
.irand = KAIR_DIVERGING, \
.weight = KAIR_FIN_TPDF_WEIGHT, \
.cache = tfifo_init(&__pdf->cache), \
.tabling = kair_tpdf_tabling, \
.untabling = kair_tpdf_untabling, \
.rv_register = kair_tpdf_rv_register, \
.rv_unregister = kair_tpdf_rv_unregister, \
.interpolator = linear_interpolator, \
.equilibrator = kair_tpdf_equilibrator, \
.rescaler = kair_tpdf_rescaler, \
}; \
__err = __pdf->tabling(__pdf); \
__err; \
})
#define tpdf_instant_init(pdf, __tlevel, period) \
({ \
int __err = 0; \
typeof((pdf) + 1) __pdf = (pdf); \
sprintf(__pdf->alias, "%s_tpdf", #__tlevel); \
*__pdf = (struct tpdf) { \
.pos = LIST_HEAD_INIT(__pdf->pos), \
.qlvl = KAIR_QUANT_STEP, \
.win_size = ((u64)period << KAIR_WINDOW_SIZE_SHIFT), \
.pc_cnt = 0, \
.irand = KAIR_DIVERGING, \
.weight = KAIR_INST_TPDF_WEIGHT, \
.cache = tfifo_init(&__pdf->cache), \
.tabling = kair_tpdf_tabling, \
.untabling = kair_tpdf_untabling, \
.rv_register = kair_tpdf_rv_register, \
.rv_unregister = kair_tpdf_rv_unregister, \
.interpolator = linear_interpolator, \
.equilibrator = kair_tpdf_equilibrator, \
.rescaler = kair_tpdf_rescaler, \
}; \
__err = __pdf->tabling(__pdf); \
__err; \
})
/**
* @kair_l1_pdf_init() : instant generative TPDF covering only about past 1 second.
* @kair_l2_pdf_init() : generative TPDF convering only about past 1 minute.
* @kair_l3_pdf_init() : ever lasting generative TPDF
**/
#define kair_l1_pdf_init(pdf) tpdf_instant_init((pdf), l1, KAIR_TPDF_CUMSUM)
#define kair_l2_pdf_init(pdf) tpdf_finite_init((pdf), l2, KAIR_TPDF_CUMSUM << 1)
#define kair_l3_pdf_init(pdf) tpdf_infinite_init((pdf), l3)
/**
* KAIR window information extractors
**/
#define is_kair_window_infty(pdf) \
({ \
typeof((pdf) + 1) __pdf = (pdf); \
(__pdf->win_size & KAIR_WINDOW_INF_MASK); \
})
#define kair_window_size(pdf) \
({ \
typeof((pdf) + 1) __pdf = (pdf); \
((__pdf->win_size & KAIR_WINDOW_SIZE_MASK) >> KAIR_WINDOW_SIZE_SHIFT); \
})
#define kair_windowing_cnt(pdf) \
({ \
typeof((pdf) + 1) __pdf = (pdf); \
(__pdf->win_size & KAIR_WINDOW_CNT_MASK); \
})
#define kair_window_need_rescale(pdf) \
({ \
typeof((pdf) + 1) __pdf = (pdf); \
(__pdf->win_size & KAIR_WINDOW_NEED_RESCALE); \
})
#define kair_window_rescale_done(pdf) \
({ \
typeof((pdf) + 1) __pdf = (pdf); \
__pdf->win_size &= ~KAIR_WINDOW_NEED_RESCALE; \
})
/*.............................................................................
*.............................................................................
*................ """ KAIR Major External interfaces """ ....................
*.............................................................................
*...........................................................................*/
struct kair_class *kair_obj_creator(const char *alias,
unsigned int resilience,
unsigned int max_capa,
unsigned int min_capa,
struct kairistics *kairistic);
void kair_obj_destructor(struct kair_class *self);

72
include/linux/tfifo.h Normal file
View file

@ -0,0 +1,72 @@
/**
* Tiny FIFO implemenation - simplest FIFO only for handling 1-dimensional
* single unstructured unsigned integer data array in circular manner.
**/
#include <linux/slab.h>
struct __tfifo {
unsigned int *data;
unsigned int length;
unsigned int in;
unsigned int out;
};
/**
* For now, we only consider of 'unsigned integer' use case.
**/
#define __entity_size (sizeof(unsigned int))
#define tfifo_init(fifo) \
({ \
typeof((fifo) + 1) __tmp = (fifo); \
*__tmp = (struct __tfifo) { \
.data = NULL, \
.length = 0, \
.in = 0, \
.out = 0 }; \
})
#define tfifo_alloc(fifo, len) \
({ \
bool __ret = true; \
typeof((fifo) + 1) __tmp = (fifo); \
__tmp->data = kmalloc(__entity_size * (len + 1), GFP_KERNEL); \
if (__tmp->data) \
__tmp->length = (len + 1); \
else \
__ret = false; \
__ret; \
})
#define is_tfifo_empty(fifo) \
({ \
typeof((fifo) + 1) __tmp = (fifo); \
bool __ret = (__tmp->in == __tmp->out) ? true : false; \
__ret; \
})
#define tfifo_free(fifo) \
({ \
typeof((fifo) + 1) __tmp = (fifo); \
if (__tmp->data) kfree(__tmp->data); \
__tmp->length = 0; \
__tmp->in = 0; \
__tmp->out = 0; \
})
#define tfifo_in(fifo, val) \
({ \
typeof((fifo) + 1) __tmp = (fifo); \
__tmp->data[__tmp->in] = (val); \
__tmp->in = (__tmp->in + 1) % __tmp->length; \
})
#define tfifo_out(fifo) \
({ \
unsigned int __ret; \
typeof((fifo) + 1) __tmp = (fifo); \
__ret = __tmp->data[__tmp->out]; \
__tmp->out = (__tmp->out + 1) % __tmp->length; \
__ret; \
})

View file

@ -223,6 +223,59 @@ DEFINE_EVENT(cpu, cpu_capacity,
TP_ARGS(capacity, cpu_id)
);
TRACE_EVENT(sugov_kair_freq,
TP_PROTO(unsigned int cpu, unsigned long util, unsigned long max,
int l1_rand, unsigned int legacy_freq, unsigned int freq),
TP_ARGS(cpu, util, max, l1_rand, legacy_freq, freq),
TP_STRUCT__entry(
__field( unsigned int, cpu)
__field( unsigned long, util)
__field( unsigned long, max)
__field( int, l1_rand)
__field( unsigned int, legacy_freq)
__field( unsigned int, freq)
),
TP_fast_assign(
__entry->cpu = cpu;
__entry->util = util;
__entry->max = max;
__entry->l1_rand = l1_rand;
__entry->legacy_freq = legacy_freq;
__entry->freq = freq;
),
TP_printk("cpu=%u util=%lu max=%lu l1_rand=%d legacy_freq=%u kair_freq=%u",
__entry->cpu,
__entry->util,
__entry->max,
__entry->l1_rand,
__entry->legacy_freq,
__entry->freq)
);
TRACE_EVENT(cpu_frequency_sugov,
TP_PROTO(unsigned int freq, unsigned long util, unsigned int cpu_id),
TP_ARGS(freq, util, cpu_id),
TP_STRUCT__entry(
__field( u32, freq )
__field( u32, util )
__field( u32, cpu_id )
),
TP_fast_assign(
__entry->freq = freq;
__entry->util = util;
__entry->cpu_id = cpu_id;
),
TP_printk("freq=%lu util=%lu cpu_id=%lu",
(unsigned long)__entry->freq,
(unsigned long)__entry->util,
(unsigned long)__entry->cpu_id)
);
TRACE_EVENT(device_pm_callback_start,
TP_PROTO(struct device *dev, const char *pm_ops, int event),

View file

@ -23,6 +23,18 @@
#include "tune.h"
#include "ems/ems.h"
#ifdef CONFIG_SCHED_KAIR_GLUE
#include <linux/kair.h>
/**
* 2nd argument of kair_obj_creator() experimentally decided by KAIR client
* itself, which represents how much variant the random variable registered to
* the KAIR instance can behave at most, in terms of referencing d2u_decl_cmtpdf
* table(maximum index of d2u_decl_cmtpdf table).
**/
#define UTILAVG_KAIR_VARIANCE 16
DECLARE_KAIRISTICS(cpufreq, 32, 25, 24, 25);
#endif
unsigned long boosted_cpu_util(int cpu, unsigned long other_util);
/* Stub out fast switch routines present on mainline to reduce the backport
@ -37,6 +49,9 @@ struct sugov_tunables {
struct gov_attr_set attr_set;
unsigned int up_rate_limit_us;
unsigned int down_rate_limit_us;
#ifdef CONFIG_SCHED_KAIR_GLUE
bool fb_legacy;
#endif
bool iowait_boost_enable;
bool exp_util;
};
@ -65,6 +80,9 @@ struct sugov_policy {
bool need_freq_update;
bool limits_changed;
#ifdef CONFIG_SCHED_KAIR_GLUE
bool be_stochastic;
#endif
};
struct sugov_cpu {
@ -76,6 +94,14 @@ struct sugov_cpu {
unsigned int iowait_boost_max;
u64 last_update;
#ifdef CONFIG_SCHED_KAIR_GLUE
/**
* KAIR instance which should be referenced in percpu manner,
* and data accordingly to handle the target job intensity.
**/
struct kair_class *util_vessel;
unsigned long cached_util;
#endif
/* The fields below are only needed when sharing a policy. */
unsigned long util;
unsigned long max;
@ -120,9 +146,13 @@ static bool sugov_should_update_freq(struct sugov_policy *sg_policy, u64 time)
return true;
}
delta_ns = time - sg_policy->last_freq_update_time;
/* No need to recalculate next freq for min_rate_limit_us
* at least. However we might still decide to further rate
* limit once frequency change direction is decided, according
* to the separate rate limits.
*/
/* No need to recalculate next freq for min_rate_limit_us at least */
delta_ns = time - sg_policy->last_freq_update_time;
return delta_ns >= sg_policy->min_rate_limit_ns;
}
@ -211,12 +241,76 @@ static unsigned int get_next_freq(struct sugov_policy *sg_policy,
struct cpufreq_policy *policy = sg_policy->policy;
unsigned int freq = arch_scale_freq_invariant() ?
policy->max : policy->cur;
#ifdef CONFIG_SCHED_KAIR_GLUE
struct sugov_cpu *sg_cpu;
struct kair_class *vessel;
unsigned int delta_max, delta_min;
int util_delta;
unsigned int legacy_freq;
#ifdef KAIR_CLUSTER_TRAVERSING
unsigned int each;
unsigned int sigma_cpu = policy->cpu;
randomness most_rand = 0;
#endif
int cur_rand = KAIR_DIVERGING;
RV_DECLARE(rv);
#endif
if (sg_policy->tunables->exp_util)
freq = (freq + (freq >> 2)) * int_sqrt(util * 100 / max) / 10;
else
freq = (freq + (freq >> 2)) * util / max;
#ifdef CONFIG_SCHED_KAIR_GLUE
legacy_freq = freq;
if (sg_policy->tunables->fb_legacy)
goto skip_betting;
#ifndef KAIR_CLUSTER_TRAVERSING
sg_cpu = &per_cpu(sugov_cpu, policy->cpu);
vessel = sg_cpu->util_vessel;
if (!vessel)
goto skip_betting;
cur_rand = vessel->job_inferer(vessel);
if (cur_rand == KAIR_DIVERGING)
goto skip_betting;
#else
for_each_cpu(each, policy->cpus) {
sg_cpu = &per_cpu(sugov_cpu, each);
vessel = sg_cpu->util_vessel;
if (vessel) {
cur_rand = vessel->job_inferer(vessel);
if (cur_rand == KAIR_DIVERGING)
goto skip_betting;
else {
if (cur_rand > (int)most_rand) {
most_rand = (randomness)cur_rand;
sigma_cpu = each;
}
}
} else
goto skip_betting;
}
sg_cpu = &per_cpu(sugov_cpu, sigma_cpu);
vessel = sg_cpu->util_vessel;
#endif
util_delta = sg_cpu->util - sg_cpu->cached_util;
delta_max = sg_cpu->max - sg_cpu->cached_util;
delta_min = sg_cpu->cached_util;
RV_SET(rv, util_delta, delta_max, delta_min);
freq = vessel->cap_bettor(vessel, &rv, freq);
skip_betting:
trace_sugov_kair_freq(policy->cpu, util, max, cur_rand, legacy_freq, freq);
#endif
if (freq == sg_policy->cached_raw_freq && !sg_policy->need_freq_update)
return sg_policy->next_freq;
@ -269,6 +363,23 @@ static void sugov_get_util(unsigned long *util, unsigned long *max, u64 time)
#endif
}
#ifdef CONFIG_SCHED_KAIR_GLUE
static inline void sugov_util_collapse(struct sugov_cpu *sg_cpu)
{
struct kair_class *vessel = sg_cpu->util_vessel;
int util_delta = min(sg_cpu->max, sg_cpu->util) - sg_cpu->cached_util;
unsigned int delta_max = sg_cpu->max - sg_cpu->cached_util;
unsigned int delta_min = sg_cpu->cached_util;
RV_DECLARE(job);
if (vessel) {
RV_SET(job, util_delta, delta_max, delta_min);
vessel->job_learner(vessel, &job);
}
}
#endif
static void sugov_set_iowait_boost(struct sugov_cpu *sg_cpu, u64 time,
unsigned int flags)
{
@ -436,10 +547,16 @@ static void sugov_update_shared(struct update_util_data *hook, u64 time,
raw_spin_lock(&sg_policy->update_lock);
#ifdef CONFIG_SCHED_KAIR_GLUE
sg_cpu->cached_util = min(max, sg_cpu->max ?
mult_frac(sg_cpu->util, max, sg_cpu->max) : sg_cpu->util);
#endif
sg_cpu->util = util;
sg_cpu->max = max;
sg_cpu->flags = flags;
#ifdef CONFIG_SCHED_KAIR_GLUE
sugov_util_collapse(sg_cpu);
#endif
sugov_set_iowait_boost(sg_cpu, time, flags);
sg_cpu->last_update = time;
@ -665,14 +782,40 @@ static ssize_t exp_util_store(struct gov_attr_set *attr_set, const char *buf,
return count;
}
#ifdef CONFIG_SCHED_KAIR_GLUE
static ssize_t fb_legacy_show(struct gov_attr_set *attr_set, char *buf)
{
struct sugov_tunables *tunables = to_sugov_tunables(attr_set);
return scnprintf(buf, PAGE_SIZE, "%u\n", tunables->fb_legacy);
}
static ssize_t fb_legacy_store(struct gov_attr_set *attr_set, const char *buf,
size_t count)
{
struct sugov_tunables *tunables = to_sugov_tunables(attr_set);
if (kstrtobool(buf, &tunables->fb_legacy))
return -EINVAL;
return count;
}
#endif
static struct governor_attr up_rate_limit_us = __ATTR_RW(up_rate_limit_us);
static struct governor_attr down_rate_limit_us = __ATTR_RW(down_rate_limit_us);
#ifdef CONFIG_SCHED_KAIR_GLUE
static struct governor_attr fb_legacy = __ATTR_RW(fb_legacy);
#endif
static struct governor_attr iowait_boost_enable = __ATTR_RW(iowait_boost_enable);
static struct governor_attr exp_util = __ATTR_RW(exp_util);
static struct attribute *sugov_attributes[] = {
&up_rate_limit_us.attr,
&down_rate_limit_us.attr,
#ifdef CONFIG_SCHED_KAIR_GLUE
&fb_legacy.attr,
#endif
&iowait_boost_enable.attr,
&exp_util.attr,
NULL
@ -875,6 +1018,10 @@ static int sugov_init(struct cpufreq_policy *policy)
tunables->down_rate_limit_us *= lat;
}
}
#ifdef CONFIG_SCHED_KAIR_GLUE
tunables->fb_legacy = false;
sg_policy->be_stochastic = false;
#endif
tunables->iowait_boost_enable = policy->iowait_boost_enable;
tunables->exp_util = false;
@ -918,6 +1065,9 @@ static void sugov_exit(struct cpufreq_policy *policy)
struct sugov_policy *sg_policy = policy->governor_data;
struct sugov_tunables *tunables = sg_policy->tunables;
unsigned int count;
#ifdef CONFIG_SCHED_KAIR_GLUE
struct sugov_cpu *sg_cpu = &per_cpu(sugov_cpu, policy->cpu);
#endif
mutex_lock(&global_tunables_lock);
@ -928,6 +1078,15 @@ static void sugov_exit(struct cpufreq_policy *policy)
sugov_tunables_free(tunables);
}
#ifdef CONFIG_SCHED_KAIR_GLUE
if (sg_cpu->util_vessel) {
sg_cpu->util_vessel->finalizer(sg_cpu->util_vessel);
kair_obj_destructor(sg_cpu->util_vessel);
sg_cpu->util_vessel = NULL;
}
sg_policy->be_stochastic = false;
#endif
sugov_kthread_stop(sg_policy);
sugov_policy_free(sg_policy);
mutex_unlock(&global_tunables_lock);
@ -939,6 +1098,9 @@ static int sugov_start(struct cpufreq_policy *policy)
{
struct sugov_policy *sg_policy = policy->governor_data;
unsigned int cpu;
#ifdef CONFIG_SCHED_KAIR_GLUE
char alias[KAIR_ALIAS_LEN];
#endif
sg_policy->up_rate_delay_ns =
sg_policy->tunables->up_rate_limit_us * NSEC_PER_USEC;
@ -955,13 +1117,46 @@ static int sugov_start(struct cpufreq_policy *policy)
for_each_cpu(cpu, policy->cpus) {
struct sugov_cpu *sg_cpu = &per_cpu(sugov_cpu, cpu);
#ifdef CONFIG_SCHED_KAIR_GLUE
if (cpu != policy->cpu) {
memset(sg_cpu, 0, sizeof(*sg_cpu));
goto skip_subcpus;
}
if (!sg_policy->be_stochastic) {
memset(alias, 0, KAIR_ALIAS_LEN);
sprintf(alias, "govern%d", cpu);
memset(sg_cpu, 0, sizeof(*sg_cpu));
sg_cpu->util_vessel =
kair_obj_creator(alias,
UTILAVG_KAIR_VARIANCE,
policy->cpuinfo.max_freq,
policy->cpuinfo.min_freq,
&kairistic_cpufreq);
if (sg_cpu->util_vessel->initializer(sg_cpu->util_vessel) < 0) {
sg_cpu->util_vessel->finalizer(sg_cpu->util_vessel);
kair_obj_destructor(sg_cpu->util_vessel);
sg_cpu->util_vessel = NULL;
}
} else {
struct kair_class *vptr = sg_cpu->util_vessel;
memset(sg_cpu, 0, sizeof(*sg_cpu));
sg_cpu->util_vessel = vptr;
}
skip_subcpus:
#else
memset(sg_cpu, 0, sizeof(*sg_cpu));
#endif
sg_cpu->sg_policy = sg_policy;
sg_cpu->flags = 0;
sugov_start_slack(cpu);
sg_cpu->iowait_boost_max = policy->cpuinfo.max_freq;
}
#ifdef CONFIG_SCHED_KAIR_GLUE
sg_policy->be_stochastic = true;
#endif
for_each_cpu(cpu, policy->cpus) {
struct sugov_cpu *sg_cpu = &per_cpu(sugov_cpu, cpu);
@ -985,6 +1180,15 @@ static void sugov_stop(struct cpufreq_policy *policy)
synchronize_sched();
#ifdef CONFIG_SCHED_KAIR_GLUE
for_each_cpu(cpu, policy->cpus) {
struct sugov_cpu *sg_cpu = &per_cpu(sugov_cpu, cpu);
if (sg_cpu->util_vessel) {
sg_cpu->util_vessel->stopper(sg_cpu->util_vessel);
}
}
#endif
if (!policy->fast_switch_enabled) {
irq_work_sync(&sg_policy->irq_work);
kthread_cancel_work_sync(&sg_policy->work);

View file

@ -569,4 +569,19 @@ config STACKDEPOT
config SBITMAP
bool
config KAIR
bool
default y
help
Library to support KAIR class as the realization of Elastic Capacity
Vessel abstract theory.
config SCHED_KAIR_GLUE
bool
depends on KAIR
default y
help
Glue interface sticking edge of CFS scheduler and smoother governor
working based on KAIR concept strategy.
endmenu

View file

@ -237,5 +237,5 @@ obj-$(CONFIG_UBSAN) += ubsan.o
UBSAN_SANITIZE_ubsan.o := n
obj-$(CONFIG_SBITMAP) += sbitmap.o
obj-$(CONFIG_WIREGUARD) += siphash.o
obj-$(CONFIG_WIREGUARD) += siphash.o
obj-$(CONFIG_KAIR) += kair.o

72
lib/d2u_decl_cmtpdf.h Normal file
View file

@ -0,0 +1,72 @@
/**
*
* Precalculated gradually delta-to-uniform declining
* centered-mean, x KAIR_TPDF_CUMSUM-scaled, KAIR_QUANT_STEP-quantized
* 0-biased Sunchul TPDF tables.
*
**/
const unsigned int d2u_decl_cmtpdf[KAIR_VARIANCE_SPAN][KAIR_QUANT_STEP] =
{{1, 1, 1, 1, 1, 1, 1, 1, 497, 1, 1, 1, 1, 1, 1, 1, 0},
{1, 1, 1, 1, 1, 1, 1, 2, 494, 2, 1, 1, 1, 1, 1, 1, 0},
{1, 1, 1, 1, 1, 1, 1, 16, 467, 16, 1, 1, 1, 1, 1, 1, 0},
{1, 1, 1, 1, 1, 1, 1, 42, 413, 42, 1, 1, 1, 1, 1, 1, 0},
{1, 1, 1, 1, 1, 1, 1, 68, 361, 68, 1, 1, 1, 1, 1, 1, 0},
{1, 1, 1, 1, 1, 1, 3, 88, 320, 88, 3, 1, 1, 1, 1, 1, 0},
{1, 1, 1, 1, 1, 1, 5, 101, 287, 101, 5, 1, 1, 1, 1, 1, 0},
{1, 1, 1, 1, 1, 1, 9, 110, 262, 110, 9, 1, 1, 1, 1, 1, 0},
{1, 1, 1, 1, 1, 1, 14, 116, 241, 116, 14, 1, 1, 1, 1, 1, 0},
{1, 1, 1, 1, 1, 2, 18, 119, 224, 119, 18, 2, 1, 1, 1, 1, 0},
{1, 1, 1, 1, 1, 2, 23, 121, 210, 121, 23, 2, 1, 1, 1, 1, 0},
{1, 1, 1, 1, 1, 3, 28, 121, 197, 121, 28, 3, 1, 1, 1, 1, 0},
{1, 1, 1, 1, 1, 4, 33, 120, 187, 120, 33, 4, 1, 1, 1, 1, 0},
{1, 1, 1, 1, 1, 6, 37, 119, 177, 119, 37, 6, 1, 1, 1, 1, 0},
{1, 1, 1, 1, 1, 7, 41, 118, 169, 118, 41, 7, 1, 1, 1, 1, 0},
{1, 1, 1, 1, 2, 9, 44, 116, 162, 116, 44, 9, 2, 1, 1, 1, 0},
{1, 1, 1, 1, 2, 11, 47, 115, 155, 115, 47, 11, 2, 1, 1, 1, 0},
{1, 1, 1, 1, 3, 13, 49, 113, 149, 113, 49, 13, 3, 1, 1, 1, 0},
{1, 1, 1, 1, 3, 15, 51, 111, 143, 111, 51, 15, 3, 1, 1, 1, 0},
{1, 1, 1, 1, 4, 17, 53, 109, 138, 109, 53, 17, 4, 1, 1, 1, 0},
{1, 1, 1, 1, 5, 18, 55, 107, 134, 107, 55, 18, 5, 1, 1, 1, 0},
{1, 1, 1, 2, 5, 20, 56, 105, 129, 105, 56, 20, 5, 2, 1, 1, 0},
{1, 1, 1, 2, 6, 22, 57, 103, 126, 103, 57, 22, 6, 2, 1, 1, 0},
{1, 1, 1, 2, 7, 23, 58, 101, 122, 101, 58, 23, 7, 2, 1, 1, 0},
{1, 1, 1, 2, 8, 25, 59, 99, 118, 99, 59, 25, 8, 2, 1, 1, 0},
{1, 1, 1, 3, 9, 26, 60, 98, 115, 98, 60, 26, 9, 3, 1, 1, 0},
{1, 1, 1, 3, 10, 28, 60, 96, 112, 96, 60, 28, 10, 3, 1, 1, 0},
{1, 1, 1, 3, 11, 29, 60, 94, 110, 94, 60, 29, 11, 3, 1, 1, 0},
{1, 1, 2, 4, 12, 30, 61, 93, 107, 93, 61, 30, 12, 4, 2, 1, 0},
{1, 1, 2, 4, 13, 31, 61, 91, 104, 91, 61, 31, 13, 4, 2, 1, 0},
{1, 1, 2, 5, 13, 32, 61, 90, 102, 90, 61, 32, 13, 5, 2, 1, 0},
{1, 1, 2, 5, 14, 33, 61, 88, 100, 88, 61, 33, 14, 5, 2, 1, 1},
{1, 1, 2, 6, 15, 34, 61, 87, 98, 87, 61, 34, 15, 6, 2, 1, 1},
{1, 1, 2, 6, 16, 35, 61, 85, 96, 85, 61, 35, 16, 6, 2, 1, 1},
{1, 1, 3, 7, 17, 35, 61, 84, 94, 84, 61, 35, 17, 7, 3, 1, 1},
{1, 1, 3, 7, 18, 36, 61, 83, 92, 83, 61, 36, 18, 7, 3, 1, 1},
{1, 2, 3, 8, 18, 37, 60, 82, 90, 82, 60, 37, 18, 8, 3, 2, 1},
{1, 2, 4, 8, 19, 37, 60, 80, 89, 80, 60, 37, 19, 8, 4, 2, 1},
{1, 2, 4, 9, 20, 38, 60, 79, 87, 79, 60, 38, 20, 9, 4, 2, 1},
{1, 2, 4, 10, 21, 38, 60, 78, 86, 78, 60, 38, 21, 10, 4, 2, 1},
{1, 2, 4, 10, 21, 39, 59, 77, 84, 77, 59, 39, 21, 10, 4, 2, 1},
{1, 2, 5, 11, 22, 39, 59, 76, 83, 76, 59, 39, 22, 11, 5, 2, 1},
{1, 2, 5, 11, 22, 39, 59, 75, 81, 75, 59, 39, 22, 11, 5, 2, 1},
{1, 3, 5, 12, 23, 39, 58, 74, 80, 74, 58, 39, 23, 12, 5, 3, 1},
{2, 3, 6, 12, 23, 40, 58, 73, 79, 73, 58, 40, 23, 12, 6, 3, 1},
{2, 3, 6, 13, 24, 40, 58, 72, 78, 72, 58, 40, 24, 13, 6, 3, 1},
{2, 3, 6, 13, 24, 40, 57, 71, 77, 71, 57, 40, 24, 13, 6, 3, 1},
{2, 3, 7, 14, 25, 40, 57, 70, 75, 70, 57, 40, 25, 14, 7, 3, 1},
{2, 3, 7, 14, 25, 41, 57, 70, 74, 70, 57, 41, 25, 14, 7, 3, 1},
{2, 4, 7, 15, 26, 41, 56, 69, 73, 69, 56, 41, 26, 15, 7, 4, 1},
{2, 4, 8, 15, 26, 41, 56, 68, 72, 68, 56, 41, 26, 15, 8, 4, 2},
{2, 4, 8, 15, 27, 41, 56, 67, 72, 67, 56, 41, 27, 15, 8, 4, 2},
{2, 4, 9, 16, 27, 41, 55, 66, 71, 66, 55, 41, 27, 16, 9, 4, 2},
{2, 5, 9, 16, 27, 41, 55, 66, 70, 66, 55, 41, 27, 16, 9, 5, 2},
{3, 5, 9, 17, 28, 41, 55, 65, 69, 65, 55, 41, 28, 17, 9, 5, 2},
{3, 5, 10, 17, 28, 41, 54, 64, 68, 64, 54, 41, 28, 17, 10, 5, 2},
{3, 5, 10, 17, 28, 41, 54, 64, 67, 64, 54, 41, 28, 17, 10, 5, 2},
{3, 6, 10, 18, 28, 41, 54, 63, 66, 63, 54, 41, 28, 18, 10, 6, 2},
{3, 6, 11, 18, 29, 41, 53, 62, 66, 62, 53, 41, 29, 18, 11, 6, 3},
{3, 6, 11, 18, 29, 41, 53, 62, 65, 62, 53, 41, 29, 18, 11, 6, 3},
{3, 6, 11, 19, 29, 41, 53, 61, 64, 61, 53, 41, 29, 19, 11, 6, 3},
{4, 7, 12, 19, 29, 41, 52, 61, 64, 61, 52, 41, 29, 19, 12, 7, 3},
{4, 7, 12, 19, 30, 41, 52, 60, 63, 60, 52, 41, 30, 19, 12, 7, 3},
{4, 7, 12, 20, 30, 41, 52, 60, 62, 60, 52, 41, 30, 20, 12, 7, 3}};

1079
lib/kair.c Normal file

File diff suppressed because it is too large Load diff