oolong/mali_kbase_js_policy_cfs.c

/*
 *
 * (C) COPYRIGHT 2011-2015 ARM Limited. All rights reserved.
 *
 * This program is free software and is provided to you under the terms of the
 * GNU General Public License version 2 as published by the Free Software
 * Foundation, and any use by you of this program is subject to the terms
 * of such GNU licence.
 *
 * A copy of the licence is included with the program, and can also be obtained
 * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
 * Boston, MA  02110-1301, USA.
 *
 */

/*
 * Job Scheduler: Completely Fair Policy Implementation
 */

#include <mali_kbase.h>
#include <mali_kbase_js.h>
#include <mali_kbase_js_policy_cfs.h>
#include <linux/version.h>
#include <linux/sched/rt.h>

/**
 * Define for when dumping is enabled.
 * This should not be based on the instrumentation level as whether dumping is enabled for a particular level is down to the integrator.
 * However this is being used for now as otherwise the cinstr headers would be needed.
 */
#define CINSTR_DUMPING_ENABLED (2 == MALI_INSTRUMENTATION_LEVEL)

/* Fixed point constants used for runtime weight calculations */
#define WEIGHT_FIXEDPOINT_SHIFT 10
#define WEIGHT_TABLE_SIZE       40
#define WEIGHT_0_NICE           (WEIGHT_TABLE_SIZE/2)
#define WEIGHT_0_VAL            (1 << WEIGHT_FIXEDPOINT_SHIFT)

#define PROCESS_PRIORITY_MIN (-20)
#define PROCESS_PRIORITY_MAX  (19)

/* Defines for easy asserts 'is scheduled'/'is queued'/'is neither queued norscheduled' */
#define KBASEP_JS_CHECKFLAG_QUEUED       (1u << 0) /**< Check the queued state */
#define KBASEP_JS_CHECKFLAG_SCHEDULED    (1u << 1) /**< Check the scheduled state */
#define KBASEP_JS_CHECKFLAG_IS_QUEUED    (1u << 2) /**< Expect queued state to be set */
#define KBASEP_JS_CHECKFLAG_IS_SCHEDULED (1u << 3) /**< Expect scheduled state to be set */

enum {
	KBASEP_JS_CHECK_NOTQUEUED = KBASEP_JS_CHECKFLAG_QUEUED,
	KBASEP_JS_CHECK_NOTSCHEDULED = KBASEP_JS_CHECKFLAG_SCHEDULED,
	KBASEP_JS_CHECK_QUEUED = KBASEP_JS_CHECKFLAG_QUEUED | KBASEP_JS_CHECKFLAG_IS_QUEUED,
	KBASEP_JS_CHECK_SCHEDULED = KBASEP_JS_CHECKFLAG_SCHEDULED | KBASEP_JS_CHECKFLAG_IS_SCHEDULED
};

typedef u32 kbasep_js_check;

/*
 * Private Functions
 */

/* Table autogenerated using util built from: base/tools/gen_cfs_weight_of_prio/ */

/* weight = 1.25 */
static const int weight_of_priority[] = {
	/*  -20 */ 11, 14, 18, 23,
	/*  -16 */ 29, 36, 45, 56,
	/*  -12 */ 70, 88, 110, 137,
	/*   -8 */ 171, 214, 268, 335,
	/*   -4 */ 419, 524, 655, 819,
	/*    0 */ 1024, 1280, 1600, 2000,
	/*    4 */ 2500, 3125, 3906, 4883,
	/*    8 */ 6104, 7630, 9538, 11923,
	/*   12 */ 14904, 18630, 23288, 29110,
	/*   16 */ 36388, 45485, 56856, 71070
};

/*
 * Note: There is nothing to stop the priority of the ctx containing
 * ctx_info changing during or immediately after this function is called
 * (because its jsctx_mutex cannot be held during IRQ). Therefore, this
 * function should only be seen as a heuristic guide as to the priority weight
 * of the context.
 */
static u64 priority_weight(struct kbasep_js_policy_cfs_ctx *ctx_info, u64 time_us)
{
	u64 time_delta_us;
	int priority;

	priority = ctx_info->process_priority;

	/* Adjust runtime_us using priority weight if required */
	if (priority != 0 && time_us != 0) {
		int clamped_priority;

		/* Clamp values to min..max weights */
		if (priority > PROCESS_PRIORITY_MAX)
			clamped_priority = PROCESS_PRIORITY_MAX;
		else if (priority < PROCESS_PRIORITY_MIN)
			clamped_priority = PROCESS_PRIORITY_MIN;
		else
			clamped_priority = priority;

		/* Fixed point multiplication */
		time_delta_us = (time_us * weight_of_priority[WEIGHT_0_NICE + clamped_priority]);
		/* Remove fraction */
		time_delta_us = time_delta_us >> WEIGHT_FIXEDPOINT_SHIFT;
		/* Make sure the time always increases */
		if (time_delta_us == 0)
			time_delta_us++;
	} else {
		time_delta_us = time_us;
	}

	return time_delta_us;
}

#if KBASE_TRACE_ENABLE
static int kbasep_js_policy_trace_get_refcnt_nolock(struct kbase_device *kbdev, struct kbase_context *kctx)
{
	struct kbasep_js_device_data *js_devdata;
	int as_nr;
	int refcnt = 0;

	js_devdata = &kbdev->js_data;

	as_nr = kctx->as_nr;
	if (as_nr != KBASEP_AS_NR_INVALID) {
		struct kbasep_js_per_as_data *js_per_as_data;

		js_per_as_data = &js_devdata->runpool_irq.per_as_data[as_nr];

		refcnt = js_per_as_data->as_busy_refcount;
	}

	return refcnt;
}

static inline int kbasep_js_policy_trace_get_refcnt(struct kbase_device *kbdev, struct kbase_context *kctx)
{
	unsigned long flags;
	struct kbasep_js_device_data *js_devdata;
	int refcnt = 0;

	js_devdata = &kbdev->js_data;

	spin_lock_irqsave(&js_devdata->runpool_irq.lock, flags);
	refcnt = kbasep_js_policy_trace_get_refcnt_nolock(kbdev, kctx);
	spin_unlock_irqrestore(&js_devdata->runpool_irq.lock, flags);

	return refcnt;
}
#else				/* KBASE_TRACE_ENABLE  */
static inline int kbasep_js_policy_trace_get_refcnt(struct kbase_device *kbdev, struct kbase_context *kctx)
{
	CSTD_UNUSED(kbdev);
	CSTD_UNUSED(kctx);
	return 0;
}
#endif				/* KBASE_TRACE_ENABLE  */

/*
 * Non-private functions
 */

int kbasep_js_policy_init(struct kbase_device *kbdev)
{
	struct kbasep_js_device_data *js_devdata;
	struct kbasep_js_policy_cfs *policy_info;

	KBASE_DEBUG_ASSERT(kbdev != NULL);
	js_devdata = &kbdev->js_data;
	policy_info = &js_devdata->policy.cfs;

	atomic64_set(&policy_info->least_runtime_us, KBASEP_JS_RUNTIME_EMPTY);
	atomic64_set(&policy_info->rt_least_runtime_us, KBASEP_JS_RUNTIME_EMPTY);

	policy_info->head_runtime_us = 0;

	return 0;
}

void kbasep_js_policy_term(union kbasep_js_policy *js_policy)
{
	CSTD_UNUSED(js_policy);
}

int kbasep_js_policy_init_ctx(struct kbase_device *kbdev, struct kbase_context *kctx)
{
	struct kbasep_js_device_data *js_devdata;
	struct kbasep_js_policy_cfs_ctx *ctx_info;
	struct kbasep_js_policy_cfs *policy_info;
	int policy;

	KBASE_DEBUG_ASSERT(kbdev != NULL);
	KBASE_DEBUG_ASSERT(kctx != NULL);

	js_devdata = &kbdev->js_data;
	policy_info = &kbdev->js_data.policy.cfs;
	ctx_info = &kctx->jctx.sched_info.runpool.policy_ctx.cfs;

	KBASE_TRACE_ADD_REFCOUNT(kbdev, JS_POLICY_INIT_CTX, kctx, NULL, 0u, kbasep_js_policy_trace_get_refcnt(kbdev, kctx));

	policy = current->policy;
	if (policy == SCHED_FIFO || policy == SCHED_RR) {
		ctx_info->process_rt_policy = true;
		ctx_info->process_priority = (((MAX_RT_PRIO - 1) - current->rt_priority) / 5) - 20;
	} else {
		ctx_info->process_rt_policy = false;
		ctx_info->process_priority = (current->static_prio - MAX_RT_PRIO) - 20;
	}

	/* Initial runtime (relative to least-run context runtime)
	 *
	 * This uses the Policy Queue's most up-to-date head_runtime_us by using the
	 * queue mutex to issue memory barriers - also ensure future updates to
	 * head_runtime_us occur strictly after this context is initialized */
	mutex_lock(&js_devdata->queue_mutex);

	/* No need to hold the the runpool_irq.lock here, because we're initializing
	 * the value, and the context is definitely not being updated in the
	 * runpool at this point. The queue_mutex ensures the memory barrier. */
	ctx_info->runtime_us = policy_info->head_runtime_us + priority_weight(ctx_info, (u64) js_devdata->cfs_ctx_runtime_init_slices * (u64) (js_devdata->ctx_timeslice_ns / 1000u));

	mutex_unlock(&js_devdata->queue_mutex);

	return 0;
}

void kbasep_js_policy_term_ctx(union kbasep_js_policy *js_policy, struct kbase_context *kctx)
{
	struct kbasep_js_policy_cfs_ctx *ctx_info;
	struct kbasep_js_policy_cfs *policy_info;
	struct kbase_device *kbdev;

	KBASE_DEBUG_ASSERT(js_policy != NULL);
	KBASE_DEBUG_ASSERT(kctx != NULL);

	policy_info = &js_policy->cfs;
	ctx_info = &kctx->jctx.sched_info.runpool.policy_ctx.cfs;

	kbdev = container_of(js_policy, struct kbase_device, js_data.policy);
	KBASE_TRACE_ADD_REFCOUNT(kbdev, JS_POLICY_TERM_CTX, kctx, NULL, 0u, kbasep_js_policy_trace_get_refcnt(kbdev, kctx));

	/* No work to do */
}

/*
 * Job Chain Management
 */

void kbasep_js_policy_log_job_result(union kbasep_js_policy *js_policy, struct kbase_jd_atom *katom, u64 time_spent_us)
{
	struct kbasep_js_policy_cfs_ctx *ctx_info;
	struct kbase_context *parent_ctx;

	KBASE_DEBUG_ASSERT(js_policy != NULL);
	KBASE_DEBUG_ASSERT(katom != NULL);
	CSTD_UNUSED(js_policy);

	parent_ctx = katom->kctx;
	KBASE_DEBUG_ASSERT(parent_ctx != NULL);

	ctx_info = &parent_ctx->jctx.sched_info.runpool.policy_ctx.cfs;

	ctx_info->runtime_us += priority_weight(ctx_info, time_spent_us);

	katom->time_spent_us += time_spent_us;
}

bool kbasep_js_policy_ctx_has_priority(union kbasep_js_policy *js_policy, struct kbase_context *current_ctx, struct kbase_context *new_ctx)
{
	struct kbasep_js_policy_cfs_ctx *current_ctx_info;
	struct kbasep_js_policy_cfs_ctx *new_ctx_info;

	KBASE_DEBUG_ASSERT(current_ctx != NULL);
	KBASE_DEBUG_ASSERT(new_ctx != NULL);
	CSTD_UNUSED(js_policy);

	current_ctx_info = &current_ctx->jctx.sched_info.runpool.policy_ctx.cfs;
	new_ctx_info = &new_ctx->jctx.sched_info.runpool.policy_ctx.cfs;

	if (!current_ctx_info->process_rt_policy && new_ctx_info->process_rt_policy)
		return true;

	if (current_ctx_info->process_rt_policy ==
			new_ctx_info->process_rt_policy)
		return true;

	return false;
}