← all fixtures

c-sched-ext-scx-many-many-moves-some-deletes-some-adds

C View before/after files on GitHub Upstream commit sched-ext-scx@991b73e scheds/rust/scx_lavd/src/bpf/lavd.bpf.h

Excellent test case to demonstrate why move tracking is important.

Before

…lines 1–209 unchanged (209 lines)
210struct cpu_ctx *get_cpu_ctx_task(const struct task_struct *p);
211
212/*
213 * CPU context
214 */
215struct cpu_ctx {
216 /*
217 * Information used to keep track of CPU utilization
218 */
219 volatile u32 avg_util; /* average of the CPU utilization */
220 volatile u32 cur_util; /* CPU utilization of the current interval */
221 volatile u32 avg_sc_util; /* average of the scaled CPU utilization, which is capacity and frequency invariant. */
222 volatile u32 cur_sc_util; /* the scaled CPU utilization of the current interval, which is capacity and frequency invariant. */
223 volatile u64 idle_total; /* total idle time so far */
224 volatile u64 idle_start_clk; /* when the CPU becomes idle */
225
226 /*
227 * Information used to keep track of load
228 */
229 volatile u64 tot_svc_time; /* total service time on a CPU scaled by tasks' weights */
230 volatile u64 tot_sc_time; /* total scaled CPU time, which is capacity and frequency invariant. */
231 volatile u64 cpu_release_clk; /* when the CPU is taken by higher-priority scheduler class */
232
233 /*
234 * Information used to keep track of latency criticality
235 */
236 volatile u32 max_lat_cri; /* maximum latency criticality */
237 volatile u32 nr_sched; /* number of schedules */
238 volatile u64 sum_lat_cri; /* sum of latency criticality */
239
240 /*
241 * Information used to keep track of performance criticality
242 */
243 volatile u64 sum_perf_cri; /* sum of performance criticality */
244 volatile u32 min_perf_cri; /* mininum performance criticality */
245 volatile u32 max_perf_cri; /* maximum performance criticality */
246
247 /*
248 * Information of a current running task for preemption
249 */
250 volatile u64 running_clk; /* when a task starts running */
251 volatile u64 est_stopping_clk; /* estimated stopping time */
252 volatile u64 flags; /* cached copy of task's flags */
253 volatile u32 nr_pinned_tasks; /* the number of pinned tasks waiting for running on this CPU */
254 volatile s32 futex_op; /* futex op in futex V1 */
255 volatile u16 lat_cri; /* latency criticality */
256 volatile u8 is_online; /* is this CPU online? */
257
258 /*
259 * Information for CPU frequency scaling
260 */
261 u32 cpuperf_cur; /* CPU's current performance target */
262
263 /*
264 * Fields for core compaction
265 *
266 */
267 u16 cpu_id; /* cpu id */
268 u16 capacity; /* CPU capacity based on 1024 */
269 u8 big_core; /* is it a big core? */
270 u8 turbo_core; /* is it a turbo core? */
271 u8 llc_id; /* llc domain id */
272 u8 cpdom_id; /* compute domain id */
273 u8 cpdom_alt_id; /* compute domain id of anternative type */
274 u8 cpdom_poll_pos; /* index to check if a DSQ of a compute domain is starving */
275
276 /*
277 * Information for statistics.
278 */
279 volatile u32 nr_preempt;
280 volatile u32 nr_x_migration;
281 volatile u32 nr_perf_cri;
282 volatile u32 nr_lat_cri;
283
284 /*
285 * Information for cpu hotplug
286 */
287 u64 online_clk; /* when a CPU becomes online */
288 u64 offline_clk; /* when a CPU becomes offline */
289
290 /*
291 * Temporary cpu masks
292 */
293 struct bpf_cpumask __kptr *tmp_a_mask; /* for active set */
294 struct bpf_cpumask __kptr *tmp_o_mask; /* for overflow set */
295 struct bpf_cpumask __kptr *tmp_l_mask; /* for online cpumask */
296 struct bpf_cpumask __kptr *tmp_i_mask; /* for idle cpumask */
297 struct bpf_cpumask __kptr *tmp_t_mask;
298 struct bpf_cpumask __kptr *tmp_t2_mask;
299 struct bpf_cpumask __kptr *tmp_t3_mask;
300} __attribute__((aligned(CACHELINE_SIZE)));
301
…lines 302–496 unchanged (195 lines)

After

…lines 1–209 unchanged (209 lines)
210struct cpu_ctx *get_cpu_ctx_task(const struct task_struct *p);
211
212/*
213 * CPU context
214 */
215struct cpu_ctx {
216 /* --- cacheline 0 boundary (0 bytes) --- */
217 volatile u64 flags; /* cached copy of task's flags */
218 volatile u64 tot_svc_time; /* total service time on a CPU scaled by tasks' weights */
219 volatile u64 tot_sc_time; /* total scaled CPU time, which is capacity and frequency invariant. */
220 volatile u64 est_stopping_clk; /* estimated stopping time */
221 volatile u64 running_clk; /* when a task starts running */
222 volatile u16 lat_cri; /* latency criticality */
223 volatile u32 max_lat_cri; /* maximum latency criticality */
224 volatile u64 sum_lat_cri; /* sum of latency criticality */
225 volatile u64 sum_perf_cri; /* sum of performance criticality */
226
227 /* --- cacheline 1 boundary (64 bytes) --- */
228 volatile u32 min_perf_cri; /* mininum performance criticality */
229 volatile u32 max_perf_cri; /* maximum performance criticality */
230 volatile u32 nr_sched; /* number of schedules */
231 volatile u32 nr_preempt;
232 volatile u32 nr_x_migration;
233 volatile u32 nr_perf_cri;
234 volatile u32 nr_lat_cri;
235 volatile u32 nr_pinned_tasks; /* the number of pinned tasks waiting for running on this CPU */
236 volatile s32 futex_op; /* futex op in futex V1 */
237 volatile u32 avg_util; /* average of the CPU utilization */
238 volatile u32 cur_util; /* CPU utilization of the current interval */
239 u32 cpuperf_cur; /* CPU's current performance target */
240 volatile u32 avg_sc_util; /* average of the scaled CPU utilization, which is capacity and frequency invariant. */
241 volatile u32 cur_sc_util; /* the scaled CPU utilization of the current interval, which is capacity and frequency invariant. */
242
243 volatile u64 cpu_release_clk; /* when the CPU is taken by higher-priority scheduler class */
244
245 /* --- cacheline 2 boundary (128 bytes) --- */
246 /*
247 * Idle tracking (read-mostly)
248 */
249 volatile u64 idle_total; /* total idle time so far */
250 volatile u64 idle_start_clk; /* when the CPU becomes idle */
251 u64 online_clk; /* when a CPU becomes online */
252 u64 offline_clk; /* when a CPU becomes offline */
253
254 /*
255 * Fields for core compaction (read-only)
256 */
257 u16 cpu_id; /* cpu id */
258 u16 capacity; /* CPU capacity based on 1024 */
259 u8 big_core; /* is it a big core? */
260 u8 turbo_core; /* is it a turbo core? */
261 u8 llc_id; /* llc domain id */
262 u8 cpdom_id; /* compute domain id */
263 u8 cpdom_alt_id; /* compute domain id of anternative type */
264 u8 is_online; /* is this CPU online? */
265
266 /*
267 * Temporary cpu masks (read-only)
268 */
269 struct bpf_cpumask __kptr *tmp_a_mask; /* for active set */
270 struct bpf_cpumask __kptr *tmp_o_mask; /* for overflow set */
271 /* --- cacheline 3 boundary (192 bytes) --- */
272 struct bpf_cpumask __kptr *tmp_l_mask; /* for online cpumask */
273 struct bpf_cpumask __kptr *tmp_i_mask; /* for idle cpumask */
274 struct bpf_cpumask __kptr *tmp_t_mask;
275 struct bpf_cpumask __kptr *tmp_t2_mask;
276 struct bpf_cpumask __kptr *tmp_t3_mask;
277} __attribute__((aligned(CACHELINE_SIZE)));
…lines 278–473 unchanged (196 lines)

Before

translation_unit
comment "/* SPDX-License-Identifier: GPL-2.0 */"
comment "/*\n * Copyright (c) 2023, 2024 Valve Corporation.\n * Author:"...
preproc_ifdef
#ifndef "#ifndef"
identifier "__LAVD_H"
preproc_def
#define "#define"
identifier "__LAVD_H"
preproc_include
#include "#include"
system_lib_string "<scx/common.bpf.h>"
preproc_include
#include "#include"
system_lib_string "<scx/bpf_arena_common.bpf.h>"
preproc_include
#include "#include"
system_lib_string "<lib/sdt_task.h>"
preproc_include
#include "#include"
system_lib_string "<lib/atq.h>"
comment "/*\n * common macros\n */"
preproc_def
#define "#define"
identifier "U64_MAX"
preproc_arg "((u64)~0ULL)"
preproc_def
#define "#define"
identifier "S64_MAX"
preproc_arg "((s64)(U64_MAX >> 1))"
preproc_def
#define "#define"
identifier "U32_MAX"
preproc_arg "((u32)~0U)"
preproc_def
#define "#define"
identifier "S32_MAX"
preproc_arg "((s32)(U32_MAX >> 1))"
preproc_def
#define "#define"
identifier "MAX_RT_PRIO"
preproc_arg "100"
preproc_def
#define "#define"
identifier "LAVD_SHIFT"
preproc_arg "10"
preproc_def
#define "#define"
identifier "LAVD_SCALE"
preproc_arg "(1L << LAVD_SHIFT)"
preproc_function_def
#define "#define"
identifier "p2s"
preproc_params
( "("
identifier "percent"
) ")"
preproc_arg "(((percent) << LAVD_SHIFT) / 100)"
preproc_function_def
#define "#define"
identifier "s2p"
preproc_params
( "("
identifier "scale"
) ")"
preproc_arg "(((scale) * 100) >> LAVD_SHIFT)"
preproc_function_def
#define "#define"
identifier "cpdom_to_dsq"
preproc_params
( "("
identifier "cpdom_id"
) ")"
preproc_arg "((cpdom_id) | LAVD_DSQ_TYPE_CPDOM << LAVD_DSQ_TYPE_SHFT)"
preproc_function_def
#define "#define"
identifier "dsq_to_cpdom"
preproc_params
( "("
identifier "dsq_id"
) ")"
preproc_arg "((dsq_id) & LAVD_DSQ_ID_MASK)"
preproc_function_def
#define "#define"
identifier "dsq_to_cpu"
preproc_params
( "("
identifier "dsq_id"
) ")"
preproc_arg "((dsq_id) & LAVD_DSQ_ID_MASK)"
preproc_function_def
#define "#define"
identifier "dsq_type"
preproc_params
( "("
identifier "dsq_id"
) ")"
preproc_arg "(((dsq_id) & LAVD_DSQ_TYPE_MASK) >> LAVD_DSQ_TYPE_SHFT)"
comment "/*\n * DSQ (dispatch queue) IDs are 64bit of the format:\n * "...
enum_specifier (+46 nodes collapsed)
; ";"
comment "/*\n * common constants\n */"
enum_specifier (+290 nodes collapsed)
; ";"
enum_specifier (+138 nodes collapsed)
; ";"
comment "/*\n * Task context\n */"
struct_specifier (+187 nodes collapsed)
; ";"
comment "/*\n * Compute domain context\n * - system > numa node > llc d"...
struct_specifier (+142 nodes collapsed)
; ";"
preproc_function_def
#define "#define"
identifier "get_neighbor_id"
preproc_params
( "("
identifier "cpdomc"
, ","
identifier "d"
, ","
identifier "i"
) ")"
preproc_arg "((cpdomc)->neighbor_ids[((d) * LAVD_CPDOM_MAX_NR) + (i)])"
declaration
storage_class_specifier
extern "extern"
struct_specifier
struct "struct"
type_identifier "cpdom_ctx"
array_declarator
identifier "cpdom_ctxs"
[ "["
identifier "LAVD_CPDOM_MAX_NR"
] "]"
; ";"
declaration
storage_class_specifier
extern "extern"
struct_specifier
struct "struct"
type_identifier "bpf_cpumask"
array_declarator
identifier "cpdom_cpumask"
[ "["
identifier "LAVD_CPDOM_MAX_NR"
] "]"
; ";"
declaration
storage_class_specifier
extern "extern"
primitive_type "int"
identifier "nr_cpdoms"
; ";"
type_definition
typedef "typedef"
struct_specifier
struct "struct"
type_identifier "task_ctx"
ERROR
type_identifier "__arena"
type_identifier "task_ctx"
; ";"
declaration
type_identifier "u64"
function_declarator
identifier "get_task_ctx_internal"
parameter_list
( "("
parameter_declaration
struct_specifier
struct "struct"
type_identifier "task_struct"
pointer_declarator
* "*"
identifier "p"
) ")"
; ";"
preproc_function_def
#define "#define"
identifier "get_task_ctx"
preproc_params
( "("
identifier "p"
) ")"
preproc_arg "((task_ctx *)get_task_ctx_internal((p)))"
declaration
struct_specifier
struct "struct"
type_identifier "cpu_ctx"
pointer_declarator
* "*"
function_declarator
identifier "get_cpu_ctx"
parameter_list
( "("
parameter_declaration
primitive_type "void"
) ")"
; ";"
declaration
struct_specifier
struct "struct"
type_identifier "cpu_ctx"
pointer_declarator
* "*"
function_declarator
identifier "get_cpu_ctx_id"
parameter_list
( "("
parameter_declaration
type_identifier "s32"
identifier "cpu_id"
) ")"
; ";"
declaration (+21 nodes collapsed)
comment "/*\n * CPU context\n */"
struct_specifier
struct "struct"
type_identifier "cpu_ctx"
field_declaration_list
{ "{"
comment "/* \n\t * Information used to keep track of CPU utilization\n\t "...
field_declaration
type_qualifier
volatile "volatile"
type_identifier "u32"
field_identifier "avg_util"
; ";"
comment "/* average of the CPU utilization */"
field_declaration
type_qualifier
volatile "volatile"
type_identifier "u32"
field_identifier "cur_util"
; ";"
comment "/* CPU utilization of the current interval */"
field_declaration
type_qualifier
volatile "volatile"
type_identifier "u32"
field_identifier "avg_sc_util"
; ";"
comment "/* average of the scaled CPU utilization, which is capacity "...
field_declaration
type_qualifier
volatile "volatile"
type_identifier "u32"
field_identifier "cur_sc_util"
; ";"
comment "/* the scaled CPU utilization of the current interval, which"...
field_declaration
type_qualifier
volatile "volatile"
type_identifier "u64"
field_identifier "idle_total"
; ";"
comment "/* total idle time so far */"
field_declaration
type_qualifier
volatile "volatile"
type_identifier "u64"
field_identifier "idle_start_clk"
; ";"
comment "/* when the CPU becomes idle */"
comment "/*\n\t * Information used to keep track of load\n\t */"
field_declaration
type_qualifier
volatile "volatile"
type_identifier "u64"
field_identifier "tot_svc_time"
; ";"
comment "/* total service time on a CPU scaled by tasks' weights */"
field_declaration
type_qualifier
volatile "volatile"
type_identifier "u64"
field_identifier "tot_sc_time"
; ";"
comment "/* total scaled CPU time, which is capacity and frequency in"...
field_declaration
type_qualifier
volatile "volatile"
type_identifier "u64"
field_identifier "cpu_release_clk"
; ";"
comment "/* when the CPU is taken by higher-priority scheduler class "...
comment "/*\n\t * Information used to keep track of latency criticality"...
field_declaration
type_qualifier
volatile "volatile"
type_identifier "u32"
field_identifier "max_lat_cri"
; ";"
comment "/* maximum latency criticality */"
field_declaration
type_qualifier
volatile "volatile"
type_identifier "u32"
field_identifier "nr_sched"
; ";"
comment "/* number of schedules */"
field_declaration
type_qualifier
volatile "volatile"
type_identifier "u64"
field_identifier "sum_lat_cri"
; ";"
comment "/* sum of latency criticality */"
comment "/*\n\t * Information used to keep track of performance critica"...
field_declaration
type_qualifier
volatile "volatile"
type_identifier "u64"
field_identifier "sum_perf_cri"
; ";"
comment "/* sum of performance criticality */"
field_declaration
type_qualifier
volatile "volatile"
type_identifier "u32"
field_identifier "min_perf_cri"
; ";"
comment "/* mininum performance criticality */"
field_declaration
type_qualifier
volatile "volatile"
type_identifier "u32"
field_identifier "max_perf_cri"
; ";"
comment "/* maximum performance criticality */"
comment "/*\n\t * Information of a current running task for preemption\n"...
field_declaration
type_qualifier
volatile "volatile"
type_identifier "u64"
field_identifier "running_clk"
; ";"
comment "/* when a task starts running */"
field_declaration
type_qualifier
volatile "volatile"
type_identifier "u64"
field_identifier "est_stopping_clk"
; ";"
comment "/* estimated stopping time */"
field_declaration
type_qualifier
volatile "volatile"
type_identifier "u64"
field_identifier "flags"
; ";"
comment "/* cached copy of task's flags */"
field_declaration
type_qualifier
volatile "volatile"
type_identifier "u32"
field_identifier "nr_pinned_tasks"
; ";"
comment "/* the number of pinned tasks waiting for running on this CP"...
field_declaration
type_qualifier
volatile "volatile"
type_identifier "s32"
field_identifier "futex_op"
; ";"
comment "/* futex op in futex V1 */"
field_declaration
type_qualifier
volatile "volatile"
type_identifier "u16"
field_identifier "lat_cri"
; ";"
comment "/* latency criticality */"
field_declaration
type_qualifier
volatile "volatile"
type_identifier "u8"
field_identifier "is_online"
; ";"
comment "/* is this CPU online? */"
comment "/*\n\t * Information for CPU frequency scaling\n\t */"
field_declaration
type_identifier "u32"
field_identifier "cpuperf_cur"
; ";"
comment "/* CPU's current performance target */"
comment "/*\n\t * Fields for core compaction\n\t *\n\t */"
field_declaration
type_identifier "u16"
field_identifier "cpu_id"
; ";"
comment "/* cpu id */"
field_declaration
type_identifier "u16"
field_identifier "capacity"
; ";"
comment "/* CPU capacity based on 1024 */"
field_declaration
type_identifier "u8"
field_identifier "big_core"
; ";"
comment "/* is it a big core? */"
field_declaration
type_identifier "u8"
field_identifier "turbo_core"
; ";"
comment "/* is it a turbo core? */"
field_declaration
type_identifier "u8"
field_identifier "llc_id"
; ";"
comment "/* llc domain id */"
field_declaration
type_identifier "u8"
field_identifier "cpdom_id"
; ";"
comment "/* compute domain id */"
field_declaration
type_identifier "u8"
field_identifier "cpdom_alt_id"
; ";"
comment "/* compute domain id of anternative type */"
field_declaration
type_identifier "u8"
field_identifier "cpdom_poll_pos"
; ";"
comment "/* index to check if a DSQ of a compute domain is starving *"...
comment "/*\n\t * Information for statistics.\n\t */"
field_declaration
type_qualifier
volatile "volatile"
type_identifier "u32"
field_identifier "nr_preempt"
; ";"
field_declaration
type_qualifier
volatile "volatile"
type_identifier "u32"
field_identifier "nr_x_migration"
; ";"
field_declaration
type_qualifier
volatile "volatile"
type_identifier "u32"
field_identifier "nr_perf_cri"
; ";"
field_declaration
type_qualifier
volatile "volatile"
type_identifier "u32"
field_identifier "nr_lat_cri"
; ";"
comment "/*\n\t * Information for cpu hotplug\n\t */"
field_declaration
type_identifier "u64"
field_identifier "online_clk"
; ";"
comment "/* when a CPU becomes online */"
field_declaration
type_identifier "u64"
field_identifier "offline_clk"
; ";"
comment "/* when a CPU becomes offline */"
comment "/*\n\t * Temporary cpu masks\n\t */"
field_declaration
struct_specifier
struct "struct"
type_identifier "bpf_cpumask"
ERROR
field_identifier "__kptr"
pointer_declarator
* "*"
field_identifier "tmp_a_mask"
; ";"
comment "/* for active set */"
field_declaration
struct_specifier
struct "struct"
type_identifier "bpf_cpumask"
ERROR
field_identifier "__kptr"
pointer_declarator
* "*"
field_identifier "tmp_o_mask"
; ";"
comment "/* for overflow set */"
field_declaration
struct_specifier
struct "struct"
type_identifier "bpf_cpumask"
ERROR
field_identifier "__kptr"
pointer_declarator
* "*"
field_identifier "tmp_l_mask"
; ";"
comment "/* for online cpumask */"
field_declaration
struct_specifier
struct "struct"
type_identifier "bpf_cpumask"
ERROR
field_identifier "__kptr"
pointer_declarator
* "*"
field_identifier "tmp_i_mask"
; ";"
comment "/* for idle cpumask */"
field_declaration
struct_specifier
struct "struct"
type_identifier "bpf_cpumask"
ERROR
field_identifier "__kptr"
pointer_declarator
* "*"
field_identifier "tmp_t_mask"
; ";"
field_declaration
struct_specifier
struct "struct"
type_identifier "bpf_cpumask"
ERROR
field_identifier "__kptr"
pointer_declarator
* "*"
field_identifier "tmp_t2_mask"
; ";"
field_declaration
struct_specifier
struct "struct"
type_identifier "bpf_cpumask"
ERROR
field_identifier "__kptr"
pointer_declarator
* "*"
field_identifier "tmp_t3_mask"
; ";"
} "}"
attribute_specifier
__attribute__ "__attribute__"
( "("
argument_list
( "("
call_expression
identifier "aligned"
argument_list
( "("
identifier "CACHELINE_SIZE"
) ")"
) ")"
) ")"
; ";"
declaration
storage_class_specifier
extern "extern"
type_qualifier
const "const"
type_qualifier
volatile "volatile"
type_identifier "u64"
identifier "nr_llcs"
; ";"
comment "/* number of LLC domains */"
declaration
type_qualifier
const "const"
storage_class_specifier
extern "extern"
type_qualifier
volatile "volatile"
type_identifier "u32"
identifier "nr_cpu_ids"
; ";"
declaration
storage_class_specifier
extern "extern"
type_qualifier
volatile "volatile"
type_identifier "u64"
identifier "nr_cpus_onln"
; ";"
comment "/* current number of online CPUs */"
declaration
storage_class_specifier
extern "extern"
type_qualifier
const "const"
type_qualifier
volatile "volatile"
type_identifier "u16"
array_declarator
identifier "cpu_capacity"
[ "["
identifier "LAVD_CPU_ID_MAX"
] "]"
; ";"
declaration
storage_class_specifier
extern "extern"
type_qualifier
const "const"
type_qualifier
volatile "volatile"
type_identifier "u8"
array_declarator
identifier "cpu_big"
[ "["
identifier "LAVD_CPU_ID_MAX"
] "]"
; ";"
declaration
storage_class_specifier
extern "extern"
type_qualifier
const "const"
type_qualifier
volatile "volatile"
type_identifier "u8"
array_declarator
identifier "cpu_turbo"
[ "["
identifier "LAVD_CPU_ID_MAX"
] "]"
; ";"
comment "/* Logging helpers. */"
declaration
storage_class_specifier
extern "extern"
type_qualifier
const "const"
type_qualifier
volatile "volatile"
primitive_type "bool"
identifier "no_wake_sync"
; ";"
declaration
storage_class_specifier
extern "extern"
type_qualifier
const "const"
type_qualifier
volatile "volatile"
primitive_type "bool"
identifier "no_slice_boost"
; ";"
declaration
storage_class_specifier
extern "extern"
type_qualifier
const "const"
type_qualifier
volatile "volatile"
type_identifier "u8"
identifier "verbose"
; ";"
preproc_function_def
#define "#define"
identifier "debugln"
preproc_params
( "("
identifier "fmt"
, ","
... "..."
) ")"
preproc_arg "({\t\t\t\t\t\t\t\t\t\\\n\tif (verbose > 0)\t\t\t\t\t\t\\\n\t\tbpf_printk(\"[%s:%d] "...
preproc_function_def
#define "#define"
identifier "traceln"
preproc_params
( "("
identifier "fmt"
, ","
... "..."
) ")"
preproc_arg "({\t\t\t\t\t\t\t\t\t\\\n\tif (verbose > 1)\t\t\t\t\t\t\\\n\t\tbpf_printk(\"[%s:%d] "...
comment "/* Arithmetic helpers. */"
preproc_ifdef
#ifndef "#ifndef"
identifier "min"
preproc_function_def
#define "#define"
identifier "min"
preproc_params
( "("
identifier "X"
, ","
identifier "Y"
) ")"
preproc_arg "(((X) < (Y)) ? (X) : (Y))"
#endif "#endif"
preproc_ifdef
#ifndef "#ifndef"
identifier "max"
preproc_function_def
#define "#define"
identifier "max"
preproc_params
( "("
identifier "X"
, ","
identifier "Y"
) ")"
preproc_arg "(((X) < (Y)) ? (Y) : (X))"
#endif "#endif"
preproc_ifdef
#ifndef "#ifndef"
identifier "clamp"
preproc_function_def
#define "#define"
identifier "clamp"
preproc_params
( "("
identifier "val"
, ","
identifier "lo"
, ","
identifier "hi"
) ")"
preproc_arg "min(max(val, lo), hi)"
#endif "#endif"
declaration
type_identifier "u64"
function_declarator
identifier "calc_avg"
parameter_list
( "("
parameter_declaration
type_identifier "u64"
identifier "old_val"
, ","
parameter_declaration
type_identifier "u64"
identifier "new_val"
) ")"
; ";"
declaration
type_identifier "u64"
function_declarator
identifier "calc_asym_avg"
parameter_list
( "("
parameter_declaration
type_identifier "u64"
identifier "old_val"
, ","
parameter_declaration
type_identifier "u64"
identifier "new_val"
) ")"
; ";"
comment "/* Bitmask helpers. */"
function_definition (+69 nodes collapsed)
comment "/* System statistics module .*/"
declaration
storage_class_specifier
extern "extern"
struct_specifier
struct "struct"
type_identifier "sys_stat"
identifier "sys_stat"
; ";"
declaration
type_identifier "s32"
function_declarator
identifier "init_sys_stat"
parameter_list
( "("
parameter_declaration
type_identifier "u64"
identifier "now"
) ")"
; ";"
declaration
primitive_type "int"
function_declarator
identifier "update_sys_stat"
parameter_list
( "("
parameter_declaration
primitive_type "void"
) ")"
; ";"
declaration
storage_class_specifier
extern "extern"
type_qualifier
volatile "volatile"
type_identifier "u64"
identifier "performance_mode_ns"
; ";"
declaration
storage_class_specifier
extern "extern"
type_qualifier
volatile "volatile"
type_identifier "u64"
identifier "balanced_mode_ns"
; ";"
declaration
storage_class_specifier
extern "extern"
type_qualifier
volatile "volatile"
type_identifier "u64"
identifier "powersave_mode_ns"
; ";"
comment "/* Helpers from util.bpf.c for querying CPU/task state. */"
declaration
storage_class_specifier
extern "extern"
type_qualifier
const "const"
type_qualifier
volatile "volatile"
primitive_type "bool"
identifier "per_cpu_dsq"
; ";"
declaration
storage_class_specifier
extern "extern"
type_qualifier
const "const"
type_qualifier
volatile "volatile"
type_identifier "u64"
identifier "pinned_slice_ns"
; ";"
declaration
storage_class_specifier
extern "extern"
type_qualifier
volatile "volatile"
primitive_type "bool"
identifier "reinit_cpumask_for_performance"
; ";"
declaration
storage_class_specifier
extern "extern"
type_qualifier
volatile "volatile"
primitive_type "bool"
identifier "no_preemption"
; ";"
declaration
storage_class_specifier
extern "extern"
type_qualifier
volatile "volatile"
primitive_type "bool"
identifier "no_core_compaction"
; ";"
declaration
storage_class_specifier
extern "extern"
type_qualifier
volatile "volatile"
primitive_type "bool"
identifier "no_freq_scaling"
; ";"
declaration
primitive_type "bool"
function_declarator
identifier "test_cpu_flag"
parameter_list
( "("
parameter_declaration
struct_specifier
struct "struct"
type_identifier "cpu_ctx"
pointer_declarator
* "*"
identifier "cpuc"
, ","
parameter_declaration
type_identifier "u64"
identifier "flag"
) ")"
; ";"
declaration
primitive_type "void"
function_declarator
identifier "set_cpu_flag"
parameter_list
( "("
parameter_declaration
struct_specifier
struct "struct"
type_identifier "cpu_ctx"
pointer_declarator
* "*"
identifier "cpuc"
, ","
parameter_declaration
type_identifier "u64"
identifier "flag"
) ")"
; ";"
declaration
primitive_type "void"
function_declarator
identifier "reset_cpu_flag"
parameter_list
( "("
parameter_declaration
struct_specifier
struct "struct"
type_identifier "cpu_ctx"
pointer_declarator
* "*"
identifier "cpuc"
, ","
parameter_declaration
type_identifier "u64"
identifier "flag"
) ")"
; ";"
declaration
primitive_type "bool"
function_declarator
identifier "is_lock_holder"
parameter_list
( "("
parameter_declaration
type_identifier "task_ctx"
pointer_declarator
* "*"
identifier "taskc"
) ")"
; ";"
declaration
primitive_type "bool"
function_declarator
identifier "is_lock_holder_running"
parameter_list
( "("
parameter_declaration
struct_specifier
struct "struct"
type_identifier "cpu_ctx"
pointer_declarator
* "*"
identifier "cpuc"
) ")"
; ";"
declaration
primitive_type "bool"
function_declarator
identifier "have_scheduled"
parameter_list
( "("
parameter_declaration
type_identifier "task_ctx"
pointer_declarator
* "*"
identifier "taskc"
) ")"
; ";"
declaration
primitive_type "bool"
function_declarator
identifier "have_pending_tasks"
parameter_list
( "("
parameter_declaration
struct_specifier
struct "struct"
type_identifier "cpu_ctx"
pointer_declarator
* "*"
identifier "cpuc"
) ")"
; ";"
declaration
primitive_type "bool"
function_declarator
identifier "can_boost_slice"
parameter_list
( "("
parameter_declaration
primitive_type "void"
) ")"
; ";"
declaration
primitive_type "bool"
function_declarator
identifier "is_lat_cri"
parameter_list
( "("
parameter_declaration
type_identifier "task_ctx"
pointer_declarator
* "*"
identifier "taskc"
) ")"
; ";"
declaration
type_identifier "u16"
function_declarator
identifier "get_nice_prio"
parameter_list
( "("
parameter_declaration
struct_specifier
struct "struct"
type_identifier "task_struct"
pointer_declarator
* "*"
identifier "p"
) ")"
; ";"
declaration
type_identifier "u32"
function_declarator
identifier "cpu_to_dsq"
parameter_list
( "("
parameter_declaration
type_identifier "u32"
identifier "cpu"
) ")"
; ";"
declaration
primitive_type "void"
function_declarator
identifier "set_task_flag"
parameter_list
( "("
parameter_declaration
type_identifier "task_ctx"
pointer_declarator
* "*"
identifier "taskc"
, ","
parameter_declaration
type_identifier "u64"
identifier "flag"
) ")"
; ";"
declaration
primitive_type "void"
function_declarator
identifier "reset_task_flag"
parameter_list
( "("
parameter_declaration
type_identifier "task_ctx"
pointer_declarator
* "*"
identifier "taskc"
, ","
parameter_declaration
type_identifier "u64"
identifier "flag"
) ")"
; ";"
declaration
primitive_type "bool"
function_declarator
identifier "test_task_flag"
parameter_list
( "("
parameter_declaration
type_identifier "task_ctx"
pointer_declarator
* "*"
identifier "taskc"
, ","
parameter_declaration
type_identifier "u64"
identifier "flag"
) ")"
; ";"
declaration
primitive_type "void"
function_declarator
identifier "reset_task_flag"
parameter_list
( "("
parameter_declaration
type_identifier "task_ctx"
pointer_declarator
* "*"
identifier "taskc"
, ","
parameter_declaration
type_identifier "u64"
identifier "flag"
) ")"
; ";"
function_definition (+23 nodes collapsed)
function_definition (+21 nodes collapsed)
function_definition (+22 nodes collapsed)
declaration
type_identifier "s32"
function_declarator
identifier "nr_queued_on_cpu"
parameter_list
( "("
parameter_declaration
struct_specifier
struct "struct"
type_identifier "cpu_ctx"
pointer_declarator
* "*"
identifier "cpuc"
) ")"
; ";"
declaration (+23 nodes collapsed)
declaration
storage_class_specifier
extern "extern"
struct_specifier
struct "struct"
type_identifier "bpf_cpumask"
ERROR
identifier "__kptr"
pointer_declarator
* "*"
identifier "turbo_cpumask"
; ";"
comment "/* CPU mask for turbo CPUs */"
declaration
storage_class_specifier
extern "extern"
struct_specifier
struct "struct"
type_identifier "bpf_cpumask"
ERROR
identifier "__kptr"
pointer_declarator
* "*"
identifier "big_cpumask"
; ";"
comment "/* CPU mask for big CPUs */"
declaration
storage_class_specifier
extern "extern"
struct_specifier
struct "struct"
type_identifier "bpf_cpumask"
ERROR
identifier "__kptr"
pointer_declarator
* "*"
identifier "little_cpumask"
; ";"
comment "/* CPU mask for little CPUs */"
declaration
storage_class_specifier
extern "extern"
struct_specifier
struct "struct"
type_identifier "bpf_cpumask"
ERROR
identifier "__kptr"
pointer_declarator
* "*"
identifier "active_cpumask"
; ";"
comment "/* CPU mask for active CPUs */"
declaration
storage_class_specifier
extern "extern"
struct_specifier
struct "struct"
type_identifier "bpf_cpumask"
ERROR
identifier "__kptr"
pointer_declarator
* "*"
identifier "ovrflw_cpumask"
; ";"
comment "/* CPU mask for overflow CPUs */"
comment "/* Power management helpers. */"
declaration
primitive_type "int"
function_declarator
identifier "do_core_compaction"
parameter_list
( "("
parameter_declaration
primitive_type "void"
) ")"
; ";"
declaration
primitive_type "int"
function_declarator
identifier "update_thr_perf_cri"
parameter_list
( "("
parameter_declaration
primitive_type "void"
) ")"
; ";"
declaration
primitive_type "int"
function_declarator
identifier "reinit_active_cpumask_for_performance"
parameter_list
( "("
parameter_declaration
primitive_type "void"
) ")"
; ";"
declaration
primitive_type "bool"
function_declarator
identifier "is_perf_cri"
parameter_list
( "("
parameter_declaration
type_identifier "task_ctx"
pointer_declarator
* "*"
identifier "taskc"
) ")"
; ";"
declaration
storage_class_specifier
extern "extern"
primitive_type "bool"
identifier "have_little_core"
; ";"
declaration
storage_class_specifier
extern "extern"
primitive_type "bool"
identifier "have_turbo_core"
; ";"
declaration
storage_class_specifier
extern "extern"
type_qualifier
const "const"
type_qualifier
volatile "volatile"
primitive_type "bool"
identifier "is_smt_active"
; ";"
declaration
storage_class_specifier
extern "extern"
type_identifier "u64"
identifier "total_capacity"
; ";"
declaration
storage_class_specifier
extern "extern"
type_identifier "u64"
identifier "one_little_capacity"
; ";"
declaration
storage_class_specifier
extern "extern"
type_identifier "u32"
identifier "cur_big_core_scale"
; ";"
declaration
storage_class_specifier
extern "extern"
type_identifier "u32"
identifier "default_big_core_scale"
; ";"
declaration
primitive_type "int"
function_declarator
identifier "init_autopilot_caps"
parameter_list
( "("
parameter_declaration
primitive_type "void"
) ")"
; ";"
declaration
primitive_type "int"
function_declarator
identifier "update_autopilot_high_cap"
parameter_list
( "("
parameter_declaration
primitive_type "void"
) ")"
; ";"
declaration
type_identifier "u64"
function_declarator
identifier "scale_cap_freq"
parameter_list
( "("
parameter_declaration
type_identifier "u64"
identifier "dur"
, ","
parameter_declaration
type_identifier "s32"
identifier "cpu"
) ")"
; ";"
declaration
primitive_type "int"
function_declarator
identifier "reset_cpuperf_target"
parameter_list
( "("
parameter_declaration
struct_specifier
struct "struct"
type_identifier "cpu_ctx"
pointer_declarator
* "*"
identifier "cpuc"
) ")"
; ";"
declaration
primitive_type "int"
function_declarator
identifier "update_cpuperf_target"
parameter_list
( "("
parameter_declaration
struct_specifier
struct "struct"
type_identifier "cpu_ctx"
pointer_declarator
* "*"
identifier "cpuc"
) ")"
; ";"
declaration
type_identifier "u16"
function_declarator
identifier "get_cpuperf_cap"
parameter_list
( "("
parameter_declaration
type_identifier "s32"
identifier "cpu"
) ")"
; ";"
declaration
primitive_type "int"
function_declarator
identifier "reset_suspended_duration"
parameter_list
( "("
parameter_declaration
struct_specifier
struct "struct"
type_identifier "cpu_ctx"
pointer_declarator
* "*"
identifier "cpuc"
) ")"
; ";"
declaration
type_identifier "u64"
function_declarator
identifier "get_suspended_duration_and_reset"
parameter_list
( "("
parameter_declaration
struct_specifier
struct "struct"
type_identifier "cpu_ctx"
pointer_declarator
* "*"
identifier "cpuc"
) ")"
; ";"
declaration
type_qualifier
const "const"
type_qualifier
volatile "volatile"
type_identifier "u16"
pointer_declarator
* "*"
function_declarator
identifier "get_cpu_order"
parameter_list
( "("
parameter_declaration
primitive_type "void"
) ")"
; ";"
comment "/* Load balancer helpers. */"
declaration
primitive_type "int"
function_declarator
identifier "plan_x_cpdom_migration"
parameter_list
( "("
parameter_declaration
primitive_type "void"
) ")"
; ";"
comment "/* Preemption management helpers. */"
declaration (+27 nodes collapsed)
comment "/* Futex lock-related helpers. */"
declaration (+21 nodes collapsed)
comment "/* Scheduler introspection-related helpers. */"
declaration
type_identifier "u64"
function_declarator
identifier "get_est_stopping_clk"
parameter_list
( "("
parameter_declaration
type_identifier "task_ctx"
pointer_declarator
* "*"
identifier "taskc"
, ","
parameter_declaration
type_identifier "u64"
identifier "now"
) ")"
; ";"
declaration (+21 nodes collapsed)
declaration
primitive_type "void"
function_declarator
identifier "reset_cpu_preemption_info"
parameter_list
( "("
parameter_declaration
struct_specifier
struct "struct"
type_identifier "cpu_ctx"
pointer_declarator
* "*"
identifier "cpuc"
, ","
parameter_declaration
primitive_type "bool"
identifier "released"
) ")"
; ";"
declaration
primitive_type "int"
function_declarator
identifier "shrink_boosted_slice_remote"
parameter_list
( "("
parameter_declaration
struct_specifier
struct "struct"
type_identifier "cpu_ctx"
pointer_declarator
* "*"
identifier "cpuc"
, ","
parameter_declaration
type_identifier "u64"
identifier "now"
) ")"
; ";"
declaration (+27 nodes collapsed)
declaration (+23 nodes collapsed)
declaration (+29 nodes collapsed)
declaration
storage_class_specifier
extern "extern"
type_qualifier
volatile "volatile"
primitive_type "bool"
identifier "is_monitored"
; ";"
#endif "#endif"
comment "/* __LAVD_H */"

After

translation_unit
comment "/* SPDX-License-Identifier: GPL-2.0 */"
comment "/*\n * Copyright (c) 2023, 2024 Valve Corporation.\n * Author:"...
preproc_ifdef
#ifndef "#ifndef"
identifier "__LAVD_H"
preproc_def
#define "#define"
identifier "__LAVD_H"
preproc_include
#include "#include"
system_lib_string "<scx/common.bpf.h>"
preproc_include
#include "#include"
system_lib_string "<scx/bpf_arena_common.bpf.h>"
preproc_include
#include "#include"
system_lib_string "<lib/sdt_task.h>"
preproc_include
#include "#include"
system_lib_string "<lib/atq.h>"
comment "/*\n * common macros\n */"
preproc_def
#define "#define"
identifier "U64_MAX"
preproc_arg "((u64)~0ULL)"
preproc_def
#define "#define"
identifier "S64_MAX"
preproc_arg "((s64)(U64_MAX >> 1))"
preproc_def
#define "#define"
identifier "U32_MAX"
preproc_arg "((u32)~0U)"
preproc_def
#define "#define"
identifier "S32_MAX"
preproc_arg "((s32)(U32_MAX >> 1))"
preproc_def
#define "#define"
identifier "MAX_RT_PRIO"
preproc_arg "100"
preproc_def
#define "#define"
identifier "LAVD_SHIFT"
preproc_arg "10"
preproc_def
#define "#define"
identifier "LAVD_SCALE"
preproc_arg "(1L << LAVD_SHIFT)"
preproc_function_def
#define "#define"
identifier "p2s"
preproc_params
( "("
identifier "percent"
) ")"
preproc_arg "(((percent) << LAVD_SHIFT) / 100)"
preproc_function_def
#define "#define"
identifier "s2p"
preproc_params
( "("
identifier "scale"
) ")"
preproc_arg "(((scale) * 100) >> LAVD_SHIFT)"
preproc_function_def
#define "#define"
identifier "cpdom_to_dsq"
preproc_params
( "("
identifier "cpdom_id"
) ")"
preproc_arg "((cpdom_id) | LAVD_DSQ_TYPE_CPDOM << LAVD_DSQ_TYPE_SHFT)"
preproc_function_def
#define "#define"
identifier "dsq_to_cpdom"
preproc_params
( "("
identifier "dsq_id"
) ")"
preproc_arg "((dsq_id) & LAVD_DSQ_ID_MASK)"
preproc_function_def
#define "#define"
identifier "dsq_to_cpu"
preproc_params
( "("
identifier "dsq_id"
) ")"
preproc_arg "((dsq_id) & LAVD_DSQ_ID_MASK)"
preproc_function_def
#define "#define"
identifier "dsq_type"
preproc_params
( "("
identifier "dsq_id"
) ")"
preproc_arg "(((dsq_id) & LAVD_DSQ_TYPE_MASK) >> LAVD_DSQ_TYPE_SHFT)"
comment "/*\n * DSQ (dispatch queue) IDs are 64bit of the format:\n * "...
enum_specifier (+46 nodes collapsed)
; ";"
comment "/*\n * common constants\n */"
enum_specifier (+290 nodes collapsed)
; ";"
enum_specifier (+138 nodes collapsed)
; ";"
comment "/*\n * Task context\n */"
struct_specifier (+187 nodes collapsed)
; ";"
comment "/*\n * Compute domain context\n * - system > numa node > llc d"...
struct_specifier (+142 nodes collapsed)
; ";"
preproc_function_def
#define "#define"
identifier "get_neighbor_id"
preproc_params
( "("
identifier "cpdomc"
, ","
identifier "d"
, ","
identifier "i"
) ")"
preproc_arg "((cpdomc)->neighbor_ids[((d) * LAVD_CPDOM_MAX_NR) + (i)])"
declaration
storage_class_specifier
extern "extern"
struct_specifier
struct "struct"
type_identifier "cpdom_ctx"
array_declarator
identifier "cpdom_ctxs"
[ "["
identifier "LAVD_CPDOM_MAX_NR"
] "]"
; ";"
declaration
storage_class_specifier
extern "extern"
struct_specifier
struct "struct"
type_identifier "bpf_cpumask"
array_declarator
identifier "cpdom_cpumask"
[ "["
identifier "LAVD_CPDOM_MAX_NR"
] "]"
; ";"
declaration
storage_class_specifier
extern "extern"
primitive_type "int"
identifier "nr_cpdoms"
; ";"
type_definition
typedef "typedef"
struct_specifier
struct "struct"
type_identifier "task_ctx"
ERROR
type_identifier "__arena"
type_identifier "task_ctx"
; ";"
declaration
type_identifier "u64"
function_declarator
identifier "get_task_ctx_internal"
parameter_list
( "("
parameter_declaration
struct_specifier
struct "struct"
type_identifier "task_struct"
pointer_declarator
* "*"
identifier "p"
) ")"
; ";"
preproc_function_def
#define "#define"
identifier "get_task_ctx"
preproc_params
( "("
identifier "p"
) ")"
preproc_arg "((task_ctx *)get_task_ctx_internal((p)))"
declaration
struct_specifier
struct "struct"
type_identifier "cpu_ctx"
pointer_declarator
* "*"
function_declarator
identifier "get_cpu_ctx"
parameter_list
( "("
parameter_declaration
primitive_type "void"
) ")"
; ";"
declaration
struct_specifier
struct "struct"
type_identifier "cpu_ctx"
pointer_declarator
* "*"
function_declarator
identifier "get_cpu_ctx_id"
parameter_list
( "("
parameter_declaration
type_identifier "s32"
identifier "cpu_id"
) ")"
; ";"
declaration (+21 nodes collapsed)
comment "/*\n * CPU context\n */"
struct_specifier
struct "struct"
type_identifier "cpu_ctx"
field_declaration_list
{ "{"
comment "/* --- cacheline 0 boundary (0 bytes) --- */"
field_declaration
type_qualifier
volatile "volatile"
type_identifier "u64"
field_identifier "flags"
; ";"
comment "/* cached copy of task's flags */"
field_declaration
type_qualifier
volatile "volatile"
type_identifier "u64"
field_identifier "tot_svc_time"
; ";"
comment "/* total service time on a CPU scaled by tasks' weights */"
field_declaration
type_qualifier
volatile "volatile"
type_identifier "u64"
field_identifier "tot_sc_time"
; ";"
comment "/* total scaled CPU time, which is capacity and frequency in"...
field_declaration
type_qualifier
volatile "volatile"
type_identifier "u64"
field_identifier "est_stopping_clk"
; ";"
comment "/* estimated stopping time */"
field_declaration
type_qualifier
volatile "volatile"
type_identifier "u64"
field_identifier "running_clk"
; ";"
comment "/* when a task starts running */"
field_declaration
type_qualifier
volatile "volatile"
type_identifier "u16"
field_identifier "lat_cri"
; ";"
comment "/* latency criticality */"
field_declaration
type_qualifier
volatile "volatile"
type_identifier "u32"
field_identifier "max_lat_cri"
; ";"
comment "/* maximum latency criticality */"
field_declaration
type_qualifier
volatile "volatile"
type_identifier "u64"
field_identifier "sum_lat_cri"
; ";"
comment "/* sum of latency criticality */"
field_declaration
type_qualifier
volatile "volatile"
type_identifier "u64"
field_identifier "sum_perf_cri"
; ";"
comment "/* sum of performance criticality */"
comment "/* --- cacheline 1 boundary (64 bytes) --- */"
field_declaration
type_qualifier
volatile "volatile"
type_identifier "u32"
field_identifier "min_perf_cri"
; ";"
comment "/* mininum performance criticality */"
field_declaration
type_qualifier
volatile "volatile"
type_identifier "u32"
field_identifier "max_perf_cri"
; ";"
comment "/* maximum performance criticality */"
field_declaration
type_qualifier
volatile "volatile"
type_identifier "u32"
field_identifier "nr_sched"
; ";"
comment "/* number of schedules */"
field_declaration
type_qualifier
volatile "volatile"
type_identifier "u32"
field_identifier "nr_preempt"
; ";"
field_declaration
type_qualifier
volatile "volatile"
type_identifier "u32"
field_identifier "nr_x_migration"
; ";"
field_declaration
type_qualifier
volatile "volatile"
type_identifier "u32"
field_identifier "nr_perf_cri"
; ";"
field_declaration
type_qualifier
volatile "volatile"
type_identifier "u32"
field_identifier "nr_lat_cri"
; ";"
field_declaration
type_qualifier
volatile "volatile"
type_identifier "u32"
field_identifier "nr_pinned_tasks"
; ";"
comment "/* the number of pinned tasks waiting for running on this CP"...
field_declaration
type_qualifier
volatile "volatile"
type_identifier "s32"
field_identifier "futex_op"
; ";"
comment "/* futex op in futex V1 */"
field_declaration
type_qualifier
volatile "volatile"
type_identifier "u32"
field_identifier "avg_util"
; ";"
comment "/* average of the CPU utilization */"
field_declaration
type_qualifier
volatile "volatile"
type_identifier "u32"
field_identifier "cur_util"
; ";"
comment "/* CPU utilization of the current interval */"
field_declaration
type_identifier "u32"
field_identifier "cpuperf_cur"
; ";"
comment "/* CPU's current performance target */"
field_declaration
type_qualifier
volatile "volatile"
type_identifier "u32"
field_identifier "avg_sc_util"
; ";"
comment "/* average of the scaled CPU utilization, which is capacity "...
field_declaration
type_qualifier
volatile "volatile"
type_identifier "u32"
field_identifier "cur_sc_util"
; ";"
comment "/* the scaled CPU utilization of the current interval, which"...
field_declaration
type_qualifier
volatile "volatile"
type_identifier "u64"
field_identifier "cpu_release_clk"
; ";"
comment "/* when the CPU is taken by higher-priority scheduler class "...
comment "/* --- cacheline 2 boundary (128 bytes) --- */"
comment "/*\n\t * Idle tracking (read-mostly)\n\t */"
field_declaration
type_qualifier
volatile "volatile"
type_identifier "u64"
field_identifier "idle_total"
; ";"
comment "/* total idle time so far */"
field_declaration
type_qualifier
volatile "volatile"
type_identifier "u64"
field_identifier "idle_start_clk"
; ";"
comment "/* when the CPU becomes idle */"
field_declaration
type_identifier "u64"
field_identifier "online_clk"
; ";"
comment "/* when a CPU becomes online */"
field_declaration
type_identifier "u64"
field_identifier "offline_clk"
; ";"
comment "/* when a CPU becomes offline */"
comment "/*\n\t * Fields for core compaction (read-only)\n\t */"
field_declaration
type_identifier "u16"
field_identifier "cpu_id"
; ";"
comment "/* cpu id */"
field_declaration
type_identifier "u16"
field_identifier "capacity"
; ";"
comment "/* CPU capacity based on 1024 */"
field_declaration
type_identifier "u8"
field_identifier "big_core"
; ";"
comment "/* is it a big core? */"
field_declaration
type_identifier "u8"
field_identifier "turbo_core"
; ";"
comment "/* is it a turbo core? */"
field_declaration
type_identifier "u8"
field_identifier "llc_id"
; ";"
comment "/* llc domain id */"
field_declaration
type_identifier "u8"
field_identifier "cpdom_id"
; ";"
comment "/* compute domain id */"
field_declaration
type_identifier "u8"
field_identifier "cpdom_alt_id"
; ";"
comment "/* compute domain id of anternative type */"
field_declaration
type_identifier "u8"
field_identifier "is_online"
; ";"
comment "/* is this CPU online? */"
comment "/*\n\t * Temporary cpu masks (read-only)\n\t */"
field_declaration
struct_specifier
struct "struct"
type_identifier "bpf_cpumask"
ERROR
field_identifier "__kptr"
pointer_declarator
* "*"
field_identifier "tmp_a_mask"
; ";"
comment "/* for active set */"
field_declaration
struct_specifier
struct "struct"
type_identifier "bpf_cpumask"
ERROR
field_identifier "__kptr"
pointer_declarator
* "*"
field_identifier "tmp_o_mask"
; ";"
comment "/* for overflow set */"
comment "/* --- cacheline 3 boundary (192 bytes) --- */"
field_declaration
struct_specifier
struct "struct"
type_identifier "bpf_cpumask"
ERROR
field_identifier "__kptr"
pointer_declarator
* "*"
field_identifier "tmp_l_mask"
; ";"
comment "/* for online cpumask */"
field_declaration
struct_specifier
struct "struct"
type_identifier "bpf_cpumask"
ERROR
field_identifier "__kptr"
pointer_declarator
* "*"
field_identifier "tmp_i_mask"
; ";"
comment "/* for idle cpumask */"
field_declaration
struct_specifier
struct "struct"
type_identifier "bpf_cpumask"
ERROR
field_identifier "__kptr"
pointer_declarator
* "*"
field_identifier "tmp_t_mask"
; ";"
field_declaration
struct_specifier
struct "struct"
type_identifier "bpf_cpumask"
ERROR
field_identifier "__kptr"
pointer_declarator
* "*"
field_identifier "tmp_t2_mask"
; ";"
field_declaration
struct_specifier
struct "struct"
type_identifier "bpf_cpumask"
ERROR
field_identifier "__kptr"
pointer_declarator
* "*"
field_identifier "tmp_t3_mask"
; ";"
} "}"
attribute_specifier
__attribute__ "__attribute__"
( "("
argument_list
( "("
call_expression
identifier "aligned"
argument_list
( "("
identifier "CACHELINE_SIZE"
) ")"
) ")"
) ")"
; ";"
declaration
storage_class_specifier
extern "extern"
type_qualifier
const "const"
type_qualifier
volatile "volatile"
type_identifier "u64"
identifier "nr_llcs"
; ";"
comment "/* number of LLC domains */"
declaration
type_qualifier
const "const"
storage_class_specifier
extern "extern"
type_qualifier
volatile "volatile"
type_identifier "u32"
identifier "nr_cpu_ids"
; ";"
declaration
storage_class_specifier
extern "extern"
type_qualifier
volatile "volatile"
type_identifier "u64"
identifier "nr_cpus_onln"
; ";"
comment "/* current number of online CPUs */"
declaration
storage_class_specifier
extern "extern"
type_qualifier
const "const"
type_qualifier
volatile "volatile"
type_identifier "u16"
array_declarator
identifier "cpu_capacity"
[ "["
identifier "LAVD_CPU_ID_MAX"
] "]"
; ";"
declaration
storage_class_specifier
extern "extern"
type_qualifier
const "const"
type_qualifier
volatile "volatile"
type_identifier "u8"
array_declarator
identifier "cpu_big"
[ "["
identifier "LAVD_CPU_ID_MAX"
] "]"
; ";"
declaration
storage_class_specifier
extern "extern"
type_qualifier
const "const"
type_qualifier
volatile "volatile"
type_identifier "u8"
array_declarator
identifier "cpu_turbo"
[ "["
identifier "LAVD_CPU_ID_MAX"
] "]"
; ";"
comment "/* Logging helpers. */"
declaration
storage_class_specifier
extern "extern"
type_qualifier
const "const"
type_qualifier
volatile "volatile"
primitive_type "bool"
identifier "no_wake_sync"
; ";"
declaration
storage_class_specifier
extern "extern"
type_qualifier
const "const"
type_qualifier
volatile "volatile"
primitive_type "bool"
identifier "no_slice_boost"
; ";"
declaration
storage_class_specifier
extern "extern"
type_qualifier
const "const"
type_qualifier
volatile "volatile"
type_identifier "u8"
identifier "verbose"
; ";"
preproc_function_def
#define "#define"
identifier "debugln"
preproc_params
( "("
identifier "fmt"
, ","
... "..."
) ")"
preproc_arg "({\t\t\t\t\t\t\t\t\t\\\n\tif (verbose > 0)\t\t\t\t\t\t\\\n\t\tbpf_printk(\"[%s:%d] "...
preproc_function_def
#define "#define"
identifier "traceln"
preproc_params
( "("
identifier "fmt"
, ","
... "..."
) ")"
preproc_arg "({\t\t\t\t\t\t\t\t\t\\\n\tif (verbose > 1)\t\t\t\t\t\t\\\n\t\tbpf_printk(\"[%s:%d] "...
comment "/* Arithmetic helpers. */"
preproc_ifdef
#ifndef "#ifndef"
identifier "min"
preproc_function_def
#define "#define"
identifier "min"
preproc_params
( "("
identifier "X"
, ","
identifier "Y"
) ")"
preproc_arg "(((X) < (Y)) ? (X) : (Y))"
#endif "#endif"
preproc_ifdef
#ifndef "#ifndef"
identifier "max"
preproc_function_def
#define "#define"
identifier "max"
preproc_params
( "("
identifier "X"
, ","
identifier "Y"
) ")"
preproc_arg "(((X) < (Y)) ? (Y) : (X))"
#endif "#endif"
preproc_ifdef
#ifndef "#ifndef"
identifier "clamp"
preproc_function_def
#define "#define"
identifier "clamp"
preproc_params
( "("
identifier "val"
, ","
identifier "lo"
, ","
identifier "hi"
) ")"
preproc_arg "min(max(val, lo), hi)"
#endif "#endif"
declaration
type_identifier "u64"
function_declarator
identifier "calc_avg"
parameter_list
( "("
parameter_declaration
type_identifier "u64"
identifier "old_val"
, ","
parameter_declaration
type_identifier "u64"
identifier "new_val"
) ")"
; ";"
declaration
type_identifier "u64"
function_declarator
identifier "calc_asym_avg"
parameter_list
( "("
parameter_declaration
type_identifier "u64"
identifier "old_val"
, ","
parameter_declaration
type_identifier "u64"
identifier "new_val"
) ")"
; ";"
comment "/* Bitmask helpers. */"
function_definition (+69 nodes collapsed)
comment "/* System statistics module .*/"
declaration
storage_class_specifier
extern "extern"
struct_specifier
struct "struct"
type_identifier "sys_stat"
identifier "sys_stat"
; ";"
declaration
type_identifier "s32"
function_declarator
identifier "init_sys_stat"
parameter_list
( "("
parameter_declaration
type_identifier "u64"
identifier "now"
) ")"
; ";"
declaration
primitive_type "int"
function_declarator
identifier "update_sys_stat"
parameter_list
( "("
parameter_declaration
primitive_type "void"
) ")"
; ";"
declaration
storage_class_specifier
extern "extern"
type_qualifier
volatile "volatile"
type_identifier "u64"
identifier "performance_mode_ns"
; ";"
declaration
storage_class_specifier
extern "extern"
type_qualifier
volatile "volatile"
type_identifier "u64"
identifier "balanced_mode_ns"
; ";"
declaration
storage_class_specifier
extern "extern"
type_qualifier
volatile "volatile"
type_identifier "u64"
identifier "powersave_mode_ns"
; ";"
comment "/* Helpers from util.bpf.c for querying CPU/task state. */"
declaration
storage_class_specifier
extern "extern"
type_qualifier
const "const"
type_qualifier
volatile "volatile"
primitive_type "bool"
identifier "per_cpu_dsq"
; ";"
declaration
storage_class_specifier
extern "extern"
type_qualifier
const "const"
type_qualifier
volatile "volatile"
type_identifier "u64"
identifier "pinned_slice_ns"
; ";"
declaration
storage_class_specifier
extern "extern"
type_qualifier
volatile "volatile"
primitive_type "bool"
identifier "reinit_cpumask_for_performance"
; ";"
declaration
storage_class_specifier
extern "extern"
type_qualifier
volatile "volatile"
primitive_type "bool"
identifier "no_preemption"
; ";"
declaration
storage_class_specifier
extern "extern"
type_qualifier
volatile "volatile"
primitive_type "bool"
identifier "no_core_compaction"
; ";"
declaration
storage_class_specifier
extern "extern"
type_qualifier
volatile "volatile"
primitive_type "bool"
identifier "no_freq_scaling"
; ";"
declaration
primitive_type "bool"
function_declarator
identifier "test_cpu_flag"
parameter_list
( "("
parameter_declaration
struct_specifier
struct "struct"
type_identifier "cpu_ctx"
pointer_declarator
* "*"
identifier "cpuc"
, ","
parameter_declaration
type_identifier "u64"
identifier "flag"
) ")"
; ";"
declaration
primitive_type "void"
function_declarator
identifier "set_cpu_flag"
parameter_list
( "("
parameter_declaration
struct_specifier
struct "struct"
type_identifier "cpu_ctx"
pointer_declarator
* "*"
identifier "cpuc"
, ","
parameter_declaration
type_identifier "u64"
identifier "flag"
) ")"
; ";"
declaration
primitive_type "void"
function_declarator
identifier "reset_cpu_flag"
parameter_list
( "("
parameter_declaration
struct_specifier
struct "struct"
type_identifier "cpu_ctx"
pointer_declarator
* "*"
identifier "cpuc"
, ","
parameter_declaration
type_identifier "u64"
identifier "flag"
) ")"
; ";"
declaration
primitive_type "bool"
function_declarator
identifier "is_lock_holder"
parameter_list
( "("
parameter_declaration
type_identifier "task_ctx"
pointer_declarator
* "*"
identifier "taskc"
) ")"
; ";"
declaration
primitive_type "bool"
function_declarator
identifier "is_lock_holder_running"
parameter_list
( "("
parameter_declaration
struct_specifier
struct "struct"
type_identifier "cpu_ctx"
pointer_declarator
* "*"
identifier "cpuc"
) ")"
; ";"
declaration
primitive_type "bool"
function_declarator
identifier "have_scheduled"
parameter_list
( "("
parameter_declaration
type_identifier "task_ctx"
pointer_declarator
* "*"
identifier "taskc"
) ")"
; ";"
declaration
primitive_type "bool"
function_declarator
identifier "have_pending_tasks"
parameter_list
( "("
parameter_declaration
struct_specifier
struct "struct"
type_identifier "cpu_ctx"
pointer_declarator
* "*"
identifier "cpuc"
) ")"
; ";"
declaration
primitive_type "bool"
function_declarator
identifier "can_boost_slice"
parameter_list
( "("
parameter_declaration
primitive_type "void"
) ")"
; ";"
declaration
primitive_type "bool"
function_declarator
identifier "is_lat_cri"
parameter_list
( "("
parameter_declaration
type_identifier "task_ctx"
pointer_declarator
* "*"
identifier "taskc"
) ")"
; ";"
declaration
type_identifier "u16"
function_declarator
identifier "get_nice_prio"
parameter_list
( "("
parameter_declaration
struct_specifier
struct "struct"
type_identifier "task_struct"
pointer_declarator
* "*"
identifier "p"
) ")"
; ";"
declaration
type_identifier "u32"
function_declarator
identifier "cpu_to_dsq"
parameter_list
( "("
parameter_declaration
type_identifier "u32"
identifier "cpu"
) ")"
; ";"
declaration
primitive_type "void"
function_declarator
identifier "set_task_flag"
parameter_list
( "("
parameter_declaration
type_identifier "task_ctx"
pointer_declarator
* "*"
identifier "taskc"
, ","
parameter_declaration
type_identifier "u64"
identifier "flag"
) ")"
; ";"
declaration
primitive_type "void"
function_declarator
identifier "reset_task_flag"
parameter_list
( "("
parameter_declaration
type_identifier "task_ctx"
pointer_declarator
* "*"
identifier "taskc"
, ","
parameter_declaration
type_identifier "u64"
identifier "flag"
) ")"
; ";"
declaration
primitive_type "bool"
function_declarator
identifier "test_task_flag"
parameter_list
( "("
parameter_declaration
type_identifier "task_ctx"
pointer_declarator
* "*"
identifier "taskc"
, ","
parameter_declaration
type_identifier "u64"
identifier "flag"
) ")"
; ";"
declaration
primitive_type "void"
function_declarator
identifier "reset_task_flag"
parameter_list
( "("
parameter_declaration
type_identifier "task_ctx"
pointer_declarator
* "*"
identifier "taskc"
, ","
parameter_declaration
type_identifier "u64"
identifier "flag"
) ")"
; ";"
function_definition (+23 nodes collapsed)
function_definition (+21 nodes collapsed)
function_definition (+22 nodes collapsed)
declaration
type_identifier "s32"
function_declarator
identifier "nr_queued_on_cpu"
parameter_list
( "("
parameter_declaration
struct_specifier
struct "struct"
type_identifier "cpu_ctx"
pointer_declarator
* "*"
identifier "cpuc"
) ")"
; ";"
declaration (+23 nodes collapsed)
declaration
storage_class_specifier
extern "extern"
struct_specifier
struct "struct"
type_identifier "bpf_cpumask"
ERROR
identifier "__kptr"
pointer_declarator
* "*"
identifier "turbo_cpumask"
; ";"
comment "/* CPU mask for turbo CPUs */"
declaration
storage_class_specifier
extern "extern"
struct_specifier
struct "struct"
type_identifier "bpf_cpumask"
ERROR
identifier "__kptr"
pointer_declarator
* "*"
identifier "big_cpumask"
; ";"
comment "/* CPU mask for big CPUs */"
declaration
storage_class_specifier
extern "extern"
struct_specifier
struct "struct"
type_identifier "bpf_cpumask"
ERROR
identifier "__kptr"
pointer_declarator
* "*"
identifier "little_cpumask"
; ";"
comment "/* CPU mask for little CPUs */"
declaration
storage_class_specifier
extern "extern"
struct_specifier
struct "struct"
type_identifier "bpf_cpumask"
ERROR
identifier "__kptr"
pointer_declarator
* "*"
identifier "active_cpumask"
; ";"
comment "/* CPU mask for active CPUs */"
declaration
storage_class_specifier
extern "extern"
struct_specifier
struct "struct"
type_identifier "bpf_cpumask"
ERROR
identifier "__kptr"
pointer_declarator
* "*"
identifier "ovrflw_cpumask"
; ";"
comment "/* CPU mask for overflow CPUs */"
comment "/* Power management helpers. */"
declaration
primitive_type "int"
function_declarator
identifier "do_core_compaction"
parameter_list
( "("
parameter_declaration
primitive_type "void"
) ")"
; ";"
declaration
primitive_type "int"
function_declarator
identifier "update_thr_perf_cri"
parameter_list
( "("
parameter_declaration
primitive_type "void"
) ")"
; ";"
declaration
primitive_type "int"
function_declarator
identifier "reinit_active_cpumask_for_performance"
parameter_list
( "("
parameter_declaration
primitive_type "void"
) ")"
; ";"
declaration
primitive_type "bool"
function_declarator
identifier "is_perf_cri"
parameter_list
( "("
parameter_declaration
type_identifier "task_ctx"
pointer_declarator
* "*"
identifier "taskc"
) ")"
; ";"
declaration
storage_class_specifier
extern "extern"
primitive_type "bool"
identifier "have_little_core"
; ";"
declaration
storage_class_specifier
extern "extern"
primitive_type "bool"
identifier "have_turbo_core"
; ";"
declaration
storage_class_specifier
extern "extern"
type_qualifier
const "const"
type_qualifier
volatile "volatile"
primitive_type "bool"
identifier "is_smt_active"
; ";"
declaration
storage_class_specifier
extern "extern"
type_identifier "u64"
identifier "total_capacity"
; ";"
declaration
storage_class_specifier
extern "extern"
type_identifier "u64"
identifier "one_little_capacity"
; ";"
declaration
storage_class_specifier
extern "extern"
type_identifier "u32"
identifier "cur_big_core_scale"
; ";"
declaration
storage_class_specifier
extern "extern"
type_identifier "u32"
identifier "default_big_core_scale"
; ";"
declaration
primitive_type "int"
function_declarator
identifier "init_autopilot_caps"
parameter_list
( "("
parameter_declaration
primitive_type "void"
) ")"
; ";"
declaration
primitive_type "int"
function_declarator
identifier "update_autopilot_high_cap"
parameter_list
( "("
parameter_declaration
primitive_type "void"
) ")"
; ";"
declaration
type_identifier "u64"
function_declarator
identifier "scale_cap_freq"
parameter_list
( "("
parameter_declaration
type_identifier "u64"
identifier "dur"
, ","
parameter_declaration
type_identifier "s32"
identifier "cpu"
) ")"
; ";"
declaration
primitive_type "int"
function_declarator
identifier "reset_cpuperf_target"
parameter_list
( "("
parameter_declaration
struct_specifier
struct "struct"
type_identifier "cpu_ctx"
pointer_declarator
* "*"
identifier "cpuc"
) ")"
; ";"
declaration
primitive_type "int"
function_declarator
identifier "update_cpuperf_target"
parameter_list
( "("
parameter_declaration
struct_specifier
struct "struct"
type_identifier "cpu_ctx"
pointer_declarator
* "*"
identifier "cpuc"
) ")"
; ";"
declaration
type_identifier "u16"
function_declarator
identifier "get_cpuperf_cap"
parameter_list
( "("
parameter_declaration
type_identifier "s32"
identifier "cpu"
) ")"
; ";"
declaration
primitive_type "int"
function_declarator
identifier "reset_suspended_duration"
parameter_list
( "("
parameter_declaration
struct_specifier
struct "struct"
type_identifier "cpu_ctx"
pointer_declarator
* "*"
identifier "cpuc"
) ")"
; ";"
declaration
type_identifier "u64"
function_declarator
identifier "get_suspended_duration_and_reset"
parameter_list
( "("
parameter_declaration
struct_specifier
struct "struct"
type_identifier "cpu_ctx"
pointer_declarator
* "*"
identifier "cpuc"
) ")"
; ";"
declaration
type_qualifier
const "const"
type_qualifier
volatile "volatile"
type_identifier "u16"
pointer_declarator
* "*"
function_declarator
identifier "get_cpu_order"
parameter_list
( "("
parameter_declaration
primitive_type "void"
) ")"
; ";"
comment "/* Load balancer helpers. */"
declaration
primitive_type "int"
function_declarator
identifier "plan_x_cpdom_migration"
parameter_list
( "("
parameter_declaration
primitive_type "void"
) ")"
; ";"
comment "/* Preemption management helpers. */"
declaration (+27 nodes collapsed)
comment "/* Futex lock-related helpers. */"
declaration (+21 nodes collapsed)
comment "/* Scheduler introspection-related helpers. */"
declaration
type_identifier "u64"
function_declarator
identifier "get_est_stopping_clk"
parameter_list
( "("
parameter_declaration
type_identifier "task_ctx"
pointer_declarator
* "*"
identifier "taskc"
, ","
parameter_declaration
type_identifier "u64"
identifier "now"
) ")"
; ";"
declaration (+21 nodes collapsed)
declaration
primitive_type "void"
function_declarator
identifier "reset_cpu_preemption_info"
parameter_list
( "("
parameter_declaration
struct_specifier
struct "struct"
type_identifier "cpu_ctx"
pointer_declarator
* "*"
identifier "cpuc"
, ","
parameter_declaration
primitive_type "bool"
identifier "released"
) ")"
; ";"
declaration
primitive_type "int"
function_declarator
identifier "shrink_boosted_slice_remote"
parameter_list
( "("
parameter_declaration
struct_specifier
struct "struct"
type_identifier "cpu_ctx"
pointer_declarator
* "*"
identifier "cpuc"
, ","
parameter_declaration
type_identifier "u64"
identifier "now"
) ")"
; ";"
declaration (+27 nodes collapsed)
declaration (+23 nodes collapsed)
declaration (+29 nodes collapsed)
declaration
storage_class_specifier
extern "extern"
type_qualifier
volatile "volatile"
primitive_type "bool"
identifier "is_monitored"
; ";"
#endif "#endif"
comment "/* __LAVD_H */"