File Manager
#!/usr/bin/env python3
#
# Copyright (c) 2022,2023 Red Hat, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at:
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
#
# Script information:
# -------------------
# This script allows a developer to quickly identify if the issue at hand
# might be related to the kernel running out of resources or if it really is
# an Open vSwitch issue.
#
# For documentation see the kernel_delay.rst file.
#
#
# Dependencies:
# -------------
# You need to install the BCC package for your specific platform or build it
# yourself using the following instructions:
# https://raw.githubusercontent.com/iovisor/bcc/master/INSTALL.md
#
# Python needs the following additional packages installed:
# - pytz
# - psutil
#
# You can either install your distribution specific package or use pip:
# pip install pytz psutil
#
import argparse
import datetime
import os
import pytz
import psutil
import re
import sys
import time
import ctypes as ct
try:
from bcc import BPF, USDT, USDTException
from bcc.syscall import syscalls, syscall_name
except ModuleNotFoundError:
print("ERROR: Can't find the BPF Compiler Collection (BCC) tools!")
sys.exit(os.EX_OSFILE)
from enum import IntEnum
#
# Actual eBPF source code
#
EBPF_SOURCE = """
#include <linux/irq.h>
#include <linux/sched.h>
#define MONITOR_PID <MONITOR_PID>
enum {
<EVENT_ENUM>
};
struct event_t {
u64 ts;
u32 tid;
u32 id;
int user_stack_id;
int kernel_stack_id;
u32 syscall;
u64 entry_ts;
};
BPF_RINGBUF_OUTPUT(events, <BUFFER_PAGE_CNT>);
BPF_STACK_TRACE(stack_traces, <STACK_TRACE_SIZE>);
BPF_TABLE("percpu_array", uint32_t, uint64_t, dropcnt, 1);
BPF_TABLE("percpu_array", uint32_t, uint64_t, trigger_miss, 1);
BPF_ARRAY(capture_on, u64, 1);
static inline bool capture_enabled(u64 pid_tgid) {
int key = 0;
u64 *ret;
if ((pid_tgid >> 32) != MONITOR_PID)
return false;
ret = capture_on.lookup(&key);
return ret && *ret == 1;
}
static inline bool capture_enabled__() {
int key = 0;
u64 *ret;
ret = capture_on.lookup(&key);
return ret && *ret == 1;
}
static struct event_t *get_event(uint32_t id) {
struct event_t *event = events.ringbuf_reserve(sizeof(struct event_t));
if (!event) {
dropcnt.increment(0);
return NULL;
}
event->id = id;
event->ts = bpf_ktime_get_ns();
event->tid = bpf_get_current_pid_tgid();
return event;
}
static int start_trigger() {
int key = 0;
u64 *val = capture_on.lookup(&key);
/* If the value is -1 we can't start as we are still processing the
* results in userspace. */
if (!val || *val != 0) {
trigger_miss.increment(0);
return 0;
}
struct event_t *event = get_event(EVENT_START_TRIGGER);
if (event) {
events.ringbuf_submit(event, 0);
*val = 1;
} else {
trigger_miss.increment(0);
}
return 0;
}
static int stop_trigger() {
int key = 0;
u64 *val = capture_on.lookup(&key);
if (!val || *val != 1)
return 0;
struct event_t *event = get_event(EVENT_STOP_TRIGGER);
if (event)
events.ringbuf_submit(event, 0);
if (val)
*val = -1;
return 0;
}
<START_TRIGGER>
<STOP_TRIGGER>
/*
* For the syscall monitor the following probes get installed.
*/
struct syscall_data_t {
u64 count;
u64 total_ns;
u64 worst_ns;
};
struct syscall_data_key_t {
u32 pid;
u32 tid;
u32 syscall;
};
BPF_HASH(syscall_start, u64, u64);
BPF_HASH(syscall_data, struct syscall_data_key_t, struct syscall_data_t);
TRACEPOINT_PROBE(raw_syscalls, sys_enter) {
u64 pid_tgid = bpf_get_current_pid_tgid();
if (!capture_enabled(pid_tgid))
return 0;
u64 t = bpf_ktime_get_ns();
syscall_start.update(&pid_tgid, &t);
return 0;
}
TRACEPOINT_PROBE(raw_syscalls, sys_exit) {
struct syscall_data_t *val, zero = {};
struct syscall_data_key_t key;
u64 pid_tgid = bpf_get_current_pid_tgid();
if (!capture_enabled(pid_tgid))
return 0;
key.pid = pid_tgid >> 32;
key.tid = (u32)pid_tgid;
key.syscall = args->id;
u64 *start_ns = syscall_start.lookup(&pid_tgid);
if (!start_ns)
return 0;
val = syscall_data.lookup_or_try_init(&key, &zero);
if (val) {
u64 delta = bpf_ktime_get_ns() - *start_ns;
val->count++;
val->total_ns += delta;
if (val->worst_ns == 0 || delta > val->worst_ns)
val->worst_ns = delta;
if (<SYSCALL_TRACE_EVENTS>) {
struct event_t *event = get_event(EVENT_SYSCALL);
if (event) {
event->syscall = args->id;
event->entry_ts = *start_ns;
if (<STACK_TRACE_ENABLED>) {
event->user_stack_id = stack_traces.get_stackid(
args, BPF_F_USER_STACK);
event->kernel_stack_id = stack_traces.get_stackid(
args, 0);
}
events.ringbuf_submit(event, 0);
}
}
}
return 0;
}
/*
* For measuring the thread run time, we need the following.
*/
struct run_time_data_t {
u64 count;
u64 total_ns;
u64 max_ns;
u64 min_ns;
};
struct pid_tid_key_t {
u32 pid;
u32 tid;
};
BPF_HASH(run_start, u64, u64);
BPF_HASH(run_data, struct pid_tid_key_t, struct run_time_data_t);
static inline void thread_start_run(u64 pid_tgid, u64 ktime)
{
run_start.update(&pid_tgid, &ktime);
}
static inline void thread_stop_run(u32 pid, u32 tgid, u64 ktime)
{
u64 pid_tgid = (u64) tgid << 32 | pid;
u64 *start_ns = run_start.lookup(&pid_tgid);
if (!start_ns || *start_ns == 0)
return;
struct run_time_data_t *val, zero = {};
struct pid_tid_key_t key = { .pid = tgid,
.tid = pid };
val = run_data.lookup_or_try_init(&key, &zero);
if (val) {
u64 delta = ktime - *start_ns;
val->count++;
val->total_ns += delta;
if (val->max_ns == 0 || delta > val->max_ns)
val->max_ns = delta;
if (val->min_ns == 0 || delta < val->min_ns)
val->min_ns = delta;
}
*start_ns = 0;
}
/*
* For measuring the thread-ready delay, we need the following.
*/
struct ready_data_t {
u64 count;
u64 total_ns;
u64 worst_ns;
};
BPF_HASH(ready_start, u64, u64);
BPF_HASH(ready_data, struct pid_tid_key_t, struct ready_data_t);
static inline int sched_wakeup__(u32 pid, u32 tgid)
{
u64 pid_tgid = (u64) tgid << 32 | pid;
if (!capture_enabled(pid_tgid))
return 0;
u64 t = bpf_ktime_get_ns();
ready_start.update(&pid_tgid, &t);
return 0;
}
RAW_TRACEPOINT_PROBE(sched_wakeup)
{
struct task_struct *t = (struct task_struct *)ctx->args[0];
return sched_wakeup__(t->pid, t->tgid);
}
RAW_TRACEPOINT_PROBE(sched_wakeup_new)
{
struct task_struct *t = (struct task_struct *)ctx->args[0];
return sched_wakeup__(t->pid, t->tgid);
}
RAW_TRACEPOINT_PROBE(sched_switch)
{
struct task_struct *prev = (struct task_struct *)ctx->args[1];
struct task_struct *next= (struct task_struct *)ctx->args[2];
u64 ktime = 0;
if (!capture_enabled__())
return 0;
if (prev-><STATE_FIELD> == TASK_RUNNING && prev->tgid == MONITOR_PID)
sched_wakeup__(prev->pid, prev->tgid);
if (prev->tgid == MONITOR_PID) {
ktime = bpf_ktime_get_ns();
thread_stop_run(prev->pid, prev->tgid, ktime);
}
u64 pid_tgid = (u64)next->tgid << 32 | next->pid;
if (next->tgid != MONITOR_PID)
return 0;
if (ktime == 0)
ktime = bpf_ktime_get_ns();
u64 *start_ns = ready_start.lookup(&pid_tgid);
if (start_ns && *start_ns != 0) {
struct ready_data_t *val, zero = {};
struct pid_tid_key_t key = { .pid = next->tgid,
.tid = next->pid };
val = ready_data.lookup_or_try_init(&key, &zero);
if (val) {
u64 delta = ktime - *start_ns;
val->count++;
val->total_ns += delta;
if (val->worst_ns == 0 || delta > val->worst_ns)
val->worst_ns = delta;
}
*start_ns = 0;
}
thread_start_run(pid_tgid, ktime);
return 0;
}
/*
* For measuring the hard irq time, we need the following.
*/
struct hardirq_start_data_t {
u64 start_ns;
char irq_name[32];
};
struct hardirq_data_t {
u64 count;
u64 total_ns;
u64 worst_ns;
};
struct hardirq_data_key_t {
u32 pid;
u32 tid;
char irq_name[32];
};
BPF_HASH(hardirq_start, u64, struct hardirq_start_data_t);
BPF_HASH(hardirq_data, struct hardirq_data_key_t, struct hardirq_data_t);
TRACEPOINT_PROBE(irq, irq_handler_entry)
{
u64 pid_tgid = bpf_get_current_pid_tgid();
if (!capture_enabled(pid_tgid))
return 0;
struct hardirq_start_data_t data = {};
data.start_ns = bpf_ktime_get_ns();
TP_DATA_LOC_READ_STR(&data.irq_name, name, sizeof(data.irq_name));
hardirq_start.update(&pid_tgid, &data);
return 0;
}
TRACEPOINT_PROBE(irq, irq_handler_exit)
{
u64 pid_tgid = bpf_get_current_pid_tgid();
if (!capture_enabled(pid_tgid))
return 0;
struct hardirq_start_data_t *data;
data = hardirq_start.lookup(&pid_tgid);
if (!data || data->start_ns == 0)
return 0;
if (args->ret != IRQ_NONE) {
struct hardirq_data_t *val, zero = {};
struct hardirq_data_key_t key = { .pid = pid_tgid >> 32,
.tid = (u32)pid_tgid };
bpf_probe_read_kernel(&key.irq_name, sizeof(key.irq_name),
data->irq_name);
val = hardirq_data.lookup_or_try_init(&key, &zero);
if (val) {
u64 delta = bpf_ktime_get_ns() - data->start_ns;
val->count++;
val->total_ns += delta;
if (val->worst_ns == 0 || delta > val->worst_ns)
val->worst_ns = delta;
}
}
data->start_ns = 0;
return 0;
}
/*
* For measuring the soft irq time, we need the following.
*/
struct softirq_start_data_t {
u64 start_ns;
u32 vec_nr;
};
struct softirq_data_t {
u64 count;
u64 total_ns;
u64 worst_ns;
};
struct softirq_data_key_t {
u32 pid;
u32 tid;
u32 vec_nr;
};
BPF_HASH(softirq_start, u64, struct softirq_start_data_t);
BPF_HASH(softirq_data, struct softirq_data_key_t, struct softirq_data_t);
TRACEPOINT_PROBE(irq, softirq_entry)
{
u64 pid_tgid = bpf_get_current_pid_tgid();
if (!capture_enabled(pid_tgid))
return 0;
struct softirq_start_data_t data = {};
data.start_ns = bpf_ktime_get_ns();
data.vec_nr = args->vec;
softirq_start.update(&pid_tgid, &data);
return 0;
}
TRACEPOINT_PROBE(irq, softirq_exit)
{
u64 pid_tgid = bpf_get_current_pid_tgid();
if (!capture_enabled(pid_tgid))
return 0;
struct softirq_start_data_t *data;
data = softirq_start.lookup(&pid_tgid);
if (!data || data->start_ns == 0)
return 0;
struct softirq_data_t *val, zero = {};
struct softirq_data_key_t key = { .pid = pid_tgid >> 32,
.tid = (u32)pid_tgid,
.vec_nr = data->vec_nr};
val = softirq_data.lookup_or_try_init(&key, &zero);
if (val) {
u64 delta = bpf_ktime_get_ns() - data->start_ns;
val->count++;
val->total_ns += delta;
if (val->worst_ns == 0 || delta > val->worst_ns)
val->worst_ns = delta;
}
data->start_ns = 0;
return 0;
}
"""
#
# time_ns()
#
try:
from time import time_ns
except ImportError:
# For compatibility with Python <= v3.6.
def time_ns():
now = datetime.datetime.now()
return int(now.timestamp() * 1e9)
#
# Probe class to use for the start/stop triggers
#
class Probe(object):
'''
The goal for this object is to support as many as possible
probe/events as supported by BCC. See
https://github.com/iovisor/bcc/blob/master/docs/reference_guide.md#events--arguments
'''
def __init__(self, probe, pid=None):
self.pid = pid
self.text_probe = probe
self._parse_text_probe()
def __str__(self):
if self.probe_type == "usdt":
return "[{}]; {}:{}:{}".format(self.text_probe, self.probe_type,
self.usdt_provider, self.usdt_probe)
elif self.probe_type == "trace":
return "[{}]; {}:{}:{}".format(self.text_probe, self.probe_type,
self.trace_system, self.trace_event)
elif self.probe_type == "kprobe" or self.probe_type == "kretprobe":
return "[{}]; {}:{}".format(self.text_probe, self.probe_type,
self.kprobe_function)
elif self.probe_type == "uprobe" or self.probe_type == "uretprobe":
return "[{}]; {}:{}".format(self.text_probe, self.probe_type,
self.uprobe_function)
else:
return "[{}] <{}:unknown probe>".format(self.text_probe,
self.probe_type)
def _raise(self, error):
raise ValueError("[{}]; {}".format(self.text_probe, error))
def _verify_kprobe_probe(self):
# Nothing to verify for now, just return.
return
def _verify_trace_probe(self):
# Nothing to verify for now, just return.
return
def _verify_uprobe_probe(self):
# Nothing to verify for now, just return.
return
def _verify_usdt_probe(self):
if not self.pid:
self._raise("USDT probes need a valid PID.")
usdt = USDT(pid=self.pid)
for probe in usdt.enumerate_probes():
if probe.provider.decode("utf-8") == self.usdt_provider and \
probe.name.decode("utf-8") == self.usdt_probe:
return
self._raise("Can't find UDST probe '{}:{}'".format(self.usdt_provider,
self.usdt_probe))
def _parse_text_probe(self):
'''
The text probe format is defined as follows:
<probe_type>:<probe_specific>
Types:
USDT: u|usdt:<provider>:<probe>
TRACE: t|trace:<system>:<event>
KPROBE: k|kprobe:<kernel_function>
KRETPROBE: kr|kretprobe:<kernel_function>
UPROBE: up|uprobe:<function>
URETPROBE: ur|uretprobe:<function>
'''
args = self.text_probe.split(":")
if len(args) <= 1:
self._raise("Can't extract probe type.")
if args[0] not in ["k", "kprobe", "kr", "kretprobe", "t", "trace",
"u", "usdt", "up", "uprobe", "ur", "uretprobe"]:
self._raise("Invalid probe type '{}'".format(args[0]))
self.probe_type = "kprobe" if args[0] == "k" else args[0]
self.probe_type = "kretprobe" if args[0] == "kr" else self.probe_type
self.probe_type = "trace" if args[0] == "t" else self.probe_type
self.probe_type = "usdt" if args[0] == "u" else self.probe_type
self.probe_type = "uprobe" if args[0] == "up" else self.probe_type
self.probe_type = "uretprobe" if args[0] == "ur" else self.probe_type
if self.probe_type == "usdt":
if len(args) != 3:
self._raise("Invalid number of arguments for USDT")
self.usdt_provider = args[1]
self.usdt_probe = args[2]
self._verify_usdt_probe()
elif self.probe_type == "trace":
if len(args) != 3:
self._raise("Invalid number of arguments for TRACE")
self.trace_system = args[1]
self.trace_event = args[2]
self._verify_trace_probe()
elif self.probe_type == "kprobe" or self.probe_type == "kretprobe":
if len(args) != 2:
self._raise("Invalid number of arguments for K(RET)PROBE")
self.kprobe_function = args[1]
self._verify_kprobe_probe()
elif self.probe_type == "uprobe" or self.probe_type == "uretprobe":
if len(args) != 2:
self._raise("Invalid number of arguments for U(RET)PROBE")
self.uprobe_function = args[1]
self._verify_uprobe_probe()
def _get_kprobe_c_code(self, function_name, function_content):
#
# The kprobe__* do not require a function name, so it's
# ignored in the code generation.
#
return """
int {}__{}(struct pt_regs *ctx) {{
{}
}}
""".format(self.probe_type, self.kprobe_function, function_content)
def _get_trace_c_code(self, function_name, function_content):
#
# The TRACEPOINT_PROBE() do not require a function name, so it's
# ignored in the code generation.
#
return """
TRACEPOINT_PROBE({},{}) {{
{}
}}
""".format(self.trace_system, self.trace_event, function_content)
def _get_uprobe_c_code(self, function_name, function_content):
return """
int {}(struct pt_regs *ctx) {{
{}
}}
""".format(function_name, function_content)
def _get_usdt_c_code(self, function_name, function_content):
return """
int {}(struct pt_regs *ctx) {{
{}
}}
""".format(function_name, function_content)
def get_c_code(self, function_name, function_content):
if self.probe_type == "kprobe" or self.probe_type == "kretprobe":
return self._get_kprobe_c_code(function_name, function_content)
elif self.probe_type == "trace":
return self._get_trace_c_code(function_name, function_content)
elif self.probe_type == "uprobe" or self.probe_type == "uretprobe":
return self._get_uprobe_c_code(function_name, function_content)
elif self.probe_type == "usdt":
return self._get_usdt_c_code(function_name, function_content)
return ""
def probe_name(self):
if self.probe_type == "kprobe" or self.probe_type == "kretprobe":
return "{}".format(self.kprobe_function)
elif self.probe_type == "trace":
return "{}:{}".format(self.trace_system,
self.trace_event)
elif self.probe_type == "uprobe" or self.probe_type == "uretprobe":
return "{}".format(self.uprobe_function)
elif self.probe_type == "usdt":
return "{}:{}".format(self.usdt_provider,
self.usdt_probe)
return ""
#
# event_to_dict()
#
def event_to_dict(event):
return dict([(field, getattr(event, field))
for (field, _) in event._fields_
if isinstance(getattr(event, field), (int, bytes))])
#
# Event enum
#
Event = IntEnum("Event", ["SYSCALL", "START_TRIGGER", "STOP_TRIGGER"],
start=0)
#
# process_event()
#
def process_event(ctx, data, size):
global start_trigger_ts
global stop_trigger_ts
event = bpf["events"].event(data)
if event.id == Event.SYSCALL:
syscall_events.append({"tid": event.tid,
"ts_entry": event.entry_ts,
"ts_exit": event.ts,
"syscall": event.syscall,
"user_stack_id": event.user_stack_id,
"kernel_stack_id": event.kernel_stack_id})
elif event.id == Event.START_TRIGGER:
#
# This event would have started the trigger already, so all we need to
# do is record the start timestamp.
#
start_trigger_ts = event.ts
elif event.id == Event.STOP_TRIGGER:
#
# This event would have stopped the trigger already, so all we need to
# do is record the start timestamp.
stop_trigger_ts = event.ts
#
# next_power_of_two()
#
def next_power_of_two(val):
np = 1
while np < val:
np *= 2
return np
#
# unsigned_int()
#
def unsigned_int(value):
try:
value = int(value)
except ValueError:
raise argparse.ArgumentTypeError("must be an integer")
if value < 0:
raise argparse.ArgumentTypeError("must be positive")
return value
#
# unsigned_nonzero_int()
#
def unsigned_nonzero_int(value):
value = unsigned_int(value)
if value == 0:
raise argparse.ArgumentTypeError("must be nonzero")
return value
#
# get_thread_name()
#
def get_thread_name(pid, tid):
try:
with open(f"/proc/{pid}/task/{tid}/comm", encoding="utf8") as f:
return f.readline().strip("\n")
except FileNotFoundError:
pass
return f"<unknown:{pid}/{tid}>"
#
# get_vec_nr_name()
#
def get_vec_nr_name(vec_nr):
known_vec_nr = ["hi", "timer", "net_tx", "net_rx", "block", "irq_poll",
"tasklet", "sched", "hrtimer", "rcu"]
if vec_nr < 0 or vec_nr > len(known_vec_nr):
return f"<unknown:{vec_nr}>"
return known_vec_nr[vec_nr]
#
# start/stop/reset capture
#
def start_capture():
bpf["capture_on"][ct.c_int(0)] = ct.c_int(1)
def stop_capture(force=False):
if force:
bpf["capture_on"][ct.c_int(0)] = ct.c_int(0xffff)
else:
bpf["capture_on"][ct.c_int(0)] = ct.c_int(0)
def capture_running():
return bpf["capture_on"][ct.c_int(0)].value == 1
def reset_capture():
bpf["syscall_start"].clear()
bpf["syscall_data"].clear()
bpf["run_start"].clear()
bpf["run_data"].clear()
bpf["ready_start"].clear()
bpf["ready_data"].clear()
bpf["hardirq_start"].clear()
bpf["hardirq_data"].clear()
bpf["softirq_start"].clear()
bpf["softirq_data"].clear()
bpf["stack_traces"].clear()
#
# Display timestamp
#
def print_timestamp(msg):
ltz = datetime.datetime.now()
utc = ltz.astimezone(pytz.utc)
time_string = "{} @{} ({} UTC)".format(
msg, ltz.isoformat(), utc.strftime("%H:%M:%S"))
print(time_string)
#
# process_results()
#
def process_results(syscall_events=None, trigger_delta=None):
if trigger_delta:
print_timestamp("# Triggered sample dump, stop-start delta {:,} ns".
format(trigger_delta))
else:
print_timestamp("# Sample dump")
#
# First get a list of all threads we need to report on.
#
threads_syscall = {k.tid for k, _ in bpf["syscall_data"].items()
if k.syscall != 0xffffffff}
threads_run = {k.tid for k, _ in bpf["run_data"].items()
if k.pid != 0xffffffff}
threads_ready = {k.tid for k, _ in bpf["ready_data"].items()
if k.pid != 0xffffffff}
threads_hardirq = {k.tid for k, _ in bpf["hardirq_data"].items()
if k.pid != 0xffffffff}
threads_softirq = {k.tid for k, _ in bpf["softirq_data"].items()
if k.pid != 0xffffffff}
threads = sorted(threads_syscall | threads_run | threads_ready |
threads_hardirq | threads_softirq,
key=lambda x: get_thread_name(options.pid, x))
#
# Print header...
#
print("{:10} {:16} {}".format("TID", "THREAD", "<RESOURCE SPECIFIC>"))
print("{:10} {:16} {}".format("-" * 10, "-" * 16, "-" * 76))
indent = 28 * " "
#
# Print all events/statistics per threads.
#
poll_id = [k for k, v in syscalls.items() if v == b"poll"][0]
for thread in threads:
if thread != threads[0]:
print("")
#
# SYSCALL_STATISTICS
#
print("{:10} {:16} {}\n{}{:20} {:>6} {:>10} {:>16} {:>16}".format(
thread, get_thread_name(options.pid, thread),
"[SYSCALL STATISTICS]", indent,
"NAME", "NUMBER", "COUNT", "TOTAL ns", "MAX ns"))
total_count = 0
total_ns = 0
for k, v in sorted(filter(lambda t: t[0].tid == thread,
bpf["syscall_data"].items()),
key=lambda kv: -kv[1].total_ns):
print("{}{:20.20} {:6} {:10} {:16,} {:16,}".format(
indent, syscall_name(k.syscall).decode("utf-8"), k.syscall,
v.count, v.total_ns, v.worst_ns))
if k.syscall != poll_id:
total_count += v.count
total_ns += v.total_ns
if total_count > 0:
print("{}{:20.20} {:6} {:10} {:16,}".format(
indent, "TOTAL( - poll):", "", total_count, total_ns))
#
# THREAD RUN STATISTICS
#
print("\n{:10} {:16} {}\n{}{:10} {:>16} {:>16} {:>16}".format(
"", "", "[THREAD RUN STATISTICS]", indent,
"SCHED_CNT", "TOTAL ns", "MIN ns", "MAX ns"))
for k, v in filter(lambda t: t[0].tid == thread,
bpf["run_data"].items()):
print("{}{:10} {:16,} {:16,} {:16,}".format(
indent, v.count, v.total_ns, v.min_ns, v.max_ns))
#
# THREAD READY STATISTICS
#
print("\n{:10} {:16} {}\n{}{:10} {:>16} {:>16}".format(
"", "", "[THREAD READY STATISTICS]", indent,
"SCHED_CNT", "TOTAL ns", "MAX ns"))
for k, v in filter(lambda t: t[0].tid == thread,
bpf["ready_data"].items()):
print("{}{:10} {:16,} {:16,}".format(
indent, v.count, v.total_ns, v.worst_ns))
#
# HARD IRQ STATISTICS
#
total_ns = 0
total_count = 0
header_printed = False
for k, v in sorted(filter(lambda t: t[0].tid == thread,
bpf["hardirq_data"].items()),
key=lambda kv: -kv[1].total_ns):
if not header_printed:
print("\n{:10} {:16} {}\n{}{:20} {:>10} {:>16} {:>16}".
format("", "", "[HARD IRQ STATISTICS]", indent,
"NAME", "COUNT", "TOTAL ns", "MAX ns"))
header_printed = True
print("{}{:20.20} {:10} {:16,} {:16,}".format(
indent, k.irq_name.decode("utf-8"),
v.count, v.total_ns, v.worst_ns))
total_count += v.count
total_ns += v.total_ns
if total_count > 0:
print("{}{:20.20} {:10} {:16,}".format(
indent, "TOTAL:", total_count, total_ns))
#
# SOFT IRQ STATISTICS
#
total_ns = 0
total_count = 0
header_printed = False
for k, v in sorted(filter(lambda t: t[0].tid == thread,
bpf["softirq_data"].items()),
key=lambda kv: -kv[1].total_ns):
if not header_printed:
print("\n{:10} {:16} {}\n"
"{}{:20} {:>7} {:>10} {:>16} {:>16}".
format("", "", "[SOFT IRQ STATISTICS]", indent,
"NAME", "VECT_NR", "COUNT", "TOTAL ns", "MAX ns"))
header_printed = True
print("{}{:20.20} {:>7} {:10} {:16,} {:16,}".format(
indent, get_vec_nr_name(k.vec_nr), k.vec_nr,
v.count, v.total_ns, v.worst_ns))
total_count += v.count
total_ns += v.total_ns
if total_count > 0:
print("{}{:20.20} {:7} {:10} {:16,}".format(
indent, "TOTAL:", "", total_count, total_ns))
#
# Print events
#
lost_stack_traces = 0
if syscall_events:
stack_traces = bpf.get_table("stack_traces")
print("\n\n# SYSCALL EVENTS:"
"\n{}{:>19} {:>19} {:>10} {:16} {:>10} {}".format(
2 * " ", "ENTRY (ns)", "EXIT (ns)", "TID", "COMM",
"DELTA (us)", "SYSCALL"))
print("{}{:19} {:19} {:10} {:16} {:10} {}".format(
2 * " ", "-" * 19, "-" * 19, "-" * 10, "-" * 16,
"-" * 10, "-" * 16))
for event in syscall_events:
print("{}{:19} {:19} {:10} {:16} {:10,} {}".format(
" " * 2,
event["ts_entry"], event["ts_exit"], event["tid"],
get_thread_name(options.pid, event["tid"]),
int((event["ts_exit"] - event["ts_entry"]) / 1000),
syscall_name(event["syscall"]).decode("utf-8")))
#
# Not sure where to put this, but I'll add some info on stack
# traces here... Userspace stack traces are very limited due to
# the fact that bcc does not support dwarf backtraces. As OVS
# gets compiled without frame pointers we will not see much.
# If however, OVS does get built with frame pointers, we should not
# use the BPF_STACK_TRACE_BUILDID as it does not seem to handle
# the debug symbols correctly. Also, note that for kernel
# traces you should not use BPF_STACK_TRACE_BUILDID, so two
# buffers are needed.
#
# Some info on manual dwarf walk support:
# https://github.com/iovisor/bcc/issues/3515
# https://github.com/iovisor/bcc/pull/4463
#
if options.stack_trace_size == 0:
continue
if event["kernel_stack_id"] < 0 or event["user_stack_id"] < 0:
lost_stack_traces += 1
kernel_stack = stack_traces.walk(event["kernel_stack_id"]) \
if event["kernel_stack_id"] >= 0 else []
user_stack = stack_traces.walk(event["user_stack_id"]) \
if event["user_stack_id"] >= 0 else []
for addr in kernel_stack:
print("{}{}".format(
" " * 10,
bpf.ksym(addr, show_module=True,
show_offset=True).decode("utf-8", "replace")))
for addr in user_stack:
addr_str = bpf.sym(addr, options.pid, show_module=True,
show_offset=True).decode("utf-8", "replace")
if addr_str == "[unknown]":
addr_str += " 0x{:x}".format(addr)
print("{}{}".format(" " * 10, addr_str))
#
# Print any footer messages.
#
if lost_stack_traces > 0:
print("\n#WARNING: We where not able to display {} stack traces!\n"
"# Consider increasing the stack trace size using\n"
"# the '--stack-trace-size' option.\n"
"# Note that this can also happen due to a stack id\n"
"# collision.".format(lost_stack_traces))
#
# main()
#
def main():
#
# Don't like these globals, but ctx passing does not seem to work with the
# existing open_ring_buffer() API :(
#
global bpf
global options
global syscall_events
global start_trigger_ts
global stop_trigger_ts
start_trigger_ts = 0
stop_trigger_ts = 0
#
# Argument parsing
#
parser = argparse.ArgumentParser()
parser.add_argument("-D", "--debug",
help="Enable eBPF debugging",
type=int, const=0x3f, default=0, nargs="?")
parser.add_argument("-p", "--pid", metavar="VSWITCHD_PID",
help="ovs-vswitch's PID",
type=unsigned_int, default=None)
parser.add_argument("-s", "--syscall-events", metavar="DURATION_NS",
help="Record syscall events that take longer than "
"DURATION_NS. Omit the duration value to record all "
"syscall events",
type=unsigned_int, const=0, default=None, nargs="?")
parser.add_argument("--buffer-page-count",
help="Number of BPF ring buffer pages, default 1024",
type=unsigned_int, default=1024, metavar="NUMBER")
parser.add_argument("--sample-count",
help="Number of sample runs, default 1",
type=unsigned_nonzero_int, default=1, metavar="RUNS")
parser.add_argument("--sample-interval",
help="Delay between sample runs, default 0",
type=float, default=0, metavar="SECONDS")
parser.add_argument("--sample-time",
help="Sample time, default 0.5 seconds",
type=float, default=0.5, metavar="SECONDS")
parser.add_argument("--skip-syscall-poll-events",
help="Skip poll() syscalls with --syscall-events",
action="store_true")
parser.add_argument("--stack-trace-size",
help="Number of unique stack traces that can be "
"recorded, default 4096. 0 to disable",
type=unsigned_int, default=4096)
parser.add_argument("--start-trigger", metavar="TRIGGER",
help="Start trigger, see documentation for details",
type=str, default=None)
parser.add_argument("--stop-trigger", metavar="TRIGGER",
help="Stop trigger, see documentation for details",
type=str, default=None)
parser.add_argument("--trigger-delta", metavar="DURATION_NS",
help="Only report event when the trigger duration > "
"DURATION_NS, default 0 (all events)",
type=unsigned_int, const=0, default=0, nargs="?")
options = parser.parse_args()
#
# Find the PID of the ovs-vswitchd daemon if not specified.
#
if not options.pid:
for proc in psutil.process_iter():
if "ovs-vswitchd" in proc.name():
if options.pid:
print("ERROR: Multiple ovs-vswitchd daemons running, "
"use the -p option!")
sys.exit(os.EX_NOINPUT)
options.pid = proc.pid
#
# Error checking on input parameters.
#
if not options.pid:
print("ERROR: Failed to find ovs-vswitchd's PID!")
sys.exit(os.EX_UNAVAILABLE)
options.buffer_page_count = next_power_of_two(options.buffer_page_count)
#
# Make sure we are running as root, or else we can not attach the probes.
#
if os.geteuid() != 0:
print("ERROR: We need to run as root to attached probes!")
sys.exit(os.EX_NOPERM)
#
# Setup any of the start stop triggers
#
if options.start_trigger is not None:
try:
start_trigger = Probe(options.start_trigger, pid=options.pid)
except ValueError as e:
print(f"ERROR: Invalid start trigger {str(e)}")
sys.exit(os.EX_CONFIG)
else:
start_trigger = None
if options.stop_trigger is not None:
try:
stop_trigger = Probe(options.stop_trigger, pid=options.pid)
except ValueError as e:
print(f"ERROR: Invalid stop trigger {str(e)}")
sys.exit(os.EX_CONFIG)
else:
stop_trigger = None
#
# Attach probe to running process.
#
source = EBPF_SOURCE.replace("<EVENT_ENUM>", "\n".join(
[" EVENT_{} = {},".format(
event.name, event.value) for event in Event]))
source = source.replace("<BUFFER_PAGE_CNT>",
str(options.buffer_page_count))
source = source.replace("<MONITOR_PID>", str(options.pid))
if BPF.kernel_struct_has_field(b"task_struct", b"state") == 1:
source = source.replace("<STATE_FIELD>", "state")
else:
source = source.replace("<STATE_FIELD>", "__state")
poll_id = [k for k, v in syscalls.items() if v == b"poll"][0]
if options.syscall_events is None:
syscall_trace_events = "false"
elif options.syscall_events == 0:
if not options.skip_syscall_poll_events:
syscall_trace_events = "true"
else:
syscall_trace_events = f"args->id != {poll_id}"
else:
syscall_trace_events = "delta > {}".format(options.syscall_events)
if options.skip_syscall_poll_events:
syscall_trace_events += f" && args->id != {poll_id}"
source = source.replace("<SYSCALL_TRACE_EVENTS>",
syscall_trace_events)
source = source.replace("<STACK_TRACE_SIZE>",
str(options.stack_trace_size))
source = source.replace("<STACK_TRACE_ENABLED>", "true"
if options.stack_trace_size > 0 else "false")
#
# Handle start/stop probes
#
if start_trigger:
source = source.replace("<START_TRIGGER>",
start_trigger.get_c_code(
"start_trigger_probe",
"return start_trigger();"))
else:
source = source.replace("<START_TRIGGER>", "")
if stop_trigger:
source = source.replace("<STOP_TRIGGER>",
stop_trigger.get_c_code(
"stop_trigger_probe",
"return stop_trigger();"))
else:
source = source.replace("<STOP_TRIGGER>", "")
#
# Setup usdt or other probes that need handling trough the BFP class.
#
usdt = USDT(pid=int(options.pid))
try:
if start_trigger and start_trigger.probe_type == "usdt":
usdt.enable_probe(probe=start_trigger.probe_name(),
fn_name="start_trigger_probe")
if stop_trigger and stop_trigger.probe_type == "usdt":
usdt.enable_probe(probe=stop_trigger.probe_name(),
fn_name="stop_trigger_probe")
except USDTException as e:
print("ERROR: {}".format(
(re.sub("^", " " * 7, str(e), flags=re.MULTILINE)).strip().
replace("--with-dtrace or --enable-dtrace",
"--enable-usdt-probes")))
sys.exit(os.EX_OSERR)
bpf = BPF(text=source, usdt_contexts=[usdt], debug=options.debug)
if start_trigger:
try:
if start_trigger.probe_type == "uprobe":
bpf.attach_uprobe(name=f"/proc/{options.pid}/exe",
sym=start_trigger.probe_name(),
fn_name="start_trigger_probe",
pid=options.pid)
if start_trigger.probe_type == "uretprobe":
bpf.attach_uretprobe(name=f"/proc/{options.pid}/exe",
sym=start_trigger.probe_name(),
fn_name="start_trigger_probe",
pid=options.pid)
except Exception as e:
print("ERROR: Failed attaching uprobe start trigger "
f"'{start_trigger.probe_name()}';\n {str(e)}")
sys.exit(os.EX_OSERR)
if stop_trigger:
try:
if stop_trigger.probe_type == "uprobe":
bpf.attach_uprobe(name=f"/proc/{options.pid}/exe",
sym=stop_trigger.probe_name(),
fn_name="stop_trigger_probe",
pid=options.pid)
if stop_trigger.probe_type == "uretprobe":
bpf.attach_uretprobe(name=f"/proc/{options.pid}/exe",
sym=stop_trigger.probe_name(),
fn_name="stop_trigger_probe",
pid=options.pid)
except Exception as e:
print("ERROR: Failed attaching uprobe stop trigger"
f"'{stop_trigger.probe_name()}';\n {str(e)}")
sys.exit(os.EX_OSERR)
#
# If no triggers are configured use the delay configuration
#
bpf["events"].open_ring_buffer(process_event)
sample_count = 0
while sample_count < options.sample_count:
sample_count += 1
syscall_events = []
if not options.start_trigger:
print_timestamp("# Start sampling")
start_capture()
stop_time = -1 if options.stop_trigger else \
time_ns() + options.sample_time * 1000000000
else:
# For start triggers the stop time depends on the start trigger
# time, or depends on the stop trigger if configured.
stop_time = -1 if options.stop_trigger else 0
while True:
keyboard_interrupt = False
try:
last_start_ts = start_trigger_ts
last_stop_ts = stop_trigger_ts
if stop_time > 0:
delay = int((stop_time - time_ns()) / 1000000)
if delay <= 0:
break
else:
delay = -1
bpf.ring_buffer_poll(timeout=delay)
if stop_time <= 0 and last_start_ts != start_trigger_ts:
print_timestamp(
"# Start sampling (trigger@{})".format(
start_trigger_ts))
if not options.stop_trigger:
stop_time = time_ns() + \
options.sample_time * 1000000000
if last_stop_ts != stop_trigger_ts:
break
except KeyboardInterrupt:
keyboard_interrupt = True
break
if options.stop_trigger and not capture_running():
print_timestamp("# Stop sampling (trigger@{})".format(
stop_trigger_ts))
else:
print_timestamp("# Stop sampling")
if stop_trigger_ts != 0 and start_trigger_ts != 0:
trigger_delta = stop_trigger_ts - start_trigger_ts
else:
trigger_delta = None
if not trigger_delta or trigger_delta >= options.trigger_delta:
stop_capture(force=True) # Prevent a new trigger to start.
process_results(syscall_events=syscall_events,
trigger_delta=trigger_delta)
elif trigger_delta:
sample_count -= 1
print_timestamp("# Sample dump skipped, delta {:,} ns".format(
trigger_delta))
reset_capture()
stop_capture()
if keyboard_interrupt:
break
if options.sample_interval > 0:
time.sleep(options.sample_interval)
#
# Report lost events.
#
dropcnt = bpf.get_table("dropcnt")
for k in dropcnt.keys():
count = dropcnt.sum(k).value
if k.value == 0 and count > 0:
print("\n# WARNING: Not all events were captured, {} were "
"dropped!\n# Increase the BPF ring buffer size "
"with the --buffer-page-count option.".format(count))
if options.sample_count > 1:
trigger_miss = bpf.get_table("trigger_miss")
for k in trigger_miss.keys():
count = trigger_miss.sum(k).value
if k.value == 0 and count > 0:
print("\n# WARNING: Not all start triggers were successful. "
"{} were missed due to\n# slow userspace "
"processing!".format(count))
#
# Start main() as the default entry point...
#
if __name__ == "__main__":
main()
File Manager Version 1.0, Coded By Lucas
Email: hehe@yahoo.com