xref: /spdk/scripts/core-collector.sh (revision 6a6e9354060f8c91c76ba46642c71c1a89c2c549)
145c42ac2SMichal Berger#!/usr/bin/env bash
2eb53c232Spaul luse#  SPDX-License-Identifier: BSD-3-Clause
3eb53c232Spaul luse#  Copyright (C) 2020 Intel Corporation
4eb53c232Spaul luse#  All rights reserved.
5eb53c232Spaul luse#
645c42ac2SMichal Berger# We don't want to tell kernel to include %e or %E since these
745c42ac2SMichal Berger# can include whitespaces or other funny characters, and working
845c42ac2SMichal Berger# with those on the cmdline would be a nightmare. Use procfs for
945c42ac2SMichal Berger# the remaining pieces we want to gather:
109545f9acSMichal Berger# |$rootdir/scripts/core-collector.sh %P %s %t $output_dir
1145c42ac2SMichal Berger
12*6a6e9354SMichal Bergerrootdir=$(readlink -f "$(dirname "$0")/../")
13*6a6e9354SMichal Berger
14*6a6e9354SMichal Bergermaps_to_json() {
15*6a6e9354SMichal Berger	local _maps=("${maps[@]}")
16*6a6e9354SMichal Berger	local mem_regions=() mem
17*6a6e9354SMichal Berger
18*6a6e9354SMichal Berger	mem_regions=("/proc/$core_pid/map_files/"*)
19*6a6e9354SMichal Berger
20*6a6e9354SMichal Berger	for mem in "${!mem_regions[@]}"; do
21*6a6e9354SMichal Berger		_maps[mem]=\"${_maps[mem]}@${mem_regions[mem]##*/}\"
22*6a6e9354SMichal Berger	done
23*6a6e9354SMichal Berger
24*6a6e9354SMichal Berger	local IFS=","
25*6a6e9354SMichal Berger	echo "${_maps[*]}"
26*6a6e9354SMichal Berger}
27*6a6e9354SMichal Berger
2845c42ac2SMichal Bergercore_meta() {
2945c42ac2SMichal Berger	jq . <<- CORE
3045c42ac2SMichal Berger		{
3145c42ac2SMichal Berger		  "$exe_comm": {
3245c42ac2SMichal Berger		    "ts": "$core_time",
3345c42ac2SMichal Berger		    "size": "$core_size bytes",
3445c42ac2SMichal Berger		    "PID": $core_pid,
3545c42ac2SMichal Berger		    "signal": "$core_sig ($core_sig_name)",
3645c42ac2SMichal Berger		    "path": "$exe_path",
37eda45b2aSMichal Berger		    "cwd": "$cwd_path",
386d8f4833SMichal Berger		    "statm": "$statm",
39*6a6e9354SMichal Berger		    "filter": "$(coredump_filter)",
40*6a6e9354SMichal Berger		    "mapped": [ $(maps_to_json) ]
4145c42ac2SMichal Berger		  }
4245c42ac2SMichal Berger		}
4345c42ac2SMichal Berger	CORE
4445c42ac2SMichal Berger}
4545c42ac2SMichal Berger
4645c42ac2SMichal Bergerbt() { hash gdb && gdb -batch -ex "thread apply all bt full" "$1" "$2" 2>&1; }
4745c42ac2SMichal Berger
4845c42ac2SMichal Bergerstderr() {
4945c42ac2SMichal Berger	exec 2> "$core.stderr.txt"
5045c42ac2SMichal Berger	set -x
5145c42ac2SMichal Berger}
5245c42ac2SMichal Berger
536d8f4833SMichal Bergercoredump_filter() {
546d8f4833SMichal Berger	local bitmap bit
556d8f4833SMichal Berger	local _filter filter
566d8f4833SMichal Berger
576d8f4833SMichal Berger	bitmap[0]=anon-priv-mappings
586d8f4833SMichal Berger	bitmap[1]=anon-shared-mappings
596d8f4833SMichal Berger	bitmap[2]=file-priv-mappings
606d8f4833SMichal Berger	bitmap[3]=file-shared-mappings
616d8f4833SMichal Berger	bitmap[4]=elf-headers
626d8f4833SMichal Berger	bitmap[5]=priv-hp
636d8f4833SMichal Berger	bitmap[6]=shared-hp
646d8f4833SMichal Berger	bitmap[7]=priv-DAX
656d8f4833SMichal Berger	bitmap[8]=shared-DAX
666d8f4833SMichal Berger
676d8f4833SMichal Berger	_filter=0x$(< "/proc/$core_pid/coredump_filter")
686d8f4833SMichal Berger
696d8f4833SMichal Berger	for bit in "${!bitmap[@]}"; do
706d8f4833SMichal Berger		((_filter & 1 << bit)) || continue
716d8f4833SMichal Berger		filter=${filter:+$filter,}${bitmap[bit]}
726d8f4833SMichal Berger	done
736d8f4833SMichal Berger
746d8f4833SMichal Berger	echo "$filter"
756d8f4833SMichal Berger}
766d8f4833SMichal Berger
77*6a6e9354SMichal Bergerfilter_process() {
78*6a6e9354SMichal Berger	# Did the process sit in our repo?
79*6a6e9354SMichal Berger	[[ $cwd_path == "$rootdir"* ]] && return 0
80*6a6e9354SMichal Berger
81*6a6e9354SMichal Berger	# Did we load our fio plugins?
82*6a6e9354SMichal Berger	[[ ${maps[*]} == *"$rootdir/build/fio/spdk_nvme"* ]] && return 0
83*6a6e9354SMichal Berger	[[ ${maps[*]} == *"$rootdir/build/fio/spdk_bdev"* ]] && return 0
84*6a6e9354SMichal Berger
85*6a6e9354SMichal Berger	# Do we depend on it?
86*6a6e9354SMichal Berger	local crit_binaries=() bin
87*6a6e9354SMichal Berger
88*6a6e9354SMichal Berger	crit_binaries+=("nvme")
89*6a6e9354SMichal Berger	crit_binaries+=("qemu-system*")
90*6a6e9354SMichal Berger	# Add more if needed
91*6a6e9354SMichal Berger
92*6a6e9354SMichal Berger	for bin in "${crit_binaries[@]}"; do
93*6a6e9354SMichal Berger		# The below SC is intentional
94*6a6e9354SMichal Berger		# shellcheck disable=SC2053
95*6a6e9354SMichal Berger		[[ ${exe_path##*/} == $bin ]] && return 0
96*6a6e9354SMichal Berger	done
97*6a6e9354SMichal Berger
98*6a6e9354SMichal Berger	return 1
99*6a6e9354SMichal Berger}
100*6a6e9354SMichal Berger
10145c42ac2SMichal Bergerargs+=(core_pid)
10245c42ac2SMichal Bergerargs+=(core_sig)
10345c42ac2SMichal Bergerargs+=(core_ts)
10445c42ac2SMichal Berger
10545c42ac2SMichal Bergerread -r "${args[@]}" <<< "$*"
10645c42ac2SMichal Berger
10745c42ac2SMichal Bergerexe_path=$(readlink -f "/proc/$core_pid/exe")
108eda45b2aSMichal Bergercwd_path=$(readlink -f "/proc/$core_pid/cwd")
10945c42ac2SMichal Bergerexe_comm=$(< "/proc/$core_pid/comm")
11045c42ac2SMichal Bergerstatm=$(< "/proc/$core_pid/statm")
11145c42ac2SMichal Bergercore_time=$(date -d@"$core_ts")
11245c42ac2SMichal Bergercore_sig_name=$(kill -l "$core_sig")
113*6a6e9354SMichal Bergermapfile -t maps < <(readlink -f "/proc/$core_pid/map_files/"*)
11445c42ac2SMichal Berger
115*6a6e9354SMichal Berger# Filter out processes that we don't care about
116*6a6e9354SMichal Bergerfilter_process || exit 0
117*6a6e9354SMichal Berger
118*6a6e9354SMichal Bergercore=$(< "$rootdir/.coredump_path")/${exe_path##*/}_$core_pid.core
11945c42ac2SMichal Bergerstderr
12045c42ac2SMichal Berger
12145c42ac2SMichal Berger# RLIMIT_CORE is not enforced when core is piped to us. To make
12245c42ac2SMichal Berger# sure we won't attempt to overload underlying storage, copy
12345c42ac2SMichal Berger# only the reasonable amount of bytes (systemd defaults to 2G
1249545f9acSMichal Berger# so let's follow that).
1259545f9acSMichal Bergerrlimit=$((1024 * 1024 * 1024 * 2))
12645c42ac2SMichal Berger
12745c42ac2SMichal Berger# Clear path for lz
12845c42ac2SMichal Bergerrm -f "$core"{,.{bin,bt,gz,json}}
12945c42ac2SMichal Berger
13045c42ac2SMichal Berger# Slurp the core
13145c42ac2SMichal Bergerhead -c "$rlimit" <&0 > "$core"
13245c42ac2SMichal Bergercore_size=$(wc -c < "$core")
13345c42ac2SMichal Berger
13445c42ac2SMichal Berger# Compress it
13545c42ac2SMichal Bergergzip -c "$core" > "$core.gz"
13645c42ac2SMichal Berger
13745c42ac2SMichal Berger# Save the binary
13845c42ac2SMichal Bergercp "$exe_path" "$core.bin"
13945c42ac2SMichal Berger
14045c42ac2SMichal Berger# Save the backtrace
14145c42ac2SMichal Bergerbt "$exe_path" "$core" > "$core.bt.txt"
14245c42ac2SMichal Berger
14345c42ac2SMichal Berger# Save the metadata of the core
14445c42ac2SMichal Bergercore_meta > "$core.json"
14545c42ac2SMichal Berger
14645c42ac2SMichal Berger# Nuke the original core
14745c42ac2SMichal Bergerrm "$core"
148