1# 2# CDDL HEADER START 3# 4# The contents of this file are subject to the terms of the 5# Common Development and Distribution License (the "License"). 6# You may not use this file except in compliance with the License. 7# 8# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9# or https://opensource.org/licenses/CDDL-1.0. 10# See the License for the specific language governing permissions 11# and limitations under the License. 12# 13# When distributing Covered Code, include this CDDL HEADER in each 14# file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15# If applicable, add the following below this CDDL HEADER, with the 16# fields enclosed by brackets "[]" replaced with your own identifying 17# information: Portions Copyright [yyyy] [name of copyright owner] 18# 19# CDDL HEADER END 20# 21 22# 23# Copyright (c) 2025, Klara, Inc. 24# 25 26# 27# This file provides the following helpers to read kstats from tests. 28# 29# kstat [-g] <stat> 30# kstat_pool [-g] <pool> <stat> 31# kstat_dataset [-N] <dataset | pool/objsetid> <stat> 32# 33# `kstat` and `kstat_pool` return the value of of the given <stat>, either 34# a global or pool-specific state. 35# 36# $ kstat dbgmsg 37# timestamp message 38# 1736848201 spa_history.c:304:spa_history_log_sync(): txg 14734896 ... 39# 1736848201 spa_history.c:330:spa_history_log_sync(): ioctl ... 40# ... 41# 42# $ kstat_pool garden state 43# ONLINE 44# 45# To get a single stat within a group or collection, separate the name with 46# '.' characters. 47# 48# $ kstat dbufstats.cache_target_bytes 49# 3215780693 50# 51# $ kstat_pool crayon iostats.arc_read_bytes 52# 253671670784 53# 54# -g is "group" mode. If the kstat is a group or collection, all stats in that 55# group are returned, one stat per line, key and value separated by a space. 56# 57# $ kstat -g dbufstats 58# cache_count 1792 59# cache_size_bytes 87720376 60# cache_size_bytes_max 305187768 61# cache_target_bytes 97668555 62# ... 63# 64# $ kstat_pool -g crayon iostats 65# trim_extents_written 0 66# trim_bytes_written 0 67# trim_extents_skipped 0 68# trim_bytes_skipped 0 69# ... 70# 71# `kstat_dataset` accesses the per-dataset group kstat. The dataset can be 72# specified by name: 73# 74# $ kstat_dataset crayon/home/robn nunlinks 75# 2628514 76# 77# or, with the -N switch, as <pool>/<objsetID>: 78# 79# $ kstat_dataset -N crayon/7 writes 80# 125135 81# 82 83#################### 84# Public interface 85 86# 87# kstat [-g] <stat> 88# 89function kstat 90{ 91 typeset -i want_group=0 92 93 OPTIND=1 94 while getopts "g" opt ; do 95 case $opt in 96 'g') want_group=1 ;; 97 *) log_fail "kstat: invalid option '$opt'" ;; 98 esac 99 done 100 shift $(expr $OPTIND - 1) 101 102 typeset stat=$1 103 104 $_kstat_os 'global' '' "$stat" $want_group 105} 106 107# 108# kstat_pool [-g] <pool> <stat> 109# 110function kstat_pool 111{ 112 typeset -i want_group=0 113 114 OPTIND=1 115 while getopts "g" opt ; do 116 case $opt in 117 'g') want_group=1 ;; 118 *) log_fail "kstat_pool: invalid option '$opt'" ;; 119 esac 120 done 121 shift $(expr $OPTIND - 1) 122 123 typeset pool=$1 124 typeset stat=$2 125 126 $_kstat_os 'pool' "$pool" "$stat" $want_group 127} 128 129# 130# kstat_dataset [-N] <dataset | pool/objsetid> <stat> 131# 132function kstat_dataset 133{ 134 typeset -i opt_objsetid=0 135 136 OPTIND=1 137 while getopts "N" opt ; do 138 case $opt in 139 'N') opt_objsetid=1 ;; 140 *) log_fail "kstat_dataset: invalid option '$opt'" ;; 141 esac 142 done 143 shift $(expr $OPTIND - 1) 144 145 typeset dsarg=$1 146 typeset stat=$2 147 148 if [[ $opt_objsetid == 0 ]] ; then 149 typeset pool="${dsarg%%/*}" # clear first / -> end 150 typeset objsetid=$($_resolve_dsname_os "$pool" "$dsarg") 151 if [[ -z "$objsetid" ]] ; then 152 log_fail "kstat_dataset: dataset not found: $dsarg" 153 fi 154 dsarg="$pool/$objsetid" 155 fi 156 157 $_kstat_os 'dataset' "$dsarg" "$stat" 0 158} 159 160#################### 161# Platform-specific interface 162 163# 164# Implementation notes 165# 166# There's not a lot of uniformity between platforms, so I've written to a rough 167# imagined model that seems to fit the majority of OpenZFS kstats. 168# 169# The main platform entry points look like this: 170# 171# _kstat_freebsd <scope> <object> <stat> <want_group> 172# _kstat_linux <scope> <object> <stat> <want_group> 173# 174# - scope: one of 'global', 'pool', 'dataset'. The "kind" of object the kstat 175# is attached to. 176# - object: name of the scoped object 177# global: empty string 178# pool: pool name 179# dataset: <pool>/<objsetId> pair 180# - stat: kstat name to get 181# - want_group: 0 to get the single value for the kstat, 1 to treat the kstat 182# as a group and get all the stat names+values under it. group 183# kstats cannot have values, and stat kstats cannot have 184# children (by definition) 185# 186# Stat values can have multiple lines, so be prepared for those. 187# 188# These functions either succeed and produce the requested output, or call 189# log_fail. They should never output empty, or 0, or anything else. 190# 191# Output: 192# 193# - want_group=0: the single stat value, followed by newline 194# - want_group=1: One stat per line, <name><SP><value><newline> 195# 196 197# 198# To support kstat_dataset(), platforms also need to provide a dataset 199# name->object id resolver function. 200# 201# _resolve_dsname_freebsd <pool> <dsname> 202# _resolve_dsname_linux <pool> <dsname> 203# 204# - pool: pool name. always the first part of the dataset name 205# - dsname: dataset name, in the standard <pool>/<some>/<dataset> format. 206# 207# Output is <objsetID>. objsetID is a decimal integer, > 0 208# 209 210#################### 211# FreeBSD 212 213# 214# All kstats are accessed through sysctl. We model "groups" as interior nodes 215# in the stat tree, which are normally opaque. Because sysctl has no filtering 216# options, and requesting any node produces all nodes below it, we have to 217# always get the name and value, and then consider the output to understand 218# if we got a group or a single stat, and post-process accordingly. 219# 220# Scopes are mostly mapped directly to known locations in the tree, but there 221# are a handful of stats that are out of position, so we need to adjust. 222# 223 224# 225# _kstat_freebsd <scope> <object> <stat> <want_group> 226# 227function _kstat_freebsd 228{ 229 typeset scope=$1 230 typeset obj=$2 231 typeset stat=$3 232 typeset -i want_group=$4 233 234 typeset oid="" 235 case "$scope" in 236 global) 237 oid="kstat.zfs.misc.$stat" 238 ;; 239 pool) 240 # For reasons unknown, the "multihost", "txgs" and "reads" 241 # pool-specific kstats are directly under kstat.zfs.<pool>, 242 # rather than kstat.zfs.<pool>.misc like the other pool kstats. 243 # Adjust for that here. 244 case "$stat" in 245 multihost|txgs|reads) 246 oid="kstat.zfs.$obj.$stat" 247 ;; 248 *) 249 oid="kstat.zfs.$obj.misc.$stat" 250 ;; 251 esac 252 ;; 253 dataset) 254 typeset pool="" 255 typeset -i objsetid=0 256 _split_pool_objsetid $obj pool objsetid 257 oid=$(printf 'kstat.zfs.%s.dataset.objset-0x%x.%s' \ 258 $pool $objsetid $stat) 259 ;; 260 esac 261 262 # Calling sysctl on a "group" node will return everything under that 263 # node, so we have to inspect the first line to make sure we are 264 # getting back what we expect. For a single value, the key will have 265 # the name we requested, while for a group, the key will not have the 266 # name (group nodes are "opaque", not returned by sysctl by default. 267 268 if [[ $want_group == 0 ]] ; then 269 sysctl -e "$oid" | awk -v oid="$oid" -v oidre="^$oid=" ' 270 NR == 1 && $0 !~ oidre { exit 1 } 271 NR == 1 { print substr($0, length(oid)+2) ; next } 272 { print } 273 ' 274 else 275 sysctl -e "$oid" | awk -v oid="$oid" -v oidre="^$oid=" ' 276 NR == 1 && $0 ~ oidre { exit 2 } 277 { 278 sub("^" oid "\.", "") 279 sub("=", " ") 280 print 281 } 282 ' 283 fi 284 285 typeset -i err=$? 286 case $err in 287 0) return ;; 288 1) log_fail "kstat: can't get value for group kstat: $oid" ;; 289 2) log_fail "kstat: not a group kstat: $oid" ;; 290 esac 291 292 log_fail "kstat: unknown error: $oid" 293} 294 295# 296# _resolve_dsname_freebsd <pool> <dsname> 297# 298function _resolve_dsname_freebsd 299{ 300 # we're searching for: 301 # 302 # kstat.zfs.shed.dataset.objset-0x8087.dataset_name: shed/poudriere 303 # 304 # We split on '.', then get the hex objsetid from field 5. 305 # 306 # We convert hex to decimal in the shell because there isn't a _simple_ 307 # portable way to do it in awk and this code is already too intense to 308 # do it a complicated way. 309 typeset pool=$1 310 typeset dsname=$2 311 sysctl -e kstat.zfs.$pool | \ 312 awk -F '.' -v dsnamere="=$dsname$" ' 313 /\.objset-0x[0-9a-f]+\.dataset_name=/ && $6 ~ dsnamere { 314 print substr($5, 8) 315 exit 316 } 317 ' | xargs printf %d 318} 319 320#################### 321# Linux 322 323# 324# kstats all live under /proc/spl/kstat/zfs. They have a flat structure: global 325# at top-level, pool in a directory, and dataset in a objset- file inside the 326# pool dir. 327# 328# Groups are challenge. A single stat can be the entire text of a file, or 329# a single line that must be extracted from a "group" file. The only way to 330# recognise a group from the outside is to look for its header. This naturally 331# breaks if a raw file had a matching header, or if a group file chooses to 332# hid its header. Fortunately OpenZFS does none of these things at the moment. 333# 334 335# 336# _kstat_linux <scope> <object> <stat> <want_group> 337# 338function _kstat_linux 339{ 340 typeset scope=$1 341 typeset obj=$2 342 typeset stat=$3 343 typeset -i want_group=$4 344 345 typeset singlestat="" 346 347 if [[ $scope == 'dataset' ]] ; then 348 typeset pool="" 349 typeset -i objsetid=0 350 _split_pool_objsetid $obj pool objsetid 351 stat=$(printf 'objset-0x%x.%s' $objsetid $stat) 352 obj=$pool 353 scope='pool' 354 fi 355 356 typeset path="" 357 if [[ $scope == 'global' ]] ; then 358 path="/proc/spl/kstat/zfs/$stat" 359 else 360 path="/proc/spl/kstat/zfs/$obj/$stat" 361 fi 362 363 if [[ ! -e "$path" && $want_group -eq 0 ]] ; then 364 # This single stat doesn't have its own file, but the wanted 365 # stat could be in a group kstat file, which we now need to 366 # find. To do this, we split a single stat name into two parts: 367 # the file that would contain the stat, and the key within that 368 # file to match on. This works by converting all bar the last 369 # '.' separator to '/', then splitting on the remaining '.' 370 # separator. If there are no '.' separators, the second arg 371 # returned will be empty. 372 # 373 # foo -> (foo) 374 # foo.bar -> (foo, bar) 375 # foo.bar.baz -> (foo/bar, baz) 376 # foo.bar.baz.quux -> (foo/bar/baz, quux) 377 # 378 # This is how we will target single stats within a larger NAMED 379 # kstat file, eg dbufstats.cache_target_bytes. 380 typeset -a split=($(echo "$stat" | \ 381 sed -E 's/^(.+)\.([^\.]+)$/\1 \2/ ; s/\./\//g')) 382 typeset statfile=${split[0]} 383 singlestat=${split[1]:-""} 384 385 if [[ $scope == 'global' ]] ; then 386 path="/proc/spl/kstat/zfs/$statfile" 387 else 388 path="/proc/spl/kstat/zfs/$obj/$statfile" 389 fi 390 fi 391 if [[ ! -r "$path" ]] ; then 392 log_fail "kstat: can't read $path" 393 fi 394 395 if [[ $want_group == 1 ]] ; then 396 # "group" (NAMED) kstats on Linux start: 397 # 398 # $ cat /proc/spl/kstat/zfs/crayon/iostats 399 # 70 1 0x01 26 7072 8577844978 661416318663496 400 # name type data 401 # trim_extents_written 4 0 402 # trim_bytes_written 4 0 403 # 404 # The second value on the first row is the ks_type. Group 405 # mode only works for type 1, KSTAT_TYPE_NAMED. So we check 406 # for that, and eject if it's the wrong type. Otherwise, we 407 # skip the header row and process the values. 408 awk ' 409 NR == 1 && ! /^[0-9]+ 1 / { exit 2 } 410 NR < 3 { next } 411 { print $1 " " $NF } 412 ' "$path" 413 elif [[ -n $singlestat ]] ; then 414 # single stat. must be a single line within a group stat, so 415 # we look for the header again as above. 416 awk -v singlestat="$singlestat" \ 417 -v singlestatre="^$singlestat " ' 418 NR == 1 && /^[0-9]+ [^1] / { exit 2 } 419 NR < 3 { next } 420 $0 ~ singlestatre { print $NF ; exit 0 } 421 ENDFILE { exit 3 } 422 ' "$path" 423 else 424 # raw stat. dump contents, exclude group stats 425 awk ' 426 NR == 1 && /^[0-9]+ 1 / { exit 1 } 427 { print } 428 ' "$path" 429 fi 430 431 typeset -i err=$? 432 case $err in 433 0) return ;; 434 1) log_fail "kstat: can't get value for group kstat: $path" ;; 435 2) log_fail "kstat: not a group kstat: $path" ;; 436 3) log_fail "kstat: stat not found in group: $path $singlestat" ;; 437 esac 438 439 log_fail "kstat: unknown error: $path" 440} 441 442# 443# _resolve_dsname_linux <pool> <dsname> 444# 445function _resolve_dsname_linux 446{ 447 # We look inside all: 448 # 449 # /proc/spl/kstat/zfs/crayon/objset-0x113 450 # 451 # and check the dataset_name field inside. If we get a match, we split 452 # the filename on /, then extract the hex objsetid. 453 # 454 # We convert hex to decimal in the shell because there isn't a _simple_ 455 # portable way to do it in awk and this code is already too intense to 456 # do it a complicated way. 457 typeset pool=$1 458 typeset dsname=$2 459 awk -v dsname="$dsname" ' 460 $1 == "dataset_name" && $3 == dsname { 461 split(FILENAME, a, "/") 462 print substr(a[7], 8) 463 exit 464 } 465 ' /proc/spl/kstat/zfs/$pool/objset-0x* | xargs printf %d 466} 467 468#################### 469 470# 471# _split_pool_objsetid <obj> <*pool> <*objsetid> 472# 473# Splits pool/objsetId string in <obj> and fills <pool> and <objsetid>. 474# 475function _split_pool_objsetid 476{ 477 typeset obj=$1 478 typeset -n pool=$2 479 typeset -n objsetid=$3 480 481 pool="${obj%%/*}" # clear first / -> end 482 typeset osidarg="${obj#*/}" # clear start -> first / 483 484 # ensure objsetid arg does not contain a /. we're about to convert it, 485 # but ksh will treat it as an expression, and a / will give a 486 # divide-by-zero 487 if [[ "${osidarg%%/*}" != "$osidarg" ]] ; then 488 log_fail "kstat: invalid objsetid: $osidarg" 489 fi 490 491 typeset -i id=$osidarg 492 if [[ $id -le 0 ]] ; then 493 log_fail "kstat: invalid objsetid: $osidarg" 494 fi 495 objsetid=$id 496} 497 498#################### 499 500# 501# Per-platform function selection. 502# 503# To avoid needing platform check throughout, we store the names of the 504# platform functions and call through them. 505# 506if is_freebsd ; then 507 _kstat_os='_kstat_freebsd' 508 _resolve_dsname_os='_resolve_dsname_freebsd' 509elif is_linux ; then 510 _kstat_os='_kstat_linux' 511 _resolve_dsname_os='_resolve_dsname_linux' 512else 513 _kstat_os='_kstat_unknown_platform_implement_me' 514 _resolve_dsname_os='_resolve_dsname_unknown_platform_implement_me' 515fi 516 517