1; RUN: opt < %s -passes=pseudo-probe,sample-profile -sample-profile-use-profi -sample-profile-file=%S/Inputs/profile-inference-rebalance-large.prof | opt -passes='print<branch-prob>' -disable-output 2>&1 | FileCheck %s 2; RUN: opt < %s -passes=pseudo-probe,sample-profile -sample-profile-use-profi -sample-profile-file=%S/Inputs/profile-inference-rebalance-large.prof | opt -passes='print<block-freq>' -disable-output 2>&1 | FileCheck %s --check-prefix=CHECK2 3 4; The test verifies that counts can rebalanced in switch statements that contain 5; both 'known' and 'unknown' basic blocks. 6; 7; +---------+ 8; +----------------- | b15 [?] | 9; | +---------+ 10; | ^ 11; | | 12; | | 13; | +---------+ +--------------+ +---------+ 14; | | b13 [?] | <-- | b11 [3300] | --> | b14 [?] | 15; | +---------+ +--------------+ +---------+ 16; | | | | | 17; | | | | | 18; | | v | | 19; | | +---------+ | | 20; | | | b12 [0] | | | 21; | | +---------+ | | 22; | | | | | 23; | | | | | 24; | | v v | 25; | | +--------------+ | 26; | +-----------> | | <-----+ 27; | | b16 [3300] | 28; +----------------> | | 29; +--------------+ 30 31@yydebug = dso_local global i32 0, align 4 32 33; Function Attrs: nounwind uwtable 34define dso_local i32 @foo1(i32 %0, i32 %1) #0 { 35b11: 36 call void @llvm.pseudoprobe(i64 7682762345278052905, i64 1, i32 0, i64 -1) 37 %cmp = icmp ne i32 %0, 0 38 switch i32 %1, label %b12 [ 39 i32 1, label %b13 40 i32 2, label %b14 41 i32 3, label %b15 42 i32 4, label %b16 43 ] 44; CHECK: edge %b11 -> %b12 probability is 0x00000000 / 0x80000000 = 0.00% 45; CHECK: edge %b11 -> %b13 probability is 0x20000000 / 0x80000000 = 25.00% 46; CHECK: edge %b11 -> %b14 probability is 0x20000000 / 0x80000000 = 25.00% 47; CHECK: edge %b11 -> %b15 probability is 0x20000000 / 0x80000000 = 25.00% 48; CHECK: edge %b11 -> %b16 probability is 0x20000000 / 0x80000000 = 25.00% 49; CHECK2: - b11: float = {{.*}}, int = {{.*}}, count = 3300 50 51b12: 52 call void @llvm.pseudoprobe(i64 7682762345278052905, i64 2, i32 0, i64 -1) 53 br label %b16 54; CHECK2: - b12: float = {{.*}}, int = {{.*}}, count = 0 55 56b13: 57 call void @llvm.pseudoprobe(i64 7682762345278052905, i64 3, i32 0, i64 -1) 58 br label %b16 59; CHECK2: - b13: float = {{.*}}, int = {{.*}}, count = 825 60 61b14: 62 call void @llvm.pseudoprobe(i64 7682762345278052905, i64 4, i32 0, i64 -1) 63 br label %b16 64; CHECK2: - b14: float = {{.*}}, int = {{.*}}, count = 825 65 66b15: 67 call void @llvm.pseudoprobe(i64 7682762345278052905, i64 5, i32 0, i64 -1) 68 br label %b16 69; CHECK2: - b15: float = {{.*}}, int = {{.*}}, count = 825 70 71b16: 72 call void @llvm.pseudoprobe(i64 7682762345278052905, i64 6, i32 0, i64 -1) 73 ret i32 %1 74; CHECK2: - b16: float = {{.*}}, int = {{.*}}, count = 3300 75} 76 77 78; The test verifies that counts can rebalanced even when control-flow ends at 79; a basic block with an unknown count. 80; 81; +-----------+ 82; | b21 [128] | -+ 83; +-----------+ | 84; | | 85; v | 86; +-----------+ | 87; | b22 [128] | | 88; +-----------+ | 89; | | 90; v | 91; +-----------+ | 92; +------------ | b23 [128] | <+ 93; | +-----------+ 94; | | 95; v v 96; +---------+ +-----------+ 97; | b26 [?] | <-- | b24 [128] | 98; +---------+ +-----------+ 99; | | 100; | v 101; | +-----------+ 102; | | b25 [?] | 103; | +-----------+ 104; | | 105; | v 106; | +-----------+ 107; +-----------> | b27 [?] | -+ 108; +-----------+ | 109; | | 110; v | 111; +-----------+ | 112; | b28 [?] | | 113; +-----------+ | 114; | | 115; v | 116; +-----------+ | 117; | b29 [?] | <+ 118; +-----------+ 119 120define dso_local i32 @foo2(i32 %0, i32 %1) #0 { 121b21: 122 call void @llvm.pseudoprobe(i64 2494702099028631698, i64 1, i32 0, i64 -1) 123 %cmp = icmp ne i32 %0, 0 124 br i1 %cmp, label %b22, label %b23 125; CHECK: edge %b21 -> %b22 probability is 0x80000000 / 0x80000000 = 100.00% [HOT edge] 126; CHECK: edge %b21 -> %b23 probability is 0x00000000 / 0x80000000 = 0.00% 127; CHECK2: - b21: float = {{.*}}, int = {{.*}}, count = 128 128 129b22: 130 call void @llvm.pseudoprobe(i64 2494702099028631698, i64 2, i32 0, i64 -1) 131 br label %b23 132 133b23: 134 call void @llvm.pseudoprobe(i64 2494702099028631698, i64 3, i32 0, i64 -1) 135 br i1 %cmp, label %b24, label %b26 136; CHECK: edge %b23 -> %b24 probability is 0x80000000 / 0x80000000 = 100.00% [HOT edge] 137; CHECK: edge %b23 -> %b26 probability is 0x00000000 / 0x80000000 = 0.00% 138; CHECK2: - b23: float = {{.*}}, int = {{.*}}, count = 128 139 140b24: 141 call void @llvm.pseudoprobe(i64 2494702099028631698, i64 4, i32 0, i64 -1) 142 br i1 %cmp, label %b25, label %b26 143; CHECK: edge %b24 -> %b25 probability is 0x40000000 / 0x80000000 = 50.00% 144; CHECK: edge %b24 -> %b26 probability is 0x40000000 / 0x80000000 = 50.00% 145 146b25: 147 call void @llvm.pseudoprobe(i64 2494702099028631698, i64 5, i32 0, i64 -1) 148 br label %b27 149; CHECK2: - b25: float = {{.*}}, int = {{.*}}, count = 64 150 151b26: 152 call void @llvm.pseudoprobe(i64 2494702099028631698, i64 6, i32 0, i64 -1) 153 br label %b27 154; CHECK2: - b26: float = {{.*}}, int = {{.*}}, count = 64 155 156b27: 157 call void @llvm.pseudoprobe(i64 2494702099028631698, i64 7, i32 0, i64 -1) 158 br i1 %cmp, label %b28, label %b29 159; CHECK: edge %b27 -> %b28 probability is 0x40000000 / 0x80000000 = 50.00% 160; CHECK: edge %b27 -> %b29 probability is 0x40000000 / 0x80000000 = 50.00% 161; CHECK2: - b27: float = {{.*}}, int = {{.*}}, count = 128 162 163b28: 164 call void @llvm.pseudoprobe(i64 2494702099028631698, i64 8, i32 0, i64 -1) 165 br label %b29 166; CHECK2: - b28: float = {{.*}}, int = {{.*}}, count = 64 167 168b29: 169 call void @llvm.pseudoprobe(i64 2494702099028631698, i64 9, i32 0, i64 -1) 170 ret i32 %1 171; CHECK2: - b29: float = {{.*}}, int = {{.*}}, count = 128 172} 173 174 175; The test verifies a flexible mode of rebalancing in which some jumps to known 176; basic blocks are ignored. 177; 178; +------------+ 179; | b31 [500] | 180; +------------+ 181; | 182; v 183; +---------+ +------------+ 184; | b33 [?] | <-- | b32 [1500] | <-----+ 185; +---------+ +------------+ | 186; | | | 187; | v | 188; | +------------+ +-----------+ 189; | | b34 [1200] | --> | b36 [900] | 190; | +------------+ +-----------+ 191; | | 192; | v 193; | +------------+ 194; | | b35 [?] | 195; | +------------+ 196; | | 197; | v 198; | +------------+ 199; +-----------> | b37 [?] | -+ 200; +------------+ | 201; | | 202; v | 203; +------------+ | 204; | b38 [?] | | 205; +------------+ | 206; | | 207; v | 208; +------------+ | 209; | b39 [500] | <+ 210; +------------+ 211; 212 213define dso_local i32 @foo3(i32 %0, i32 %1) #0 { 214b31: 215 call void @llvm.pseudoprobe(i64 -7908226060800700466, i64 1, i32 0, i64 -1) 216 %cmp = icmp ne i32 %0, 0 217 br label %b32 218; CHECK2: - b31: float = {{.*}}, int = {{.*}}, count = 500 219 220b32: 221 call void @llvm.pseudoprobe(i64 -7908226060800700466, i64 2, i32 0, i64 -1) 222 br i1 %cmp, label %b33, label %b34 223; CHECK: edge %b32 -> %b33 probability is 0x1999999a / 0x80000000 = 20.00% 224; CHECK: edge %b32 -> %b34 probability is 0x66666666 / 0x80000000 = 80.00% 225; CHECK2: - b32: float = {{.*}}, int = {{.*}}, count = 1500 226 227b33: 228 call void @llvm.pseudoprobe(i64 -7908226060800700466, i64 3, i32 0, i64 -1) 229 br label %b37 230; CHECK2: - b33: float = {{.*}}, int = {{.*}}, count = 300 231 232b34: 233 call void @llvm.pseudoprobe(i64 -7908226060800700466, i64 4, i32 0, i64 -1) 234 br i1 %cmp, label %b35, label %b36 235; CHECK: edge %b34 -> %b35 probability is 0x15555555 / 0x80000000 = 16.67% 236; CHECK: edge %b34 -> %b36 probability is 0x6aaaaaab / 0x80000000 = 83.33% [HOT edge] 237; CHECK2: - b34: float = {{.*}}, int = {{.*}}, count = 1200 238 239b35: 240 call void @llvm.pseudoprobe(i64 -7908226060800700466, i64 5, i32 0, i64 -1) 241 br label %b37 242; CHECK2: - b35: float = {{.*}}, int = {{.*}}, count = 200 243 244b36: 245 call void @llvm.pseudoprobe(i64 -7908226060800700466, i64 6, i32 0, i64 -1) 246 br label %b32 247; CHECK2: - b36: float = {{.*}}, int = {{.*}}, count = 1000 248 249b37: 250 call void @llvm.pseudoprobe(i64 -7908226060800700466, i64 7, i32 0, i64 -1) 251 br i1 %cmp, label %b38, label %b39 252; CHECK: edge %b37 -> %b38 probability is 0x40000000 / 0x80000000 = 50.00% 253; CHECK: edge %b37 -> %b39 probability is 0x40000000 / 0x80000000 = 50.00% 254; CHECK2: - b37: float = {{.*}}, int = {{.*}}, count = 500 255 256b38: 257 call void @llvm.pseudoprobe(i64 -7908226060800700466, i64 8, i32 0, i64 -1) 258 br label %b39 259; CHECK2: - b38: float = {{.*}}, int = {{.*}}, count = 250 260 261b39: 262 call void @llvm.pseudoprobe(i64 -7908226060800700466, i64 9, i32 0, i64 -1) 263 ret i32 %1 264; CHECK2: - b39: float = {{.*}}, int = {{.*}}, count = 500 265} 266 267 268; The test verifies that flow rebalancer can ignore 'unlikely' jumps. 269; 270; +-----------+ 271; | b41 [400] | -+ 272; +-----------+ | 273; | | 274; | | 275; v | 276; +-----------+ | 277; | b42 [?] | | 278; +-----------+ | 279; | | 280; | | 281; v v 282; +---------++---------+ +---------------------------+ +---------++---------+ 283; | b48 [?] || b46 [?] | <-- | | --> | b47 [?] || b49 [?] | 284; +---------++---------+ | | +---------++---------+ 285; | ^ | | | | ^ 286; | | | | b43 [400] | | | 287; | +-------+-------------| | | | 288; | | | | | | 289; | | | | ------+----------+ 290; | | +---------------------------+ | 291; | | | | | 292; | | | | | 293; | | v v | 294; | | +-----------+ +---------+ | 295; | | | b44 [?] | | b45 [?] | | 296; | | +-----------+ +---------+ | 297; | | | | | 298; | | | | | 299; | | v v | 300; | | +---------------------------+ | 301; | +-----------> | | <-----+ 302; | | b410 [400] | 303; | | | 304; +----------------------> | | 305; +---------------------------+ 306 307 308define dso_local void @foo4(i32 %0, i32 %1) #0 { 309b41: 310 call void @llvm.pseudoprobe(i64 -6882312132165544686, i64 1, i32 0, i64 -1) 311 %cmp = icmp ne i32 %0, 0 312 br i1 %cmp, label %b42, label %b43 313; CHECK: edge %b41 -> %b42 probability is 0x40000000 / 0x80000000 = 50.00% 314; CHECK: edge %b41 -> %b43 probability is 0x40000000 / 0x80000000 = 50.00% 315; CHECK2: - b41: float = {{.*}}, int = {{.*}}, count = 400 316 317b42: 318 call void @llvm.pseudoprobe(i64 -6882312132165544686, i64 2, i32 0, i64 -1) 319 br label %b43 320; CHECK2: - b42: float = {{.*}}, int = {{.*}}, count = 200 321 322b43: 323 call void @llvm.pseudoprobe(i64 -6882312132165544686, i64 3, i32 0, i64 -1) 324 switch i32 %1, label %b49 [ 325 i32 1, label %b44 326 i32 2, label %b45 327 i32 3, label %b46 328 i32 4, label %b47 329 i32 5, label %b48 330 ] 331; CHECK: edge %b43 -> %b49 probability is 0x00000000 / 0x80000000 = 0.00% 332; CHECK: edge %b43 -> %b44 probability is 0x1999999a / 0x80000000 = 20.00% 333; CHECK: edge %b43 -> %b45 probability is 0x1999999a / 0x80000000 = 20.00% 334; CHECK: edge %b43 -> %b46 probability is 0x1999999a / 0x80000000 = 20.00% 335; CHECK: edge %b43 -> %b47 probability is 0x1999999a / 0x80000000 = 20.00% 336; CHECK: edge %b43 -> %b48 probability is 0x1999999a / 0x80000000 = 20.00% 337; CHECK2: - b43: float = {{.*}}, int = {{.*}}, count = 400 338 339b44: 340 call void @llvm.pseudoprobe(i64 -6882312132165544686, i64 4, i32 0, i64 -1) 341 br label %b410 342; CHECK2: - b44: float = {{.*}}, int = {{.*}}, count = 80 343 344b45: 345 call void @llvm.pseudoprobe(i64 -6882312132165544686, i64 5, i32 0, i64 -1) 346 br label %b410 347; CHECK2: - b45: float = {{.*}}, int = {{.*}}, count = 80 348 349b46: 350 call void @llvm.pseudoprobe(i64 -6882312132165544686, i64 6, i32 0, i64 -1) 351 br label %b410 352; CHECK2: - b46: float = {{.*}}, int = {{.*}}, count = 80 353 354b47: 355 call void @llvm.pseudoprobe(i64 -6882312132165544686, i64 7, i32 0, i64 -1) 356 br label %b410 357; CHECK2: - b47: float = {{.*}}, int = {{.*}}, count = 80 358 359b48: 360 call void @llvm.pseudoprobe(i64 -6882312132165544686, i64 8, i32 0, i64 -1) 361 br label %b410 362; CHECK2: - b48: float = {{.*}}, int = {{.*}}, count = 80 363 364b49: 365 call void @llvm.pseudoprobe(i64 -6882312132165544686, i64 9, i32 0, i64 -1) 366 unreachable 367; CHECK2: - b49: float = {{.*}}, int = {{.*}}, count = 0 368 369b410: 370 call void @llvm.pseudoprobe(i64 -6882312132165544686, i64 10, i32 0, i64 -1) 371 ret void 372; CHECK2: - b410: float = {{.*}}, int = {{.*}}, count = 400 373} 374 375 376; Function Attrs: inaccessiblememonly nounwind willreturn 377declare void @llvm.pseudoprobe(i64, i64, i32, i64) #4 378 379attributes #0 = { noinline nounwind uwtable "use-sample-profile" } 380attributes #4 = { inaccessiblememonly nounwind willreturn } 381 382!llvm.pseudo_probe_desc = !{!7, !8, !9, !10} 383 384!7 = !{i64 7682762345278052905, i64 157181141624, !"foo1", null} 385!8 = !{i64 2494702099028631698, i64 208782362068, !"foo2", null} 386!9 = !{i64 -7908226060800700466, i64 189901498683, !"foo3", null} 387!10 = !{i64 -6882312132165544686, i64 241030178952, !"foo4", null} 388