; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
; RUN: llc < %s -mtriple=i686-unknown -mattr=+popcnt | FileCheck %s -check-prefixes=X86-POPCOUNT
; RUN: llc < %s -mtriple=x86_64-unknown -mattr=+popcnt | FileCheck %s -check-prefixes=X64-POPCOUNT
; RUN: llc < %s -mtriple=i686-unknown -mattr=-popcnt | FileCheck %s -check-prefixes=X86-NO-POPCOUNT
; RUN: llc < %s -mtriple=i686-unknown -mattr=+sse2 -mattr=-popcnt | FileCheck %s -check-prefixes=X86-NO-POPCOUNT
; RUN: llc < %s -mtriple=x86_64-unknown -mattr=-popcnt | FileCheck %s -check-prefixes=X64-NO-POPCOUNT

declare i8 @llvm.ctpop.i8(i8) nounwind readnone
declare i16 @llvm.ctpop.i16(i16) nounwind readnone
declare i32 @llvm.ctpop.i32(i32) nounwind readnone
declare i64 @llvm.ctpop.i64(i64) nounwind readnone

; PR79823
; CTPOP tests where we know only some bits are non-zero.
; FIXME: Move shifted masked values to lsb to use smaller bitsize special cases.

define i64 @ctpop_mask2(i64 %x) nounwind readnone {
; X86-POPCOUNT-LABEL: ctpop_mask2:
; X86-POPCOUNT:       # %bb.0:
; X86-POPCOUNT-NEXT:    movl {{[0-9]+}}(%esp), %eax
; X86-POPCOUNT-NEXT:    andl $3, %eax
; X86-POPCOUNT-NEXT:    popcntl %eax, %eax
; X86-POPCOUNT-NEXT:    xorl %edx, %edx
; X86-POPCOUNT-NEXT:    retl
;
; X64-POPCOUNT-LABEL: ctpop_mask2:
; X64-POPCOUNT:       # %bb.0:
; X64-POPCOUNT-NEXT:    andl $3, %edi
; X64-POPCOUNT-NEXT:    popcntl %edi, %eax
; X64-POPCOUNT-NEXT:    retq
;
; X86-NO-POPCOUNT-LABEL: ctpop_mask2:
; X86-NO-POPCOUNT:       # %bb.0:
; X86-NO-POPCOUNT-NEXT:    movl {{[0-9]+}}(%esp), %eax
; X86-NO-POPCOUNT-NEXT:    andl $3, %eax
; X86-NO-POPCOUNT-NEXT:    movl %eax, %ecx
; X86-NO-POPCOUNT-NEXT:    shrl %ecx
; X86-NO-POPCOUNT-NEXT:    subl %ecx, %eax
; X86-NO-POPCOUNT-NEXT:    xorl %edx, %edx
; X86-NO-POPCOUNT-NEXT:    retl
;
; X64-NO-POPCOUNT-LABEL: ctpop_mask2:
; X64-NO-POPCOUNT:       # %bb.0:
; X64-NO-POPCOUNT-NEXT:    movq %rdi, %rax
; X64-NO-POPCOUNT-NEXT:    andl $3, %eax
; X64-NO-POPCOUNT-NEXT:    movl %eax, %ecx
; X64-NO-POPCOUNT-NEXT:    shrl %ecx
; X64-NO-POPCOUNT-NEXT:    subl %ecx, %eax
; X64-NO-POPCOUNT-NEXT:    retq
  %mask = and i64 %x, 3
  %count = tail call i64 @llvm.ctpop.i64(i64 %mask)
  ret i64 %count
}

define i32 @ctpop_shifted_mask2(i32 %x) nounwind readnone {
; X86-POPCOUNT-LABEL: ctpop_shifted_mask2:
; X86-POPCOUNT:       # %bb.0:
; X86-POPCOUNT-NEXT:    movl $1572864, %eax # imm = 0x180000
; X86-POPCOUNT-NEXT:    andl {{[0-9]+}}(%esp), %eax
; X86-POPCOUNT-NEXT:    popcntl %eax, %eax
; X86-POPCOUNT-NEXT:    retl
;
; X64-POPCOUNT-LABEL: ctpop_shifted_mask2:
; X64-POPCOUNT:       # %bb.0:
; X64-POPCOUNT-NEXT:    andl $1572864, %edi # imm = 0x180000
; X64-POPCOUNT-NEXT:    popcntl %edi, %eax
; X64-POPCOUNT-NEXT:    retq
;
; X86-NO-POPCOUNT-LABEL: ctpop_shifted_mask2:
; X86-NO-POPCOUNT:       # %bb.0:
; X86-NO-POPCOUNT-NEXT:    movl $1572864, %eax # imm = 0x180000
; X86-NO-POPCOUNT-NEXT:    andl {{[0-9]+}}(%esp), %eax
; X86-NO-POPCOUNT-NEXT:    movl %eax, %ecx
; X86-NO-POPCOUNT-NEXT:    shrl $20, %ecx
; X86-NO-POPCOUNT-NEXT:    shrl $19, %eax
; X86-NO-POPCOUNT-NEXT:    subl %ecx, %eax
; X86-NO-POPCOUNT-NEXT:    retl
;
; X64-NO-POPCOUNT-LABEL: ctpop_shifted_mask2:
; X64-NO-POPCOUNT:       # %bb.0:
; X64-NO-POPCOUNT-NEXT:    movl %edi, %eax
; X64-NO-POPCOUNT-NEXT:    andl $1572864, %eax # imm = 0x180000
; X64-NO-POPCOUNT-NEXT:    movl %eax, %ecx
; X64-NO-POPCOUNT-NEXT:    shrl $20, %ecx
; X64-NO-POPCOUNT-NEXT:    shrl $19, %eax
; X64-NO-POPCOUNT-NEXT:    subl %ecx, %eax
; X64-NO-POPCOUNT-NEXT:    retq
  %mask = and i32 %x, 1572864 ; 3 << 19
  %count = tail call i32 @llvm.ctpop.i32(i32 %mask)
  ret i32 %count
}

define i32 @ctpop_mask3(i32 %x) nounwind readnone {
; X86-POPCOUNT-LABEL: ctpop_mask3:
; X86-POPCOUNT:       # %bb.0:
; X86-POPCOUNT-NEXT:    movl {{[0-9]+}}(%esp), %eax
; X86-POPCOUNT-NEXT:    andl $5, %eax
; X86-POPCOUNT-NEXT:    popcntl %eax, %eax
; X86-POPCOUNT-NEXT:    retl
;
; X64-POPCOUNT-LABEL: ctpop_mask3:
; X64-POPCOUNT:       # %bb.0:
; X64-POPCOUNT-NEXT:    andl $5, %edi
; X64-POPCOUNT-NEXT:    popcntl %edi, %eax
; X64-POPCOUNT-NEXT:    retq
;
; X86-NO-POPCOUNT-LABEL: ctpop_mask3:
; X86-NO-POPCOUNT:       # %bb.0:
; X86-NO-POPCOUNT-NEXT:    movl {{[0-9]+}}(%esp), %ecx
; X86-NO-POPCOUNT-NEXT:    andl $5, %ecx
; X86-NO-POPCOUNT-NEXT:    addl %ecx, %ecx
; X86-NO-POPCOUNT-NEXT:    movl $59796, %eax # imm = 0xE994
; X86-NO-POPCOUNT-NEXT:    # kill: def $cl killed $cl killed $ecx
; X86-NO-POPCOUNT-NEXT:    shrl %cl, %eax
; X86-NO-POPCOUNT-NEXT:    andl $3, %eax
; X86-NO-POPCOUNT-NEXT:    retl
;
; X64-NO-POPCOUNT-LABEL: ctpop_mask3:
; X64-NO-POPCOUNT:       # %bb.0:
; X64-NO-POPCOUNT-NEXT:    # kill: def $edi killed $edi def $rdi
; X64-NO-POPCOUNT-NEXT:    andl $5, %edi
; X64-NO-POPCOUNT-NEXT:    leal (%rdi,%rdi), %ecx
; X64-NO-POPCOUNT-NEXT:    movl $59796, %eax # imm = 0xE994
; X64-NO-POPCOUNT-NEXT:    # kill: def $cl killed $cl killed $ecx
; X64-NO-POPCOUNT-NEXT:    shrl %cl, %eax
; X64-NO-POPCOUNT-NEXT:    andl $3, %eax
; X64-NO-POPCOUNT-NEXT:    retq
  %mask = and i32 %x, 5 ; 0b101
  %count = tail call i32 @llvm.ctpop.i32(i32 %mask)
  ret i32 %count
}

define i16 @ctpop_shifted_mask3(i16 %x) nounwind readnone {
; X86-POPCOUNT-LABEL: ctpop_shifted_mask3:
; X86-POPCOUNT:       # %bb.0:
; X86-POPCOUNT-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
; X86-POPCOUNT-NEXT:    andl $14, %eax
; X86-POPCOUNT-NEXT:    popcntl %eax, %eax
; X86-POPCOUNT-NEXT:    # kill: def $ax killed $ax killed $eax
; X86-POPCOUNT-NEXT:    retl
;
; X64-POPCOUNT-LABEL: ctpop_shifted_mask3:
; X64-POPCOUNT:       # %bb.0:
; X64-POPCOUNT-NEXT:    andl $14, %edi
; X64-POPCOUNT-NEXT:    popcntl %edi, %eax
; X64-POPCOUNT-NEXT:    # kill: def $ax killed $ax killed $eax
; X64-POPCOUNT-NEXT:    retq
;
; X86-NO-POPCOUNT-LABEL: ctpop_shifted_mask3:
; X86-NO-POPCOUNT:       # %bb.0:
; X86-NO-POPCOUNT-NEXT:    movzwl {{[0-9]+}}(%esp), %ecx
; X86-NO-POPCOUNT-NEXT:    andl $14, %ecx
; X86-NO-POPCOUNT-NEXT:    movl $59796, %eax # imm = 0xE994
; X86-NO-POPCOUNT-NEXT:    # kill: def $cl killed $cl killed $ecx
; X86-NO-POPCOUNT-NEXT:    shrl %cl, %eax
; X86-NO-POPCOUNT-NEXT:    andl $3, %eax
; X86-NO-POPCOUNT-NEXT:    # kill: def $ax killed $ax killed $eax
; X86-NO-POPCOUNT-NEXT:    retl
;
; X64-NO-POPCOUNT-LABEL: ctpop_shifted_mask3:
; X64-NO-POPCOUNT:       # %bb.0:
; X64-NO-POPCOUNT-NEXT:    movl %edi, %ecx
; X64-NO-POPCOUNT-NEXT:    andl $14, %ecx
; X64-NO-POPCOUNT-NEXT:    movl $59796, %eax # imm = 0xE994
; X64-NO-POPCOUNT-NEXT:    # kill: def $cl killed $cl killed $ecx
; X64-NO-POPCOUNT-NEXT:    shrl %cl, %eax
; X64-NO-POPCOUNT-NEXT:    andl $3, %eax
; X64-NO-POPCOUNT-NEXT:    # kill: def $ax killed $ax killed $eax
; X64-NO-POPCOUNT-NEXT:    retq
  %mask = and i16 %x, 14 ; 7 << 1
  %count = tail call i16 @llvm.ctpop.i16(i16 %mask)
  ret i16 %count
}

define i64 @ctpop_mask4(i64 %x) nounwind readnone {
; X86-POPCOUNT-LABEL: ctpop_mask4:
; X86-POPCOUNT:       # %bb.0:
; X86-POPCOUNT-NEXT:    movl {{[0-9]+}}(%esp), %eax
; X86-POPCOUNT-NEXT:    andl $15, %eax
; X86-POPCOUNT-NEXT:    popcntl %eax, %eax
; X86-POPCOUNT-NEXT:    xorl %edx, %edx
; X86-POPCOUNT-NEXT:    retl
;
; X64-POPCOUNT-LABEL: ctpop_mask4:
; X64-POPCOUNT:       # %bb.0:
; X64-POPCOUNT-NEXT:    andl $15, %edi
; X64-POPCOUNT-NEXT:    popcntl %edi, %eax
; X64-POPCOUNT-NEXT:    retq
;
; X86-NO-POPCOUNT-LABEL: ctpop_mask4:
; X86-NO-POPCOUNT:       # %bb.0:
; X86-NO-POPCOUNT-NEXT:    movl {{[0-9]+}}(%esp), %eax
; X86-NO-POPCOUNT-NEXT:    andl $15, %eax
; X86-NO-POPCOUNT-NEXT:    imull $134480385, %eax, %eax # imm = 0x8040201
; X86-NO-POPCOUNT-NEXT:    shrl $3, %eax
; X86-NO-POPCOUNT-NEXT:    andl $17895697, %eax # imm = 0x1111111
; X86-NO-POPCOUNT-NEXT:    imull $286331153, %eax, %eax # imm = 0x11111111
; X86-NO-POPCOUNT-NEXT:    shrl $28, %eax
; X86-NO-POPCOUNT-NEXT:    xorl %edx, %edx
; X86-NO-POPCOUNT-NEXT:    retl
;
; X64-NO-POPCOUNT-LABEL: ctpop_mask4:
; X64-NO-POPCOUNT:       # %bb.0:
; X64-NO-POPCOUNT-NEXT:    andl $15, %edi
; X64-NO-POPCOUNT-NEXT:    leal (,%rdi,4), %ecx
; X64-NO-POPCOUNT-NEXT:    movabsq $4841987667533046032, %rax # imm = 0x4332322132212110
; X64-NO-POPCOUNT-NEXT:    # kill: def $cl killed $cl killed $ecx
; X64-NO-POPCOUNT-NEXT:    shrq %cl, %rax
; X64-NO-POPCOUNT-NEXT:    andl $7, %eax
; X64-NO-POPCOUNT-NEXT:    retq
  %mask = and i64 %x, 15
  %count = tail call i64 @llvm.ctpop.i64(i64 %mask)
  ret i64 %count
}

define i32 @ctpop_shifted_mask4(i32 %x) nounwind readnone {
; X86-POPCOUNT-LABEL: ctpop_shifted_mask4:
; X86-POPCOUNT:       # %bb.0:
; X86-POPCOUNT-NEXT:    movl $7680, %eax # imm = 0x1E00
; X86-POPCOUNT-NEXT:    andl {{[0-9]+}}(%esp), %eax
; X86-POPCOUNT-NEXT:    popcntl %eax, %eax
; X86-POPCOUNT-NEXT:    retl
;
; X64-POPCOUNT-LABEL: ctpop_shifted_mask4:
; X64-POPCOUNT:       # %bb.0:
; X64-POPCOUNT-NEXT:    andl $7680, %edi # imm = 0x1E00
; X64-POPCOUNT-NEXT:    popcntl %edi, %eax
; X64-POPCOUNT-NEXT:    retq
;
; X86-NO-POPCOUNT-LABEL: ctpop_shifted_mask4:
; X86-NO-POPCOUNT:       # %bb.0:
; X86-NO-POPCOUNT-NEXT:    movl {{[0-9]+}}(%esp), %eax
; X86-NO-POPCOUNT-NEXT:    shrl $9, %eax
; X86-NO-POPCOUNT-NEXT:    andl $15, %eax
; X86-NO-POPCOUNT-NEXT:    imull $134480385, %eax, %eax # imm = 0x8040201
; X86-NO-POPCOUNT-NEXT:    shrl $3, %eax
; X86-NO-POPCOUNT-NEXT:    andl $17895697, %eax # imm = 0x1111111
; X86-NO-POPCOUNT-NEXT:    imull $286331153, %eax, %eax # imm = 0x11111111
; X86-NO-POPCOUNT-NEXT:    shrl $28, %eax
; X86-NO-POPCOUNT-NEXT:    retl
;
; X64-NO-POPCOUNT-LABEL: ctpop_shifted_mask4:
; X64-NO-POPCOUNT:       # %bb.0:
; X64-NO-POPCOUNT-NEXT:    movl %edi, %ecx
; X64-NO-POPCOUNT-NEXT:    shrl $7, %ecx
; X64-NO-POPCOUNT-NEXT:    andl $60, %ecx
; X64-NO-POPCOUNT-NEXT:    movabsq $4841987667533046032, %rax # imm = 0x4332322132212110
; X64-NO-POPCOUNT-NEXT:    # kill: def $cl killed $cl killed $ecx
; X64-NO-POPCOUNT-NEXT:    shrq %cl, %rax
; X64-NO-POPCOUNT-NEXT:    andl $7, %eax
; X64-NO-POPCOUNT-NEXT:    # kill: def $eax killed $eax killed $rax
; X64-NO-POPCOUNT-NEXT:    retq
  %mask = and i32 %x, 7680 ; 15 << 9
  %count = tail call i32 @llvm.ctpop.i32(i32 %mask)
  ret i32 %count
}

define i64 @ctpop_mask5(i64 %x) nounwind readnone {
; X86-POPCOUNT-LABEL: ctpop_mask5:
; X86-POPCOUNT:       # %bb.0:
; X86-POPCOUNT-NEXT:    movl {{[0-9]+}}(%esp), %eax
; X86-POPCOUNT-NEXT:    andl $31, %eax
; X86-POPCOUNT-NEXT:    popcntl %eax, %eax
; X86-POPCOUNT-NEXT:    xorl %edx, %edx
; X86-POPCOUNT-NEXT:    retl
;
; X64-POPCOUNT-LABEL: ctpop_mask5:
; X64-POPCOUNT:       # %bb.0:
; X64-POPCOUNT-NEXT:    andl $31, %edi
; X64-POPCOUNT-NEXT:    popcntl %edi, %eax
; X64-POPCOUNT-NEXT:    retq
;
; X86-NO-POPCOUNT-LABEL: ctpop_mask5:
; X86-NO-POPCOUNT:       # %bb.0:
; X86-NO-POPCOUNT-NEXT:    movl {{[0-9]+}}(%esp), %eax
; X86-NO-POPCOUNT-NEXT:    andl $31, %eax
; X86-NO-POPCOUNT-NEXT:    imull $134480385, %eax, %eax # imm = 0x8040201
; X86-NO-POPCOUNT-NEXT:    shrl $3, %eax
; X86-NO-POPCOUNT-NEXT:    andl $286331153, %eax # imm = 0x11111111
; X86-NO-POPCOUNT-NEXT:    imull $286331153, %eax, %eax # imm = 0x11111111
; X86-NO-POPCOUNT-NEXT:    shrl $28, %eax
; X86-NO-POPCOUNT-NEXT:    xorl %edx, %edx
; X86-NO-POPCOUNT-NEXT:    retl
;
; X64-NO-POPCOUNT-LABEL: ctpop_mask5:
; X64-NO-POPCOUNT:       # %bb.0:
; X64-NO-POPCOUNT-NEXT:    andl $31, %edi
; X64-NO-POPCOUNT-NEXT:    imull $134480385, %edi, %eax # imm = 0x8040201
; X64-NO-POPCOUNT-NEXT:    shrl $3, %eax
; X64-NO-POPCOUNT-NEXT:    andl $286331153, %eax # imm = 0x11111111
; X64-NO-POPCOUNT-NEXT:    imull $286331153, %eax, %eax # imm = 0x11111111
; X64-NO-POPCOUNT-NEXT:    shrl $28, %eax
; X64-NO-POPCOUNT-NEXT:    retq
  %mask = and i64 %x, 31
  %count = tail call i64 @llvm.ctpop.i64(i64 %mask)
  ret i64 %count
}

define i32 @ctpop_shifted_mask5(i32 %x) nounwind readnone {
; X86-POPCOUNT-LABEL: ctpop_shifted_mask5:
; X86-POPCOUNT:       # %bb.0:
; X86-POPCOUNT-NEXT:    movl $11776, %eax # imm = 0x2E00
; X86-POPCOUNT-NEXT:    andl {{[0-9]+}}(%esp), %eax
; X86-POPCOUNT-NEXT:    popcntl %eax, %eax
; X86-POPCOUNT-NEXT:    retl
;
; X64-POPCOUNT-LABEL: ctpop_shifted_mask5:
; X64-POPCOUNT:       # %bb.0:
; X64-POPCOUNT-NEXT:    andl $11776, %edi # imm = 0x2E00
; X64-POPCOUNT-NEXT:    popcntl %edi, %eax
; X64-POPCOUNT-NEXT:    retq
;
; X86-NO-POPCOUNT-LABEL: ctpop_shifted_mask5:
; X86-NO-POPCOUNT:       # %bb.0:
; X86-NO-POPCOUNT-NEXT:    movl {{[0-9]+}}(%esp), %eax
; X86-NO-POPCOUNT-NEXT:    shrl $9, %eax
; X86-NO-POPCOUNT-NEXT:    andl $23, %eax
; X86-NO-POPCOUNT-NEXT:    imull $134480385, %eax, %eax # imm = 0x8040201
; X86-NO-POPCOUNT-NEXT:    shrl $3, %eax
; X86-NO-POPCOUNT-NEXT:    andl $286331153, %eax # imm = 0x11111111
; X86-NO-POPCOUNT-NEXT:    imull $286331153, %eax, %eax # imm = 0x11111111
; X86-NO-POPCOUNT-NEXT:    shrl $28, %eax
; X86-NO-POPCOUNT-NEXT:    retl
;
; X64-NO-POPCOUNT-LABEL: ctpop_shifted_mask5:
; X64-NO-POPCOUNT:       # %bb.0:
; X64-NO-POPCOUNT-NEXT:    shrl $9, %edi
; X64-NO-POPCOUNT-NEXT:    andl $23, %edi
; X64-NO-POPCOUNT-NEXT:    imull $134480385, %edi, %eax # imm = 0x8040201
; X64-NO-POPCOUNT-NEXT:    shrl $3, %eax
; X64-NO-POPCOUNT-NEXT:    andl $286331153, %eax # imm = 0x11111111
; X64-NO-POPCOUNT-NEXT:    imull $286331153, %eax, %eax # imm = 0x11111111
; X64-NO-POPCOUNT-NEXT:    shrl $28, %eax
; X64-NO-POPCOUNT-NEXT:    retq
  %mask = and i32 %x, 11776 ; 23 << 9
  %count = tail call i32 @llvm.ctpop.i32(i32 %mask)
  ret i32 %count
}

define i32 @ctpop_mask6(i32 %x) nounwind readnone {
; X86-POPCOUNT-LABEL: ctpop_mask6:
; X86-POPCOUNT:       # %bb.0:
; X86-POPCOUNT-NEXT:    movl {{[0-9]+}}(%esp), %eax
; X86-POPCOUNT-NEXT:    andl $63, %eax
; X86-POPCOUNT-NEXT:    popcntl %eax, %eax
; X86-POPCOUNT-NEXT:    retl
;
; X64-POPCOUNT-LABEL: ctpop_mask6:
; X64-POPCOUNT:       # %bb.0:
; X64-POPCOUNT-NEXT:    andl $63, %edi
; X64-POPCOUNT-NEXT:    popcntl %edi, %eax
; X64-POPCOUNT-NEXT:    retq
;
; X86-NO-POPCOUNT-LABEL: ctpop_mask6:
; X86-NO-POPCOUNT:       # %bb.0:
; X86-NO-POPCOUNT-NEXT:    movl {{[0-9]+}}(%esp), %eax
; X86-NO-POPCOUNT-NEXT:    andl $63, %eax
; X86-NO-POPCOUNT-NEXT:    imull $134480385, %eax, %eax # imm = 0x8040201
; X86-NO-POPCOUNT-NEXT:    shrl $3, %eax
; X86-NO-POPCOUNT-NEXT:    andl $286331153, %eax # imm = 0x11111111
; X86-NO-POPCOUNT-NEXT:    imull $286331153, %eax, %eax # imm = 0x11111111
; X86-NO-POPCOUNT-NEXT:    shrl $28, %eax
; X86-NO-POPCOUNT-NEXT:    retl
;
; X64-NO-POPCOUNT-LABEL: ctpop_mask6:
; X64-NO-POPCOUNT:       # %bb.0:
; X64-NO-POPCOUNT-NEXT:    andl $63, %edi
; X64-NO-POPCOUNT-NEXT:    imull $134480385, %edi, %eax # imm = 0x8040201
; X64-NO-POPCOUNT-NEXT:    shrl $3, %eax
; X64-NO-POPCOUNT-NEXT:    andl $286331153, %eax # imm = 0x11111111
; X64-NO-POPCOUNT-NEXT:    imull $286331153, %eax, %eax # imm = 0x11111111
; X64-NO-POPCOUNT-NEXT:    shrl $28, %eax
; X64-NO-POPCOUNT-NEXT:    retq
  %mask = and i32 %x, 63
  %count = tail call i32 @llvm.ctpop.i32(i32 %mask)
  ret i32 %count
}

define i64 @ctpop_shifted_mask6(i64 %x) nounwind readnone {
; X86-POPCOUNT-LABEL: ctpop_shifted_mask6:
; X86-POPCOUNT:       # %bb.0:
; X86-POPCOUNT-NEXT:    movl $26112, %eax # imm = 0x6600
; X86-POPCOUNT-NEXT:    andl {{[0-9]+}}(%esp), %eax
; X86-POPCOUNT-NEXT:    popcntl %eax, %eax
; X86-POPCOUNT-NEXT:    xorl %edx, %edx
; X86-POPCOUNT-NEXT:    retl
;
; X64-POPCOUNT-LABEL: ctpop_shifted_mask6:
; X64-POPCOUNT:       # %bb.0:
; X64-POPCOUNT-NEXT:    andl $26112, %edi # imm = 0x6600
; X64-POPCOUNT-NEXT:    popcntl %edi, %eax
; X64-POPCOUNT-NEXT:    retq
;
; X86-NO-POPCOUNT-LABEL: ctpop_shifted_mask6:
; X86-NO-POPCOUNT:       # %bb.0:
; X86-NO-POPCOUNT-NEXT:    movl {{[0-9]+}}(%esp), %eax
; X86-NO-POPCOUNT-NEXT:    shrl $9, %eax
; X86-NO-POPCOUNT-NEXT:    andl $51, %eax
; X86-NO-POPCOUNT-NEXT:    imull $134480385, %eax, %eax # imm = 0x8040201
; X86-NO-POPCOUNT-NEXT:    shrl $3, %eax
; X86-NO-POPCOUNT-NEXT:    andl $286331153, %eax # imm = 0x11111111
; X86-NO-POPCOUNT-NEXT:    imull $286331153, %eax, %eax # imm = 0x11111111
; X86-NO-POPCOUNT-NEXT:    shrl $28, %eax
; X86-NO-POPCOUNT-NEXT:    xorl %edx, %edx
; X86-NO-POPCOUNT-NEXT:    retl
;
; X64-NO-POPCOUNT-LABEL: ctpop_shifted_mask6:
; X64-NO-POPCOUNT:       # %bb.0:
; X64-NO-POPCOUNT-NEXT:    shrl $9, %edi
; X64-NO-POPCOUNT-NEXT:    andl $51, %edi
; X64-NO-POPCOUNT-NEXT:    imull $134480385, %edi, %eax # imm = 0x8040201
; X64-NO-POPCOUNT-NEXT:    shrl $3, %eax
; X64-NO-POPCOUNT-NEXT:    andl $286331153, %eax # imm = 0x11111111
; X64-NO-POPCOUNT-NEXT:    imull $286331153, %eax, %eax # imm = 0x11111111
; X64-NO-POPCOUNT-NEXT:    shrl $28, %eax
; X64-NO-POPCOUNT-NEXT:    retq
  %mask = and i64 %x, 26112 ; 51 << 9
  %count = tail call i64 @llvm.ctpop.i64(i64 %mask)
  ret i64 %count
}

define i16 @ctpop_mask7(i16 %x) nounwind readnone {
; X86-POPCOUNT-LABEL: ctpop_mask7:
; X86-POPCOUNT:       # %bb.0:
; X86-POPCOUNT-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
; X86-POPCOUNT-NEXT:    andl $127, %eax
; X86-POPCOUNT-NEXT:    popcntl %eax, %eax
; X86-POPCOUNT-NEXT:    # kill: def $ax killed $ax killed $eax
; X86-POPCOUNT-NEXT:    retl
;
; X64-POPCOUNT-LABEL: ctpop_mask7:
; X64-POPCOUNT:       # %bb.0:
; X64-POPCOUNT-NEXT:    andl $127, %edi
; X64-POPCOUNT-NEXT:    popcntl %edi, %eax
; X64-POPCOUNT-NEXT:    # kill: def $ax killed $ax killed $eax
; X64-POPCOUNT-NEXT:    retq
;
; X86-NO-POPCOUNT-LABEL: ctpop_mask7:
; X86-NO-POPCOUNT:       # %bb.0:
; X86-NO-POPCOUNT-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
; X86-NO-POPCOUNT-NEXT:    andl $127, %eax
; X86-NO-POPCOUNT-NEXT:    imull $134480385, %eax, %eax # imm = 0x8040201
; X86-NO-POPCOUNT-NEXT:    shrl $3, %eax
; X86-NO-POPCOUNT-NEXT:    andl $286331153, %eax # imm = 0x11111111
; X86-NO-POPCOUNT-NEXT:    imull $286331153, %eax, %eax # imm = 0x11111111
; X86-NO-POPCOUNT-NEXT:    shrl $28, %eax
; X86-NO-POPCOUNT-NEXT:    # kill: def $ax killed $ax killed $eax
; X86-NO-POPCOUNT-NEXT:    retl
;
; X64-NO-POPCOUNT-LABEL: ctpop_mask7:
; X64-NO-POPCOUNT:       # %bb.0:
; X64-NO-POPCOUNT-NEXT:    andl $127, %edi
; X64-NO-POPCOUNT-NEXT:    imull $134480385, %edi, %eax # imm = 0x8040201
; X64-NO-POPCOUNT-NEXT:    shrl $3, %eax
; X64-NO-POPCOUNT-NEXT:    andl $286331153, %eax # imm = 0x11111111
; X64-NO-POPCOUNT-NEXT:    imull $286331153, %eax, %eax # imm = 0x11111111
; X64-NO-POPCOUNT-NEXT:    shrl $28, %eax
; X64-NO-POPCOUNT-NEXT:    # kill: def $ax killed $ax killed $eax
; X64-NO-POPCOUNT-NEXT:    retq
  %mask = and i16 %x, 127
  %count = tail call i16 @llvm.ctpop.i16(i16 %mask)
  ret i16 %count
}

define i32 @ctpop_shift_mask7(i32 %x) nounwind readnone {
; X86-POPCOUNT-LABEL: ctpop_shift_mask7:
; X86-POPCOUNT:       # %bb.0:
; X86-POPCOUNT-NEXT:    movl $1040384, %eax # imm = 0xFE000
; X86-POPCOUNT-NEXT:    andl {{[0-9]+}}(%esp), %eax
; X86-POPCOUNT-NEXT:    popcntl %eax, %eax
; X86-POPCOUNT-NEXT:    retl
;
; X64-POPCOUNT-LABEL: ctpop_shift_mask7:
; X64-POPCOUNT:       # %bb.0:
; X64-POPCOUNT-NEXT:    andl $1040384, %edi # imm = 0xFE000
; X64-POPCOUNT-NEXT:    popcntl %edi, %eax
; X64-POPCOUNT-NEXT:    retq
;
; X86-NO-POPCOUNT-LABEL: ctpop_shift_mask7:
; X86-NO-POPCOUNT:       # %bb.0:
; X86-NO-POPCOUNT-NEXT:    movl {{[0-9]+}}(%esp), %eax
; X86-NO-POPCOUNT-NEXT:    shrl $13, %eax
; X86-NO-POPCOUNT-NEXT:    andl $127, %eax
; X86-NO-POPCOUNT-NEXT:    imull $134480385, %eax, %eax # imm = 0x8040201
; X86-NO-POPCOUNT-NEXT:    shrl $3, %eax
; X86-NO-POPCOUNT-NEXT:    andl $286331153, %eax # imm = 0x11111111
; X86-NO-POPCOUNT-NEXT:    imull $286331153, %eax, %eax # imm = 0x11111111
; X86-NO-POPCOUNT-NEXT:    shrl $28, %eax
; X86-NO-POPCOUNT-NEXT:    retl
;
; X64-NO-POPCOUNT-LABEL: ctpop_shift_mask7:
; X64-NO-POPCOUNT:       # %bb.0:
; X64-NO-POPCOUNT-NEXT:    shrl $13, %edi
; X64-NO-POPCOUNT-NEXT:    andl $127, %edi
; X64-NO-POPCOUNT-NEXT:    imull $134480385, %edi, %eax # imm = 0x8040201
; X64-NO-POPCOUNT-NEXT:    shrl $3, %eax
; X64-NO-POPCOUNT-NEXT:    andl $286331153, %eax # imm = 0x11111111
; X64-NO-POPCOUNT-NEXT:    imull $286331153, %eax, %eax # imm = 0x11111111
; X64-NO-POPCOUNT-NEXT:    shrl $28, %eax
; X64-NO-POPCOUNT-NEXT:    retq
  %mask = and i32 %x, 1040384 ; 127 << 13
  %count = tail call i32 @llvm.ctpop.i32(i32 %mask)
  ret i32 %count
}

define i32 @ctpop_mask8(i32 %x) nounwind readnone {
; X86-POPCOUNT-LABEL: ctpop_mask8:
; X86-POPCOUNT:       # %bb.0:
; X86-POPCOUNT-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
; X86-POPCOUNT-NEXT:    popcntl %eax, %eax
; X86-POPCOUNT-NEXT:    retl
;
; X64-POPCOUNT-LABEL: ctpop_mask8:
; X64-POPCOUNT:       # %bb.0:
; X64-POPCOUNT-NEXT:    movzbl %dil, %eax
; X64-POPCOUNT-NEXT:    popcntl %eax, %eax
; X64-POPCOUNT-NEXT:    retq
;
; X86-NO-POPCOUNT-LABEL: ctpop_mask8:
; X86-NO-POPCOUNT:       # %bb.0:
; X86-NO-POPCOUNT-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
; X86-NO-POPCOUNT-NEXT:    imull $134480385, %eax, %eax # imm = 0x8040201
; X86-NO-POPCOUNT-NEXT:    shrl $3, %eax
; X86-NO-POPCOUNT-NEXT:    andl $286331153, %eax # imm = 0x11111111
; X86-NO-POPCOUNT-NEXT:    imull $286331153, %eax, %eax # imm = 0x11111111
; X86-NO-POPCOUNT-NEXT:    shrl $28, %eax
; X86-NO-POPCOUNT-NEXT:    retl
;
; X64-NO-POPCOUNT-LABEL: ctpop_mask8:
; X64-NO-POPCOUNT:       # %bb.0:
; X64-NO-POPCOUNT-NEXT:    movzbl %dil, %eax
; X64-NO-POPCOUNT-NEXT:    imull $134480385, %eax, %eax # imm = 0x8040201
; X64-NO-POPCOUNT-NEXT:    shrl $3, %eax
; X64-NO-POPCOUNT-NEXT:    andl $286331153, %eax # imm = 0x11111111
; X64-NO-POPCOUNT-NEXT:    imull $286331153, %eax, %eax # imm = 0x11111111
; X64-NO-POPCOUNT-NEXT:    shrl $28, %eax
; X64-NO-POPCOUNT-NEXT:    retq
  %mask = and i32 %x, 255
  %count = tail call i32 @llvm.ctpop.i32(i32 %mask)
  ret i32 %count
}

define i64 @ctpop_shifted_mask8(i64 %x) nounwind readnone {
; X86-POPCOUNT-LABEL: ctpop_shifted_mask8:
; X86-POPCOUNT:       # %bb.0:
; X86-POPCOUNT-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
; X86-POPCOUNT-NEXT:    popcntl %eax, %eax
; X86-POPCOUNT-NEXT:    xorl %edx, %edx
; X86-POPCOUNT-NEXT:    retl
;
; X64-POPCOUNT-LABEL: ctpop_shifted_mask8:
; X64-POPCOUNT:       # %bb.0:
; X64-POPCOUNT-NEXT:    andl $65280, %edi # imm = 0xFF00
; X64-POPCOUNT-NEXT:    popcntl %edi, %eax
; X64-POPCOUNT-NEXT:    retq
;
; X86-NO-POPCOUNT-LABEL: ctpop_shifted_mask8:
; X86-NO-POPCOUNT:       # %bb.0:
; X86-NO-POPCOUNT-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
; X86-NO-POPCOUNT-NEXT:    imull $134480385, %eax, %eax # imm = 0x8040201
; X86-NO-POPCOUNT-NEXT:    shrl $3, %eax
; X86-NO-POPCOUNT-NEXT:    andl $286331153, %eax # imm = 0x11111111
; X86-NO-POPCOUNT-NEXT:    imull $286331153, %eax, %eax # imm = 0x11111111
; X86-NO-POPCOUNT-NEXT:    shrl $28, %eax
; X86-NO-POPCOUNT-NEXT:    xorl %edx, %edx
; X86-NO-POPCOUNT-NEXT:    retl
;
; X64-NO-POPCOUNT-LABEL: ctpop_shifted_mask8:
; X64-NO-POPCOUNT:       # %bb.0:
; X64-NO-POPCOUNT-NEXT:    movq %rdi, %rax
; X64-NO-POPCOUNT-NEXT:    movzbl %ah, %eax
; X64-NO-POPCOUNT-NEXT:    imull $134480385, %eax, %eax # imm = 0x8040201
; X64-NO-POPCOUNT-NEXT:    shrl $3, %eax
; X64-NO-POPCOUNT-NEXT:    andl $286331153, %eax # imm = 0x11111111
; X64-NO-POPCOUNT-NEXT:    imull $286331153, %eax, %eax # imm = 0x11111111
; X64-NO-POPCOUNT-NEXT:    shrl $28, %eax
; X64-NO-POPCOUNT-NEXT:    retq
  %mask = and i64 %x, 65280 ; 255 << 8
  %count = tail call i64 @llvm.ctpop.i64(i64 %mask)
  ret i64 %count
}

define i32 @ctpop_mask16(i32 %x) nounwind readnone {
; X86-POPCOUNT-LABEL: ctpop_mask16:
; X86-POPCOUNT:       # %bb.0:
; X86-POPCOUNT-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
; X86-POPCOUNT-NEXT:    popcntl %eax, %eax
; X86-POPCOUNT-NEXT:    retl
;
; X64-POPCOUNT-LABEL: ctpop_mask16:
; X64-POPCOUNT:       # %bb.0:
; X64-POPCOUNT-NEXT:    movzwl %di, %eax
; X64-POPCOUNT-NEXT:    popcntl %eax, %eax
; X64-POPCOUNT-NEXT:    retq
;
; X86-NO-POPCOUNT-LABEL: ctpop_mask16:
; X86-NO-POPCOUNT:       # %bb.0:
; X86-NO-POPCOUNT-NEXT:    movzwl {{[0-9]+}}(%esp), %eax
; X86-NO-POPCOUNT-NEXT:    movl %eax, %ecx
; X86-NO-POPCOUNT-NEXT:    shrl %ecx
; X86-NO-POPCOUNT-NEXT:    andl $21845, %ecx # imm = 0x5555
; X86-NO-POPCOUNT-NEXT:    subl %ecx, %eax
; X86-NO-POPCOUNT-NEXT:    movl %eax, %ecx
; X86-NO-POPCOUNT-NEXT:    andl $858993459, %ecx # imm = 0x33333333
; X86-NO-POPCOUNT-NEXT:    shrl $2, %eax
; X86-NO-POPCOUNT-NEXT:    andl $858993459, %eax # imm = 0x33333333
; X86-NO-POPCOUNT-NEXT:    addl %ecx, %eax
; X86-NO-POPCOUNT-NEXT:    movl %eax, %ecx
; X86-NO-POPCOUNT-NEXT:    shrl $4, %ecx
; X86-NO-POPCOUNT-NEXT:    addl %eax, %ecx
; X86-NO-POPCOUNT-NEXT:    andl $252645135, %ecx # imm = 0xF0F0F0F
; X86-NO-POPCOUNT-NEXT:    imull $16843009, %ecx, %eax # imm = 0x1010101
; X86-NO-POPCOUNT-NEXT:    shrl $24, %eax
; X86-NO-POPCOUNT-NEXT:    retl
;
; X64-NO-POPCOUNT-LABEL: ctpop_mask16:
; X64-NO-POPCOUNT:       # %bb.0:
; X64-NO-POPCOUNT-NEXT:    movzwl %di, %eax
; X64-NO-POPCOUNT-NEXT:    shrl %edi
; X64-NO-POPCOUNT-NEXT:    andl $21845, %edi # imm = 0x5555
; X64-NO-POPCOUNT-NEXT:    subl %edi, %eax
; X64-NO-POPCOUNT-NEXT:    movl %eax, %ecx
; X64-NO-POPCOUNT-NEXT:    andl $858993459, %ecx # imm = 0x33333333
; X64-NO-POPCOUNT-NEXT:    shrl $2, %eax
; X64-NO-POPCOUNT-NEXT:    andl $858993459, %eax # imm = 0x33333333
; X64-NO-POPCOUNT-NEXT:    addl %ecx, %eax
; X64-NO-POPCOUNT-NEXT:    movl %eax, %ecx
; X64-NO-POPCOUNT-NEXT:    shrl $4, %ecx
; X64-NO-POPCOUNT-NEXT:    addl %eax, %ecx
; X64-NO-POPCOUNT-NEXT:    andl $252645135, %ecx # imm = 0xF0F0F0F
; X64-NO-POPCOUNT-NEXT:    imull $16843009, %ecx, %eax # imm = 0x1010101
; X64-NO-POPCOUNT-NEXT:    shrl $24, %eax
; X64-NO-POPCOUNT-NEXT:    retq
  %mask = and i32 %x, 65535
  %count = tail call i32 @llvm.ctpop.i32(i32 %mask)
  ret i32 %count
}

define i64 @ctpop_shifted_mask16(i64 %x) nounwind readnone {
; X86-POPCOUNT-LABEL: ctpop_shifted_mask16:
; X86-POPCOUNT:       # %bb.0:
; X86-POPCOUNT-NEXT:    movl $524280, %eax # imm = 0x7FFF8
; X86-POPCOUNT-NEXT:    andl {{[0-9]+}}(%esp), %eax
; X86-POPCOUNT-NEXT:    popcntl %eax, %eax
; X86-POPCOUNT-NEXT:    xorl %edx, %edx
; X86-POPCOUNT-NEXT:    retl
;
; X64-POPCOUNT-LABEL: ctpop_shifted_mask16:
; X64-POPCOUNT:       # %bb.0:
; X64-POPCOUNT-NEXT:    movabsq $2251765453946880, %rax # imm = 0x7FFF800000000
; X64-POPCOUNT-NEXT:    andq %rdi, %rax
; X64-POPCOUNT-NEXT:    popcntq %rax, %rax
; X64-POPCOUNT-NEXT:    retq
;
; X86-NO-POPCOUNT-LABEL: ctpop_shifted_mask16:
; X86-NO-POPCOUNT:       # %bb.0:
; X86-NO-POPCOUNT-NEXT:    movl {{[0-9]+}}(%esp), %ecx
; X86-NO-POPCOUNT-NEXT:    movl %ecx, %eax
; X86-NO-POPCOUNT-NEXT:    andl $524280, %eax # imm = 0x7FFF8
; X86-NO-POPCOUNT-NEXT:    shrl %ecx
; X86-NO-POPCOUNT-NEXT:    andl $87380, %ecx # imm = 0x15554
; X86-NO-POPCOUNT-NEXT:    subl %ecx, %eax
; X86-NO-POPCOUNT-NEXT:    movl %eax, %ecx
; X86-NO-POPCOUNT-NEXT:    andl $858993456, %ecx # imm = 0x33333330
; X86-NO-POPCOUNT-NEXT:    shrl $2, %eax
; X86-NO-POPCOUNT-NEXT:    andl $858993459, %eax # imm = 0x33333333
; X86-NO-POPCOUNT-NEXT:    addl %ecx, %eax
; X86-NO-POPCOUNT-NEXT:    movl %eax, %ecx
; X86-NO-POPCOUNT-NEXT:    shrl $4, %ecx
; X86-NO-POPCOUNT-NEXT:    addl %eax, %ecx
; X86-NO-POPCOUNT-NEXT:    andl $252645135, %ecx # imm = 0xF0F0F0F
; X86-NO-POPCOUNT-NEXT:    imull $16843009, %ecx, %eax # imm = 0x1010101
; X86-NO-POPCOUNT-NEXT:    shrl $24, %eax
; X86-NO-POPCOUNT-NEXT:    xorl %edx, %edx
; X86-NO-POPCOUNT-NEXT:    retl
;
; X64-NO-POPCOUNT-LABEL: ctpop_shifted_mask16:
; X64-NO-POPCOUNT:       # %bb.0:
; X64-NO-POPCOUNT-NEXT:    movabsq $2251765453946880, %rax # imm = 0x7FFF800000000
; X64-NO-POPCOUNT-NEXT:    andq %rdi, %rax
; X64-NO-POPCOUNT-NEXT:    shrq %rdi
; X64-NO-POPCOUNT-NEXT:    movabsq $375294242324480, %rcx # imm = 0x1555400000000
; X64-NO-POPCOUNT-NEXT:    andq %rdi, %rcx
; X64-NO-POPCOUNT-NEXT:    subq %rcx, %rax
; X64-NO-POPCOUNT-NEXT:    movabsq $3689348800998014976, %rcx # imm = 0x3333333000000000
; X64-NO-POPCOUNT-NEXT:    andq %rax, %rcx
; X64-NO-POPCOUNT-NEXT:    shrq $2, %rax
; X64-NO-POPCOUNT-NEXT:    movabsq $3689348813882916864, %rdx # imm = 0x3333333300000000
; X64-NO-POPCOUNT-NEXT:    andq %rax, %rdx
; X64-NO-POPCOUNT-NEXT:    addq %rcx, %rdx
; X64-NO-POPCOUNT-NEXT:    movq %rdx, %rax
; X64-NO-POPCOUNT-NEXT:    shrq $4, %rax
; X64-NO-POPCOUNT-NEXT:    addq %rdx, %rax
; X64-NO-POPCOUNT-NEXT:    movabsq $1085102592571150095, %rcx # imm = 0xF0F0F0F0F0F0F0F
; X64-NO-POPCOUNT-NEXT:    andq %rax, %rcx
; X64-NO-POPCOUNT-NEXT:    movabsq $72340172838076673, %rax # imm = 0x101010101010101
; X64-NO-POPCOUNT-NEXT:    imulq %rcx, %rax
; X64-NO-POPCOUNT-NEXT:    shrq $56, %rax
; X64-NO-POPCOUNT-NEXT:    retq
  %mask = and i64 %x, 2251765453946880 ; 65535 << 35
  %count = tail call i64 @llvm.ctpop.i64(i64 %mask)
  ret i64 %count
}