1; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 4 2; RUN: opt < %s -passes=memcpyopt -S -verify-memoryssa | FileCheck %s 3 4%buf = type [9 x i8] 5 6; We can forward `memcpy` because the copy location are the same, 7define void @forward_offset(ptr %src, ptr %dest) { 8; CHECK-LABEL: define void @forward_offset( 9; CHECK-SAME: ptr [[SRC:%.*]], ptr [[DEST:%.*]]) { 10; CHECK-NEXT: [[DEP_DEST:%.*]] = alloca [9 x i8], align 1 11; CHECK-NEXT: call void @llvm.memcpy.p0.p0.i64(ptr align 1 [[DEP_DEST]], ptr align 1 [[SRC]], i64 7, i1 false) 12; CHECK-NEXT: [[SRC_OFFSET:%.*]] = getelementptr inbounds i8, ptr [[DEP_DEST]], i64 1 13; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i8, ptr [[SRC]], i64 1 14; CHECK-NEXT: call void @llvm.memmove.p0.p0.i64(ptr align 1 [[DEST]], ptr align 1 [[TMP1]], i64 6, i1 false) 15; CHECK-NEXT: ret void 16; 17 %cpy_tmp = alloca %buf, align 1 18 call void @llvm.memcpy.p0.p0.i64(ptr align 1 %cpy_tmp, ptr align 1 %src, i64 7, i1 false) 19 %cpy_tmp_offset = getelementptr inbounds i8, ptr %cpy_tmp, i64 1 20 call void @llvm.memcpy.p0.p0.i64(ptr align 1 %dest, ptr align 1 %cpy_tmp_offset, i64 6, i1 false) 21 ret void 22} 23 24; We need to update the align value of the source of `memcpy` when forwarding. 25define void @forward_offset_align(ptr %src, ptr %dest) { 26; CHECK-LABEL: define void @forward_offset_align( 27; CHECK-SAME: ptr [[SRC:%.*]], ptr [[DEST:%.*]]) { 28; CHECK-NEXT: [[DEP_DEST:%.*]] = alloca [9 x i8], align 1 29; CHECK-NEXT: call void @llvm.memcpy.p0.p0.i64(ptr align 1 [[DEP_DEST]], ptr align 4 [[SRC]], i64 9, i1 false) 30; CHECK-NEXT: [[TMP_OFFSET:%.*]] = getelementptr inbounds i8, ptr [[DEP_DEST]], i64 3 31; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i8, ptr [[SRC]], i64 3 32; CHECK-NEXT: call void @llvm.memmove.p0.p0.i64(ptr align 1 [[DEST]], ptr align 1 [[TMP1]], i64 5, i1 false) 33; CHECK-NEXT: ret void 34; 35 %cpy_tmp = alloca %buf, align 1 36 call void @llvm.memcpy.p0.p0.i64(ptr align 1 %cpy_tmp, ptr align 4 %src, i64 9, i1 false) 37 %cpy_tmp_offset = getelementptr inbounds i8, ptr %cpy_tmp, i64 3 38 call void @llvm.memcpy.p0.p0.i64(ptr align 1 %dest, ptr align 1 %cpy_tmp_offset, i64 5, i1 false) 39 ret void 40} 41 42; We can change the align value to 2 when forwarding. 43define void @forward_offset_align_2(ptr %src, ptr %dest) { 44; CHECK-LABEL: define void @forward_offset_align_2( 45; CHECK-SAME: ptr [[SRC:%.*]], ptr [[DEST:%.*]]) { 46; CHECK-NEXT: [[DEP_DEST:%.*]] = alloca [9 x i8], align 1 47; CHECK-NEXT: call void @llvm.memcpy.p0.p0.i64(ptr align 1 [[DEP_DEST]], ptr align 4 [[SRC]], i64 9, i1 false) 48; CHECK-NEXT: [[TMP_OFFSET:%.*]] = getelementptr inbounds i8, ptr [[DEP_DEST]], i64 2 49; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i8, ptr [[SRC]], i64 2 50; CHECK-NEXT: call void @llvm.memmove.p0.p0.i64(ptr align 1 [[DEST]], ptr align 2 [[TMP1]], i64 6, i1 false) 51; CHECK-NEXT: ret void 52; 53 %cpy_tmp = alloca %buf, align 1 54 call void @llvm.memcpy.p0.p0.i64(ptr align 1 %cpy_tmp, ptr align 4 %src, i64 9, i1 false) 55 %cpy_tmp_offset = getelementptr inbounds i8, ptr %cpy_tmp, i64 2 56 call void @llvm.memcpy.p0.p0.i64(ptr align 1 %dest, ptr align 1 %cpy_tmp_offset, i64 6, i1 false) 57 ret void 58} 59 60; If the copy destination can be used as the copy source, we don't need to create a GEP instruction. 61define void @forward_offset_without_gep(ptr %src) { 62; CHECK-LABEL: define void @forward_offset_without_gep( 63; CHECK-SAME: ptr [[SRC:%.*]]) { 64; CHECK-NEXT: [[TMP:%.*]] = alloca [9 x i8], align 1 65; CHECK-NEXT: call void @llvm.memcpy.p0.p0.i64(ptr align 1 [[TMP]], ptr align 1 [[SRC]], i64 7, i1 false) 66; CHECK-NEXT: [[TMP_OFFSET:%.*]] = getelementptr inbounds i8, ptr [[TMP]], i64 1 67; CHECK-NEXT: [[DEST:%.*]] = getelementptr inbounds i8, ptr [[SRC]], i64 1 68; CHECK-NEXT: ret void 69; 70 %cpy_tmp = alloca %buf, align 1 71 call void @llvm.memcpy.p0.p0.i64(ptr align 1 %cpy_tmp, ptr align 1 %src, i64 7, i1 false) 72 %cpy_tmp_offset = getelementptr inbounds i8, ptr %cpy_tmp, i64 1 73 %dest = getelementptr inbounds i8, ptr %src, i64 1 74 call void @llvm.memcpy.p0.p0.i64(ptr align 1 %dest, ptr align 1 %cpy_tmp_offset, i64 6, i1 false) 75 ret void 76} 77 78; We need to create a GEP instruction when forwarding. 79define void @forward_offset_with_gep(ptr %src) { 80; CHECK-LABEL: define void @forward_offset_with_gep( 81; CHECK-SAME: ptr [[SRC:%.*]]) { 82; CHECK-NEXT: [[DEP_DEST:%.*]] = alloca [9 x i8], align 1 83; CHECK-NEXT: call void @llvm.memcpy.p0.p0.i64(ptr align 1 [[DEP_DEST]], ptr align 1 [[SRC]], i64 7, i1 false) 84; CHECK-NEXT: [[TMP_OFFSET:%.*]] = getelementptr inbounds i8, ptr [[DEP_DEST]], i64 1 85; CHECK-NEXT: [[DEST:%.*]] = getelementptr inbounds i8, ptr [[SRC]], i64 2 86; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i8, ptr [[SRC]], i64 1 87; CHECK-NEXT: call void @llvm.memmove.p0.p0.i64(ptr align 1 [[DEST]], ptr align 1 [[TMP1]], i64 6, i1 false) 88; CHECK-NEXT: ret void 89; 90 %cpy_tmp = alloca %buf, align 1 91 call void @llvm.memcpy.p0.p0.i64(ptr align 1 %cpy_tmp, ptr align 1 %src, i64 7, i1 false) 92 %cpy_tmp_offset = getelementptr inbounds i8, ptr %cpy_tmp, i64 1 93 %dest = getelementptr inbounds i8, ptr %src, i64 2 94 call void @llvm.memcpy.p0.p0.i64(ptr align 1 %dest, ptr align 1 %cpy_tmp_offset, i64 6, i1 false) 95 ret void 96} 97 98; Make sure we pass the right parameters when calling `memcpy`. 99define void @forward_offset_memcpy(ptr %src, ptr %dest) { 100; CHECK-LABEL: define void @forward_offset_memcpy( 101; CHECK-SAME: ptr [[SRC:%.*]], ptr [[DEST:%.*]]) { 102; CHECK-NEXT: [[DEP_DEST:%.*]] = alloca [9 x i8], align 1 103; CHECK-NEXT: call void @llvm.memcpy.p0.p0.i64(ptr align 1 [[DEP_DEST]], ptr align 1 [[SRC]], i64 7, i1 false) 104; CHECK-NEXT: [[TMP_OFFSET:%.*]] = getelementptr inbounds i8, ptr [[DEP_DEST]], i64 1 105; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i8, ptr [[SRC]], i64 1 106; CHECK-NEXT: call void @llvm.memmove.p0.p0.i64(ptr align 1 [[DEST]], ptr align 1 [[TMP1]], i64 6, i1 false) 107; CHECK-NEXT: call void @use(ptr [[DEST]]) 108; CHECK-NEXT: ret void 109; 110 %cpy_tmp = alloca %buf, align 1 111 call void @llvm.memcpy.p0.p0.i64(ptr align 1 %cpy_tmp, ptr align 1 %src, i64 7, i1 false) 112 %cpy_tmp_offset = getelementptr inbounds i8, ptr %cpy_tmp, i64 1 113 call void @llvm.memcpy.p0.p0.i64(ptr align 1 %dest, ptr align 1 %cpy_tmp_offset, i64 6, i1 false) 114 call void @use(ptr %dest) 115 ret void 116} 117 118; Make sure we pass the right parameters when calling `memcpy.inline`. 119define void @forward_offset_memcpy_inline(ptr %src, ptr %dest) { 120; CHECK-LABEL: define void @forward_offset_memcpy_inline( 121; CHECK-SAME: ptr [[SRC:%.*]], ptr [[DEST:%.*]]) { 122; CHECK-NEXT: [[DEP_DEST:%.*]] = alloca [9 x i8], align 1 123; CHECK-NEXT: call void @llvm.memcpy.p0.p0.i64(ptr align 1 [[DEP_DEST]], ptr align 1 [[SRC]], i64 7, i1 false) 124; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i8, ptr [[DEP_DEST]], i64 1 125; CHECK-NEXT: call void @llvm.memcpy.inline.p0.p0.i64(ptr align 1 [[DEST]], ptr align 1 [[TMP1]], i64 6, i1 false) 126; CHECK-NEXT: call void @use(ptr [[DEST]]) 127; CHECK-NEXT: ret void 128; 129 %cpy_tmp = alloca %buf, align 1 130 call void @llvm.memcpy.p0.p0.i64(ptr align 1 %cpy_tmp, ptr align 1 %src, i64 7, i1 false) 131 %cpy_tmp_offset = getelementptr inbounds i8, ptr %cpy_tmp, i64 1 132 call void @llvm.memcpy.inline.p0.p0.i64(ptr align 1 %dest, ptr align 1 %cpy_tmp_offset, i64 6, i1 false) 133 call void @use(ptr %dest) 134 ret void 135} 136 137; We cannot forward `memcpy` because it exceeds the size of `memcpy` it depends on. 138define void @do_not_forward_oversize_offset(ptr %src, ptr %dest) { 139; CHECK-LABEL: define void @do_not_forward_oversize_offset( 140; CHECK-SAME: ptr [[SRC:%.*]], ptr [[DEST:%.*]]) { 141; CHECK-NEXT: [[DEP_DEST:%.*]] = alloca [9 x i8], align 1 142; CHECK-NEXT: call void @llvm.memcpy.p0.p0.i64(ptr align 1 [[DEP_DEST]], ptr align 1 [[SRC]], i64 6, i1 false) 143; CHECK-NEXT: [[TMP_OFFSET:%.*]] = getelementptr inbounds i8, ptr [[DEP_DEST]], i64 1 144; CHECK-NEXT: call void @llvm.memcpy.p0.p0.i64(ptr align 1 [[DEST]], ptr align 1 [[TMP_OFFSET]], i64 6, i1 false) 145; CHECK-NEXT: ret void 146; 147 %cpy_tmp = alloca %buf, align 1 148 call void @llvm.memcpy.p0.p0.i64(ptr align 1 %cpy_tmp, ptr align 1 %src, i64 6, i1 false) 149 %cpy_tmp_offset = getelementptr inbounds i8, ptr %cpy_tmp, i64 1 150 call void @llvm.memcpy.p0.p0.i64(ptr align 1 %dest, ptr align 1 %cpy_tmp_offset, i64 6, i1 false) 151 ret void 152} 153 154; We can forward `memcpy` because the write operation does not corrupt the location to be copied. 155define void @forward_offset_and_store(ptr %src, ptr %dest) { 156; CHECK-LABEL: define void @forward_offset_and_store( 157; CHECK-SAME: ptr [[SRC:%.*]], ptr [[DEST:%.*]]) { 158; CHECK-NEXT: [[DEP_DEST:%.*]] = alloca [9 x i8], align 1 159; CHECK-NEXT: call void @llvm.memcpy.p0.p0.i64(ptr align 1 [[DEP_DEST]], ptr align 1 [[SRC]], i64 7, i1 false) 160; CHECK-NEXT: store i8 1, ptr [[SRC]], align 1 161; CHECK-NEXT: [[DEP_SRC_END:%.*]] = getelementptr inbounds i8, ptr [[SRC]], i64 6 162; CHECK-NEXT: store i8 1, ptr [[DEP_SRC_END]], align 1 163; CHECK-NEXT: [[TMP_OFFSET:%.*]] = getelementptr inbounds i8, ptr [[DEP_DEST]], i64 1 164; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i8, ptr [[SRC]], i64 1 165; CHECK-NEXT: call void @llvm.memmove.p0.p0.i64(ptr align 1 [[DEST]], ptr align 1 [[TMP1]], i64 5, i1 false) 166; CHECK-NEXT: ret void 167; 168 %cpy_tmp = alloca %buf, align 1 169 call void @llvm.memcpy.p0.p0.i64(ptr align 1 %cpy_tmp, ptr align 1 %src, i64 7, i1 false) 170 store i8 1, ptr %src, align 1 171 %src_end = getelementptr inbounds i8, ptr %src, i64 6 172 store i8 1, ptr %src_end, align 1 173 %cpy_tmp_offset = getelementptr inbounds i8, ptr %cpy_tmp, i64 1 174 call void @llvm.memcpy.p0.p0.i64(ptr align 1 %dest, ptr align 1 %cpy_tmp_offset, i64 5, i1 false) 175 ret void 176} 177 178; We cannot forward `memcpy` because the write operation alters the location to be copied. 179; Also, make sure we have removed the GEP instruction that was created temporarily. 180define void @do_not_forward_offset_and_store(ptr %src, ptr %dest) { 181; CHECK-LABEL: define void @do_not_forward_offset_and_store( 182; CHECK-SAME: ptr [[SRC:%.*]], ptr [[DEST:%.*]]) { 183; CHECK-NEXT: [[DEP_DEST:%.*]] = alloca [9 x i8], align 1 184; CHECK-NEXT: call void @llvm.memcpy.p0.p0.i64(ptr align 1 [[DEP_DEST]], ptr align 1 [[SRC]], i64 7, i1 false) 185; CHECK-NEXT: [[DEP:%.*]] = getelementptr inbounds i8, ptr [[SRC]], i64 1 186; CHECK-NEXT: store i8 1, ptr [[DEP]], align 1 187; CHECK-NEXT: [[TMP_OFFSET:%.*]] = getelementptr inbounds i8, ptr [[DEP_DEST]], i64 1 188; CHECK-NEXT: call void @llvm.memcpy.p0.p0.i64(ptr align 1 [[DEST]], ptr align 1 [[TMP_OFFSET]], i64 5, i1 false) 189; CHECK-NEXT: ret void 190; 191 %cpy_tmp = alloca %buf, align 1 192 call void @llvm.memcpy.p0.p0.i64(ptr align 1 %cpy_tmp, ptr align 1 %src, i64 7, i1 false) 193 %src_offset = getelementptr inbounds i8, ptr %src, i64 1 194 store i8 1, ptr %src_offset, align 1 195 %cpy_tmp_offset = getelementptr inbounds i8, ptr %cpy_tmp, i64 1 196 call void @llvm.memcpy.p0.p0.i64(ptr align 1 %dest, ptr align 1 %cpy_tmp_offset, i64 5, i1 false) 197 ret void 198} 199 200; Make sure we don't crash when the copy source is a constant. 201@buf = external global [32 x i8] 202 203define void @pr98675(ptr noalias %p1, ptr noalias %p2) { 204; CHECK-LABEL: define void @pr98675( 205; CHECK-SAME: ptr noalias [[P1:%.*]], ptr noalias [[P2:%.*]]) { 206; CHECK-NEXT: call void @llvm.memcpy.p0.p0.i64(ptr [[P1]], ptr @buf, i64 26, i1 false) 207; CHECK-NEXT: [[GEP:%.*]] = getelementptr i8, ptr [[P1]], i64 10 208; CHECK-NEXT: call void @llvm.memcpy.p0.p0.i64(ptr [[P2]], ptr getelementptr inbounds (i8, ptr @buf, i64 10), i64 1, i1 false) 209; CHECK-NEXT: ret void 210; 211 call void @llvm.memcpy.p0.p0.i64(ptr %p1, ptr @buf, i64 26, i1 false) 212 %gep = getelementptr i8, ptr %p1, i64 10 213 call void @llvm.memmove.p0.p0.i64(ptr %p2, ptr %gep, i64 1, i1 false) 214 ret void 215} 216 217declare void @use(ptr) 218 219declare void @llvm.memcpy.p0.p0.i64(ptr nocapture, ptr nocapture, i64, i1) 220declare void @llvm.memcpy.inline.p0.p0.i64(ptr nocapture, ptr nocapture, i64, i1) 221