1 //===--- Descriptor.cpp - Types for the constexpr VM ------------*- C++ -*-===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 9 #include "Descriptor.h" 10 #include "Boolean.h" 11 #include "Floating.h" 12 #include "FunctionPointer.h" 13 #include "IntegralAP.h" 14 #include "MemberPointer.h" 15 #include "Pointer.h" 16 #include "PrimType.h" 17 #include "Record.h" 18 19 using namespace clang; 20 using namespace clang::interp; 21 22 template <typename T> 23 static void ctorTy(Block *, std::byte *Ptr, bool, bool, bool, bool, 24 const Descriptor *) { 25 new (Ptr) T(); 26 } 27 28 template <typename T> 29 static void dtorTy(Block *, std::byte *Ptr, const Descriptor *) { 30 reinterpret_cast<T *>(Ptr)->~T(); 31 } 32 33 template <typename T> 34 static void moveTy(Block *, const std::byte *Src, std::byte *Dst, 35 const Descriptor *) { 36 // FIXME: Get rid of the const_cast. 37 auto *SrcPtr = reinterpret_cast<T *>(const_cast<std::byte *>(Src)); 38 auto *DstPtr = reinterpret_cast<T *>(Dst); 39 new (DstPtr) T(std::move(*SrcPtr)); 40 } 41 42 template <typename T> 43 static void ctorArrayTy(Block *, std::byte *Ptr, bool, bool, bool, bool, 44 const Descriptor *D) { 45 new (Ptr) InitMapPtr(std::nullopt); 46 47 Ptr += sizeof(InitMapPtr); 48 for (unsigned I = 0, NE = D->getNumElems(); I < NE; ++I) { 49 new (&reinterpret_cast<T *>(Ptr)[I]) T(); 50 } 51 } 52 53 template <typename T> 54 static void dtorArrayTy(Block *, std::byte *Ptr, const Descriptor *D) { 55 InitMapPtr &IMP = *reinterpret_cast<InitMapPtr *>(Ptr); 56 57 if (IMP) 58 IMP = std::nullopt; 59 Ptr += sizeof(InitMapPtr); 60 for (unsigned I = 0, NE = D->getNumElems(); I < NE; ++I) { 61 reinterpret_cast<T *>(Ptr)[I].~T(); 62 } 63 } 64 65 template <typename T> 66 static void moveArrayTy(Block *, const std::byte *Src, std::byte *Dst, 67 const Descriptor *D) { 68 // FIXME: Get rid of the const_cast. 69 InitMapPtr &SrcIMP = 70 *reinterpret_cast<InitMapPtr *>(const_cast<std::byte *>(Src)); 71 if (SrcIMP) { 72 // We only ever invoke the moveFunc when moving block contents to a 73 // DeadBlock. DeadBlocks don't need InitMaps, so we destroy them here. 74 SrcIMP = std::nullopt; 75 } 76 Src += sizeof(InitMapPtr); 77 Dst += sizeof(InitMapPtr); 78 for (unsigned I = 0, NE = D->getNumElems(); I < NE; ++I) { 79 auto *SrcPtr = &reinterpret_cast<T *>(const_cast<std::byte *>(Src))[I]; 80 auto *DstPtr = &reinterpret_cast<T *>(Dst)[I]; 81 new (DstPtr) T(std::move(*SrcPtr)); 82 } 83 } 84 85 static void ctorArrayDesc(Block *B, std::byte *Ptr, bool IsConst, 86 bool IsMutable, bool IsActive, bool InUnion, 87 const Descriptor *D) { 88 const unsigned NumElems = D->getNumElems(); 89 const unsigned ElemSize = 90 D->ElemDesc->getAllocSize() + sizeof(InlineDescriptor); 91 92 unsigned ElemOffset = 0; 93 for (unsigned I = 0; I < NumElems; ++I, ElemOffset += ElemSize) { 94 auto *ElemPtr = Ptr + ElemOffset; 95 auto *Desc = reinterpret_cast<InlineDescriptor *>(ElemPtr); 96 auto *ElemLoc = reinterpret_cast<std::byte *>(Desc + 1); 97 auto *SD = D->ElemDesc; 98 99 Desc->Offset = ElemOffset + sizeof(InlineDescriptor); 100 Desc->Desc = SD; 101 Desc->IsInitialized = true; 102 Desc->IsBase = false; 103 Desc->IsActive = IsActive; 104 Desc->IsConst = IsConst || D->IsConst; 105 Desc->IsFieldMutable = IsMutable || D->IsMutable; 106 Desc->InUnion = InUnion; 107 108 if (auto Fn = D->ElemDesc->CtorFn) 109 Fn(B, ElemLoc, Desc->IsConst, Desc->IsFieldMutable, IsActive, 110 Desc->InUnion || SD->isUnion(), D->ElemDesc); 111 } 112 } 113 114 static void dtorArrayDesc(Block *B, std::byte *Ptr, const Descriptor *D) { 115 const unsigned NumElems = D->getNumElems(); 116 const unsigned ElemSize = 117 D->ElemDesc->getAllocSize() + sizeof(InlineDescriptor); 118 119 unsigned ElemOffset = 0; 120 for (unsigned I = 0; I < NumElems; ++I, ElemOffset += ElemSize) { 121 auto *ElemPtr = Ptr + ElemOffset; 122 auto *Desc = reinterpret_cast<InlineDescriptor *>(ElemPtr); 123 auto *ElemLoc = reinterpret_cast<std::byte *>(Desc + 1); 124 if (auto Fn = D->ElemDesc->DtorFn) 125 Fn(B, ElemLoc, D->ElemDesc); 126 } 127 } 128 129 static void moveArrayDesc(Block *B, const std::byte *Src, std::byte *Dst, 130 const Descriptor *D) { 131 const unsigned NumElems = D->getNumElems(); 132 const unsigned ElemSize = 133 D->ElemDesc->getAllocSize() + sizeof(InlineDescriptor); 134 135 unsigned ElemOffset = 0; 136 for (unsigned I = 0; I < NumElems; ++I, ElemOffset += ElemSize) { 137 const auto *SrcPtr = Src + ElemOffset; 138 auto *DstPtr = Dst + ElemOffset; 139 140 const auto *SrcDesc = reinterpret_cast<const InlineDescriptor *>(SrcPtr); 141 const auto *SrcElemLoc = reinterpret_cast<const std::byte *>(SrcDesc + 1); 142 auto *DstDesc = reinterpret_cast<InlineDescriptor *>(DstPtr); 143 auto *DstElemLoc = reinterpret_cast<std::byte *>(DstDesc + 1); 144 145 *DstDesc = *SrcDesc; 146 if (auto Fn = D->ElemDesc->MoveFn) 147 Fn(B, SrcElemLoc, DstElemLoc, D->ElemDesc); 148 } 149 } 150 151 static void initField(Block *B, std::byte *Ptr, bool IsConst, bool IsMutable, 152 bool IsActive, bool IsUnionField, bool InUnion, 153 const Descriptor *D, unsigned FieldOffset) { 154 auto *Desc = reinterpret_cast<InlineDescriptor *>(Ptr + FieldOffset) - 1; 155 Desc->Offset = FieldOffset; 156 Desc->Desc = D; 157 Desc->IsInitialized = D->IsArray; 158 Desc->IsBase = false; 159 Desc->IsActive = IsActive && !IsUnionField; 160 Desc->InUnion = InUnion; 161 Desc->IsConst = IsConst || D->IsConst; 162 Desc->IsFieldMutable = IsMutable || D->IsMutable; 163 164 if (auto Fn = D->CtorFn) 165 Fn(B, Ptr + FieldOffset, Desc->IsConst, Desc->IsFieldMutable, 166 Desc->IsActive, InUnion || D->isUnion(), D); 167 } 168 169 static void initBase(Block *B, std::byte *Ptr, bool IsConst, bool IsMutable, 170 bool IsActive, bool InUnion, const Descriptor *D, 171 unsigned FieldOffset, bool IsVirtualBase) { 172 assert(D); 173 assert(D->ElemRecord); 174 assert(!D->ElemRecord->isUnion()); // Unions cannot be base classes. 175 176 auto *Desc = reinterpret_cast<InlineDescriptor *>(Ptr + FieldOffset) - 1; 177 Desc->Offset = FieldOffset; 178 Desc->Desc = D; 179 Desc->IsInitialized = D->IsArray; 180 Desc->IsBase = true; 181 Desc->IsVirtualBase = IsVirtualBase; 182 Desc->IsActive = IsActive && !InUnion; 183 Desc->IsConst = IsConst || D->IsConst; 184 Desc->IsFieldMutable = IsMutable || D->IsMutable; 185 Desc->InUnion = InUnion; 186 187 for (const auto &V : D->ElemRecord->bases()) 188 initBase(B, Ptr + FieldOffset, IsConst, IsMutable, IsActive, InUnion, 189 V.Desc, V.Offset, false); 190 for (const auto &F : D->ElemRecord->fields()) 191 initField(B, Ptr + FieldOffset, IsConst, IsMutable, IsActive, InUnion, 192 InUnion, F.Desc, F.Offset); 193 } 194 195 static void ctorRecord(Block *B, std::byte *Ptr, bool IsConst, bool IsMutable, 196 bool IsActive, bool InUnion, const Descriptor *D) { 197 for (const auto &V : D->ElemRecord->bases()) 198 initBase(B, Ptr, IsConst, IsMutable, IsActive, InUnion, V.Desc, V.Offset, 199 false); 200 for (const auto &F : D->ElemRecord->fields()) { 201 bool IsUnionField = D->isUnion(); 202 initField(B, Ptr, IsConst, IsMutable, IsActive, IsUnionField, 203 InUnion || IsUnionField, F.Desc, F.Offset); 204 } 205 for (const auto &V : D->ElemRecord->virtual_bases()) 206 initBase(B, Ptr, IsConst, IsMutable, IsActive, InUnion, V.Desc, V.Offset, 207 true); 208 } 209 210 static void destroyField(Block *B, std::byte *Ptr, const Descriptor *D, 211 unsigned FieldOffset) { 212 if (auto Fn = D->DtorFn) 213 Fn(B, Ptr + FieldOffset, D); 214 } 215 216 static void destroyBase(Block *B, std::byte *Ptr, const Descriptor *D, 217 unsigned FieldOffset) { 218 assert(D); 219 assert(D->ElemRecord); 220 221 for (const auto &V : D->ElemRecord->bases()) 222 destroyBase(B, Ptr + FieldOffset, V.Desc, V.Offset); 223 for (const auto &F : D->ElemRecord->fields()) 224 destroyField(B, Ptr + FieldOffset, F.Desc, F.Offset); 225 } 226 227 static void dtorRecord(Block *B, std::byte *Ptr, const Descriptor *D) { 228 for (const auto &F : D->ElemRecord->bases()) 229 destroyBase(B, Ptr, F.Desc, F.Offset); 230 for (const auto &F : D->ElemRecord->fields()) 231 destroyField(B, Ptr, F.Desc, F.Offset); 232 for (const auto &F : D->ElemRecord->virtual_bases()) 233 destroyBase(B, Ptr, F.Desc, F.Offset); 234 } 235 236 static void moveRecord(Block *B, const std::byte *Src, std::byte *Dst, 237 const Descriptor *D) { 238 assert(D); 239 assert(D->ElemRecord); 240 241 // FIXME: There might be cases where we need to move over the (v)bases as 242 // well. 243 for (const auto &F : D->ElemRecord->fields()) { 244 auto FieldOffset = F.Offset; 245 const auto *SrcDesc = 246 reinterpret_cast<const InlineDescriptor *>(Src + FieldOffset) - 1; 247 auto *DestDesc = 248 reinterpret_cast<InlineDescriptor *>(Dst + FieldOffset) - 1; 249 std::memcpy(DestDesc, SrcDesc, sizeof(InlineDescriptor)); 250 251 if (auto Fn = F.Desc->MoveFn) 252 Fn(B, Src + FieldOffset, Dst + FieldOffset, F.Desc); 253 } 254 } 255 256 static BlockCtorFn getCtorPrim(PrimType Type) { 257 // Floating types are special. They are primitives, but need their 258 // constructor called. 259 if (Type == PT_Float) 260 return ctorTy<PrimConv<PT_Float>::T>; 261 if (Type == PT_IntAP) 262 return ctorTy<PrimConv<PT_IntAP>::T>; 263 if (Type == PT_IntAPS) 264 return ctorTy<PrimConv<PT_IntAPS>::T>; 265 if (Type == PT_MemberPtr) 266 return ctorTy<PrimConv<PT_MemberPtr>::T>; 267 268 COMPOSITE_TYPE_SWITCH(Type, return ctorTy<T>, return nullptr); 269 } 270 271 static BlockDtorFn getDtorPrim(PrimType Type) { 272 // Floating types are special. They are primitives, but need their 273 // destructor called, since they might allocate memory. 274 if (Type == PT_Float) 275 return dtorTy<PrimConv<PT_Float>::T>; 276 if (Type == PT_IntAP) 277 return dtorTy<PrimConv<PT_IntAP>::T>; 278 if (Type == PT_IntAPS) 279 return dtorTy<PrimConv<PT_IntAPS>::T>; 280 if (Type == PT_MemberPtr) 281 return dtorTy<PrimConv<PT_MemberPtr>::T>; 282 283 COMPOSITE_TYPE_SWITCH(Type, return dtorTy<T>, return nullptr); 284 } 285 286 static BlockMoveFn getMovePrim(PrimType Type) { 287 if (Type == PT_Float) 288 return moveTy<PrimConv<PT_Float>::T>; 289 if (Type == PT_IntAP) 290 return moveTy<PrimConv<PT_IntAP>::T>; 291 if (Type == PT_IntAPS) 292 return moveTy<PrimConv<PT_IntAPS>::T>; 293 if (Type == PT_MemberPtr) 294 return moveTy<PrimConv<PT_MemberPtr>::T>; 295 COMPOSITE_TYPE_SWITCH(Type, return moveTy<T>, return nullptr); 296 } 297 298 static BlockCtorFn getCtorArrayPrim(PrimType Type) { 299 TYPE_SWITCH(Type, return ctorArrayTy<T>); 300 llvm_unreachable("unknown Expr"); 301 } 302 303 static BlockDtorFn getDtorArrayPrim(PrimType Type) { 304 TYPE_SWITCH(Type, return dtorArrayTy<T>); 305 llvm_unreachable("unknown Expr"); 306 } 307 308 static BlockMoveFn getMoveArrayPrim(PrimType Type) { 309 TYPE_SWITCH(Type, return moveArrayTy<T>); 310 llvm_unreachable("unknown Expr"); 311 } 312 313 /// Primitives. 314 Descriptor::Descriptor(const DeclTy &D, PrimType Type, MetadataSize MD, 315 bool IsConst, bool IsTemporary, bool IsMutable) 316 : Source(D), ElemSize(primSize(Type)), Size(ElemSize), 317 MDSize(MD.value_or(0)), AllocSize(align(Size + MDSize)), PrimT(Type), 318 IsConst(IsConst), IsMutable(IsMutable), IsTemporary(IsTemporary), 319 CtorFn(getCtorPrim(Type)), DtorFn(getDtorPrim(Type)), 320 MoveFn(getMovePrim(Type)) { 321 assert(AllocSize >= Size); 322 assert(Source && "Missing source"); 323 } 324 325 /// Primitive arrays. 326 Descriptor::Descriptor(const DeclTy &D, PrimType Type, MetadataSize MD, 327 size_t NumElems, bool IsConst, bool IsTemporary, 328 bool IsMutable) 329 : Source(D), ElemSize(primSize(Type)), Size(ElemSize * NumElems), 330 MDSize(MD.value_or(0)), 331 AllocSize(align(MDSize) + align(Size) + sizeof(InitMapPtr)), PrimT(Type), 332 IsConst(IsConst), IsMutable(IsMutable), IsTemporary(IsTemporary), 333 IsArray(true), CtorFn(getCtorArrayPrim(Type)), 334 DtorFn(getDtorArrayPrim(Type)), MoveFn(getMoveArrayPrim(Type)) { 335 assert(Source && "Missing source"); 336 assert(NumElems <= (MaxArrayElemBytes / ElemSize)); 337 } 338 339 /// Primitive unknown-size arrays. 340 Descriptor::Descriptor(const DeclTy &D, PrimType Type, MetadataSize MD, 341 bool IsTemporary, UnknownSize) 342 : Source(D), ElemSize(primSize(Type)), Size(UnknownSizeMark), 343 MDSize(MD.value_or(0)), 344 AllocSize(MDSize + sizeof(InitMapPtr) + alignof(void *)), IsConst(true), 345 IsMutable(false), IsTemporary(IsTemporary), IsArray(true), 346 CtorFn(getCtorArrayPrim(Type)), DtorFn(getDtorArrayPrim(Type)), 347 MoveFn(getMoveArrayPrim(Type)) { 348 assert(Source && "Missing source"); 349 } 350 351 /// Arrays of composite elements. 352 Descriptor::Descriptor(const DeclTy &D, const Descriptor *Elem, MetadataSize MD, 353 unsigned NumElems, bool IsConst, bool IsTemporary, 354 bool IsMutable) 355 : Source(D), ElemSize(Elem->getAllocSize() + sizeof(InlineDescriptor)), 356 Size(ElemSize * NumElems), MDSize(MD.value_or(0)), 357 AllocSize(std::max<size_t>(alignof(void *), Size) + MDSize), 358 ElemDesc(Elem), IsConst(IsConst), IsMutable(IsMutable), 359 IsTemporary(IsTemporary), IsArray(true), CtorFn(ctorArrayDesc), 360 DtorFn(dtorArrayDesc), MoveFn(moveArrayDesc) { 361 assert(Source && "Missing source"); 362 } 363 364 /// Unknown-size arrays of composite elements. 365 Descriptor::Descriptor(const DeclTy &D, const Descriptor *Elem, MetadataSize MD, 366 bool IsTemporary, UnknownSize) 367 : Source(D), ElemSize(Elem->getAllocSize() + sizeof(InlineDescriptor)), 368 Size(UnknownSizeMark), MDSize(MD.value_or(0)), 369 AllocSize(MDSize + alignof(void *)), ElemDesc(Elem), IsConst(true), 370 IsMutable(false), IsTemporary(IsTemporary), IsArray(true), 371 CtorFn(ctorArrayDesc), DtorFn(dtorArrayDesc), MoveFn(moveArrayDesc) { 372 assert(Source && "Missing source"); 373 } 374 375 /// Composite records. 376 Descriptor::Descriptor(const DeclTy &D, const Record *R, MetadataSize MD, 377 bool IsConst, bool IsTemporary, bool IsMutable) 378 : Source(D), ElemSize(std::max<size_t>(alignof(void *), R->getFullSize())), 379 Size(ElemSize), MDSize(MD.value_or(0)), AllocSize(Size + MDSize), 380 ElemRecord(R), IsConst(IsConst), IsMutable(IsMutable), 381 IsTemporary(IsTemporary), CtorFn(ctorRecord), DtorFn(dtorRecord), 382 MoveFn(moveRecord) { 383 assert(Source && "Missing source"); 384 } 385 386 /// Dummy. 387 Descriptor::Descriptor(const DeclTy &D) 388 : Source(D), ElemSize(1), Size(1), MDSize(0), AllocSize(MDSize), 389 ElemRecord(nullptr), IsConst(true), IsMutable(false), IsTemporary(false), 390 IsDummy(true) { 391 assert(Source && "Missing source"); 392 } 393 394 QualType Descriptor::getType() const { 395 if (const auto *E = asExpr()) 396 return E->getType(); 397 if (const auto *D = asValueDecl()) 398 return D->getType(); 399 if (const auto *T = dyn_cast<TypeDecl>(asDecl())) 400 return QualType(T->getTypeForDecl(), 0); 401 llvm_unreachable("Invalid descriptor type"); 402 } 403 404 QualType Descriptor::getElemQualType() const { 405 assert(isArray()); 406 QualType T = getType(); 407 if (const auto *AT = T->getAsArrayTypeUnsafe()) 408 return AT->getElementType(); 409 if (const auto *CT = T->getAs<ComplexType>()) 410 return CT->getElementType(); 411 if (const auto *CT = T->getAs<VectorType>()) 412 return CT->getElementType(); 413 llvm_unreachable("Array that's not an array/complex/vector type?"); 414 } 415 416 SourceLocation Descriptor::getLocation() const { 417 if (auto *D = Source.dyn_cast<const Decl *>()) 418 return D->getLocation(); 419 if (auto *E = Source.dyn_cast<const Expr *>()) 420 return E->getExprLoc(); 421 llvm_unreachable("Invalid descriptor type"); 422 } 423 424 bool Descriptor::isUnion() const { return isRecord() && ElemRecord->isUnion(); } 425 426 InitMap::InitMap(unsigned N) 427 : UninitFields(N), Data(std::make_unique<T[]>(numFields(N))) { 428 std::fill_n(data(), numFields(N), 0); 429 } 430 431 bool InitMap::initializeElement(unsigned I) { 432 unsigned Bucket = I / PER_FIELD; 433 T Mask = T(1) << (I % PER_FIELD); 434 if (!(data()[Bucket] & Mask)) { 435 data()[Bucket] |= Mask; 436 UninitFields -= 1; 437 } 438 return UninitFields == 0; 439 } 440 441 bool InitMap::isElementInitialized(unsigned I) const { 442 unsigned Bucket = I / PER_FIELD; 443 return data()[Bucket] & (T(1) << (I % PER_FIELD)); 444 } 445