xref: /openbsd-src/gnu/llvm/lld/ELF/ARMErrataFix.cpp (revision dfe94b169149f14cc1aee2cf6dad58a8d9a1860c)
1ece8a530Spatrick //===- ARMErrataFix.cpp ---------------------------------------------------===//
2ece8a530Spatrick //
3ece8a530Spatrick // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4ece8a530Spatrick // See https://llvm.org/LICENSE.txt for license information.
5ece8a530Spatrick // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6ece8a530Spatrick //
7ece8a530Spatrick //===----------------------------------------------------------------------===//
8ece8a530Spatrick // This file implements Section Patching for the purpose of working around the
9ece8a530Spatrick // Cortex-a8 erratum 657417 "A 32bit branch instruction that spans 2 4K regions
10ece8a530Spatrick // can result in an incorrect instruction fetch or processor deadlock." The
11ece8a530Spatrick // erratum affects all but r1p7, r2p5, r2p6, r3p1 and r3p2 revisions of the
12ece8a530Spatrick // Cortex-A8. A high level description of the patching technique is given in
13ece8a530Spatrick // the opening comment of AArch64ErrataFix.cpp.
14ece8a530Spatrick //===----------------------------------------------------------------------===//
15ece8a530Spatrick 
16ece8a530Spatrick #include "ARMErrataFix.h"
17*dfe94b16Srobert #include "InputFiles.h"
18ece8a530Spatrick #include "LinkerScript.h"
19ece8a530Spatrick #include "OutputSections.h"
20ece8a530Spatrick #include "Relocations.h"
21ece8a530Spatrick #include "Symbols.h"
22ece8a530Spatrick #include "SyntheticSections.h"
23ece8a530Spatrick #include "Target.h"
24*dfe94b16Srobert #include "lld/Common/CommonLinkerContext.h"
25ece8a530Spatrick #include "lld/Common/Strings.h"
26ece8a530Spatrick #include "llvm/Support/Endian.h"
27ece8a530Spatrick #include <algorithm>
28ece8a530Spatrick 
29ece8a530Spatrick using namespace llvm;
30ece8a530Spatrick using namespace llvm::ELF;
31ece8a530Spatrick using namespace llvm::object;
32ece8a530Spatrick using namespace llvm::support;
33ece8a530Spatrick using namespace llvm::support::endian;
34bb684c34Spatrick using namespace lld;
35bb684c34Spatrick using namespace lld::elf;
36ece8a530Spatrick 
37ece8a530Spatrick // The documented title for Erratum 657417 is:
38ece8a530Spatrick // "A 32bit branch instruction that spans two 4K regions can result in an
39ece8a530Spatrick // incorrect instruction fetch or processor deadlock". Graphically using a
40ece8a530Spatrick // 32-bit B.w instruction encoded as a pair of halfwords 0xf7fe 0xbfff
41ece8a530Spatrick // xxxxxx000 // Memory region 1 start
42ece8a530Spatrick // target:
43ece8a530Spatrick // ...
44ece8a530Spatrick // xxxxxxffe f7fe // First halfword of branch to target:
45ece8a530Spatrick // xxxxxx000 // Memory region 2 start
46ece8a530Spatrick // xxxxxx002 bfff // Second halfword of branch to target:
47ece8a530Spatrick //
48ece8a530Spatrick // The specific trigger conditions that can be detected at link time are:
49ece8a530Spatrick // - There is a 32-bit Thumb-2 branch instruction with an address of the form
50ece8a530Spatrick //   xxxxxxFFE. The first 2 bytes of the instruction are in 4KiB region 1, the
51ece8a530Spatrick //   second 2 bytes are in region 2.
52ece8a530Spatrick // - The branch instruction is one of BLX, BL, B.w BCC.w
53ece8a530Spatrick // - The instruction preceding the branch is a 32-bit non-branch instruction.
54ece8a530Spatrick // - The target of the branch is in region 1.
55ece8a530Spatrick //
56ece8a530Spatrick // The linker mitigation for the fix is to redirect any branch that meets the
57ece8a530Spatrick // erratum conditions to a patch section containing a branch to the target.
58ece8a530Spatrick //
59ece8a530Spatrick // As adding patch sections may move branches onto region boundaries the patch
60ece8a530Spatrick // must iterate until no more patches are added.
61ece8a530Spatrick //
62ece8a530Spatrick // Example, before:
63ece8a530Spatrick // 00000FFA func: NOP.w      // 32-bit Thumb function
64ece8a530Spatrick // 00000FFE       B.W func   // 32-bit branch spanning 2 regions, dest in 1st.
65ece8a530Spatrick // Example, after:
66ece8a530Spatrick // 00000FFA func: NOP.w      // 32-bit Thumb function
67ece8a530Spatrick // 00000FFE       B.w __CortexA8657417_00000FFE
68ece8a530Spatrick // 00001002       2 - bytes padding
69ece8a530Spatrick // 00001004 __CortexA8657417_00000FFE: B.w func
70ece8a530Spatrick 
71*dfe94b16Srobert class elf::Patch657417Section final : public SyntheticSection {
72ece8a530Spatrick public:
73ece8a530Spatrick   Patch657417Section(InputSection *p, uint64_t off, uint32_t instr, bool isARM);
74ece8a530Spatrick 
75ece8a530Spatrick   void writeTo(uint8_t *buf) override;
76ece8a530Spatrick 
getSize() const77ece8a530Spatrick   size_t getSize() const override { return 4; }
78ece8a530Spatrick 
79ece8a530Spatrick   // Get the virtual address of the branch instruction at patcheeOffset.
80ece8a530Spatrick   uint64_t getBranchAddr() const;
81ece8a530Spatrick 
classof(const SectionBase * d)82ece8a530Spatrick   static bool classof(const SectionBase *d) {
83ece8a530Spatrick     return d->kind() == InputSectionBase::Synthetic && d->name ==".text.patch";
84ece8a530Spatrick   }
85ece8a530Spatrick 
86ece8a530Spatrick   // The Section we are patching.
87ece8a530Spatrick   const InputSection *patchee;
88ece8a530Spatrick   // The offset of the instruction in the Patchee section we are patching.
89ece8a530Spatrick   uint64_t patcheeOffset;
90ece8a530Spatrick   // A label for the start of the Patch that we can use as a relocation target.
91ece8a530Spatrick   Symbol *patchSym;
92ece8a530Spatrick   // A decoding of the branch instruction at patcheeOffset.
93ece8a530Spatrick   uint32_t instr;
94ece8a530Spatrick   // True If the patch is to be written in ARM state, otherwise the patch will
95ece8a530Spatrick   // be written in Thumb state.
96ece8a530Spatrick   bool isARM;
97ece8a530Spatrick };
98ece8a530Spatrick 
99ece8a530Spatrick // Return true if the half-word, when taken as the first of a pair of halfwords
100ece8a530Spatrick // is the first half of a 32-bit instruction.
101ece8a530Spatrick // Reference from ARM Architecture Reference Manual ARMv7-A and ARMv7-R edition
102ece8a530Spatrick // section A6.3: 32-bit Thumb instruction encoding
103ece8a530Spatrick // |             HW1                   |               HW2                |
104ece8a530Spatrick // | 1 1 1 | op1 (2) | op2 (7) | x (4) |op|           x (15)              |
105ece8a530Spatrick // With op1 == 0b00, a 16-bit instruction is encoded.
106ece8a530Spatrick //
107ece8a530Spatrick // We test only the first halfword, looking for op != 0b00.
is32bitInstruction(uint16_t hw)108ece8a530Spatrick static bool is32bitInstruction(uint16_t hw) {
109ece8a530Spatrick   return (hw & 0xe000) == 0xe000 && (hw & 0x1800) != 0x0000;
110ece8a530Spatrick }
111ece8a530Spatrick 
112ece8a530Spatrick // Reference from ARM Architecture Reference Manual ARMv7-A and ARMv7-R edition
113ece8a530Spatrick // section A6.3.4 Branches and miscellaneous control.
114ece8a530Spatrick // |             HW1              |               HW2                |
115ece8a530Spatrick // | 1 1 1 | 1 0 | op (7) | x (4) | 1 | op1 (3) | op2 (4) | imm8 (8) |
116ece8a530Spatrick // op1 == 0x0 op != x111xxx | Conditional branch (Bcc.W)
117ece8a530Spatrick // op1 == 0x1               | Branch (B.W)
118ece8a530Spatrick // op1 == 1x0               | Branch with Link and Exchange (BLX.w)
119ece8a530Spatrick // op1 == 1x1               | Branch with Link (BL.W)
120ece8a530Spatrick 
isBcc(uint32_t instr)121ece8a530Spatrick static bool isBcc(uint32_t instr) {
122ece8a530Spatrick   return (instr & 0xf800d000) == 0xf0008000 &&
123ece8a530Spatrick          (instr & 0x03800000) != 0x03800000;
124ece8a530Spatrick }
125ece8a530Spatrick 
isB(uint32_t instr)126ece8a530Spatrick static bool isB(uint32_t instr) { return (instr & 0xf800d000) == 0xf0009000; }
127ece8a530Spatrick 
isBLX(uint32_t instr)128ece8a530Spatrick static bool isBLX(uint32_t instr) { return (instr & 0xf800d000) == 0xf000c000; }
129ece8a530Spatrick 
isBL(uint32_t instr)130ece8a530Spatrick static bool isBL(uint32_t instr) { return (instr & 0xf800d000) == 0xf000d000; }
131ece8a530Spatrick 
is32bitBranch(uint32_t instr)132ece8a530Spatrick static bool is32bitBranch(uint32_t instr) {
133ece8a530Spatrick   return isBcc(instr) || isB(instr) || isBL(instr) || isBLX(instr);
134ece8a530Spatrick }
135ece8a530Spatrick 
Patch657417Section(InputSection * p,uint64_t off,uint32_t instr,bool isARM)136ece8a530Spatrick Patch657417Section::Patch657417Section(InputSection *p, uint64_t off,
137ece8a530Spatrick                                        uint32_t instr, bool isARM)
138ece8a530Spatrick     : SyntheticSection(SHF_ALLOC | SHF_EXECINSTR, SHT_PROGBITS, 4,
139ece8a530Spatrick                        ".text.patch"),
140ece8a530Spatrick       patchee(p), patcheeOffset(off), instr(instr), isARM(isARM) {
141ece8a530Spatrick   parent = p->getParent();
142ece8a530Spatrick   patchSym = addSyntheticLocal(
143*dfe94b16Srobert       saver().save("__CortexA8657417_" + utohexstr(getBranchAddr())), STT_FUNC,
144ece8a530Spatrick       isARM ? 0 : 1, getSize(), *this);
145*dfe94b16Srobert   addSyntheticLocal(saver().save(isARM ? "$a" : "$t"), STT_NOTYPE, 0, 0, *this);
146ece8a530Spatrick }
147ece8a530Spatrick 
getBranchAddr() const148ece8a530Spatrick uint64_t Patch657417Section::getBranchAddr() const {
149ece8a530Spatrick   return patchee->getVA(patcheeOffset);
150ece8a530Spatrick }
151ece8a530Spatrick 
152ece8a530Spatrick // Given a branch instruction instr at sourceAddr work out its destination
153ece8a530Spatrick // address. This is only used when the branch instruction has no relocation.
getThumbDestAddr(uint64_t sourceAddr,uint32_t instr)154ece8a530Spatrick static uint64_t getThumbDestAddr(uint64_t sourceAddr, uint32_t instr) {
155ece8a530Spatrick   uint8_t buf[4];
156ece8a530Spatrick   write16le(buf, instr >> 16);
157ece8a530Spatrick   write16le(buf + 2, instr & 0x0000ffff);
158ece8a530Spatrick   int64_t offset;
159ece8a530Spatrick   if (isBcc(instr))
160ece8a530Spatrick     offset = target->getImplicitAddend(buf, R_ARM_THM_JUMP19);
161ece8a530Spatrick   else if (isB(instr))
162ece8a530Spatrick     offset = target->getImplicitAddend(buf, R_ARM_THM_JUMP24);
163ece8a530Spatrick   else
164ece8a530Spatrick     offset = target->getImplicitAddend(buf, R_ARM_THM_CALL);
1651cf9926bSpatrick   // A BLX instruction from Thumb to Arm may have an address that is
1661cf9926bSpatrick   // not 4-byte aligned. As Arm instructions are always 4-byte aligned
1671cf9926bSpatrick   // the instruction is calculated (from Arm ARM):
1681cf9926bSpatrick   // targetAddress = Align(PC, 4) + imm32
1691cf9926bSpatrick   // where
1701cf9926bSpatrick   //   Align(x, y) = y * (x Div y)
1711cf9926bSpatrick   // which corresponds to alignDown.
1721cf9926bSpatrick   if (isBLX(instr))
1731cf9926bSpatrick     sourceAddr = alignDown(sourceAddr, 4);
174ece8a530Spatrick   return sourceAddr + offset + 4;
175ece8a530Spatrick }
176ece8a530Spatrick 
writeTo(uint8_t * buf)177ece8a530Spatrick void Patch657417Section::writeTo(uint8_t *buf) {
178ece8a530Spatrick   // The base instruction of the patch is always a 32-bit unconditional branch.
179ece8a530Spatrick   if (isARM)
180ece8a530Spatrick     write32le(buf, 0xea000000);
181ece8a530Spatrick   else
182ece8a530Spatrick     write32le(buf, 0x9000f000);
1831cf9926bSpatrick   // If we have a relocation then apply it.
184*dfe94b16Srobert   if (!relocs().empty()) {
185*dfe94b16Srobert     target->relocateAlloc(*this, buf);
186ece8a530Spatrick     return;
187ece8a530Spatrick   }
188ece8a530Spatrick 
189ece8a530Spatrick   // If we don't have a relocation then we must calculate and write the offset
190ece8a530Spatrick   // ourselves.
191ece8a530Spatrick   // Get the destination offset from the addend in the branch instruction.
192ece8a530Spatrick   // We cannot use the instruction in the patchee section as this will have
193ece8a530Spatrick   // been altered to point to us!
194ece8a530Spatrick   uint64_t s = getThumbDestAddr(getBranchAddr(), instr);
1951cf9926bSpatrick   // A BLX changes the state of the branch in the patch to Arm state, which
1961cf9926bSpatrick   // has a PC Bias of 8, whereas in all other cases the branch is in Thumb
1971cf9926bSpatrick   // state with a PC Bias of 4.
1981cf9926bSpatrick   uint64_t pcBias = isBLX(instr) ? 8 : 4;
1991cf9926bSpatrick   uint64_t p = getVA(pcBias);
200bb684c34Spatrick   target->relocateNoSym(buf, isARM ? R_ARM_JUMP24 : R_ARM_THM_JUMP24, s - p);
201ece8a530Spatrick }
202ece8a530Spatrick 
203ece8a530Spatrick // Given a branch instruction spanning two 4KiB regions, at offset off from the
204ece8a530Spatrick // start of isec, return true if the destination of the branch is within the
205ece8a530Spatrick // first of the two 4Kib regions.
branchDestInFirstRegion(const InputSection * isec,uint64_t off,uint32_t instr,const Relocation * r)206ece8a530Spatrick static bool branchDestInFirstRegion(const InputSection *isec, uint64_t off,
207ece8a530Spatrick                                     uint32_t instr, const Relocation *r) {
208ece8a530Spatrick   uint64_t sourceAddr = isec->getVA(0) + off;
209ece8a530Spatrick   assert((sourceAddr & 0xfff) == 0xffe);
210*dfe94b16Srobert   uint64_t destAddr;
211ece8a530Spatrick   // If there is a branch relocation at the same offset we must use this to
212ece8a530Spatrick   // find the destination address as the branch could be indirected via a thunk
213ece8a530Spatrick   // or the PLT.
214ece8a530Spatrick   if (r) {
215ece8a530Spatrick     uint64_t dst = (r->expr == R_PLT_PC) ? r->sym->getPltVA() : r->sym->getVA();
216ece8a530Spatrick     // Account for Thumb PC bias, usually cancelled to 0 by addend of -4.
217ece8a530Spatrick     destAddr = dst + r->addend + 4;
218ece8a530Spatrick   } else {
219ece8a530Spatrick     // If there is no relocation, we must have an intra-section branch
220ece8a530Spatrick     // We must extract the offset from the addend manually.
221ece8a530Spatrick     destAddr = getThumbDestAddr(sourceAddr, instr);
222ece8a530Spatrick   }
223ece8a530Spatrick 
224ece8a530Spatrick   return (destAddr & 0xfffff000) == (sourceAddr & 0xfffff000);
225ece8a530Spatrick }
226ece8a530Spatrick 
227ece8a530Spatrick // Return true if a branch can reach a patch section placed after isec.
228ece8a530Spatrick // The Bcc.w instruction has a range of 1 MiB, all others have 16 MiB.
patchInRange(const InputSection * isec,uint64_t off,uint32_t instr)229ece8a530Spatrick static bool patchInRange(const InputSection *isec, uint64_t off,
230ece8a530Spatrick                          uint32_t instr) {
231ece8a530Spatrick 
232ece8a530Spatrick   // We need the branch at source to reach a patch section placed immediately
233ece8a530Spatrick   // after isec. As there can be more than one patch in the patch section we
234ece8a530Spatrick   // add 0x100 as contingency to account for worst case of 1 branch every 4KiB
235ece8a530Spatrick   // for a 1 MiB range.
236ece8a530Spatrick   return target->inBranchRange(
237ece8a530Spatrick       isBcc(instr) ? R_ARM_THM_JUMP19 : R_ARM_THM_JUMP24, isec->getVA(off),
238ece8a530Spatrick       isec->getVA() + isec->getSize() + 0x100);
239ece8a530Spatrick }
240ece8a530Spatrick 
241ece8a530Spatrick struct ScanResult {
242ece8a530Spatrick   // Offset of branch within its InputSection.
243ece8a530Spatrick   uint64_t off;
244ece8a530Spatrick   // Cached decoding of the branch instruction.
245ece8a530Spatrick   uint32_t instr;
246ece8a530Spatrick   // Branch relocation at off. Will be nullptr if no relocation exists.
247ece8a530Spatrick   Relocation *rel;
248ece8a530Spatrick };
249ece8a530Spatrick 
250ece8a530Spatrick // Detect the erratum sequence, returning the offset of the branch instruction
251ece8a530Spatrick // and a decoding of the branch. If the erratum sequence is not found then
252ece8a530Spatrick // return an offset of 0 for the branch. 0 is a safe value to use for no patch
253ece8a530Spatrick // as there must be at least one 32-bit non-branch instruction before the
254ece8a530Spatrick // branch so the minimum offset for a patch is 4.
scanCortexA8Errata657417(InputSection * isec,uint64_t & off,uint64_t limit)255ece8a530Spatrick static ScanResult scanCortexA8Errata657417(InputSection *isec, uint64_t &off,
256ece8a530Spatrick                                            uint64_t limit) {
257ece8a530Spatrick   uint64_t isecAddr = isec->getVA(0);
258ece8a530Spatrick   // Advance Off so that (isecAddr + off) modulo 0x1000 is at least 0xffa. We
259ece8a530Spatrick   // need to check for a 32-bit instruction immediately before a 32-bit branch
260ece8a530Spatrick   // at 0xffe modulo 0x1000.
261ece8a530Spatrick   off = alignTo(isecAddr + off, 0x1000, 0xffa) - isecAddr;
262ece8a530Spatrick   if (off >= limit || limit - off < 8) {
263ece8a530Spatrick     // Need at least 2 4-byte sized instructions to trigger erratum.
264ece8a530Spatrick     off = limit;
265ece8a530Spatrick     return {0, 0, nullptr};
266ece8a530Spatrick   }
267ece8a530Spatrick 
268ece8a530Spatrick   ScanResult scanRes = {0, 0, nullptr};
269*dfe94b16Srobert   const uint8_t *buf = isec->content().begin();
270ece8a530Spatrick   // ARMv7-A Thumb 32-bit instructions are encoded 2 consecutive
271ece8a530Spatrick   // little-endian halfwords.
272ece8a530Spatrick   const ulittle16_t *instBuf = reinterpret_cast<const ulittle16_t *>(buf + off);
273ece8a530Spatrick   uint16_t hw11 = *instBuf++;
274ece8a530Spatrick   uint16_t hw12 = *instBuf++;
275ece8a530Spatrick   uint16_t hw21 = *instBuf++;
276ece8a530Spatrick   uint16_t hw22 = *instBuf++;
277ece8a530Spatrick   if (is32bitInstruction(hw11) && is32bitInstruction(hw21)) {
278ece8a530Spatrick     uint32_t instr1 = (hw11 << 16) | hw12;
279ece8a530Spatrick     uint32_t instr2 = (hw21 << 16) | hw22;
280ece8a530Spatrick     if (!is32bitBranch(instr1) && is32bitBranch(instr2)) {
281ece8a530Spatrick       // Find a relocation for the branch if it exists. This will be used
282ece8a530Spatrick       // to determine the target.
283ece8a530Spatrick       uint64_t branchOff = off + 4;
284*dfe94b16Srobert       auto relIt = llvm::find_if(isec->relocs(), [=](const Relocation &r) {
285ece8a530Spatrick         return r.offset == branchOff &&
286ece8a530Spatrick                (r.type == R_ARM_THM_JUMP19 || r.type == R_ARM_THM_JUMP24 ||
287ece8a530Spatrick                 r.type == R_ARM_THM_CALL);
288ece8a530Spatrick       });
289*dfe94b16Srobert       if (relIt != isec->relocs().end())
290ece8a530Spatrick         scanRes.rel = &(*relIt);
291ece8a530Spatrick       if (branchDestInFirstRegion(isec, branchOff, instr2, scanRes.rel)) {
292ece8a530Spatrick         if (patchInRange(isec, branchOff, instr2)) {
293ece8a530Spatrick           scanRes.off = branchOff;
294ece8a530Spatrick           scanRes.instr = instr2;
295ece8a530Spatrick         } else {
296ece8a530Spatrick           warn(toString(isec->file) +
297ece8a530Spatrick                ": skipping cortex-a8 657417 erratum sequence, section " +
298ece8a530Spatrick                isec->name + " is too large to patch");
299ece8a530Spatrick         }
300ece8a530Spatrick       }
301ece8a530Spatrick     }
302ece8a530Spatrick   }
303ece8a530Spatrick   off += 0x1000;
304ece8a530Spatrick   return scanRes;
305ece8a530Spatrick }
306ece8a530Spatrick 
init()307ece8a530Spatrick void ARMErr657417Patcher::init() {
308ece8a530Spatrick   // The Arm ABI permits a mix of ARM, Thumb and Data in the same
309ece8a530Spatrick   // InputSection. We must only scan Thumb instructions to avoid false
310ece8a530Spatrick   // matches. We use the mapping symbols in the InputObjects to identify this
311ece8a530Spatrick   // data, caching the results in sectionMap so we don't have to recalculate
312ece8a530Spatrick   // it each pass.
313ece8a530Spatrick 
314ece8a530Spatrick   // The ABI Section 4.5.5 Mapping symbols; defines local symbols that describe
315ece8a530Spatrick   // half open intervals [Symbol Value, Next Symbol Value) of code and data
316ece8a530Spatrick   // within sections. If there is no next symbol then the half open interval is
317ece8a530Spatrick   // [Symbol Value, End of section). The type, code or data, is determined by
318ece8a530Spatrick   // the mapping symbol name, $a for Arm code, $t for Thumb code, $d for data.
319ece8a530Spatrick   auto isArmMapSymbol = [](const Symbol *s) {
320ece8a530Spatrick     return s->getName() == "$a" || s->getName().startswith("$a.");
321ece8a530Spatrick   };
322ece8a530Spatrick   auto isThumbMapSymbol = [](const Symbol *s) {
323ece8a530Spatrick     return s->getName() == "$t" || s->getName().startswith("$t.");
324ece8a530Spatrick   };
325ece8a530Spatrick   auto isDataMapSymbol = [](const Symbol *s) {
326ece8a530Spatrick     return s->getName() == "$d" || s->getName().startswith("$d.");
327ece8a530Spatrick   };
328ece8a530Spatrick 
329ece8a530Spatrick   // Collect mapping symbols for every executable InputSection.
330*dfe94b16Srobert   for (ELFFileBase *file : ctx.objectFiles) {
331*dfe94b16Srobert     for (Symbol *s : file->getLocalSymbols()) {
332ece8a530Spatrick       auto *def = dyn_cast<Defined>(s);
333ece8a530Spatrick       if (!def)
334ece8a530Spatrick         continue;
335ece8a530Spatrick       if (!isArmMapSymbol(def) && !isThumbMapSymbol(def) &&
336ece8a530Spatrick           !isDataMapSymbol(def))
337ece8a530Spatrick         continue;
338ece8a530Spatrick       if (auto *sec = dyn_cast_or_null<InputSection>(def->section))
339ece8a530Spatrick         if (sec->flags & SHF_EXECINSTR)
340ece8a530Spatrick           sectionMap[sec].push_back(def);
341ece8a530Spatrick     }
342ece8a530Spatrick   }
343ece8a530Spatrick   // For each InputSection make sure the mapping symbols are in sorted in
344ece8a530Spatrick   // ascending order and are in alternating Thumb, non-Thumb order.
345ece8a530Spatrick   for (auto &kv : sectionMap) {
346ece8a530Spatrick     std::vector<const Defined *> &mapSyms = kv.second;
347ece8a530Spatrick     llvm::stable_sort(mapSyms, [](const Defined *a, const Defined *b) {
348ece8a530Spatrick       return a->value < b->value;
349ece8a530Spatrick     });
350ece8a530Spatrick     mapSyms.erase(std::unique(mapSyms.begin(), mapSyms.end(),
351ece8a530Spatrick                               [=](const Defined *a, const Defined *b) {
352ece8a530Spatrick                                 return (isThumbMapSymbol(a) ==
353ece8a530Spatrick                                         isThumbMapSymbol(b));
354ece8a530Spatrick                               }),
355ece8a530Spatrick                   mapSyms.end());
356ece8a530Spatrick     // Always start with a Thumb Mapping Symbol
357ece8a530Spatrick     if (!mapSyms.empty() && !isThumbMapSymbol(mapSyms.front()))
358ece8a530Spatrick       mapSyms.erase(mapSyms.begin());
359ece8a530Spatrick   }
360ece8a530Spatrick   initialized = true;
361ece8a530Spatrick }
362ece8a530Spatrick 
insertPatches(InputSectionDescription & isd,std::vector<Patch657417Section * > & patches)363ece8a530Spatrick void ARMErr657417Patcher::insertPatches(
364ece8a530Spatrick     InputSectionDescription &isd, std::vector<Patch657417Section *> &patches) {
365ece8a530Spatrick   uint64_t spacing = 0x100000 - 0x7500;
366ece8a530Spatrick   uint64_t isecLimit;
367ece8a530Spatrick   uint64_t prevIsecLimit = isd.sections.front()->outSecOff;
368ece8a530Spatrick   uint64_t patchUpperBound = prevIsecLimit + spacing;
369ece8a530Spatrick   uint64_t outSecAddr = isd.sections.front()->getParent()->addr;
370ece8a530Spatrick 
371ece8a530Spatrick   // Set the outSecOff of patches to the place where we want to insert them.
372ece8a530Spatrick   // We use a similar strategy to initial thunk placement, using 1 MiB as the
373ece8a530Spatrick   // range of the Thumb-2 conditional branch with a contingency accounting for
374ece8a530Spatrick   // thunk generation.
375ece8a530Spatrick   auto patchIt = patches.begin();
376ece8a530Spatrick   auto patchEnd = patches.end();
377ece8a530Spatrick   for (const InputSection *isec : isd.sections) {
378ece8a530Spatrick     isecLimit = isec->outSecOff + isec->getSize();
379ece8a530Spatrick     if (isecLimit > patchUpperBound) {
380ece8a530Spatrick       for (; patchIt != patchEnd; ++patchIt) {
381ece8a530Spatrick         if ((*patchIt)->getBranchAddr() - outSecAddr >= prevIsecLimit)
382ece8a530Spatrick           break;
383ece8a530Spatrick         (*patchIt)->outSecOff = prevIsecLimit;
384ece8a530Spatrick       }
385ece8a530Spatrick       patchUpperBound = prevIsecLimit + spacing;
386ece8a530Spatrick     }
387ece8a530Spatrick     prevIsecLimit = isecLimit;
388ece8a530Spatrick   }
389ece8a530Spatrick   for (; patchIt != patchEnd; ++patchIt)
390ece8a530Spatrick     (*patchIt)->outSecOff = isecLimit;
391ece8a530Spatrick 
392ece8a530Spatrick   // Merge all patch sections. We use the outSecOff assigned above to
393ece8a530Spatrick   // determine the insertion point. This is ok as we only merge into an
394ece8a530Spatrick   // InputSectionDescription once per pass, and at the end of the pass
395ece8a530Spatrick   // assignAddresses() will recalculate all the outSecOff values.
396*dfe94b16Srobert   SmallVector<InputSection *, 0> tmp;
397ece8a530Spatrick   tmp.reserve(isd.sections.size() + patches.size());
398ece8a530Spatrick   auto mergeCmp = [](const InputSection *a, const InputSection *b) {
399ece8a530Spatrick     if (a->outSecOff != b->outSecOff)
400ece8a530Spatrick       return a->outSecOff < b->outSecOff;
401ece8a530Spatrick     return isa<Patch657417Section>(a) && !isa<Patch657417Section>(b);
402ece8a530Spatrick   };
403ece8a530Spatrick   std::merge(isd.sections.begin(), isd.sections.end(), patches.begin(),
404ece8a530Spatrick              patches.end(), std::back_inserter(tmp), mergeCmp);
405ece8a530Spatrick   isd.sections = std::move(tmp);
406ece8a530Spatrick }
407ece8a530Spatrick 
408ece8a530Spatrick // Given a branch instruction described by ScanRes redirect it to a patch
409ece8a530Spatrick // section containing an unconditional branch instruction to the target.
410ece8a530Spatrick // Ensure that this patch section is 4-byte aligned so that the branch cannot
411ece8a530Spatrick // span two 4 KiB regions. Place the patch section so that it is always after
412ece8a530Spatrick // isec so the branch we are patching always goes forwards.
implementPatch(ScanResult sr,InputSection * isec,std::vector<Patch657417Section * > & patches)413ece8a530Spatrick static void implementPatch(ScanResult sr, InputSection *isec,
414ece8a530Spatrick                            std::vector<Patch657417Section *> &patches) {
415ece8a530Spatrick 
416ece8a530Spatrick   log("detected cortex-a8-657419 erratum sequence starting at " +
417ece8a530Spatrick       utohexstr(isec->getVA(sr.off)) + " in unpatched output.");
418ece8a530Spatrick   Patch657417Section *psec;
419ece8a530Spatrick   // We have two cases to deal with.
420ece8a530Spatrick   // Case 1. There is a relocation at patcheeOffset to a symbol. The
421ece8a530Spatrick   // unconditional branch in the patch must have a relocation so that any
422ece8a530Spatrick   // further redirection via the PLT or a Thunk happens as normal. At
423ece8a530Spatrick   // patcheeOffset we redirect the existing relocation to a Symbol defined at
424ece8a530Spatrick   // the start of the patch section.
425ece8a530Spatrick   //
426ece8a530Spatrick   // Case 2. There is no relocation at patcheeOffset. We are unlikely to have
427ece8a530Spatrick   // a symbol that we can use as a target for a relocation in the patch section.
428ece8a530Spatrick   // Luckily we know that the destination cannot be indirected via the PLT or
429ece8a530Spatrick   // a Thunk so we can just write the destination directly.
430ece8a530Spatrick   if (sr.rel) {
431ece8a530Spatrick     // Case 1. We have an existing relocation to redirect to patch and a
432ece8a530Spatrick     // Symbol target.
433ece8a530Spatrick 
434ece8a530Spatrick     // Create a branch relocation for the unconditional branch in the patch.
435ece8a530Spatrick     // This can be redirected via the PLT or Thunks.
436ece8a530Spatrick     RelType patchRelType = R_ARM_THM_JUMP24;
437ece8a530Spatrick     int64_t patchRelAddend = sr.rel->addend;
438ece8a530Spatrick     bool destIsARM = false;
439ece8a530Spatrick     if (isBL(sr.instr) || isBLX(sr.instr)) {
440ece8a530Spatrick       // The final target of the branch may be ARM or Thumb, if the target
441ece8a530Spatrick       // is ARM then we write the patch in ARM state to avoid a state change
442ece8a530Spatrick       // Thunk from the patch to the target.
443ece8a530Spatrick       uint64_t dstSymAddr = (sr.rel->expr == R_PLT_PC) ? sr.rel->sym->getPltVA()
444ece8a530Spatrick                                                        : sr.rel->sym->getVA();
445ece8a530Spatrick       destIsARM = (dstSymAddr & 1) == 0;
446ece8a530Spatrick     }
447ece8a530Spatrick     psec = make<Patch657417Section>(isec, sr.off, sr.instr, destIsARM);
448ece8a530Spatrick     if (destIsARM) {
449ece8a530Spatrick       // The patch will be in ARM state. Use an ARM relocation and account for
450ece8a530Spatrick       // the larger ARM PC-bias of 8 rather than Thumb's 4.
451ece8a530Spatrick       patchRelType = R_ARM_JUMP24;
452ece8a530Spatrick       patchRelAddend -= 4;
453ece8a530Spatrick     }
454*dfe94b16Srobert     psec->addReloc(
455ece8a530Spatrick         Relocation{sr.rel->expr, patchRelType, 0, patchRelAddend, sr.rel->sym});
456ece8a530Spatrick     // Redirect the existing branch relocation to the patch.
457ece8a530Spatrick     sr.rel->expr = R_PC;
458ece8a530Spatrick     sr.rel->addend = -4;
459ece8a530Spatrick     sr.rel->sym = psec->patchSym;
460ece8a530Spatrick   } else {
461ece8a530Spatrick     // Case 2. We do not have a relocation to the patch. Add a relocation of the
462ece8a530Spatrick     // appropriate type to the patch at patcheeOffset.
463ece8a530Spatrick 
464ece8a530Spatrick     // The destination is ARM if we have a BLX.
465ece8a530Spatrick     psec = make<Patch657417Section>(isec, sr.off, sr.instr, isBLX(sr.instr));
466ece8a530Spatrick     RelType type;
467ece8a530Spatrick     if (isBcc(sr.instr))
468ece8a530Spatrick       type = R_ARM_THM_JUMP19;
469ece8a530Spatrick     else if (isB(sr.instr))
470ece8a530Spatrick       type = R_ARM_THM_JUMP24;
471ece8a530Spatrick     else
472ece8a530Spatrick       type = R_ARM_THM_CALL;
473*dfe94b16Srobert     isec->addReloc(Relocation{R_PC, type, sr.off, -4, psec->patchSym});
474ece8a530Spatrick   }
475ece8a530Spatrick   patches.push_back(psec);
476ece8a530Spatrick }
477ece8a530Spatrick 
478ece8a530Spatrick // Scan all the instructions in InputSectionDescription, for each instance of
479ece8a530Spatrick // the erratum sequence create a Patch657417Section. We return the list of
480ece8a530Spatrick // Patch657417Sections that need to be applied to the InputSectionDescription.
481ece8a530Spatrick std::vector<Patch657417Section *>
patchInputSectionDescription(InputSectionDescription & isd)482ece8a530Spatrick ARMErr657417Patcher::patchInputSectionDescription(
483ece8a530Spatrick     InputSectionDescription &isd) {
484ece8a530Spatrick   std::vector<Patch657417Section *> patches;
485ece8a530Spatrick   for (InputSection *isec : isd.sections) {
486ece8a530Spatrick     // LLD doesn't use the erratum sequence in SyntheticSections.
487ece8a530Spatrick     if (isa<SyntheticSection>(isec))
488ece8a530Spatrick       continue;
489ece8a530Spatrick     // Use sectionMap to make sure we only scan Thumb code and not Arm or inline
490ece8a530Spatrick     // data. We have already sorted mapSyms in ascending order and removed
491ece8a530Spatrick     // consecutive mapping symbols of the same type. Our range of executable
492ece8a530Spatrick     // instructions to scan is therefore [thumbSym->value, nonThumbSym->value)
493ece8a530Spatrick     // or [thumbSym->value, section size).
494ece8a530Spatrick     std::vector<const Defined *> &mapSyms = sectionMap[isec];
495ece8a530Spatrick 
496ece8a530Spatrick     auto thumbSym = mapSyms.begin();
497ece8a530Spatrick     while (thumbSym != mapSyms.end()) {
498ece8a530Spatrick       auto nonThumbSym = std::next(thumbSym);
499ece8a530Spatrick       uint64_t off = (*thumbSym)->value;
500*dfe94b16Srobert       uint64_t limit = nonThumbSym == mapSyms.end() ? isec->content().size()
501ece8a530Spatrick                                                     : (*nonThumbSym)->value;
502ece8a530Spatrick 
503ece8a530Spatrick       while (off < limit) {
504ece8a530Spatrick         ScanResult sr = scanCortexA8Errata657417(isec, off, limit);
505ece8a530Spatrick         if (sr.off)
506ece8a530Spatrick           implementPatch(sr, isec, patches);
507ece8a530Spatrick       }
508ece8a530Spatrick       if (nonThumbSym == mapSyms.end())
509ece8a530Spatrick         break;
510ece8a530Spatrick       thumbSym = std::next(nonThumbSym);
511ece8a530Spatrick     }
512ece8a530Spatrick   }
513ece8a530Spatrick   return patches;
514ece8a530Spatrick }
515ece8a530Spatrick 
createFixes()516ece8a530Spatrick bool ARMErr657417Patcher::createFixes() {
517ece8a530Spatrick   if (!initialized)
518ece8a530Spatrick     init();
519ece8a530Spatrick 
520ece8a530Spatrick   bool addressesChanged = false;
521ece8a530Spatrick   for (OutputSection *os : outputSections) {
522ece8a530Spatrick     if (!(os->flags & SHF_ALLOC) || !(os->flags & SHF_EXECINSTR))
523ece8a530Spatrick       continue;
524*dfe94b16Srobert     for (SectionCommand *cmd : os->commands)
525*dfe94b16Srobert       if (auto *isd = dyn_cast<InputSectionDescription>(cmd)) {
526ece8a530Spatrick         std::vector<Patch657417Section *> patches =
527ece8a530Spatrick             patchInputSectionDescription(*isd);
528ece8a530Spatrick         if (!patches.empty()) {
529ece8a530Spatrick           insertPatches(*isd, patches);
530ece8a530Spatrick           addressesChanged = true;
531ece8a530Spatrick         }
532ece8a530Spatrick       }
533ece8a530Spatrick   }
534ece8a530Spatrick   return addressesChanged;
535ece8a530Spatrick }
536