samitolvanen updated this revision to Diff 428208.
samitolvanen added a comment.
- Handle FP, LR, and XZR register arguments in the AArch64 `llvm.kcfi.check`
lowering.
Repository:
rG LLVM Github Monorepo
CHANGES SINCE LAST ACTION
https://reviews.llvm.org/D119296/new/
https://reviews.llvm.org/D119296
Files:
clang/docs/ClangCommandLineReference.rst
clang/docs/ControlFlowIntegrity.rst
clang/docs/UsersManual.rst
clang/include/clang/Basic/Features.def
clang/include/clang/Basic/Sanitizers.def
clang/lib/CodeGen/CGExpr.cpp
clang/lib/CodeGen/CodeGenFunction.h
clang/lib/CodeGen/CodeGenModule.cpp
clang/lib/CodeGen/CodeGenModule.h
clang/lib/Driver/SanitizerArgs.cpp
clang/lib/Driver/ToolChain.cpp
clang/test/CodeGen/kcfi.c
clang/test/Driver/fsanitize.c
llvm/include/llvm/CodeGen/AsmPrinter.h
llvm/include/llvm/IR/Intrinsics.td
llvm/include/llvm/MC/MCObjectFileInfo.h
llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp
llvm/lib/MC/MCObjectFileInfo.cpp
llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp
llvm/lib/Target/AArch64/AArch64InstrInfo.td
llvm/lib/Target/X86/X86AsmPrinter.cpp
llvm/lib/Target/X86/X86AsmPrinter.h
llvm/lib/Target/X86/X86InstrCompiler.td
llvm/lib/Target/X86/X86MCInstLower.cpp
llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp
llvm/test/CodeGen/AArch64/kcfi.ll
llvm/test/CodeGen/X86/kcfi.ll
llvm/test/Transforms/InstCombine/kcfi_check.ll
Index: llvm/test/Transforms/InstCombine/kcfi_check.ll
===================================================================
--- /dev/null
+++ llvm/test/Transforms/InstCombine/kcfi_check.ll
@@ -0,0 +1,35 @@
+; RUN: opt < %s -passes=instcombine -S | FileCheck %s
+
+define void @f1() #0 prefix i32 10 {
+ ret void
+}
+
+declare void @f2() #0 prefix i32 11
+
+define internal void @f3() {
+ ret void
+}
+
+define void @g(ptr noundef %x) {
+ ; CHECK: call void @llvm.kcfi.check(ptr %x, i32 10)
+ call void @llvm.kcfi.check(ptr %x, i32 10)
+
+ ; CHECK-NOT: call void @llvm.kcfi.check(ptr nonnull @f1, i32 10)
+ ; CHECK: call void @llvm.kcfi.check(ptr nonnull @f1, i32 11)
+ call void @llvm.kcfi.check(ptr nonnull @f1, i32 10)
+ call void @llvm.kcfi.check(ptr nonnull @f1, i32 11)
+
+ ; CHECK: call void @llvm.kcfi.check(ptr nonnull @f2, i32 10)
+ ; CHECK-NOT: call void @llvm.kcfi.check(ptr nonnull @f2, i32 11)
+ call void @llvm.kcfi.check(ptr nonnull @f2, i32 10)
+ call void @llvm.kcfi.check(ptr nonnull @f2, i32 11)
+
+ ; CHECK: call void @llvm.kcfi.check(ptr nonnull @f3, i32 10)
+ call void @llvm.kcfi.check(ptr nonnull @f3, i32 10)
+ ret void
+}
+
+; CHECK: declare void @llvm.kcfi.check(ptr, i32 immarg)
+declare void @llvm.kcfi.check(ptr, i32 immarg)
+
+attributes #0 = { "kcfi" }
Index: llvm/test/CodeGen/X86/kcfi.ll
===================================================================
--- /dev/null
+++ llvm/test/CodeGen/X86/kcfi.ll
@@ -0,0 +1,39 @@
+; RUN: llc -mtriple=x86_64-unknown-linux-gnu < %s | FileCheck %s
+; RUN: llc -mtriple=x86_64-unknown-linux-gnu -stop-before=finalize-isel < %s | FileCheck %s --check-prefix=ISEL
+
+; CHECK: .type __cfi_f1,@function
+; CHECK-LABEL: __cfi_f1:
+; CHECK-NEXT: int3
+; CHECK-NEXT: int3
+; CHECK-NEXT: movl $12345678, %eax
+; CHECK-NEXT: int3
+; CHECK-NEXT: int3
+; CHECK-LABEL: .L__cfi_func_end0:
+; CHECK-NEXT: .size __cfi_f1, .L__cfi_func_end0-__cfi_f1
+define void @f1(ptr noundef %x) #0 prefix i32 12345678 {
+
+; CHECK-LABEL: f1:
+; CHECK: # %bb.0:
+; CHECK: cmpl $12345678, -6(%rdi) # imm = 0xBC614E
+; CHECK-NEXT: je .Ltmp0
+; CHECK-NEXT: .Ltmp1:
+; CHECK-NEXT: ud2
+; CHECK-NEXT: .section .kcfi_traps,"awo",@progbits,.text
+; CHECK-NEXT: .quad .Ltmp1
+; CHECK-NEXT: .text
+; CHECK-NEXT: .Ltmp0:
+; CHECK-NEXT: callq *%rdi
+
+; ISEL: name: f1
+; ISEL: body:
+; ISEL: KCFI_CHECK %[[#CALL:]], 12345678, implicit-def dead $eflags
+; ISEL: CALL64r %[[#CALL]], csr_64, implicit $rsp, implicit $ssp, implicit-def $rsp, implicit-def $ssp
+
+ call void @llvm.kcfi.check(ptr %x, i32 12345678)
+ call void %x()
+ ret void
+}
+
+declare void @llvm.kcfi.check(ptr, i32 immarg)
+
+attributes #0 = { "kcfi" }
Index: llvm/test/CodeGen/AArch64/kcfi.ll
===================================================================
--- /dev/null
+++ llvm/test/CodeGen/AArch64/kcfi.ll
@@ -0,0 +1,30 @@
+; RUN: llc -mtriple=aarch64-- < %s | FileCheck %s
+; RUN: llc -mtriple=aarch64-- -stop-before=finalize-isel < %s | FileCheck %s --check-prefix=ISEL
+
+; CHECK: .word 12345678
+define void @f1(ptr noundef %x) #0 prefix i32 12345678 {
+
+; CHECK-LABEL: f1:
+; CHECK: // %bb.0:
+; CHECK: ldur w16, [x0, #-4]
+; CHECK-NEXT: movk w17, #24910
+; CHECK-NEXT: movk w17, #188, lsl #16
+; CHECK-NEXT: cmp w16, w17
+; CHECK-NEXT: b.eq .Ltmp0
+; CHECK-NEXT: brk #0x8220
+; CHECK-NEXT: .Ltmp0:
+; CHECK-NEXT: blr x0
+
+; ISEL: name: f1
+; ISEL: body:
+; ISEL: KCFI_CHECK %[[#CALL:]], 12345678, implicit-def dead $x16, implicit-def dead $x17, implicit-def dead $nzcv
+; ISEL: BLR %[[#CALL]]
+
+ call void @llvm.kcfi.check(ptr %x, i32 12345678)
+ call void %x()
+ ret void
+}
+
+declare void @llvm.kcfi.check(ptr, i32 immarg)
+
+attributes #0 = { "kcfi" }
Index: llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp
===================================================================
--- llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp
+++ llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp
@@ -2673,6 +2673,21 @@
}
break;
}
+ case Intrinsic::kcfi_check: {
+ // If the first argument to llvm.kcfi.check() is known function, and the
+ // expected hash in the second argument matches the hash in the function
+ // prefix data, the check will always pass and can be removed.
+ auto *Target = dyn_cast<Function>(CI.getArgOperand(0)->stripPointerCasts());
+
+ if (Target && Target->hasFnAttribute("kcfi") && Target->hasPrefixData()) {
+ auto *Hash = cast<ConstantInt>(Target->getPrefixData());
+ auto *Expected = cast<ConstantInt>(CI.getArgOperand(1));
+
+ if (Hash->getZExtValue() == Expected->getZExtValue())
+ return eraseInstFromFunction(CI);
+ }
+ break;
+ }
default: {
// Handle target specific intrinsics
Optional<Instruction *> V = targetInstCombineIntrinsic(*II);
Index: llvm/lib/Target/X86/X86MCInstLower.cpp
===================================================================
--- llvm/lib/Target/X86/X86MCInstLower.cpp
+++ llvm/lib/Target/X86/X86MCInstLower.cpp
@@ -1336,6 +1336,31 @@
.addExpr(Op));
}
+void X86AsmPrinter::LowerKCFI_CHECK(const MachineInstr &MI) {
+ const MachineFunction &MF = *MI.getMF();
+
+ EmitAndCountInstruction(MCInstBuilder(X86::CMP32mi)
+ .addReg(MI.getOperand(0).getReg())
+ .addImm(1)
+ .addReg(X86::NoRegister)
+ .addImm(-6)
+ .addReg(X86::NoRegister)
+ .addImm(MI.getOperand(1).getImm()));
+
+ MCSymbol *Pass = OutContext.createTempSymbol();
+ EmitAndCountInstruction(
+ MCInstBuilder(X86::JCC_1)
+ .addExpr(MCSymbolRefExpr::create(Pass, OutContext))
+ .addImm(X86::COND_E));
+
+ MCSymbol *Trap = OutContext.createTempSymbol();
+ OutStreamer->emitLabel(Trap);
+ EmitAndCountInstruction(MCInstBuilder(X86::TRAP));
+ emitKCFITrapEntry(MF, Trap);
+
+ OutStreamer->emitLabel(Pass);
+}
+
void X86AsmPrinter::LowerASAN_CHECK_MEMACCESS(const MachineInstr &MI) {
// FIXME: Make this work on non-ELF.
if (!TM.getTargetTriple().isOSBinFormatELF()) {
@@ -2618,6 +2643,9 @@
EmitAndCountInstruction(MCInstBuilder(getRetOpcode(*Subtarget)));
return;
+ case X86::KCFI_CHECK:
+ return LowerKCFI_CHECK(*MI);
+
case X86::ASAN_CHECK_MEMACCESS:
return LowerASAN_CHECK_MEMACCESS(*MI);
Index: llvm/lib/Target/X86/X86InstrCompiler.td
===================================================================
--- llvm/lib/Target/X86/X86InstrCompiler.td
+++ llvm/lib/Target/X86/X86InstrCompiler.td
@@ -256,6 +256,17 @@
"#SEH_Epilogue", []>;
}
+//===----------------------------------------------------------------------===//
+// Pseudo instructions used by KCFI.
+//===----------------------------------------------------------------------===//
+let
+ Defs = [EFLAGS] in {
+def KCFI_CHECK : PseudoI<
+ (outs), (ins GR64:$ptr, i32imm:$type),
+ [(int_kcfi_check GR64:$ptr, (i32 timm:$type))]>,
+ Sched<[]>;
+}
+
//===----------------------------------------------------------------------===//
// Pseudo instructions used by address sanitizer.
//===----------------------------------------------------------------------===//
Index: llvm/lib/Target/X86/X86AsmPrinter.h
===================================================================
--- llvm/lib/Target/X86/X86AsmPrinter.h
+++ llvm/lib/Target/X86/X86AsmPrinter.h
@@ -98,6 +98,9 @@
void LowerFENTRY_CALL(const MachineInstr &MI, X86MCInstLower &MCIL);
+ // KCFI specific lowering for X86.
+ void LowerKCFI_CHECK(const MachineInstr &MI);
+
// Address sanitizer specific lowering for X86.
void LowerASAN_CHECK_MEMACCESS(const MachineInstr &MI);
@@ -151,6 +154,7 @@
bool runOnMachineFunction(MachineFunction &MF) override;
void emitFunctionBodyStart() override;
void emitFunctionBodyEnd() override;
+ void emitKCFITypeId(const MachineFunction &MF) override;
bool shouldEmitWeakSwiftAsyncExtendedFramePointerFlags() const override {
return ShouldEmitWeakSwiftAsyncExtendedFramePointerFlags;
Index: llvm/lib/Target/X86/X86AsmPrinter.cpp
===================================================================
--- llvm/lib/Target/X86/X86AsmPrinter.cpp
+++ llvm/lib/Target/X86/X86AsmPrinter.cpp
@@ -33,6 +33,7 @@
#include "llvm/MC/MCCodeEmitter.h"
#include "llvm/MC/MCContext.h"
#include "llvm/MC/MCExpr.h"
+#include "llvm/MC/MCInstBuilder.h"
#include "llvm/MC/MCSectionCOFF.h"
#include "llvm/MC/MCSectionELF.h"
#include "llvm/MC/MCSectionMachO.h"
@@ -108,6 +109,37 @@
}
}
+void X86AsmPrinter::emitKCFITypeId(const MachineFunction &MF) {
+ // Emit a function symbol for the type identifier data.
+ MCSymbol *FnSym = OutContext.getOrCreateSymbol("__cfi_" + MF.getName());
+ if (MAI->hasDotTypeDotSizeDirective())
+ OutStreamer->emitSymbolAttribute(FnSym, MCSA_ELF_TypeFunction);
+ OutStreamer->emitLabel(FnSym);
+
+ EmitAndCountInstruction(MCInstBuilder(X86::INT3));
+ EmitAndCountInstruction(MCInstBuilder(X86::INT3));
+
+ // Embed the type hash in a mov instruction.
+ auto *Hash = cast<ConstantInt>(MF.getFunction().getPrefixData());
+
+ EmitAndCountInstruction(MCInstBuilder(X86::MOV32ri)
+ .addReg(X86::EAX)
+ .addImm(Hash->getZExtValue()));
+
+ EmitAndCountInstruction(MCInstBuilder(X86::INT3));
+ EmitAndCountInstruction(MCInstBuilder(X86::INT3));
+
+ if (MAI->hasDotTypeDotSizeDirective()) {
+ MCSymbol *EndSym = OutContext.createTempSymbol("__cfi_func_end");
+ OutStreamer->emitLabel(EndSym);
+
+ const MCExpr *SizeExp = MCBinaryExpr::createSub(
+ MCSymbolRefExpr::create(EndSym, OutContext),
+ MCSymbolRefExpr::create(FnSym, OutContext), OutContext);
+ OutStreamer->emitELFSize(FnSym, SizeExp);
+ }
+}
+
/// PrintSymbolOperand - Print a raw symbol reference operand. This handles
/// jump tables, constant pools, global address and external symbols, all of
/// which print to a label with various suffixes for relocation types etc.
Index: llvm/lib/Target/AArch64/AArch64InstrInfo.td
===================================================================
--- llvm/lib/Target/AArch64/AArch64InstrInfo.td
+++ llvm/lib/Target/AArch64/AArch64InstrInfo.td
@@ -1368,6 +1368,13 @@
def MOVbaseTLS : Pseudo<(outs GPR64:$dst), (ins),
[(set GPR64:$dst, AArch64threadpointer)]>, Sched<[WriteSys]>;
+let Defs = [ X16, X17, NZCV ] in {
+def KCFI_CHECK : Pseudo<
+ (outs), (ins GPR64noip:$ptr, i32imm:$type),
+ [(int_kcfi_check GPR64noip:$ptr, (i32 timm:$type))]>,
+ Sched<[]>;
+}
+
let Uses = [ X9 ], Defs = [ X16, X17, LR, NZCV ] in {
def HWASAN_CHECK_MEMACCESS : Pseudo<
(outs), (ins GPR64noip:$ptr, i32imm:$accessinfo),
Index: llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp
===================================================================
--- llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp
+++ llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp
@@ -111,6 +111,7 @@
typedef std::tuple<unsigned, bool, uint32_t> HwasanMemaccessTuple;
std::map<HwasanMemaccessTuple, MCSymbol *> HwasanMemaccessSymbols;
+ void LowerKCFI_CHECK(const MachineInstr &MI);
void LowerHWASAN_CHECK_MEMACCESS(const MachineInstr &MI);
void emitHwasanMemaccessSymbols(Module &M);
@@ -317,6 +318,80 @@
recordSled(CurSled, MI, Kind, 2);
}
+void AArch64AsmPrinter::LowerKCFI_CHECK(const MachineInstr &MI) {
+ Register Addr = MI.getOperand(0).getReg();
+
+ if (Addr.id() == AArch64::XZR) {
+ // Checking XZR makes no sense. Instead of emitting a load, zero X16
+ // and use it for the ESR AddrIndex below.
+ Addr = Register(AArch64::X16);
+ EmitToStreamer(*OutStreamer, MCInstBuilder(AArch64::ORRXrs)
+ .addReg(Addr)
+ .addReg(AArch64::XZR)
+ .addReg(AArch64::XZR)
+ .addImm(0));
+ } else {
+ EmitToStreamer(*OutStreamer, MCInstBuilder(AArch64::LDURWi)
+ .addReg(AArch64::W16)
+ .addReg(Addr)
+ .addImm(-4));
+ }
+
+ int64_t Type = MI.getOperand(1).getImm();
+ EmitToStreamer(*OutStreamer, MCInstBuilder(AArch64::MOVKWi)
+ .addReg(AArch64::W17)
+ .addReg(AArch64::W17)
+ .addImm(Type & 0xFFFF)
+ .addImm(0));
+ EmitToStreamer(*OutStreamer, MCInstBuilder(AArch64::MOVKWi)
+ .addReg(AArch64::W17)
+ .addReg(AArch64::W17)
+ .addImm((Type >> 16) & 0xFFFF)
+ .addImm(16));
+
+ EmitToStreamer(*OutStreamer, MCInstBuilder(AArch64::SUBSWrs)
+ .addReg(AArch64::WZR)
+ .addReg(AArch64::W16)
+ .addReg(AArch64::W17)
+ .addImm(0));
+
+ MCSymbol *Pass = OutContext.createTempSymbol();
+ EmitToStreamer(*OutStreamer,
+ MCInstBuilder(AArch64::Bcc)
+ .addImm(AArch64CC::EQ)
+ .addExpr(MCSymbolRefExpr::create(Pass, OutContext)));
+
+ assert(Addr.isPhysical() &&
+ "Unable to encode the target register for the KCFI trap");
+
+ // The base ESR is 0x8000 and the register information is encoded
+ // in bits 0-9 as follows:
+ // - 0-4: n, where the register Xn contains the target address
+ // - 5-9: m, where the register Wm contains the type hash
+ // Where n, m are in [0, 30].
+ unsigned TypeIndex = AArch64::W17 - AArch64::W0;
+ unsigned AddrIndex;
+
+ switch (Addr.id()) {
+ default:
+ AddrIndex = Addr.id() - AArch64::X0;
+ break;
+ case AArch64::FP:
+ AddrIndex = 29;
+ break;
+ case AArch64::LR:
+ AddrIndex = 30;
+ break;
+ }
+
+ assert(AddrIndex < 31 && TypeIndex < 31);
+
+ unsigned ESR = 0x8000 | ((TypeIndex & 31) << 5) | (AddrIndex & 31);
+ EmitToStreamer(*OutStreamer, MCInstBuilder(AArch64::BRK).addImm(ESR));
+
+ OutStreamer->emitLabel(Pass);
+}
+
void AArch64AsmPrinter::LowerHWASAN_CHECK_MEMACCESS(const MachineInstr &MI) {
Register Reg = MI.getOperand(0).getReg();
bool IsShort =
@@ -1433,6 +1508,10 @@
LowerPATCHABLE_TAIL_CALL(*MI);
return;
+ case AArch64::KCFI_CHECK:
+ LowerKCFI_CHECK(*MI);
+ return;
+
case AArch64::HWASAN_CHECK_MEMACCESS:
case AArch64::HWASAN_CHECK_MEMACCESS_SHORTGRANULES:
LowerHWASAN_CHECK_MEMACCESS(*MI);
Index: llvm/lib/MC/MCObjectFileInfo.cpp
===================================================================
--- llvm/lib/MC/MCObjectFileInfo.cpp
+++ llvm/lib/MC/MCObjectFileInfo.cpp
@@ -1116,6 +1116,25 @@
cast<MCSymbolELF>(TextSec.getBeginSymbol()));
}
+MCSection *
+MCObjectFileInfo::getKCFITrapSection(const MCSection &TextSec) const {
+ if (Ctx->getObjectFileType() != MCContext::IsELF)
+ return nullptr;
+
+ const MCSectionELF &ElfSec = static_cast<const MCSectionELF &>(TextSec);
+ unsigned Flags = ELF::SHF_LINK_ORDER | ELF::SHF_ALLOC | ELF::SHF_WRITE;
+ StringRef GroupName;
+ if (const MCSymbol *Group = ElfSec.getGroup()) {
+ GroupName = Group->getName();
+ Flags |= ELF::SHF_GROUP;
+ }
+
+ return Ctx->getELFSection(".kcfi_traps", ELF::SHT_PROGBITS, Flags, 0,
+ GroupName,
+ /*IsComdat=*/true, ElfSec.getUniqueID(),
+ cast<MCSymbolELF>(TextSec.getBeginSymbol()));
+}
+
MCSection *
MCObjectFileInfo::getPseudoProbeSection(const MCSection *TextSec) const {
if (Ctx->getObjectFileType() == MCContext::IsELF) {
Index: llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp
===================================================================
--- llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp
+++ llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp
@@ -924,21 +924,26 @@
// Emit the prefix data.
if (F.hasPrefixData()) {
- if (MAI->hasSubsectionsViaSymbols()) {
- // Preserving prefix data on platforms which use subsections-via-symbols
- // is a bit tricky. Here we introduce a symbol for the prefix data
- // and use the .alt_entry attribute to mark the function's real entry point
- // as an alternative entry point to the prefix-data symbol.
- MCSymbol *PrefixSym = OutContext.createLinkerPrivateTempSymbol();
- OutStreamer->emitLabel(PrefixSym);
+ bool SubsectionsViaSymbols = MAI->hasSubsectionsViaSymbols();
+
+ if (F.hasFnAttribute("kcfi"))
+ emitKCFITypeId(*MF);
+ else {
+ if (SubsectionsViaSymbols) {
+ // Preserving prefix data on platforms which use subsections-via-symbols
+ // is a bit tricky. Here we introduce a symbol for the prefix data
+ // and use the .alt_entry attribute to mark the function's real entry
+ // point as an alternative entry point to the prefix-data symbol.
+ MCSymbol *PrefixSym = OutContext.createLinkerPrivateTempSymbol();
+ OutStreamer->emitLabel(PrefixSym);
+ }
emitGlobalConstant(F.getParent()->getDataLayout(), F.getPrefixData());
+ }
- // Emit an .alt_entry directive for the actual function symbol.
+ // Emit an .alt_entry directive for the actual function symbol.
+ if (SubsectionsViaSymbols)
OutStreamer->emitSymbolAttribute(CurrentFnSym, MCSA_AltEntry);
- } else {
- emitGlobalConstant(F.getParent()->getDataLayout(), F.getPrefixData());
- }
}
// Emit M NOPs for -fpatchable-function-entry=N,M where M>0. We arbitrarily
@@ -1326,6 +1331,24 @@
OutStreamer->PopSection();
}
+void AsmPrinter::emitKCFITrapEntry(const MachineFunction &MF,
+ const MCSymbol *Symbol) {
+ MCSection *Section =
+ getObjFileLowering().getKCFITrapSection(*MF.getSection());
+
+ if (Section) {
+ OutStreamer->PushSection();
+ OutStreamer->SwitchSection(Section);
+ OutStreamer->emitSymbolValue(Symbol, getPointerSize());
+ OutStreamer->PopSection();
+ }
+}
+
+void AsmPrinter::emitKCFITypeId(const MachineFunction &MF) {
+ const Function &F = MF.getFunction();
+ emitGlobalConstant(F.getParent()->getDataLayout(), F.getPrefixData());
+}
+
void AsmPrinter::emitPseudoProbe(const MachineInstr &MI) {
if (PP) {
auto GUID = MI.getOperand(0).getImm();
Index: llvm/include/llvm/MC/MCObjectFileInfo.h
===================================================================
--- llvm/include/llvm/MC/MCObjectFileInfo.h
+++ llvm/include/llvm/MC/MCObjectFileInfo.h
@@ -16,6 +16,7 @@
#include "llvm/ADT/Optional.h"
#include "llvm/ADT/Triple.h"
#include "llvm/BinaryFormat/Swift.h"
+#include "llvm/MC/MCSection.h"
#include "llvm/Support/VersionTuple.h"
#include <array>
@@ -356,6 +357,8 @@
MCSection *getBBAddrMapSection(const MCSection &TextSec) const;
+ MCSection *getKCFITrapSection(const MCSection &TextSec) const;
+
MCSection *getPseudoProbeSection(const MCSection *TextSec) const;
MCSection *getPseudoProbeDescSection(StringRef FuncName) const;
Index: llvm/include/llvm/IR/Intrinsics.td
===================================================================
--- llvm/include/llvm/IR/Intrinsics.td
+++ llvm/include/llvm/IR/Intrinsics.td
@@ -1766,6 +1766,9 @@
def int_load_relative: DefaultAttrsIntrinsic<[llvm_ptr_ty], [llvm_ptr_ty, llvm_anyint_ty],
[IntrReadMem, IntrArgMemOnly]>;
+def int_kcfi_check :
+ Intrinsic<[], [llvm_ptr_ty, llvm_i32_ty], [ImmArg<ArgIndex<1>>]>;
+
def int_asan_check_memaccess :
Intrinsic<[],[llvm_ptr_ty, llvm_i32_ty], [ImmArg<ArgIndex<1>>]>;
Index: llvm/include/llvm/CodeGen/AsmPrinter.h
===================================================================
--- llvm/include/llvm/CodeGen/AsmPrinter.h
+++ llvm/include/llvm/CodeGen/AsmPrinter.h
@@ -400,6 +400,9 @@
void emitBBAddrMapSection(const MachineFunction &MF);
+ void emitKCFITrapEntry(const MachineFunction &MF, const MCSymbol *Symbol);
+ virtual void emitKCFITypeId(const MachineFunction &MF);
+
void emitPseudoProbe(const MachineInstr &MI);
void emitRemarksSection(remarks::RemarkStreamer &RS);
Index: clang/test/Driver/fsanitize.c
===================================================================
--- clang/test/Driver/fsanitize.c
+++ clang/test/Driver/fsanitize.c
@@ -647,6 +647,27 @@
// RUN: %clang -target x86_64-linux-gnu -fsanitize=cfi -fsanitize-stats -flto -c %s -### 2>&1 | FileCheck %s --check-prefix=CHECK-CFI-STATS
// CHECK-CFI-STATS: -fsanitize-stats
+// RUN: %clang -target x86_64-linux-gnu -fsanitize=kcfi -fsanitize=cfi -flto -fvisibility=hidden %s -### 2>&1 | FileCheck %s --check-prefix=CHECK-KCFI-NOCFI
+// CHECK-KCFI-NOCFI: error: invalid argument '-fsanitize=kcfi' not allowed with '-fsanitize=cfi'
+
+// RUN: %clang -target x86_64-linux-gnu -fsanitize=kcfi -fpatchable-function-entry=1 %s -### 2>&1 | FileCheck %s --check-prefix=CHECK-KCFI-PATCHABLE-NOM
+// CHECK-KCFI-PATCHABLE-NOM: "-fsanitize=kcfi"
+
+// RUN: %clang -target x86_64-linux-gnu -fsanitize=kcfi -fpatchable-function-entry=1,0 %s -### 2>&1 | FileCheck %s --check-prefix=CHECK-KCFI-PATCHABLE-M0
+// CHECK-KCFI-PATCHABLE-M0: "-fsanitize=kcfi"
+
+// RUN: %clang -target x86_64-linux-gnu -fsanitize=kcfi -fpatchable-function-entry=1,1 %s -### 2>&1 | FileCheck %s --check-prefix=CHECK-KCFI-PATCHABLE-M1
+// CHECK-KCFI-PATCHABLE-M1: error: invalid argument '-fsanitize=kcfi' not allowed with '-fpatchable-function-entry=1,1'
+
+// RUN: %clang -target x86_64-linux-gnu -fsanitize=kcfi -fsanitize-trap=kcfi %s -### 2>&1 | FileCheck %s --check-prefix=CHECK-KCFI-NOTRAP
+// CHECK-KCFI-NOTRAP: error: unsupported argument 'kcfi' to option '-fsanitize-trap='
+
+// RUN: %clang -target x86_64-linux-gnu -fsanitize=kcfi %s -### 2>&1 | FileCheck %s --check-prefix=CHECK-KCFI
+// CHECK-KCFI: "-fsanitize=kcfi"
+
+// RUN: %clang -target x86_64-linux-gnu -fsanitize=kcfi -fno-sanitize-recover=kcfi %s -### 2>&1 | FileCheck %s --check-prefix=CHECK-KCFI-RECOVER
+// CHECK-KCFI-RECOVER: error: unsupported argument 'kcfi' to option '-fno-sanitize-recover='
+
// RUN: %clang_cl -fsanitize=address -c -MDd -### -- %s 2>&1 | FileCheck %s -check-prefix=CHECK-ASAN-DEBUGRTL
// RUN: %clang_cl -fsanitize=address -c -MTd -### -- %s 2>&1 | FileCheck %s -check-prefix=CHECK-ASAN-DEBUGRTL
// RUN: %clang_cl -fsanitize=address -c -LDd -### -- %s 2>&1 | FileCheck %s -check-prefix=CHECK-ASAN-DEBUGRTL
Index: clang/test/CodeGen/kcfi.c
===================================================================
--- /dev/null
+++ clang/test/CodeGen/kcfi.c
@@ -0,0 +1,57 @@
+// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -emit-llvm -fsanitize=kcfi -o - %s | FileCheck --check-prefixes=CHECK %s
+#if !__has_feature(kcfi)
+#error Missing kcfi?
+#endif
+
+// CHECK: module asm ".weak __kcfi_typeid_f4"
+// CHECK: module asm ".set __kcfi_typeid_f4, [[#]]"
+
+typedef int (*fn_t)(void);
+
+// CHECK: define dso_local i32 @f1() #[[#ATTR:]] prefix i32 [[#%d,HASH:]]
+int f1(void) { return 0; }
+
+// CHECK: define dso_local i32 @f2() #[[#ATTR]] prefix i32 [[#%d,HASH2:]]
+unsigned int f2(void) { return 2; }
+
+// CHECK-LABEL: define dso_local i32 @__call(ptr{{.*}} %f)
+int __call(fn_t f) __attribute__((__no_sanitize__("kcfi"))) {
+ // CHECK-NOT: call void @llvm.kcfi_check
+ // CHECK: %call = call i32 %[[#]]()
+ return f();
+}
+
+// CHECK-LABEL: define dso_local i32 @call(ptr{{.*}} %f)
+int call(fn_t f) {
+ // CHECK: call void @llvm.kcfi.check(ptr %[[#]], i32 [[#HASH]])
+ // CHECK: %call = call i32 %[[#]]()
+ return f();
+}
+
+// CHECK-DAG: define internal i32 @f3() #[[#ATTR]] prefix i32 [[#HASH]]
+static int f3(void) { return 1; }
+
+// CHECK-DAG: declare i32 @f4() #[[#DECLATTR:]] prefix i32 [[#HASH]]
+extern int f4(void);
+
+// CHECK-DAG: declare void @llvm.kcfi.check(ptr, i32 immarg)
+
+// CHECK: define internal i32 @f5() #[[#LOCALATTR:]]
+// CHECK-NOT: prefix i32
+// CHECK-SAME: {
+static int f5(void) { return 2; }
+
+int test(void) {
+ return call(f1) +
+ __call((fn_t)f2) +
+ call(f3) +
+ call(f4) +
+ f5();
+}
+
+// CHECK: attributes #[[#ATTR]] = {{{.*}}"kcfi"
+// CHECK: attributes #[[#DECLATTR]] = {{{.*}}"kcfi"
+
+// CHECK: attributes #[[#LOCALATTR]] = {
+// CHECK-NOT: {{.*}}"kcfi"
+// CHECK-SAME: }
Index: clang/lib/Driver/ToolChain.cpp
===================================================================
--- clang/lib/Driver/ToolChain.cpp
+++ clang/lib/Driver/ToolChain.cpp
@@ -1036,6 +1036,9 @@
getTriple().getArch() == llvm::Triple::arm || getTriple().isWasm() ||
getTriple().isAArch64())
Res |= SanitizerKind::CFIICall;
+ if (getTriple().getArch() == llvm::Triple::x86_64 ||
+ getTriple().isAArch64(64))
+ Res |= SanitizerKind::KCFI;
if (getTriple().getArch() == llvm::Triple::x86_64 ||
getTriple().isAArch64(64) || getTriple().isRISCV())
Res |= SanitizerKind::ShadowCallStack;
Index: clang/lib/Driver/SanitizerArgs.cpp
===================================================================
--- clang/lib/Driver/SanitizerArgs.cpp
+++ clang/lib/Driver/SanitizerArgs.cpp
@@ -37,7 +37,8 @@
static const SanitizerMask NotAllowedWithMinimalRuntime =
SanitizerKind::Function | SanitizerKind::Vptr;
static const SanitizerMask RequiresPIE =
- SanitizerKind::DataFlow | SanitizerKind::HWAddress | SanitizerKind::Scudo;
+ SanitizerKind::DataFlow | SanitizerKind::HWAddress | SanitizerKind::Scudo |
+ SanitizerKind::KCFI;
static const SanitizerMask NeedsUnwindTables =
SanitizerKind::Address | SanitizerKind::HWAddress | SanitizerKind::Thread |
SanitizerKind::Memory | SanitizerKind::DataFlow;
@@ -58,8 +59,9 @@
SanitizerKind::FloatDivideByZero | SanitizerKind::ObjCCast;
static const SanitizerMask Unrecoverable =
SanitizerKind::Unreachable | SanitizerKind::Return;
-static const SanitizerMask AlwaysRecoverable =
- SanitizerKind::KernelAddress | SanitizerKind::KernelHWAddress;
+static const SanitizerMask AlwaysRecoverable = SanitizerKind::KernelAddress |
+ SanitizerKind::KernelHWAddress |
+ SanitizerKind::KCFI;
static const SanitizerMask NeedsLTO = SanitizerKind::CFI;
static const SanitizerMask TrappingSupported =
(SanitizerKind::Undefined & ~SanitizerKind::Vptr) | SanitizerKind::Integer |
@@ -692,6 +694,25 @@
options::OPT_fno_sanitize_cfi_canonical_jump_tables, true);
}
+ if (AllAddedKinds & SanitizerKind::KCFI && DiagnoseErrors) {
+ if (AllAddedKinds & SanitizerKind::CFI)
+ D.Diag(diag::err_drv_argument_not_allowed_with)
+ << "-fsanitize=kcfi"
+ << lastArgumentForMask(D, Args, SanitizerKind::CFI);
+
+ if (Arg *A = Args.getLastArg(options::OPT_fpatchable_function_entry_EQ)) {
+ StringRef S = A->getValue();
+ unsigned N, M;
+ // With -fpatchable-function-entry=N,M, where M > 0,
+ // llvm::AsmPrinter::emitFunctionHeader injects nops before before the
+ // KCFI type identifier, which is currently unsupported.
+ if (!S.consumeInteger(10, N) && S.consume_front(",") &&
+ !S.consumeInteger(10, M) && M > 0)
+ D.Diag(diag::err_drv_argument_not_allowed_with)
+ << "-fsanitize=kcfi" << A->getAsString(Args);
+ }
+ }
+
Stats = Args.hasFlag(options::OPT_fsanitize_stats,
options::OPT_fno_sanitize_stats, false);
Index: clang/lib/CodeGen/CodeGenModule.h
===================================================================
--- clang/lib/CodeGen/CodeGenModule.h
+++ clang/lib/CodeGen/CodeGenModule.h
@@ -1393,6 +1393,9 @@
/// Generate a cross-DSO type identifier for MD.
llvm::ConstantInt *CreateCrossDsoCfiTypeId(llvm::Metadata *MD);
+ /// Generate a KCFI type identifier for T.
+ llvm::ConstantInt *CreateKCFITypeId(QualType T);
+
/// Create a metadata identifier for the given type. This may either be an
/// MDString (for external identifiers) or a distinct unnamed MDNode (for
/// internal identifiers).
@@ -1411,6 +1414,12 @@
void CreateFunctionTypeMetadataForIcall(const FunctionDecl *FD,
llvm::Function *F);
+ /// Set type hash as prefix data to the given function
+ void SetKCFITypePrefix(const FunctionDecl *FD, llvm::Function *F);
+
+ /// Emit KCFI type identifier constants and remove unused identifiers
+ void FinalizeKCFITypePrefixes();
+
/// Whether this function's return type has no side effects, and thus may
/// be trivially discarded if it is unused.
bool MayDropFunctionReturn(const ASTContext &Context, QualType ReturnType);
@@ -1652,7 +1661,8 @@
llvm::AttrBuilder &FuncAttrs);
llvm::Metadata *CreateMetadataIdentifierImpl(QualType T, MetadataTypeMap &Map,
- StringRef Suffix);
+ StringRef Suffix,
+ bool OnlyExternal = true);
};
} // end namespace CodeGen
Index: clang/lib/CodeGen/CodeGenModule.cpp
===================================================================
--- clang/lib/CodeGen/CodeGenModule.cpp
+++ clang/lib/CodeGen/CodeGenModule.cpp
@@ -47,6 +47,7 @@
#include "clang/CodeGen/BackendUtil.h"
#include "clang/CodeGen/ConstantInitBuilder.h"
#include "clang/Frontend/FrontendDiagnostic.h"
+#include "llvm/ADT/StringExtras.h"
#include "llvm/ADT/StringSwitch.h"
#include "llvm/ADT/Triple.h"
#include "llvm/Analysis/TargetLibraryInfo.h"
@@ -65,6 +66,7 @@
#include "llvm/Support/MD5.h"
#include "llvm/Support/TimeProfiler.h"
#include "llvm/Support/X86TargetParser.h"
+#include "llvm/Support/xxhash.h"
using namespace clang;
using namespace CodeGen;
@@ -554,6 +556,8 @@
CodeGenFunction(*this).EmitCfiCheckFail();
CodeGenFunction(*this).EmitCfiCheckStub();
}
+ if (LangOpts.Sanitize.has(SanitizerKind::KCFI))
+ FinalizeKCFITypePrefixes();
emitAtAvailableLinkGuard();
if (Context.getTargetInfo().getTriple().isWasm() &&
!Context.getTargetInfo().getTriple().isOSEmscripten()) {
@@ -1641,6 +1645,15 @@
return llvm::ConstantInt::get(Int64Ty, llvm::MD5Hash(MDS->getString()));
}
+llvm::ConstantInt *CodeGenModule::CreateKCFITypeId(QualType T) {
+ if (auto *MDS = dyn_cast<llvm::MDString>(CreateMetadataIdentifierImpl(
+ T, MetadataIdMap, "", /*OnlyExternal=*/false)))
+ return llvm::ConstantInt::get(
+ Int32Ty, static_cast<uint32_t>(llvm::xxHash64(MDS->getString())));
+
+ return nullptr;
+}
+
void CodeGenModule::SetLLVMFunctionAttributes(GlobalDecl GD,
const CGFunctionInfo &Info,
llvm::Function *F, bool IsThunk) {
@@ -2234,6 +2247,60 @@
F->addTypeMetadata(0, llvm::ConstantAsMetadata::get(CrossDsoTypeId));
}
+void CodeGenModule::SetKCFITypePrefix(const FunctionDecl *FD,
+ llvm::Function *F) {
+
+ if (isa<CXXMethodDecl>(FD) && !cast<CXXMethodDecl>(FD)->isStatic())
+ return;
+
+ F->setPrefixData(CreateKCFITypeId(FD->getType()));
+ F->addFnAttr("kcfi");
+}
+
+static bool allowKCFIIdentifier(StringRef Name) {
+ // KCFI type identifier constants are only necessary for external assembly
+ // functions, which means it's safe to skip unusual names. Subset of
+ // MCAsmInfo::isAcceptableChar() and MCAsmInfoXCOFF::isAcceptableChar().
+ for (const char &C : Name) {
+ if (llvm::isAlnum(C) || C == '_' || C == '.')
+ continue;
+ return false;
+ }
+ return true;
+}
+
+void CodeGenModule::FinalizeKCFITypePrefixes() {
+ llvm::Module &M = getModule();
+ for (auto &F : M.functions()) {
+ bool AddressTaken = F.hasAddressTaken();
+
+ // Remove KCFI prefix data and attribute from non-address-taken local
+ // functions.
+ if (!AddressTaken && F.hasLocalLinkage()) {
+ F.setPrefixData(nullptr);
+ F.removeFnAttr("kcfi");
+ }
+
+ if (!AddressTaken || !F.isDeclaration() || !F.hasPrefixData())
+ continue;
+
+ // Generate a weak constant with the expected KCFI type identifier for all
+ // address-taken function declarations.
+ auto *Id = dyn_cast<llvm::ConstantInt>(F.getPrefixData());
+ if (!Id)
+ continue;
+
+ StringRef Name = F.getName();
+ if (!allowKCFIIdentifier(Name))
+ continue;
+
+ std::string Asm = (".weak __kcfi_typeid_" + Name + "\n.set __kcfi_typeid_" +
+ Name + ", " + Twine(Id->getSExtValue()) + "\n")
+ .str();
+ M.appendModuleInlineAsm(Asm);
+ }
+}
+
void CodeGenModule::SetFunctionAttributes(GlobalDecl GD, llvm::Function *F,
bool IsIncompleteFunction,
bool IsThunk) {
@@ -2316,6 +2383,9 @@
!CodeGenOpts.SanitizeCfiCanonicalJumpTables)
CreateFunctionTypeMetadataForIcall(FD, F);
+ if (LangOpts.Sanitize.has(SanitizerKind::KCFI))
+ SetKCFITypePrefix(FD, F);
+
if (getLangOpts().OpenMP && FD->hasAttr<OMPDeclareSimdDeclAttr>())
getOpenMPRuntime().emitDeclareSimdFunction(FD, F);
@@ -6600,7 +6670,8 @@
llvm::Metadata *
CodeGenModule::CreateMetadataIdentifierImpl(QualType T, MetadataTypeMap &Map,
- StringRef Suffix) {
+ StringRef Suffix,
+ bool OnlyExternal /*=true*/) {
if (auto *FnType = T->getAs<FunctionProtoType>())
T = getContext().getFunctionType(
FnType->getReturnType(), FnType->getParamTypes(),
@@ -6610,7 +6681,7 @@
if (InternalId)
return InternalId;
- if (isExternallyVisible(T->getLinkage())) {
+ if (isExternallyVisible(T->getLinkage()) || !OnlyExternal) {
std::string OutName;
llvm::raw_string_ostream Out(OutName);
getCXXABI().getMangleContext().mangleTypeName(T, Out);
Index: clang/lib/CodeGen/CodeGenFunction.h
===================================================================
--- clang/lib/CodeGen/CodeGenFunction.h
+++ clang/lib/CodeGen/CodeGenFunction.h
@@ -4604,6 +4604,8 @@
/// passing to a runtime sanitizer handler.
llvm::Constant *EmitCheckSourceLocation(SourceLocation Loc);
+ void EmitKCFICheck(llvm::Value *Ptr, llvm::ConstantInt *Hash);
+
/// Create a basic block that will either trap or call a handler function in
/// the UBSan runtime with the provided arguments, and create a conditional
/// branch to it.
Index: clang/lib/CodeGen/CGExpr.cpp
===================================================================
--- clang/lib/CodeGen/CGExpr.cpp
+++ clang/lib/CodeGen/CGExpr.cpp
@@ -3227,6 +3227,11 @@
#undef SANITIZER_CHECK
};
+void CodeGenFunction::EmitKCFICheck(llvm::Value *Ptr, llvm::ConstantInt *Hash) {
+ Builder.CreateCall(CGM.getIntrinsic(llvm::Intrinsic::kcfi_check),
+ {Ptr, Hash});
+}
+
static void emitCheckHandlerCall(CodeGenFunction &CGF,
llvm::FunctionType *FnType,
ArrayRef<llvm::Value *> FnArgs,
@@ -5355,11 +5360,11 @@
}
const auto *FnType = cast<FunctionType>(PointeeType);
+ bool IsIndirectCall = !TargetDecl || !isa<FunctionDecl>(TargetDecl);
// If we are checking indirect calls and this call is indirect, check that the
// function pointer is a member of the bit set for the function type.
- if (SanOpts.has(SanitizerKind::CFIICall) &&
- (!TargetDecl || !isa<FunctionDecl>(TargetDecl))) {
+ if (SanOpts.has(SanitizerKind::CFIICall) && IsIndirectCall) {
SanitizerScope SanScope(this);
EmitSanitizerStatReport(llvm::SanStat_CFI_ICall);
@@ -5392,6 +5397,10 @@
}
}
+ if (SanOpts.has(SanitizerKind::KCFI) && IsIndirectCall)
+ EmitKCFICheck(Callee.getFunctionPointer(),
+ CGM.CreateKCFITypeId(QualType(FnType, 0)));
+
CallArgList Args;
if (Chain)
Args.add(RValue::get(Builder.CreateBitCast(Chain, CGM.VoidPtrTy)),
Index: clang/include/clang/Basic/Sanitizers.def
===================================================================
--- clang/include/clang/Basic/Sanitizers.def
+++ clang/include/clang/Basic/Sanitizers.def
@@ -126,6 +126,9 @@
CFIDerivedCast | CFIICall | CFIMFCall | CFIUnrelatedCast |
CFINVCall | CFIVCall)
+// Kernel Control Flow Integrity
+SANITIZER("kcfi", KCFI)
+
// Safe Stack
SANITIZER("safe-stack", SafeStack)
Index: clang/include/clang/Basic/Features.def
===================================================================
--- clang/include/clang/Basic/Features.def
+++ clang/include/clang/Basic/Features.def
@@ -227,6 +227,7 @@
FEATURE(is_trivially_copyable, LangOpts.CPlusPlus)
FEATURE(is_union, LangOpts.CPlusPlus)
FEATURE(modules, LangOpts.Modules)
+FEATURE(kcfi, LangOpts.Sanitize.has(SanitizerKind::KCFI))
FEATURE(safe_stack, LangOpts.Sanitize.has(SanitizerKind::SafeStack))
FEATURE(shadow_call_stack,
LangOpts.Sanitize.has(SanitizerKind::ShadowCallStack))
Index: clang/docs/UsersManual.rst
===================================================================
--- clang/docs/UsersManual.rst
+++ clang/docs/UsersManual.rst
@@ -1692,6 +1692,8 @@
flow analysis.
- ``-fsanitize=cfi``: :doc:`control flow integrity <ControlFlowIntegrity>`
checks. Requires ``-flto``.
+ - ``-fsanitize=kcfi``: kernel indirect call forward-edge control flow
+ integrity.
- ``-fsanitize=safe-stack``: :doc:`safe stack <SafeStack>`
protection against stack-based memory corruption errors.
Index: clang/docs/ControlFlowIntegrity.rst
===================================================================
--- clang/docs/ControlFlowIntegrity.rst
+++ clang/docs/ControlFlowIntegrity.rst
@@ -306,6 +306,19 @@
library boundaries are no different from calls within a single program or
shared library.
+.. _kcfi:
+
+``-fsanitize=kcfi``
+-------------------
+
+KCFI, enabled by ``-fsanitize=kcfi``, is an alternative indirect call
+control-flow integrity scheme designed for low-level system software, such
+as operating system kernels. Unlike ``-fsanitize=cfi-icall``, it doesn't
+require ``-flto``, won't result in function pointers being replaced with jump
+table references, and never breaks cross-DSO function address equality. These
+properties make KCFI easier to adopt in low-level software. KCFI is limited to
+indirect call checking only, and isn't compatible with executable-only memory.
+
Member Function Pointer Call Checking
=====================================
Index: clang/docs/ClangCommandLineReference.rst
===================================================================
--- clang/docs/ClangCommandLineReference.rst
+++ clang/docs/ClangCommandLineReference.rst
@@ -979,6 +979,10 @@
Generalize pointers in CFI indirect call type signature checks
+.. option:: -fsanitize-kcfi-offset=<value>
+
+Set the number of bytes between the KCFI type identifier and the start of the function.
+
.. option:: -fsanitize-coverage-allowlist=<arg>, -fsanitize-coverage-whitelist=<arg>
Restrict sanitizer coverage instrumentation exclusively to modules and functions that match the provided special case list, except the blocked ones
_______________________________________________
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
