diff --git a/llvm/include/llvm/BinaryFormat/ELF.h b/llvm/include/llvm/BinaryFormat/ELF.h index 9bf5a3c99a55..1b0412bc47be 100644 --- a/llvm/include/llvm/BinaryFormat/ELF.h +++ b/llvm/include/llvm/BinaryFormat/ELF.h @@ -1,1580 +1,1574 @@ //===- llvm/BinaryFormat/ELF.h - ELF constants and structures ---*- C++ -*-===// // // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. // See https://llvm.org/LICENSE.txt for license information. // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // // This header contains common, non-processor-specific data structures and // constants for the ELF file format. // // The details of the ELF32 bits in this file are largely based on the Tool // Interface Standard (TIS) Executable and Linking Format (ELF) Specification // Version 1.2, May 1995. The ELF64 stuff is based on ELF-64 Object File Format // Version 1.5, Draft 2, May 1998 as well as OpenBSD header files. // //===----------------------------------------------------------------------===// #ifndef LLVM_BINARYFORMAT_ELF_H #define LLVM_BINARYFORMAT_ELF_H #include #include namespace llvm { namespace ELF { using Elf32_Addr = uint32_t; // Program address using Elf32_Off = uint32_t; // File offset using Elf32_Half = uint16_t; using Elf32_Word = uint32_t; using Elf32_Sword = int32_t; using Elf64_Addr = uint64_t; using Elf64_Off = uint64_t; using Elf64_Half = uint16_t; using Elf64_Word = uint32_t; using Elf64_Sword = int32_t; using Elf64_Xword = uint64_t; using Elf64_Sxword = int64_t; // Object file magic string. static const char ElfMagic[] = {0x7f, 'E', 'L', 'F', '\0'}; // e_ident size and indices. enum { EI_MAG0 = 0, // File identification index. EI_MAG1 = 1, // File identification index. EI_MAG2 = 2, // File identification index. EI_MAG3 = 3, // File identification index. EI_CLASS = 4, // File class. EI_DATA = 5, // Data encoding. EI_VERSION = 6, // File version. EI_OSABI = 7, // OS/ABI identification. EI_ABIVERSION = 8, // ABI version. EI_PAD = 9, // Start of padding bytes. EI_NIDENT = 16 // Number of bytes in e_ident. }; struct Elf32_Ehdr { unsigned char e_ident[EI_NIDENT]; // ELF Identification bytes Elf32_Half e_type; // Type of file (see ET_* below) Elf32_Half e_machine; // Required architecture for this file (see EM_*) Elf32_Word e_version; // Must be equal to 1 Elf32_Addr e_entry; // Address to jump to in order to start program Elf32_Off e_phoff; // Program header table's file offset, in bytes Elf32_Off e_shoff; // Section header table's file offset, in bytes Elf32_Word e_flags; // Processor-specific flags Elf32_Half e_ehsize; // Size of ELF header, in bytes Elf32_Half e_phentsize; // Size of an entry in the program header table Elf32_Half e_phnum; // Number of entries in the program header table Elf32_Half e_shentsize; // Size of an entry in the section header table Elf32_Half e_shnum; // Number of entries in the section header table Elf32_Half e_shstrndx; // Sect hdr table index of sect name string table bool checkMagic() const { return (memcmp(e_ident, ElfMagic, strlen(ElfMagic))) == 0; } unsigned char getFileClass() const { return e_ident[EI_CLASS]; } unsigned char getDataEncoding() const { return e_ident[EI_DATA]; } }; // 64-bit ELF header. Fields are the same as for ELF32, but with different // types (see above). struct Elf64_Ehdr { unsigned char e_ident[EI_NIDENT]; Elf64_Half e_type; Elf64_Half e_machine; Elf64_Word e_version; Elf64_Addr e_entry; Elf64_Off e_phoff; Elf64_Off e_shoff; Elf64_Word e_flags; Elf64_Half e_ehsize; Elf64_Half e_phentsize; Elf64_Half e_phnum; Elf64_Half e_shentsize; Elf64_Half e_shnum; Elf64_Half e_shstrndx; bool checkMagic() const { return (memcmp(e_ident, ElfMagic, strlen(ElfMagic))) == 0; } unsigned char getFileClass() const { return e_ident[EI_CLASS]; } unsigned char getDataEncoding() const { return e_ident[EI_DATA]; } }; // File types enum { ET_NONE = 0, // No file type ET_REL = 1, // Relocatable file ET_EXEC = 2, // Executable file ET_DYN = 3, // Shared object file ET_CORE = 4, // Core file ET_LOPROC = 0xff00, // Beginning of processor-specific codes ET_HIPROC = 0xffff // Processor-specific }; // Versioning enum { EV_NONE = 0, EV_CURRENT = 1 }; // Machine architectures // See current registered ELF machine architectures at: // http://www.uxsglobal.com/developers/gabi/latest/ch4.eheader.html enum { EM_NONE = 0, // No machine EM_M32 = 1, // AT&T WE 32100 EM_SPARC = 2, // SPARC EM_386 = 3, // Intel 386 EM_68K = 4, // Motorola 68000 EM_88K = 5, // Motorola 88000 EM_IAMCU = 6, // Intel MCU EM_860 = 7, // Intel 80860 EM_MIPS = 8, // MIPS R3000 EM_S370 = 9, // IBM System/370 EM_MIPS_RS3_LE = 10, // MIPS RS3000 Little-endian EM_PARISC = 15, // Hewlett-Packard PA-RISC EM_VPP500 = 17, // Fujitsu VPP500 EM_SPARC32PLUS = 18, // Enhanced instruction set SPARC EM_960 = 19, // Intel 80960 EM_PPC = 20, // PowerPC EM_PPC64 = 21, // PowerPC64 EM_S390 = 22, // IBM System/390 EM_SPU = 23, // IBM SPU/SPC EM_V800 = 36, // NEC V800 EM_FR20 = 37, // Fujitsu FR20 EM_RH32 = 38, // TRW RH-32 EM_RCE = 39, // Motorola RCE EM_ARM = 40, // ARM EM_ALPHA = 41, // DEC Alpha EM_SH = 42, // Hitachi SH EM_SPARCV9 = 43, // SPARC V9 EM_TRICORE = 44, // Siemens TriCore EM_ARC = 45, // Argonaut RISC Core EM_H8_300 = 46, // Hitachi H8/300 EM_H8_300H = 47, // Hitachi H8/300H EM_H8S = 48, // Hitachi H8S EM_H8_500 = 49, // Hitachi H8/500 EM_IA_64 = 50, // Intel IA-64 processor architecture EM_MIPS_X = 51, // Stanford MIPS-X EM_COLDFIRE = 52, // Motorola ColdFire EM_68HC12 = 53, // Motorola M68HC12 EM_MMA = 54, // Fujitsu MMA Multimedia Accelerator EM_PCP = 55, // Siemens PCP EM_NCPU = 56, // Sony nCPU embedded RISC processor EM_NDR1 = 57, // Denso NDR1 microprocessor EM_STARCORE = 58, // Motorola Star*Core processor EM_ME16 = 59, // Toyota ME16 processor EM_ST100 = 60, // STMicroelectronics ST100 processor EM_TINYJ = 61, // Advanced Logic Corp. TinyJ embedded processor family EM_X86_64 = 62, // AMD x86-64 architecture EM_PDSP = 63, // Sony DSP Processor EM_PDP10 = 64, // Digital Equipment Corp. PDP-10 EM_PDP11 = 65, // Digital Equipment Corp. PDP-11 EM_FX66 = 66, // Siemens FX66 microcontroller EM_ST9PLUS = 67, // STMicroelectronics ST9+ 8/16 bit microcontroller EM_ST7 = 68, // STMicroelectronics ST7 8-bit microcontroller EM_68HC16 = 69, // Motorola MC68HC16 Microcontroller EM_68HC11 = 70, // Motorola MC68HC11 Microcontroller EM_68HC08 = 71, // Motorola MC68HC08 Microcontroller EM_68HC05 = 72, // Motorola MC68HC05 Microcontroller EM_SVX = 73, // Silicon Graphics SVx EM_ST19 = 74, // STMicroelectronics ST19 8-bit microcontroller EM_VAX = 75, // Digital VAX EM_CRIS = 76, // Axis Communications 32-bit embedded processor EM_JAVELIN = 77, // Infineon Technologies 32-bit embedded processor EM_FIREPATH = 78, // Element 14 64-bit DSP Processor EM_ZSP = 79, // LSI Logic 16-bit DSP Processor EM_MMIX = 80, // Donald Knuth's educational 64-bit processor EM_HUANY = 81, // Harvard University machine-independent object files EM_PRISM = 82, // SiTera Prism EM_AVR = 83, // Atmel AVR 8-bit microcontroller EM_FR30 = 84, // Fujitsu FR30 EM_D10V = 85, // Mitsubishi D10V EM_D30V = 86, // Mitsubishi D30V EM_V850 = 87, // NEC v850 EM_M32R = 88, // Mitsubishi M32R EM_MN10300 = 89, // Matsushita MN10300 EM_MN10200 = 90, // Matsushita MN10200 EM_PJ = 91, // picoJava EM_OPENRISC = 92, // OpenRISC 32-bit embedded processor EM_ARC_COMPACT = 93, // ARC International ARCompact processor (old // spelling/synonym: EM_ARC_A5) EM_XTENSA = 94, // Tensilica Xtensa Architecture EM_VIDEOCORE = 95, // Alphamosaic VideoCore processor EM_TMM_GPP = 96, // Thompson Multimedia General Purpose Processor EM_NS32K = 97, // National Semiconductor 32000 series EM_TPC = 98, // Tenor Network TPC processor EM_SNP1K = 99, // Trebia SNP 1000 processor EM_ST200 = 100, // STMicroelectronics (www.st.com) ST200 EM_IP2K = 101, // Ubicom IP2xxx microcontroller family EM_MAX = 102, // MAX Processor EM_CR = 103, // National Semiconductor CompactRISC microprocessor EM_F2MC16 = 104, // Fujitsu F2MC16 EM_MSP430 = 105, // Texas Instruments embedded microcontroller msp430 EM_BLACKFIN = 106, // Analog Devices Blackfin (DSP) processor EM_SE_C33 = 107, // S1C33 Family of Seiko Epson processors EM_SEP = 108, // Sharp embedded microprocessor EM_ARCA = 109, // Arca RISC Microprocessor EM_UNICORE = 110, // Microprocessor series from PKU-Unity Ltd. and MPRC // of Peking University EM_EXCESS = 111, // eXcess: 16/32/64-bit configurable embedded CPU EM_DXP = 112, // Icera Semiconductor Inc. Deep Execution Processor EM_ALTERA_NIOS2 = 113, // Altera Nios II soft-core processor EM_CRX = 114, // National Semiconductor CompactRISC CRX EM_XGATE = 115, // Motorola XGATE embedded processor EM_C166 = 116, // Infineon C16x/XC16x processor EM_M16C = 117, // Renesas M16C series microprocessors EM_DSPIC30F = 118, // Microchip Technology dsPIC30F Digital Signal // Controller EM_CE = 119, // Freescale Communication Engine RISC core EM_M32C = 120, // Renesas M32C series microprocessors EM_TSK3000 = 131, // Altium TSK3000 core EM_RS08 = 132, // Freescale RS08 embedded processor EM_SHARC = 133, // Analog Devices SHARC family of 32-bit DSP // processors EM_ECOG2 = 134, // Cyan Technology eCOG2 microprocessor EM_SCORE7 = 135, // Sunplus S+core7 RISC processor EM_DSP24 = 136, // New Japan Radio (NJR) 24-bit DSP Processor EM_VIDEOCORE3 = 137, // Broadcom VideoCore III processor EM_LATTICEMICO32 = 138, // RISC processor for Lattice FPGA architecture EM_SE_C17 = 139, // Seiko Epson C17 family EM_TI_C6000 = 140, // The Texas Instruments TMS320C6000 DSP family EM_TI_C2000 = 141, // The Texas Instruments TMS320C2000 DSP family EM_TI_C5500 = 142, // The Texas Instruments TMS320C55x DSP family EM_MMDSP_PLUS = 160, // STMicroelectronics 64bit VLIW Data Signal Processor EM_CYPRESS_M8C = 161, // Cypress M8C microprocessor EM_R32C = 162, // Renesas R32C series microprocessors EM_TRIMEDIA = 163, // NXP Semiconductors TriMedia architecture family EM_HEXAGON = 164, // Qualcomm Hexagon processor EM_8051 = 165, // Intel 8051 and variants EM_STXP7X = 166, // STMicroelectronics STxP7x family of configurable // and extensible RISC processors EM_NDS32 = 167, // Andes Technology compact code size embedded RISC // processor family EM_ECOG1 = 168, // Cyan Technology eCOG1X family EM_ECOG1X = 168, // Cyan Technology eCOG1X family EM_MAXQ30 = 169, // Dallas Semiconductor MAXQ30 Core Micro-controllers EM_XIMO16 = 170, // New Japan Radio (NJR) 16-bit DSP Processor EM_MANIK = 171, // M2000 Reconfigurable RISC Microprocessor EM_CRAYNV2 = 172, // Cray Inc. NV2 vector architecture EM_RX = 173, // Renesas RX family EM_METAG = 174, // Imagination Technologies META processor // architecture EM_MCST_ELBRUS = 175, // MCST Elbrus general purpose hardware architecture EM_ECOG16 = 176, // Cyan Technology eCOG16 family EM_CR16 = 177, // National Semiconductor CompactRISC CR16 16-bit // microprocessor EM_ETPU = 178, // Freescale Extended Time Processing Unit EM_SLE9X = 179, // Infineon Technologies SLE9X core EM_L10M = 180, // Intel L10M EM_K10M = 181, // Intel K10M EM_AARCH64 = 183, // ARM AArch64 EM_AVR32 = 185, // Atmel Corporation 32-bit microprocessor family EM_STM8 = 186, // STMicroeletronics STM8 8-bit microcontroller EM_TILE64 = 187, // Tilera TILE64 multicore architecture family EM_TILEPRO = 188, // Tilera TILEPro multicore architecture family EM_CUDA = 190, // NVIDIA CUDA architecture EM_TILEGX = 191, // Tilera TILE-Gx multicore architecture family EM_CLOUDSHIELD = 192, // CloudShield architecture family EM_COREA_1ST = 193, // KIPO-KAIST Core-A 1st generation processor family EM_COREA_2ND = 194, // KIPO-KAIST Core-A 2nd generation processor family EM_ARC_COMPACT2 = 195, // Synopsys ARCompact V2 EM_OPEN8 = 196, // Open8 8-bit RISC soft processor core EM_RL78 = 197, // Renesas RL78 family EM_VIDEOCORE5 = 198, // Broadcom VideoCore V processor EM_78KOR = 199, // Renesas 78KOR family EM_56800EX = 200, // Freescale 56800EX Digital Signal Controller (DSC) EM_BA1 = 201, // Beyond BA1 CPU architecture EM_BA2 = 202, // Beyond BA2 CPU architecture EM_XCORE = 203, // XMOS xCORE processor family EM_MCHP_PIC = 204, // Microchip 8-bit PIC(r) family EM_INTEL205 = 205, // Reserved by Intel EM_INTEL206 = 206, // Reserved by Intel EM_INTEL207 = 207, // Reserved by Intel EM_INTEL208 = 208, // Reserved by Intel EM_INTEL209 = 209, // Reserved by Intel EM_KM32 = 210, // KM211 KM32 32-bit processor EM_KMX32 = 211, // KM211 KMX32 32-bit processor EM_KMX16 = 212, // KM211 KMX16 16-bit processor EM_KMX8 = 213, // KM211 KMX8 8-bit processor EM_KVARC = 214, // KM211 KVARC processor EM_CDP = 215, // Paneve CDP architecture family EM_COGE = 216, // Cognitive Smart Memory Processor EM_COOL = 217, // iCelero CoolEngine EM_NORC = 218, // Nanoradio Optimized RISC EM_CSR_KALIMBA = 219, // CSR Kalimba architecture family EM_AMDGPU = 224, // AMD GPU architecture EM_RISCV = 243, // RISC-V EM_LANAI = 244, // Lanai 32-bit processor EM_BPF = 247, // Linux kernel bpf virtual machine }; // Object file classes. enum { ELFCLASSNONE = 0, ELFCLASS32 = 1, // 32-bit object file ELFCLASS64 = 2 // 64-bit object file }; // Object file byte orderings. enum { ELFDATANONE = 0, // Invalid data encoding. ELFDATA2LSB = 1, // Little-endian object file ELFDATA2MSB = 2 // Big-endian object file }; // OS ABI identification. enum { ELFOSABI_NONE = 0, // UNIX System V ABI ELFOSABI_HPUX = 1, // HP-UX operating system ELFOSABI_NETBSD = 2, // NetBSD ELFOSABI_GNU = 3, // GNU/Linux ELFOSABI_LINUX = 3, // Historical alias for ELFOSABI_GNU. ELFOSABI_HURD = 4, // GNU/Hurd ELFOSABI_SOLARIS = 6, // Solaris ELFOSABI_AIX = 7, // AIX ELFOSABI_IRIX = 8, // IRIX ELFOSABI_FREEBSD = 9, // FreeBSD ELFOSABI_TRU64 = 10, // TRU64 UNIX ELFOSABI_MODESTO = 11, // Novell Modesto ELFOSABI_OPENBSD = 12, // OpenBSD ELFOSABI_OPENVMS = 13, // OpenVMS ELFOSABI_NSK = 14, // Hewlett-Packard Non-Stop Kernel ELFOSABI_AROS = 15, // AROS ELFOSABI_FENIXOS = 16, // FenixOS ELFOSABI_CLOUDABI = 17, // Nuxi CloudABI ELFOSABI_FIRST_ARCH = 64, // First architecture-specific OS ABI ELFOSABI_AMDGPU_HSA = 64, // AMD HSA runtime ELFOSABI_AMDGPU_PAL = 65, // AMD PAL runtime ELFOSABI_AMDGPU_MESA3D = 66, // AMD GCN GPUs (GFX6+) for MESA runtime ELFOSABI_ARM = 97, // ARM ELFOSABI_C6000_ELFABI = 64, // Bare-metal TMS320C6000 ELFOSABI_C6000_LINUX = 65, // Linux TMS320C6000 ELFOSABI_STANDALONE = 255, // Standalone (embedded) application ELFOSABI_LAST_ARCH = 255 // Last Architecture-specific OS ABI }; #define ELF_RELOC(name, value) name = value, // X86_64 relocations. enum { #include "ELFRelocs/x86_64.def" }; // i386 relocations. enum { #include "ELFRelocs/i386.def" }; // ELF Relocation types for PPC32 enum { #include "ELFRelocs/PowerPC.def" }; // Specific e_flags for PPC64 enum { // e_flags bits specifying ABI: // 1 for original ABI using function descriptors, // 2 for revised ABI without function descriptors, // 0 for unspecified or not using any features affected by the differences. EF_PPC64_ABI = 3 }; // Special values for the st_other field in the symbol table entry for PPC64. enum { STO_PPC64_LOCAL_BIT = 5, STO_PPC64_LOCAL_MASK = (7 << STO_PPC64_LOCAL_BIT) }; static inline int64_t decodePPC64LocalEntryOffset(unsigned Other) { unsigned Val = (Other & STO_PPC64_LOCAL_MASK) >> STO_PPC64_LOCAL_BIT; return ((1 << Val) >> 2) << 2; } -static inline unsigned encodePPC64LocalEntryOffset(int64_t Offset) { - unsigned Val = - (Offset >= 4 * 4 ? (Offset >= 8 * 4 ? (Offset >= 16 * 4 ? 6 : 5) : 4) - : (Offset >= 2 * 4 ? 3 : (Offset >= 1 * 4 ? 2 : 0))); - return Val << STO_PPC64_LOCAL_BIT; -} // ELF Relocation types for PPC64 enum { #include "ELFRelocs/PowerPC64.def" }; // ELF Relocation types for AArch64 enum { #include "ELFRelocs/AArch64.def" }; // ARM Specific e_flags enum : unsigned { EF_ARM_SOFT_FLOAT = 0x00000200U, // Legacy pre EABI_VER5 EF_ARM_ABI_FLOAT_SOFT = 0x00000200U, // EABI_VER5 EF_ARM_VFP_FLOAT = 0x00000400U, // Legacy pre EABI_VER5 EF_ARM_ABI_FLOAT_HARD = 0x00000400U, // EABI_VER5 EF_ARM_EABI_UNKNOWN = 0x00000000U, EF_ARM_EABI_VER1 = 0x01000000U, EF_ARM_EABI_VER2 = 0x02000000U, EF_ARM_EABI_VER3 = 0x03000000U, EF_ARM_EABI_VER4 = 0x04000000U, EF_ARM_EABI_VER5 = 0x05000000U, EF_ARM_EABIMASK = 0xFF000000U }; // ELF Relocation types for ARM enum { #include "ELFRelocs/ARM.def" }; // ARC Specific e_flags enum : unsigned { EF_ARC_MACH_MSK = 0x000000ff, EF_ARC_OSABI_MSK = 0x00000f00, E_ARC_MACH_ARC600 = 0x00000002, E_ARC_MACH_ARC601 = 0x00000004, E_ARC_MACH_ARC700 = 0x00000003, EF_ARC_CPU_ARCV2EM = 0x00000005, EF_ARC_CPU_ARCV2HS = 0x00000006, E_ARC_OSABI_ORIG = 0x00000000, E_ARC_OSABI_V2 = 0x00000200, E_ARC_OSABI_V3 = 0x00000300, E_ARC_OSABI_V4 = 0x00000400, EF_ARC_PIC = 0x00000100 }; // ELF Relocation types for ARC enum { #include "ELFRelocs/ARC.def" }; // AVR specific e_flags enum : unsigned { EF_AVR_ARCH_AVR1 = 1, EF_AVR_ARCH_AVR2 = 2, EF_AVR_ARCH_AVR25 = 25, EF_AVR_ARCH_AVR3 = 3, EF_AVR_ARCH_AVR31 = 31, EF_AVR_ARCH_AVR35 = 35, EF_AVR_ARCH_AVR4 = 4, EF_AVR_ARCH_AVR5 = 5, EF_AVR_ARCH_AVR51 = 51, EF_AVR_ARCH_AVR6 = 6, EF_AVR_ARCH_AVRTINY = 100, EF_AVR_ARCH_XMEGA1 = 101, EF_AVR_ARCH_XMEGA2 = 102, EF_AVR_ARCH_XMEGA3 = 103, EF_AVR_ARCH_XMEGA4 = 104, EF_AVR_ARCH_XMEGA5 = 105, EF_AVR_ARCH_XMEGA6 = 106, EF_AVR_ARCH_XMEGA7 = 107 }; // ELF Relocation types for AVR enum { #include "ELFRelocs/AVR.def" }; // Mips Specific e_flags enum : unsigned { EF_MIPS_NOREORDER = 0x00000001, // Don't reorder instructions EF_MIPS_PIC = 0x00000002, // Position independent code EF_MIPS_CPIC = 0x00000004, // Call object with Position independent code EF_MIPS_ABI2 = 0x00000020, // File uses N32 ABI EF_MIPS_32BITMODE = 0x00000100, // Code compiled for a 64-bit machine // in 32-bit mode EF_MIPS_FP64 = 0x00000200, // Code compiled for a 32-bit machine // but uses 64-bit FP registers EF_MIPS_NAN2008 = 0x00000400, // Uses IEE 754-2008 NaN encoding // ABI flags EF_MIPS_ABI_O32 = 0x00001000, // This file follows the first MIPS 32 bit ABI EF_MIPS_ABI_O64 = 0x00002000, // O32 ABI extended for 64-bit architecture. EF_MIPS_ABI_EABI32 = 0x00003000, // EABI in 32 bit mode. EF_MIPS_ABI_EABI64 = 0x00004000, // EABI in 64 bit mode. EF_MIPS_ABI = 0x0000f000, // Mask for selecting EF_MIPS_ABI_ variant. // MIPS machine variant EF_MIPS_MACH_NONE = 0x00000000, // A standard MIPS implementation. EF_MIPS_MACH_3900 = 0x00810000, // Toshiba R3900 EF_MIPS_MACH_4010 = 0x00820000, // LSI R4010 EF_MIPS_MACH_4100 = 0x00830000, // NEC VR4100 EF_MIPS_MACH_4650 = 0x00850000, // MIPS R4650 EF_MIPS_MACH_4120 = 0x00870000, // NEC VR4120 EF_MIPS_MACH_4111 = 0x00880000, // NEC VR4111/VR4181 EF_MIPS_MACH_SB1 = 0x008a0000, // Broadcom SB-1 EF_MIPS_MACH_OCTEON = 0x008b0000, // Cavium Networks Octeon EF_MIPS_MACH_XLR = 0x008c0000, // RMI Xlr EF_MIPS_MACH_OCTEON2 = 0x008d0000, // Cavium Networks Octeon2 EF_MIPS_MACH_OCTEON3 = 0x008e0000, // Cavium Networks Octeon3 EF_MIPS_MACH_5400 = 0x00910000, // NEC VR5400 EF_MIPS_MACH_5900 = 0x00920000, // MIPS R5900 EF_MIPS_MACH_5500 = 0x00980000, // NEC VR5500 EF_MIPS_MACH_9000 = 0x00990000, // Unknown EF_MIPS_MACH_LS2E = 0x00a00000, // ST Microelectronics Loongson 2E EF_MIPS_MACH_LS2F = 0x00a10000, // ST Microelectronics Loongson 2F EF_MIPS_MACH_LS3A = 0x00a20000, // Loongson 3A EF_MIPS_MACH = 0x00ff0000, // EF_MIPS_MACH_xxx selection mask // ARCH_ASE EF_MIPS_MICROMIPS = 0x02000000, // microMIPS EF_MIPS_ARCH_ASE_M16 = 0x04000000, // Has Mips-16 ISA extensions EF_MIPS_ARCH_ASE_MDMX = 0x08000000, // Has MDMX multimedia extensions EF_MIPS_ARCH_ASE = 0x0f000000, // Mask for EF_MIPS_ARCH_ASE_xxx flags // ARCH EF_MIPS_ARCH_1 = 0x00000000, // MIPS1 instruction set EF_MIPS_ARCH_2 = 0x10000000, // MIPS2 instruction set EF_MIPS_ARCH_3 = 0x20000000, // MIPS3 instruction set EF_MIPS_ARCH_4 = 0x30000000, // MIPS4 instruction set EF_MIPS_ARCH_5 = 0x40000000, // MIPS5 instruction set EF_MIPS_ARCH_32 = 0x50000000, // MIPS32 instruction set per linux not elf.h EF_MIPS_ARCH_64 = 0x60000000, // MIPS64 instruction set per linux not elf.h EF_MIPS_ARCH_32R2 = 0x70000000, // mips32r2, mips32r3, mips32r5 EF_MIPS_ARCH_64R2 = 0x80000000, // mips64r2, mips64r3, mips64r5 EF_MIPS_ARCH_32R6 = 0x90000000, // mips32r6 EF_MIPS_ARCH_64R6 = 0xa0000000, // mips64r6 EF_MIPS_ARCH = 0xf0000000 // Mask for applying EF_MIPS_ARCH_ variant }; // ELF Relocation types for Mips enum { #include "ELFRelocs/Mips.def" }; // Special values for the st_other field in the symbol table entry for MIPS. enum { STO_MIPS_OPTIONAL = 0x04, // Symbol whose definition is optional STO_MIPS_PLT = 0x08, // PLT entry related dynamic table record STO_MIPS_PIC = 0x20, // PIC func in an object mixes PIC/non-PIC STO_MIPS_MICROMIPS = 0x80, // MIPS Specific ISA for MicroMips STO_MIPS_MIPS16 = 0xf0 // MIPS Specific ISA for Mips16 }; // .MIPS.options section descriptor kinds enum { ODK_NULL = 0, // Undefined ODK_REGINFO = 1, // Register usage information ODK_EXCEPTIONS = 2, // Exception processing options ODK_PAD = 3, // Section padding options ODK_HWPATCH = 4, // Hardware patches applied ODK_FILL = 5, // Linker fill value ODK_TAGS = 6, // Space for tool identification ODK_HWAND = 7, // Hardware AND patches applied ODK_HWOR = 8, // Hardware OR patches applied ODK_GP_GROUP = 9, // GP group to use for text/data sections ODK_IDENT = 10, // ID information ODK_PAGESIZE = 11 // Page size information }; // Hexagon-specific e_flags enum { // Object processor version flags, bits[11:0] EF_HEXAGON_MACH_V2 = 0x00000001, // Hexagon V2 EF_HEXAGON_MACH_V3 = 0x00000002, // Hexagon V3 EF_HEXAGON_MACH_V4 = 0x00000003, // Hexagon V4 EF_HEXAGON_MACH_V5 = 0x00000004, // Hexagon V5 EF_HEXAGON_MACH_V55 = 0x00000005, // Hexagon V55 EF_HEXAGON_MACH_V60 = 0x00000060, // Hexagon V60 EF_HEXAGON_MACH_V62 = 0x00000062, // Hexagon V62 EF_HEXAGON_MACH_V65 = 0x00000065, // Hexagon V65 EF_HEXAGON_MACH_V66 = 0x00000066, // Hexagon V66 EF_HEXAGON_MACH_V67 = 0x00000067, // Hexagon V67 EF_HEXAGON_MACH_V67T = 0x00008067, // Hexagon V67T // Highest ISA version flags EF_HEXAGON_ISA_MACH = 0x00000000, // Same as specified in bits[11:0] // of e_flags EF_HEXAGON_ISA_V2 = 0x00000010, // Hexagon V2 ISA EF_HEXAGON_ISA_V3 = 0x00000020, // Hexagon V3 ISA EF_HEXAGON_ISA_V4 = 0x00000030, // Hexagon V4 ISA EF_HEXAGON_ISA_V5 = 0x00000040, // Hexagon V5 ISA EF_HEXAGON_ISA_V55 = 0x00000050, // Hexagon V55 ISA EF_HEXAGON_ISA_V60 = 0x00000060, // Hexagon V60 ISA EF_HEXAGON_ISA_V62 = 0x00000062, // Hexagon V62 ISA EF_HEXAGON_ISA_V65 = 0x00000065, // Hexagon V65 ISA EF_HEXAGON_ISA_V66 = 0x00000066, // Hexagon V66 ISA EF_HEXAGON_ISA_V67 = 0x00000067, // Hexagon V67 ISA }; // Hexagon-specific section indexes for common small data enum { SHN_HEXAGON_SCOMMON = 0xff00, // Other access sizes SHN_HEXAGON_SCOMMON_1 = 0xff01, // Byte-sized access SHN_HEXAGON_SCOMMON_2 = 0xff02, // Half-word-sized access SHN_HEXAGON_SCOMMON_4 = 0xff03, // Word-sized access SHN_HEXAGON_SCOMMON_8 = 0xff04 // Double-word-size access }; // ELF Relocation types for Hexagon enum { #include "ELFRelocs/Hexagon.def" }; // ELF Relocation type for Lanai. enum { #include "ELFRelocs/Lanai.def" }; // RISCV Specific e_flags enum : unsigned { EF_RISCV_RVC = 0x0001, EF_RISCV_FLOAT_ABI = 0x0006, EF_RISCV_FLOAT_ABI_SOFT = 0x0000, EF_RISCV_FLOAT_ABI_SINGLE = 0x0002, EF_RISCV_FLOAT_ABI_DOUBLE = 0x0004, EF_RISCV_FLOAT_ABI_QUAD = 0x0006, EF_RISCV_RVE = 0x0008 }; // ELF Relocation types for RISC-V enum { #include "ELFRelocs/RISCV.def" }; // ELF Relocation types for S390/zSeries enum { #include "ELFRelocs/SystemZ.def" }; // ELF Relocation type for Sparc. enum { #include "ELFRelocs/Sparc.def" }; // AMDGPU specific e_flags. enum : unsigned { // Processor selection mask for EF_AMDGPU_MACH_* values. EF_AMDGPU_MACH = 0x0ff, // Not specified processor. EF_AMDGPU_MACH_NONE = 0x000, // R600-based processors. // Radeon HD 2000/3000 Series (R600). EF_AMDGPU_MACH_R600_R600 = 0x001, EF_AMDGPU_MACH_R600_R630 = 0x002, EF_AMDGPU_MACH_R600_RS880 = 0x003, EF_AMDGPU_MACH_R600_RV670 = 0x004, // Radeon HD 4000 Series (R700). EF_AMDGPU_MACH_R600_RV710 = 0x005, EF_AMDGPU_MACH_R600_RV730 = 0x006, EF_AMDGPU_MACH_R600_RV770 = 0x007, // Radeon HD 5000 Series (Evergreen). EF_AMDGPU_MACH_R600_CEDAR = 0x008, EF_AMDGPU_MACH_R600_CYPRESS = 0x009, EF_AMDGPU_MACH_R600_JUNIPER = 0x00a, EF_AMDGPU_MACH_R600_REDWOOD = 0x00b, EF_AMDGPU_MACH_R600_SUMO = 0x00c, // Radeon HD 6000 Series (Northern Islands). EF_AMDGPU_MACH_R600_BARTS = 0x00d, EF_AMDGPU_MACH_R600_CAICOS = 0x00e, EF_AMDGPU_MACH_R600_CAYMAN = 0x00f, EF_AMDGPU_MACH_R600_TURKS = 0x010, // Reserved for R600-based processors. EF_AMDGPU_MACH_R600_RESERVED_FIRST = 0x011, EF_AMDGPU_MACH_R600_RESERVED_LAST = 0x01f, // First/last R600-based processors. EF_AMDGPU_MACH_R600_FIRST = EF_AMDGPU_MACH_R600_R600, EF_AMDGPU_MACH_R600_LAST = EF_AMDGPU_MACH_R600_TURKS, // AMDGCN-based processors. // AMDGCN GFX6. EF_AMDGPU_MACH_AMDGCN_GFX600 = 0x020, EF_AMDGPU_MACH_AMDGCN_GFX601 = 0x021, // AMDGCN GFX7. EF_AMDGPU_MACH_AMDGCN_GFX700 = 0x022, EF_AMDGPU_MACH_AMDGCN_GFX701 = 0x023, EF_AMDGPU_MACH_AMDGCN_GFX702 = 0x024, EF_AMDGPU_MACH_AMDGCN_GFX703 = 0x025, EF_AMDGPU_MACH_AMDGCN_GFX704 = 0x026, // AMDGCN GFX8. EF_AMDGPU_MACH_AMDGCN_GFX801 = 0x028, EF_AMDGPU_MACH_AMDGCN_GFX802 = 0x029, EF_AMDGPU_MACH_AMDGCN_GFX803 = 0x02a, EF_AMDGPU_MACH_AMDGCN_GFX810 = 0x02b, // AMDGCN GFX9. EF_AMDGPU_MACH_AMDGCN_GFX900 = 0x02c, EF_AMDGPU_MACH_AMDGCN_GFX902 = 0x02d, EF_AMDGPU_MACH_AMDGCN_GFX904 = 0x02e, EF_AMDGPU_MACH_AMDGCN_GFX906 = 0x02f, EF_AMDGPU_MACH_AMDGCN_GFX908 = 0x030, EF_AMDGPU_MACH_AMDGCN_GFX909 = 0x031, // AMDGCN GFX10. EF_AMDGPU_MACH_AMDGCN_GFX1010 = 0x033, EF_AMDGPU_MACH_AMDGCN_GFX1011 = 0x034, EF_AMDGPU_MACH_AMDGCN_GFX1012 = 0x035, // Reserved for AMDGCN-based processors. EF_AMDGPU_MACH_AMDGCN_RESERVED0 = 0x027, EF_AMDGPU_MACH_AMDGCN_RESERVED1 = 0x032, // First/last AMDGCN-based processors. EF_AMDGPU_MACH_AMDGCN_FIRST = EF_AMDGPU_MACH_AMDGCN_GFX600, EF_AMDGPU_MACH_AMDGCN_LAST = EF_AMDGPU_MACH_AMDGCN_GFX1012, // Indicates if the "xnack" target feature is enabled for all code contained // in the object. EF_AMDGPU_XNACK = 0x100, // Indicates if the "sram-ecc" target feature is enabled for all code // contained in the object. EF_AMDGPU_SRAM_ECC = 0x200, }; // ELF Relocation types for AMDGPU enum { #include "ELFRelocs/AMDGPU.def" }; // ELF Relocation types for BPF enum { #include "ELFRelocs/BPF.def" }; // MSP430 specific e_flags enum : unsigned { EF_MSP430_MACH_MSP430x11 = 11, EF_MSP430_MACH_MSP430x11x1 = 110, EF_MSP430_MACH_MSP430x12 = 12, EF_MSP430_MACH_MSP430x13 = 13, EF_MSP430_MACH_MSP430x14 = 14, EF_MSP430_MACH_MSP430x15 = 15, EF_MSP430_MACH_MSP430x16 = 16, EF_MSP430_MACH_MSP430x20 = 20, EF_MSP430_MACH_MSP430x22 = 22, EF_MSP430_MACH_MSP430x23 = 23, EF_MSP430_MACH_MSP430x24 = 24, EF_MSP430_MACH_MSP430x26 = 26, EF_MSP430_MACH_MSP430x31 = 31, EF_MSP430_MACH_MSP430x32 = 32, EF_MSP430_MACH_MSP430x33 = 33, EF_MSP430_MACH_MSP430x41 = 41, EF_MSP430_MACH_MSP430x42 = 42, EF_MSP430_MACH_MSP430x43 = 43, EF_MSP430_MACH_MSP430x44 = 44, EF_MSP430_MACH_MSP430X = 45, EF_MSP430_MACH_MSP430x46 = 46, EF_MSP430_MACH_MSP430x47 = 47, EF_MSP430_MACH_MSP430x54 = 54, }; // ELF Relocation types for MSP430 enum { #include "ELFRelocs/MSP430.def" }; #undef ELF_RELOC // Section header. struct Elf32_Shdr { Elf32_Word sh_name; // Section name (index into string table) Elf32_Word sh_type; // Section type (SHT_*) Elf32_Word sh_flags; // Section flags (SHF_*) Elf32_Addr sh_addr; // Address where section is to be loaded Elf32_Off sh_offset; // File offset of section data, in bytes Elf32_Word sh_size; // Size of section, in bytes Elf32_Word sh_link; // Section type-specific header table index link Elf32_Word sh_info; // Section type-specific extra information Elf32_Word sh_addralign; // Section address alignment Elf32_Word sh_entsize; // Size of records contained within the section }; // Section header for ELF64 - same fields as ELF32, different types. struct Elf64_Shdr { Elf64_Word sh_name; Elf64_Word sh_type; Elf64_Xword sh_flags; Elf64_Addr sh_addr; Elf64_Off sh_offset; Elf64_Xword sh_size; Elf64_Word sh_link; Elf64_Word sh_info; Elf64_Xword sh_addralign; Elf64_Xword sh_entsize; }; // Special section indices. enum { SHN_UNDEF = 0, // Undefined, missing, irrelevant, or meaningless SHN_LORESERVE = 0xff00, // Lowest reserved index SHN_LOPROC = 0xff00, // Lowest processor-specific index SHN_HIPROC = 0xff1f, // Highest processor-specific index SHN_LOOS = 0xff20, // Lowest operating system-specific index SHN_HIOS = 0xff3f, // Highest operating system-specific index SHN_ABS = 0xfff1, // Symbol has absolute value; does not need relocation SHN_COMMON = 0xfff2, // FORTRAN COMMON or C external global variables SHN_XINDEX = 0xffff, // Mark that the index is >= SHN_LORESERVE SHN_HIRESERVE = 0xffff // Highest reserved index }; // Section types. enum : unsigned { SHT_NULL = 0, // No associated section (inactive entry). SHT_PROGBITS = 1, // Program-defined contents. SHT_SYMTAB = 2, // Symbol table. SHT_STRTAB = 3, // String table. SHT_RELA = 4, // Relocation entries; explicit addends. SHT_HASH = 5, // Symbol hash table. SHT_DYNAMIC = 6, // Information for dynamic linking. SHT_NOTE = 7, // Information about the file. SHT_NOBITS = 8, // Data occupies no space in the file. SHT_REL = 9, // Relocation entries; no explicit addends. SHT_SHLIB = 10, // Reserved. SHT_DYNSYM = 11, // Symbol table. SHT_INIT_ARRAY = 14, // Pointers to initialization functions. SHT_FINI_ARRAY = 15, // Pointers to termination functions. SHT_PREINIT_ARRAY = 16, // Pointers to pre-init functions. SHT_GROUP = 17, // Section group. SHT_SYMTAB_SHNDX = 18, // Indices for SHN_XINDEX entries. // Experimental support for SHT_RELR sections. For details, see proposal // at https://groups.google.com/forum/#!topic/generic-abi/bX460iggiKg SHT_RELR = 19, // Relocation entries; only offsets. SHT_LOOS = 0x60000000, // Lowest operating system-specific type. // Android packed relocation section types. // https://android.googlesource.com/platform/bionic/+/6f12bfece5dcc01325e0abba56a46b1bcf991c69/tools/relocation_packer/src/elf_file.cc#37 SHT_ANDROID_REL = 0x60000001, SHT_ANDROID_RELA = 0x60000002, SHT_LLVM_ODRTAB = 0x6fff4c00, // LLVM ODR table. SHT_LLVM_LINKER_OPTIONS = 0x6fff4c01, // LLVM Linker Options. SHT_LLVM_CALL_GRAPH_PROFILE = 0x6fff4c02, // LLVM Call Graph Profile. SHT_LLVM_ADDRSIG = 0x6fff4c03, // List of address-significant symbols // for safe ICF. SHT_LLVM_DEPENDENT_LIBRARIES = 0x6fff4c04, // LLVM Dependent Library Specifiers. SHT_LLVM_SYMPART = 0x6fff4c05, // Symbol partition specification. SHT_LLVM_PART_EHDR = 0x6fff4c06, // ELF header for loadable partition. SHT_LLVM_PART_PHDR = 0x6fff4c07, // Phdrs for loadable partition. // Android's experimental support for SHT_RELR sections. // https://android.googlesource.com/platform/bionic/+/b7feec74547f84559a1467aca02708ff61346d2a/libc/include/elf.h#512 SHT_ANDROID_RELR = 0x6fffff00, // Relocation entries; only offsets. SHT_GNU_ATTRIBUTES = 0x6ffffff5, // Object attributes. SHT_GNU_HASH = 0x6ffffff6, // GNU-style hash table. SHT_GNU_verdef = 0x6ffffffd, // GNU version definitions. SHT_GNU_verneed = 0x6ffffffe, // GNU version references. SHT_GNU_versym = 0x6fffffff, // GNU symbol versions table. SHT_HIOS = 0x6fffffff, // Highest operating system-specific type. SHT_LOPROC = 0x70000000, // Lowest processor arch-specific type. // Fixme: All this is duplicated in MCSectionELF. Why?? // Exception Index table SHT_ARM_EXIDX = 0x70000001U, // BPABI DLL dynamic linking pre-emption map SHT_ARM_PREEMPTMAP = 0x70000002U, // Object file compatibility attributes SHT_ARM_ATTRIBUTES = 0x70000003U, SHT_ARM_DEBUGOVERLAY = 0x70000004U, SHT_ARM_OVERLAYSECTION = 0x70000005U, SHT_HEX_ORDERED = 0x70000000, // Link editor is to sort the entries in // this section based on their sizes SHT_X86_64_UNWIND = 0x70000001, // Unwind information SHT_MIPS_REGINFO = 0x70000006, // Register usage information SHT_MIPS_OPTIONS = 0x7000000d, // General options SHT_MIPS_DWARF = 0x7000001e, // DWARF debugging section. SHT_MIPS_ABIFLAGS = 0x7000002a, // ABI information. SHT_MSP430_ATTRIBUTES = 0x70000003U, SHT_RISCV_ATTRIBUTES = 0x70000003U, SHT_HIPROC = 0x7fffffff, // Highest processor arch-specific type. SHT_LOUSER = 0x80000000, // Lowest type reserved for applications. SHT_HIUSER = 0xffffffff // Highest type reserved for applications. }; // Section flags. enum : unsigned { // Section data should be writable during execution. SHF_WRITE = 0x1, // Section occupies memory during program execution. SHF_ALLOC = 0x2, // Section contains executable machine instructions. SHF_EXECINSTR = 0x4, // The data in this section may be merged. SHF_MERGE = 0x10, // The data in this section is null-terminated strings. SHF_STRINGS = 0x20, // A field in this section holds a section header table index. SHF_INFO_LINK = 0x40U, // Adds special ordering requirements for link editors. SHF_LINK_ORDER = 0x80U, // This section requires special OS-specific processing to avoid incorrect // behavior. SHF_OS_NONCONFORMING = 0x100U, // This section is a member of a section group. SHF_GROUP = 0x200U, // This section holds Thread-Local Storage. SHF_TLS = 0x400U, // Identifies a section containing compressed data. SHF_COMPRESSED = 0x800U, // This section is excluded from the final executable or shared library. SHF_EXCLUDE = 0x80000000U, // Start of target-specific flags. SHF_MASKOS = 0x0ff00000, // Bits indicating processor-specific flags. SHF_MASKPROC = 0xf0000000, /// All sections with the "d" flag are grouped together by the linker to form /// the data section and the dp register is set to the start of the section by /// the boot code. XCORE_SHF_DP_SECTION = 0x10000000, /// All sections with the "c" flag are grouped together by the linker to form /// the constant pool and the cp register is set to the start of the constant /// pool by the boot code. XCORE_SHF_CP_SECTION = 0x20000000, // If an object file section does not have this flag set, then it may not hold // more than 2GB and can be freely referred to in objects using smaller code // models. Otherwise, only objects using larger code models can refer to them. // For example, a medium code model object can refer to data in a section that // sets this flag besides being able to refer to data in a section that does // not set it; likewise, a small code model object can refer only to code in a // section that does not set this flag. SHF_X86_64_LARGE = 0x10000000, // All sections with the GPREL flag are grouped into a global data area // for faster accesses SHF_HEX_GPREL = 0x10000000, // Section contains text/data which may be replicated in other sections. // Linker must retain only one copy. SHF_MIPS_NODUPES = 0x01000000, // Linker must generate implicit hidden weak names. SHF_MIPS_NAMES = 0x02000000, // Section data local to process. SHF_MIPS_LOCAL = 0x04000000, // Do not strip this section. SHF_MIPS_NOSTRIP = 0x08000000, // Section must be part of global data area. SHF_MIPS_GPREL = 0x10000000, // This section should be merged. SHF_MIPS_MERGE = 0x20000000, // Address size to be inferred from section entry size. SHF_MIPS_ADDR = 0x40000000, // Section data is string data by default. SHF_MIPS_STRING = 0x80000000, // Make code section unreadable when in execute-only mode SHF_ARM_PURECODE = 0x20000000 }; // Section Group Flags enum : unsigned { GRP_COMDAT = 0x1, GRP_MASKOS = 0x0ff00000, GRP_MASKPROC = 0xf0000000 }; // Symbol table entries for ELF32. struct Elf32_Sym { Elf32_Word st_name; // Symbol name (index into string table) Elf32_Addr st_value; // Value or address associated with the symbol Elf32_Word st_size; // Size of the symbol unsigned char st_info; // Symbol's type and binding attributes unsigned char st_other; // Must be zero; reserved Elf32_Half st_shndx; // Which section (header table index) it's defined in // These accessors and mutators correspond to the ELF32_ST_BIND, // ELF32_ST_TYPE, and ELF32_ST_INFO macros defined in the ELF specification: unsigned char getBinding() const { return st_info >> 4; } unsigned char getType() const { return st_info & 0x0f; } void setBinding(unsigned char b) { setBindingAndType(b, getType()); } void setType(unsigned char t) { setBindingAndType(getBinding(), t); } void setBindingAndType(unsigned char b, unsigned char t) { st_info = (b << 4) + (t & 0x0f); } }; // Symbol table entries for ELF64. struct Elf64_Sym { Elf64_Word st_name; // Symbol name (index into string table) unsigned char st_info; // Symbol's type and binding attributes unsigned char st_other; // Must be zero; reserved Elf64_Half st_shndx; // Which section (header tbl index) it's defined in Elf64_Addr st_value; // Value or address associated with the symbol Elf64_Xword st_size; // Size of the symbol // These accessors and mutators are identical to those defined for ELF32 // symbol table entries. unsigned char getBinding() const { return st_info >> 4; } unsigned char getType() const { return st_info & 0x0f; } void setBinding(unsigned char b) { setBindingAndType(b, getType()); } void setType(unsigned char t) { setBindingAndType(getBinding(), t); } void setBindingAndType(unsigned char b, unsigned char t) { st_info = (b << 4) + (t & 0x0f); } }; // The size (in bytes) of symbol table entries. enum { SYMENTRY_SIZE32 = 16, // 32-bit symbol entry size SYMENTRY_SIZE64 = 24 // 64-bit symbol entry size. }; // Symbol bindings. enum { STB_LOCAL = 0, // Local symbol, not visible outside obj file containing def STB_GLOBAL = 1, // Global symbol, visible to all object files being combined STB_WEAK = 2, // Weak symbol, like global but lower-precedence STB_GNU_UNIQUE = 10, STB_LOOS = 10, // Lowest operating system-specific binding type STB_HIOS = 12, // Highest operating system-specific binding type STB_LOPROC = 13, // Lowest processor-specific binding type STB_HIPROC = 15 // Highest processor-specific binding type }; // Symbol types. enum { STT_NOTYPE = 0, // Symbol's type is not specified STT_OBJECT = 1, // Symbol is a data object (variable, array, etc.) STT_FUNC = 2, // Symbol is executable code (function, etc.) STT_SECTION = 3, // Symbol refers to a section STT_FILE = 4, // Local, absolute symbol that refers to a file STT_COMMON = 5, // An uninitialized common block STT_TLS = 6, // Thread local data object STT_GNU_IFUNC = 10, // GNU indirect function STT_LOOS = 10, // Lowest operating system-specific symbol type STT_HIOS = 12, // Highest operating system-specific symbol type STT_LOPROC = 13, // Lowest processor-specific symbol type STT_HIPROC = 15, // Highest processor-specific symbol type // AMDGPU symbol types STT_AMDGPU_HSA_KERNEL = 10 }; enum { STV_DEFAULT = 0, // Visibility is specified by binding type STV_INTERNAL = 1, // Defined by processor supplements STV_HIDDEN = 2, // Not visible to other components STV_PROTECTED = 3 // Visible in other components but not preemptable }; // Symbol number. enum { STN_UNDEF = 0 }; // Special relocation symbols used in the MIPS64 ELF relocation entries enum { RSS_UNDEF = 0, // None RSS_GP = 1, // Value of gp RSS_GP0 = 2, // Value of gp used to create object being relocated RSS_LOC = 3 // Address of location being relocated }; // Relocation entry, without explicit addend. struct Elf32_Rel { Elf32_Addr r_offset; // Location (file byte offset, or program virtual addr) Elf32_Word r_info; // Symbol table index and type of relocation to apply // These accessors and mutators correspond to the ELF32_R_SYM, ELF32_R_TYPE, // and ELF32_R_INFO macros defined in the ELF specification: Elf32_Word getSymbol() const { return (r_info >> 8); } unsigned char getType() const { return (unsigned char)(r_info & 0x0ff); } void setSymbol(Elf32_Word s) { setSymbolAndType(s, getType()); } void setType(unsigned char t) { setSymbolAndType(getSymbol(), t); } void setSymbolAndType(Elf32_Word s, unsigned char t) { r_info = (s << 8) + t; } }; // Relocation entry with explicit addend. struct Elf32_Rela { Elf32_Addr r_offset; // Location (file byte offset, or program virtual addr) Elf32_Word r_info; // Symbol table index and type of relocation to apply Elf32_Sword r_addend; // Compute value for relocatable field by adding this // These accessors and mutators correspond to the ELF32_R_SYM, ELF32_R_TYPE, // and ELF32_R_INFO macros defined in the ELF specification: Elf32_Word getSymbol() const { return (r_info >> 8); } unsigned char getType() const { return (unsigned char)(r_info & 0x0ff); } void setSymbol(Elf32_Word s) { setSymbolAndType(s, getType()); } void setType(unsigned char t) { setSymbolAndType(getSymbol(), t); } void setSymbolAndType(Elf32_Word s, unsigned char t) { r_info = (s << 8) + t; } }; // Relocation entry without explicit addend or info (relative relocations only). typedef Elf32_Word Elf32_Relr; // offset/bitmap for relative relocations // Relocation entry, without explicit addend. struct Elf64_Rel { Elf64_Addr r_offset; // Location (file byte offset, or program virtual addr). Elf64_Xword r_info; // Symbol table index and type of relocation to apply. // These accessors and mutators correspond to the ELF64_R_SYM, ELF64_R_TYPE, // and ELF64_R_INFO macros defined in the ELF specification: Elf64_Word getSymbol() const { return (r_info >> 32); } Elf64_Word getType() const { return (Elf64_Word)(r_info & 0xffffffffL); } void setSymbol(Elf64_Word s) { setSymbolAndType(s, getType()); } void setType(Elf64_Word t) { setSymbolAndType(getSymbol(), t); } void setSymbolAndType(Elf64_Word s, Elf64_Word t) { r_info = ((Elf64_Xword)s << 32) + (t & 0xffffffffL); } }; // Relocation entry with explicit addend. struct Elf64_Rela { Elf64_Addr r_offset; // Location (file byte offset, or program virtual addr). Elf64_Xword r_info; // Symbol table index and type of relocation to apply. Elf64_Sxword r_addend; // Compute value for relocatable field by adding this. // These accessors and mutators correspond to the ELF64_R_SYM, ELF64_R_TYPE, // and ELF64_R_INFO macros defined in the ELF specification: Elf64_Word getSymbol() const { return (r_info >> 32); } Elf64_Word getType() const { return (Elf64_Word)(r_info & 0xffffffffL); } void setSymbol(Elf64_Word s) { setSymbolAndType(s, getType()); } void setType(Elf64_Word t) { setSymbolAndType(getSymbol(), t); } void setSymbolAndType(Elf64_Word s, Elf64_Word t) { r_info = ((Elf64_Xword)s << 32) + (t & 0xffffffffL); } }; // Relocation entry without explicit addend or info (relative relocations only). typedef Elf64_Xword Elf64_Relr; // offset/bitmap for relative relocations // Program header for ELF32. struct Elf32_Phdr { Elf32_Word p_type; // Type of segment Elf32_Off p_offset; // File offset where segment is located, in bytes Elf32_Addr p_vaddr; // Virtual address of beginning of segment Elf32_Addr p_paddr; // Physical address of beginning of segment (OS-specific) Elf32_Word p_filesz; // Num. of bytes in file image of segment (may be zero) Elf32_Word p_memsz; // Num. of bytes in mem image of segment (may be zero) Elf32_Word p_flags; // Segment flags Elf32_Word p_align; // Segment alignment constraint }; // Program header for ELF64. struct Elf64_Phdr { Elf64_Word p_type; // Type of segment Elf64_Word p_flags; // Segment flags Elf64_Off p_offset; // File offset where segment is located, in bytes Elf64_Addr p_vaddr; // Virtual address of beginning of segment Elf64_Addr p_paddr; // Physical addr of beginning of segment (OS-specific) Elf64_Xword p_filesz; // Num. of bytes in file image of segment (may be zero) Elf64_Xword p_memsz; // Num. of bytes in mem image of segment (may be zero) Elf64_Xword p_align; // Segment alignment constraint }; // Segment types. enum { PT_NULL = 0, // Unused segment. PT_LOAD = 1, // Loadable segment. PT_DYNAMIC = 2, // Dynamic linking information. PT_INTERP = 3, // Interpreter pathname. PT_NOTE = 4, // Auxiliary information. PT_SHLIB = 5, // Reserved. PT_PHDR = 6, // The program header table itself. PT_TLS = 7, // The thread-local storage template. PT_LOOS = 0x60000000, // Lowest operating system-specific pt entry type. PT_HIOS = 0x6fffffff, // Highest operating system-specific pt entry type. PT_LOPROC = 0x70000000, // Lowest processor-specific program hdr entry type. PT_HIPROC = 0x7fffffff, // Highest processor-specific program hdr entry type. // x86-64 program header types. // These all contain stack unwind tables. PT_GNU_EH_FRAME = 0x6474e550, PT_SUNW_EH_FRAME = 0x6474e550, PT_SUNW_UNWIND = 0x6464e550, PT_GNU_STACK = 0x6474e551, // Indicates stack executability. PT_GNU_RELRO = 0x6474e552, // Read-only after relocation. PT_GNU_PROPERTY = 0x6474e553, // .note.gnu.property notes sections. PT_OPENBSD_RANDOMIZE = 0x65a3dbe6, // Fill with random data. PT_OPENBSD_WXNEEDED = 0x65a3dbe7, // Program does W^X violations. PT_OPENBSD_BOOTDATA = 0x65a41be6, // Section for boot arguments. // ARM program header types. PT_ARM_ARCHEXT = 0x70000000, // Platform architecture compatibility info // These all contain stack unwind tables. PT_ARM_EXIDX = 0x70000001, PT_ARM_UNWIND = 0x70000001, // MIPS program header types. PT_MIPS_REGINFO = 0x70000000, // Register usage information. PT_MIPS_RTPROC = 0x70000001, // Runtime procedure table. PT_MIPS_OPTIONS = 0x70000002, // Options segment. PT_MIPS_ABIFLAGS = 0x70000003, // Abiflags segment. }; // Segment flag bits. enum : unsigned { PF_X = 1, // Execute PF_W = 2, // Write PF_R = 4, // Read PF_MASKOS = 0x0ff00000, // Bits for operating system-specific semantics. PF_MASKPROC = 0xf0000000 // Bits for processor-specific semantics. }; // Dynamic table entry for ELF32. struct Elf32_Dyn { Elf32_Sword d_tag; // Type of dynamic table entry. union { Elf32_Word d_val; // Integer value of entry. Elf32_Addr d_ptr; // Pointer value of entry. } d_un; }; // Dynamic table entry for ELF64. struct Elf64_Dyn { Elf64_Sxword d_tag; // Type of dynamic table entry. union { Elf64_Xword d_val; // Integer value of entry. Elf64_Addr d_ptr; // Pointer value of entry. } d_un; }; // Dynamic table entry tags. enum { #define DYNAMIC_TAG(name, value) DT_##name = value, #include "DynamicTags.def" #undef DYNAMIC_TAG }; // DT_FLAGS values. enum { DF_ORIGIN = 0x01, // The object may reference $ORIGIN. DF_SYMBOLIC = 0x02, // Search the shared lib before searching the exe. DF_TEXTREL = 0x04, // Relocations may modify a non-writable segment. DF_BIND_NOW = 0x08, // Process all relocations on load. DF_STATIC_TLS = 0x10 // Reject attempts to load dynamically. }; // State flags selectable in the `d_un.d_val' element of the DT_FLAGS_1 entry. enum { DF_1_NOW = 0x00000001, // Set RTLD_NOW for this object. DF_1_GLOBAL = 0x00000002, // Set RTLD_GLOBAL for this object. DF_1_GROUP = 0x00000004, // Set RTLD_GROUP for this object. DF_1_NODELETE = 0x00000008, // Set RTLD_NODELETE for this object. DF_1_LOADFLTR = 0x00000010, // Trigger filtee loading at runtime. DF_1_INITFIRST = 0x00000020, // Set RTLD_INITFIRST for this object. DF_1_NOOPEN = 0x00000040, // Set RTLD_NOOPEN for this object. DF_1_ORIGIN = 0x00000080, // $ORIGIN must be handled. DF_1_DIRECT = 0x00000100, // Direct binding enabled. DF_1_TRANS = 0x00000200, DF_1_INTERPOSE = 0x00000400, // Object is used to interpose. DF_1_NODEFLIB = 0x00000800, // Ignore default lib search path. DF_1_NODUMP = 0x00001000, // Object can't be dldump'ed. DF_1_CONFALT = 0x00002000, // Configuration alternative created. DF_1_ENDFILTEE = 0x00004000, // Filtee terminates filters search. DF_1_DISPRELDNE = 0x00008000, // Disp reloc applied at build time. DF_1_DISPRELPND = 0x00010000, // Disp reloc applied at run-time. DF_1_NODIRECT = 0x00020000, // Object has no-direct binding. DF_1_IGNMULDEF = 0x00040000, DF_1_NOKSYMS = 0x00080000, DF_1_NOHDR = 0x00100000, DF_1_EDITED = 0x00200000, // Object is modified after built. DF_1_NORELOC = 0x00400000, DF_1_SYMINTPOSE = 0x00800000, // Object has individual interposers. DF_1_GLOBAUDIT = 0x01000000, // Global auditing required. DF_1_SINGLETON = 0x02000000 // Singleton symbols are used. }; // DT_MIPS_FLAGS values. enum { RHF_NONE = 0x00000000, // No flags. RHF_QUICKSTART = 0x00000001, // Uses shortcut pointers. RHF_NOTPOT = 0x00000002, // Hash size is not a power of two. RHS_NO_LIBRARY_REPLACEMENT = 0x00000004, // Ignore LD_LIBRARY_PATH. RHF_NO_MOVE = 0x00000008, // DSO address may not be relocated. RHF_SGI_ONLY = 0x00000010, // SGI specific features. RHF_GUARANTEE_INIT = 0x00000020, // Guarantee that .init will finish // executing before any non-init // code in DSO is called. RHF_DELTA_C_PLUS_PLUS = 0x00000040, // Contains Delta C++ code. RHF_GUARANTEE_START_INIT = 0x00000080, // Guarantee that .init will start // executing before any non-init // code in DSO is called. RHF_PIXIE = 0x00000100, // Generated by pixie. RHF_DEFAULT_DELAY_LOAD = 0x00000200, // Delay-load DSO by default. RHF_REQUICKSTART = 0x00000400, // Object may be requickstarted RHF_REQUICKSTARTED = 0x00000800, // Object has been requickstarted RHF_CORD = 0x00001000, // Generated by cord. RHF_NO_UNRES_UNDEF = 0x00002000, // Object contains no unresolved // undef symbols. RHF_RLD_ORDER_SAFE = 0x00004000 // Symbol table is in a safe order. }; // ElfXX_VerDef structure version (GNU versioning) enum { VER_DEF_NONE = 0, VER_DEF_CURRENT = 1 }; // VerDef Flags (ElfXX_VerDef::vd_flags) enum { VER_FLG_BASE = 0x1, VER_FLG_WEAK = 0x2, VER_FLG_INFO = 0x4 }; // Special constants for the version table. (SHT_GNU_versym/.gnu.version) enum { VER_NDX_LOCAL = 0, // Unversioned local symbol VER_NDX_GLOBAL = 1, // Unversioned global symbol VERSYM_VERSION = 0x7fff, // Version Index mask VERSYM_HIDDEN = 0x8000 // Hidden bit (non-default version) }; // ElfXX_VerNeed structure version (GNU versioning) enum { VER_NEED_NONE = 0, VER_NEED_CURRENT = 1 }; // SHT_NOTE section types enum { NT_FREEBSD_THRMISC = 7, NT_FREEBSD_PROCSTAT_PROC = 8, NT_FREEBSD_PROCSTAT_FILES = 9, NT_FREEBSD_PROCSTAT_VMMAP = 10, NT_FREEBSD_PROCSTAT_GROUPS = 11, NT_FREEBSD_PROCSTAT_UMASK = 12, NT_FREEBSD_PROCSTAT_RLIMIT = 13, NT_FREEBSD_PROCSTAT_OSREL = 14, NT_FREEBSD_PROCSTAT_PSSTRINGS = 15, NT_FREEBSD_PROCSTAT_AUXV = 16, }; // Generic note types enum : unsigned { NT_VERSION = 1, NT_ARCH = 2, NT_GNU_BUILD_ATTRIBUTE_OPEN = 0x100, NT_GNU_BUILD_ATTRIBUTE_FUNC = 0x101, }; // Core note types enum : unsigned { NT_PRSTATUS = 1, NT_FPREGSET = 2, NT_PRPSINFO = 3, NT_TASKSTRUCT = 4, NT_AUXV = 6, NT_PSTATUS = 10, NT_FPREGS = 12, NT_PSINFO = 13, NT_LWPSTATUS = 16, NT_LWPSINFO = 17, NT_WIN32PSTATUS = 18, NT_PPC_VMX = 0x100, NT_PPC_VSX = 0x102, NT_PPC_TAR = 0x103, NT_PPC_PPR = 0x104, NT_PPC_DSCR = 0x105, NT_PPC_EBB = 0x106, NT_PPC_PMU = 0x107, NT_PPC_TM_CGPR = 0x108, NT_PPC_TM_CFPR = 0x109, NT_PPC_TM_CVMX = 0x10a, NT_PPC_TM_CVSX = 0x10b, NT_PPC_TM_SPR = 0x10c, NT_PPC_TM_CTAR = 0x10d, NT_PPC_TM_CPPR = 0x10e, NT_PPC_TM_CDSCR = 0x10f, NT_386_TLS = 0x200, NT_386_IOPERM = 0x201, NT_X86_XSTATE = 0x202, NT_S390_HIGH_GPRS = 0x300, NT_S390_TIMER = 0x301, NT_S390_TODCMP = 0x302, NT_S390_TODPREG = 0x303, NT_S390_CTRS = 0x304, NT_S390_PREFIX = 0x305, NT_S390_LAST_BREAK = 0x306, NT_S390_SYSTEM_CALL = 0x307, NT_S390_TDB = 0x308, NT_S390_VXRS_LOW = 0x309, NT_S390_VXRS_HIGH = 0x30a, NT_S390_GS_CB = 0x30b, NT_S390_GS_BC = 0x30c, NT_ARM_VFP = 0x400, NT_ARM_TLS = 0x401, NT_ARM_HW_BREAK = 0x402, NT_ARM_HW_WATCH = 0x403, NT_ARM_SVE = 0x405, NT_ARM_PAC_MASK = 0x406, NT_FILE = 0x46494c45, NT_PRXFPREG = 0x46e62b7f, NT_SIGINFO = 0x53494749, }; // LLVM-specific notes. enum { NT_LLVM_HWASAN_GLOBALS = 3, }; // GNU note types enum { NT_GNU_ABI_TAG = 1, NT_GNU_HWCAP = 2, NT_GNU_BUILD_ID = 3, NT_GNU_GOLD_VERSION = 4, NT_GNU_PROPERTY_TYPE_0 = 5, }; // Property types used in GNU_PROPERTY_TYPE_0 notes. enum : unsigned { GNU_PROPERTY_STACK_SIZE = 1, GNU_PROPERTY_NO_COPY_ON_PROTECTED = 2, GNU_PROPERTY_AARCH64_FEATURE_1_AND = 0xc0000000, GNU_PROPERTY_X86_FEATURE_1_AND = 0xc0000002, GNU_PROPERTY_X86_ISA_1_NEEDED = 0xc0008000, GNU_PROPERTY_X86_FEATURE_2_NEEDED = 0xc0008001, GNU_PROPERTY_X86_ISA_1_USED = 0xc0010000, GNU_PROPERTY_X86_FEATURE_2_USED = 0xc0010001, }; // aarch64 processor feature bits. enum : unsigned { GNU_PROPERTY_AARCH64_FEATURE_1_BTI = 1 << 0, GNU_PROPERTY_AARCH64_FEATURE_1_PAC = 1 << 1, }; // x86 processor feature bits. enum : unsigned { GNU_PROPERTY_X86_FEATURE_1_IBT = 1 << 0, GNU_PROPERTY_X86_FEATURE_1_SHSTK = 1 << 1, GNU_PROPERTY_X86_ISA_1_CMOV = 1 << 0, GNU_PROPERTY_X86_ISA_1_SSE = 1 << 1, GNU_PROPERTY_X86_ISA_1_SSE2 = 1 << 2, GNU_PROPERTY_X86_ISA_1_SSE3 = 1 << 3, GNU_PROPERTY_X86_ISA_1_SSSE3 = 1 << 4, GNU_PROPERTY_X86_ISA_1_SSE4_1 = 1 << 5, GNU_PROPERTY_X86_ISA_1_SSE4_2 = 1 << 6, GNU_PROPERTY_X86_ISA_1_AVX = 1 << 7, GNU_PROPERTY_X86_ISA_1_AVX2 = 1 << 8, GNU_PROPERTY_X86_ISA_1_FMA = 1 << 9, GNU_PROPERTY_X86_ISA_1_AVX512F = 1 << 10, GNU_PROPERTY_X86_ISA_1_AVX512CD = 1 << 11, GNU_PROPERTY_X86_ISA_1_AVX512ER = 1 << 12, GNU_PROPERTY_X86_ISA_1_AVX512PF = 1 << 13, GNU_PROPERTY_X86_ISA_1_AVX512VL = 1 << 14, GNU_PROPERTY_X86_ISA_1_AVX512DQ = 1 << 15, GNU_PROPERTY_X86_ISA_1_AVX512BW = 1 << 16, GNU_PROPERTY_X86_ISA_1_AVX512_4FMAPS = 1 << 17, GNU_PROPERTY_X86_ISA_1_AVX512_4VNNIW = 1 << 18, GNU_PROPERTY_X86_ISA_1_AVX512_BITALG = 1 << 19, GNU_PROPERTY_X86_ISA_1_AVX512_IFMA = 1 << 20, GNU_PROPERTY_X86_ISA_1_AVX512_VBMI = 1 << 21, GNU_PROPERTY_X86_ISA_1_AVX512_VBMI2 = 1 << 22, GNU_PROPERTY_X86_ISA_1_AVX512_VNNI = 1 << 23, GNU_PROPERTY_X86_FEATURE_2_X86 = 1 << 0, GNU_PROPERTY_X86_FEATURE_2_X87 = 1 << 1, GNU_PROPERTY_X86_FEATURE_2_MMX = 1 << 2, GNU_PROPERTY_X86_FEATURE_2_XMM = 1 << 3, GNU_PROPERTY_X86_FEATURE_2_YMM = 1 << 4, GNU_PROPERTY_X86_FEATURE_2_ZMM = 1 << 5, GNU_PROPERTY_X86_FEATURE_2_FXSR = 1 << 6, GNU_PROPERTY_X86_FEATURE_2_XSAVE = 1 << 7, GNU_PROPERTY_X86_FEATURE_2_XSAVEOPT = 1 << 8, GNU_PROPERTY_X86_FEATURE_2_XSAVEC = 1 << 9, }; // AMDGPU-specific section indices. enum { SHN_AMDGPU_LDS = 0xff00, // Variable in LDS; symbol encoded like SHN_COMMON }; // AMD specific notes. (Code Object V2) enum { // Note types with values between 0 and 9 (inclusive) are reserved. NT_AMD_AMDGPU_HSA_METADATA = 10, NT_AMD_AMDGPU_ISA = 11, NT_AMD_AMDGPU_PAL_METADATA = 12 }; // AMDGPU specific notes. (Code Object V3) enum { // Note types with values between 0 and 31 (inclusive) are reserved. NT_AMDGPU_METADATA = 32 }; enum { GNU_ABI_TAG_LINUX = 0, GNU_ABI_TAG_HURD = 1, GNU_ABI_TAG_SOLARIS = 2, GNU_ABI_TAG_FREEBSD = 3, GNU_ABI_TAG_NETBSD = 4, GNU_ABI_TAG_SYLLABLE = 5, GNU_ABI_TAG_NACL = 6, }; constexpr const char *ELF_NOTE_GNU = "GNU"; // Android packed relocation group flags. enum { RELOCATION_GROUPED_BY_INFO_FLAG = 1, RELOCATION_GROUPED_BY_OFFSET_DELTA_FLAG = 2, RELOCATION_GROUPED_BY_ADDEND_FLAG = 4, RELOCATION_GROUP_HAS_ADDEND_FLAG = 8, }; // Compressed section header for ELF32. struct Elf32_Chdr { Elf32_Word ch_type; Elf32_Word ch_size; Elf32_Word ch_addralign; }; // Compressed section header for ELF64. struct Elf64_Chdr { Elf64_Word ch_type; Elf64_Word ch_reserved; Elf64_Xword ch_size; Elf64_Xword ch_addralign; }; // Node header for ELF32. struct Elf32_Nhdr { Elf32_Word n_namesz; Elf32_Word n_descsz; Elf32_Word n_type; }; // Node header for ELF64. struct Elf64_Nhdr { Elf64_Word n_namesz; Elf64_Word n_descsz; Elf64_Word n_type; }; // Legal values for ch_type field of compressed section header. enum { ELFCOMPRESS_ZLIB = 1, // ZLIB/DEFLATE algorithm. ELFCOMPRESS_LOOS = 0x60000000, // Start of OS-specific. ELFCOMPRESS_HIOS = 0x6fffffff, // End of OS-specific. ELFCOMPRESS_LOPROC = 0x70000000, // Start of processor-specific. ELFCOMPRESS_HIPROC = 0x7fffffff // End of processor-specific. }; } // end namespace ELF } // end namespace llvm #endif // LLVM_BINARYFORMAT_ELF_H diff --git a/llvm/include/llvm/BinaryFormat/ELFRelocs/PowerPC64.def b/llvm/include/llvm/BinaryFormat/ELFRelocs/PowerPC64.def index 719d0c9c36ac..f8c330e2cf3f 100644 --- a/llvm/include/llvm/BinaryFormat/ELFRelocs/PowerPC64.def +++ b/llvm/include/llvm/BinaryFormat/ELFRelocs/PowerPC64.def @@ -1,197 +1,199 @@ #ifndef ELF_RELOC #error "ELF_RELOC must be defined" #endif // glibc's PowerPC asm/sigcontext.h, when compiling for PPC64, has the // unfortunate behavior of including asm/elf.h, which defines R_PPC_NONE, etc. // to their corresponding integer values. As a result, we need to undef them // here before continuing. #undef R_PPC64_NONE #undef R_PPC64_ADDR32 #undef R_PPC64_ADDR24 #undef R_PPC64_ADDR16 #undef R_PPC64_ADDR16_LO #undef R_PPC64_ADDR16_HI #undef R_PPC64_ADDR16_HA #undef R_PPC64_ADDR14 #undef R_PPC64_ADDR14_BRTAKEN #undef R_PPC64_ADDR14_BRNTAKEN #undef R_PPC64_REL24 #undef R_PPC64_REL14 #undef R_PPC64_REL14_BRTAKEN #undef R_PPC64_REL14_BRNTAKEN #undef R_PPC64_GOT16 #undef R_PPC64_GOT16_LO #undef R_PPC64_GOT16_HI #undef R_PPC64_GOT16_HA #undef R_PPC64_COPY #undef R_PPC64_GLOB_DAT #undef R_PPC64_JMP_SLOT #undef R_PPC64_RELATIVE #undef R_PPC64_REL32 #undef R_PPC64_ADDR64 #undef R_PPC64_ADDR16_HIGHER #undef R_PPC64_ADDR16_HIGHERA #undef R_PPC64_ADDR16_HIGHEST #undef R_PPC64_ADDR16_HIGHESTA #undef R_PPC64_REL64 #undef R_PPC64_TOC16 #undef R_PPC64_TOC16_LO #undef R_PPC64_TOC16_HI #undef R_PPC64_TOC16_HA #undef R_PPC64_TOC #undef R_PPC64_ADDR16_DS #undef R_PPC64_ADDR16_LO_DS #undef R_PPC64_GOT16_DS #undef R_PPC64_GOT16_LO_DS #undef R_PPC64_TOC16_DS #undef R_PPC64_TOC16_LO_DS #undef R_PPC64_TLS #undef R_PPC64_DTPMOD64 #undef R_PPC64_TPREL16 #undef R_PPC64_TPREL16_LO #undef R_PPC64_TPREL16_HI #undef R_PPC64_TPREL16_HA #undef R_PPC64_TPREL64 #undef R_PPC64_DTPREL16 #undef R_PPC64_DTPREL16_LO #undef R_PPC64_DTPREL16_HI #undef R_PPC64_DTPREL16_HA #undef R_PPC64_DTPREL64 #undef R_PPC64_GOT_TLSGD16 #undef R_PPC64_GOT_TLSGD16_LO #undef R_PPC64_GOT_TLSGD16_HI #undef R_PPC64_GOT_TLSGD16_HA #undef R_PPC64_GOT_TLSLD16 #undef R_PPC64_GOT_TLSLD16_LO #undef R_PPC64_GOT_TLSLD16_HI #undef R_PPC64_GOT_TLSLD16_HA #undef R_PPC64_GOT_TPREL16_DS #undef R_PPC64_GOT_TPREL16_LO_DS #undef R_PPC64_GOT_TPREL16_HI #undef R_PPC64_GOT_TPREL16_HA #undef R_PPC64_GOT_DTPREL16_DS #undef R_PPC64_GOT_DTPREL16_LO_DS #undef R_PPC64_GOT_DTPREL16_HI #undef R_PPC64_GOT_DTPREL16_HA #undef R_PPC64_TPREL16_DS #undef R_PPC64_TPREL16_LO_DS #undef R_PPC64_TPREL16_HIGHER #undef R_PPC64_TPREL16_HIGHERA #undef R_PPC64_TPREL16_HIGHEST #undef R_PPC64_TPREL16_HIGHESTA #undef R_PPC64_DTPREL16_DS #undef R_PPC64_DTPREL16_LO_DS #undef R_PPC64_DTPREL16_HIGHER #undef R_PPC64_DTPREL16_HIGHERA #undef R_PPC64_DTPREL16_HIGHEST #undef R_PPC64_DTPREL16_HIGHESTA #undef R_PPC64_TLSGD #undef R_PPC64_TLSLD #undef R_PPC64_ADDR16_HIGH #undef R_PPC64_ADDR16_HIGHA #undef R_PPC64_TPREL16_HIGH #undef R_PPC64_TPREL16_HIGHA #undef R_PPC64_DTPREL16_HIGH #undef R_PPC64_DTPREL16_HIGHA +#undef R_PPC64_REL24_NOTOC #undef R_PPC64_IRELATIVE #undef R_PPC64_REL16 #undef R_PPC64_REL16_LO #undef R_PPC64_REL16_HI #undef R_PPC64_REL16_HA ELF_RELOC(R_PPC64_NONE, 0) ELF_RELOC(R_PPC64_ADDR32, 1) ELF_RELOC(R_PPC64_ADDR24, 2) ELF_RELOC(R_PPC64_ADDR16, 3) ELF_RELOC(R_PPC64_ADDR16_LO, 4) ELF_RELOC(R_PPC64_ADDR16_HI, 5) ELF_RELOC(R_PPC64_ADDR16_HA, 6) ELF_RELOC(R_PPC64_ADDR14, 7) ELF_RELOC(R_PPC64_ADDR14_BRTAKEN, 8) ELF_RELOC(R_PPC64_ADDR14_BRNTAKEN, 9) ELF_RELOC(R_PPC64_REL24, 10) ELF_RELOC(R_PPC64_REL14, 11) ELF_RELOC(R_PPC64_REL14_BRTAKEN, 12) ELF_RELOC(R_PPC64_REL14_BRNTAKEN, 13) ELF_RELOC(R_PPC64_GOT16, 14) ELF_RELOC(R_PPC64_GOT16_LO, 15) ELF_RELOC(R_PPC64_GOT16_HI, 16) ELF_RELOC(R_PPC64_GOT16_HA, 17) ELF_RELOC(R_PPC64_COPY, 19) ELF_RELOC(R_PPC64_GLOB_DAT, 20) ELF_RELOC(R_PPC64_JMP_SLOT, 21) ELF_RELOC(R_PPC64_RELATIVE, 22) ELF_RELOC(R_PPC64_REL32, 26) ELF_RELOC(R_PPC64_ADDR64, 38) ELF_RELOC(R_PPC64_ADDR16_HIGHER, 39) ELF_RELOC(R_PPC64_ADDR16_HIGHERA, 40) ELF_RELOC(R_PPC64_ADDR16_HIGHEST, 41) ELF_RELOC(R_PPC64_ADDR16_HIGHESTA, 42) ELF_RELOC(R_PPC64_REL64, 44) ELF_RELOC(R_PPC64_TOC16, 47) ELF_RELOC(R_PPC64_TOC16_LO, 48) ELF_RELOC(R_PPC64_TOC16_HI, 49) ELF_RELOC(R_PPC64_TOC16_HA, 50) ELF_RELOC(R_PPC64_TOC, 51) ELF_RELOC(R_PPC64_ADDR16_DS, 56) ELF_RELOC(R_PPC64_ADDR16_LO_DS, 57) ELF_RELOC(R_PPC64_GOT16_DS, 58) ELF_RELOC(R_PPC64_GOT16_LO_DS, 59) ELF_RELOC(R_PPC64_TOC16_DS, 63) ELF_RELOC(R_PPC64_TOC16_LO_DS, 64) ELF_RELOC(R_PPC64_TLS, 67) ELF_RELOC(R_PPC64_DTPMOD64, 68) ELF_RELOC(R_PPC64_TPREL16, 69) ELF_RELOC(R_PPC64_TPREL16_LO, 70) ELF_RELOC(R_PPC64_TPREL16_HI, 71) ELF_RELOC(R_PPC64_TPREL16_HA, 72) ELF_RELOC(R_PPC64_TPREL64, 73) ELF_RELOC(R_PPC64_DTPREL16, 74) ELF_RELOC(R_PPC64_DTPREL16_LO, 75) ELF_RELOC(R_PPC64_DTPREL16_HI, 76) ELF_RELOC(R_PPC64_DTPREL16_HA, 77) ELF_RELOC(R_PPC64_DTPREL64, 78) ELF_RELOC(R_PPC64_GOT_TLSGD16, 79) ELF_RELOC(R_PPC64_GOT_TLSGD16_LO, 80) ELF_RELOC(R_PPC64_GOT_TLSGD16_HI, 81) ELF_RELOC(R_PPC64_GOT_TLSGD16_HA, 82) ELF_RELOC(R_PPC64_GOT_TLSLD16, 83) ELF_RELOC(R_PPC64_GOT_TLSLD16_LO, 84) ELF_RELOC(R_PPC64_GOT_TLSLD16_HI, 85) ELF_RELOC(R_PPC64_GOT_TLSLD16_HA, 86) ELF_RELOC(R_PPC64_GOT_TPREL16_DS, 87) ELF_RELOC(R_PPC64_GOT_TPREL16_LO_DS, 88) ELF_RELOC(R_PPC64_GOT_TPREL16_HI, 89) ELF_RELOC(R_PPC64_GOT_TPREL16_HA, 90) ELF_RELOC(R_PPC64_GOT_DTPREL16_DS, 91) ELF_RELOC(R_PPC64_GOT_DTPREL16_LO_DS, 92) ELF_RELOC(R_PPC64_GOT_DTPREL16_HI, 93) ELF_RELOC(R_PPC64_GOT_DTPREL16_HA, 94) ELF_RELOC(R_PPC64_TPREL16_DS, 95) ELF_RELOC(R_PPC64_TPREL16_LO_DS, 96) ELF_RELOC(R_PPC64_TPREL16_HIGHER, 97) ELF_RELOC(R_PPC64_TPREL16_HIGHERA, 98) ELF_RELOC(R_PPC64_TPREL16_HIGHEST, 99) ELF_RELOC(R_PPC64_TPREL16_HIGHESTA, 100) ELF_RELOC(R_PPC64_DTPREL16_DS, 101) ELF_RELOC(R_PPC64_DTPREL16_LO_DS, 102) ELF_RELOC(R_PPC64_DTPREL16_HIGHER, 103) ELF_RELOC(R_PPC64_DTPREL16_HIGHERA, 104) ELF_RELOC(R_PPC64_DTPREL16_HIGHEST, 105) ELF_RELOC(R_PPC64_DTPREL16_HIGHESTA, 106) ELF_RELOC(R_PPC64_TLSGD, 107) ELF_RELOC(R_PPC64_TLSLD, 108) ELF_RELOC(R_PPC64_ADDR16_HIGH, 110) ELF_RELOC(R_PPC64_ADDR16_HIGHA, 111) ELF_RELOC(R_PPC64_TPREL16_HIGH, 112) ELF_RELOC(R_PPC64_TPREL16_HIGHA, 113) ELF_RELOC(R_PPC64_DTPREL16_HIGH, 114) ELF_RELOC(R_PPC64_DTPREL16_HIGHA, 115) +ELF_RELOC(R_PPC64_REL24_NOTOC, 116) ELF_RELOC(R_PPC64_IRELATIVE, 248) ELF_RELOC(R_PPC64_REL16, 249) ELF_RELOC(R_PPC64_REL16_LO, 250) ELF_RELOC(R_PPC64_REL16_HI, 251) ELF_RELOC(R_PPC64_REL16_HA, 252) diff --git a/llvm/include/llvm/MC/MCExpr.h b/llvm/include/llvm/MC/MCExpr.h index c0060c8610e7..386fa9c533c1 100644 --- a/llvm/include/llvm/MC/MCExpr.h +++ b/llvm/include/llvm/MC/MCExpr.h @@ -1,623 +1,624 @@ //===- MCExpr.h - Assembly Level Expressions --------------------*- C++ -*-===// // // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. // See https://llvm.org/LICENSE.txt for license information. // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// #ifndef LLVM_MC_MCEXPR_H #define LLVM_MC_MCEXPR_H #include "llvm/ADT/DenseMap.h" #include "llvm/Support/SMLoc.h" #include namespace llvm { class MCAsmInfo; class MCAsmLayout; class MCAssembler; class MCContext; class MCFixup; class MCFragment; class MCSection; class MCStreamer; class MCSymbol; class MCValue; class raw_ostream; class StringRef; using SectionAddrMap = DenseMap; /// Base class for the full range of assembler expressions which are /// needed for parsing. class MCExpr { public: enum ExprKind { Binary, ///< Binary expressions. Constant, ///< Constant expressions. SymbolRef, ///< References to labels and assigned expressions. Unary, ///< Unary expressions. Target ///< Target specific expression. }; private: ExprKind Kind; SMLoc Loc; bool evaluateAsAbsolute(int64_t &Res, const MCAssembler *Asm, const MCAsmLayout *Layout, const SectionAddrMap *Addrs, bool InSet) const; protected: explicit MCExpr(ExprKind Kind, SMLoc Loc) : Kind(Kind), Loc(Loc) {} bool evaluateAsRelocatableImpl(MCValue &Res, const MCAssembler *Asm, const MCAsmLayout *Layout, const MCFixup *Fixup, const SectionAddrMap *Addrs, bool InSet) const; public: MCExpr(const MCExpr &) = delete; MCExpr &operator=(const MCExpr &) = delete; /// \name Accessors /// @{ ExprKind getKind() const { return Kind; } SMLoc getLoc() const { return Loc; } /// @} /// \name Utility Methods /// @{ void print(raw_ostream &OS, const MCAsmInfo *MAI, bool InParens = false) const; void dump() const; /// @} /// \name Expression Evaluation /// @{ /// Try to evaluate the expression to an absolute value. /// /// \param Res - The absolute value, if evaluation succeeds. /// \param Layout - The assembler layout object to use for evaluating symbol /// values. If not given, then only non-symbolic expressions will be /// evaluated. /// \return - True on success. bool evaluateAsAbsolute(int64_t &Res, const MCAsmLayout &Layout, const SectionAddrMap &Addrs) const; bool evaluateAsAbsolute(int64_t &Res) const; bool evaluateAsAbsolute(int64_t &Res, const MCAssembler &Asm) const; bool evaluateAsAbsolute(int64_t &Res, const MCAssembler *Asm) const; bool evaluateAsAbsolute(int64_t &Res, const MCAsmLayout &Layout) const; bool evaluateKnownAbsolute(int64_t &Res, const MCAsmLayout &Layout) const; /// Try to evaluate the expression to a relocatable value, i.e. an /// expression of the fixed form (a - b + constant). /// /// \param Res - The relocatable value, if evaluation succeeds. /// \param Layout - The assembler layout object to use for evaluating values. /// \param Fixup - The Fixup object if available. /// \return - True on success. bool evaluateAsRelocatable(MCValue &Res, const MCAsmLayout *Layout, const MCFixup *Fixup) const; /// Try to evaluate the expression to the form (a - b + constant) where /// neither a nor b are variables. /// /// This is a more aggressive variant of evaluateAsRelocatable. The intended /// use is for when relocations are not available, like the .size directive. bool evaluateAsValue(MCValue &Res, const MCAsmLayout &Layout) const; /// Find the "associated section" for this expression, which is /// currently defined as the absolute section for constants, or /// otherwise the section associated with the first defined symbol in the /// expression. MCFragment *findAssociatedFragment() const; /// @} }; inline raw_ostream &operator<<(raw_ostream &OS, const MCExpr &E) { E.print(OS, nullptr); return OS; } //// Represent a constant integer expression. class MCConstantExpr : public MCExpr { int64_t Value; bool PrintInHex = false; unsigned SizeInBytes = 0; explicit MCConstantExpr(int64_t Value) : MCExpr(MCExpr::Constant, SMLoc()), Value(Value) {} MCConstantExpr(int64_t Value, bool PrintInHex) : MCExpr(MCExpr::Constant, SMLoc()), Value(Value), PrintInHex(PrintInHex) {} MCConstantExpr(int64_t Value, bool PrintInHex, unsigned SizeInBytes) : MCExpr(MCExpr::Constant, SMLoc()), Value(Value), PrintInHex(PrintInHex), SizeInBytes(SizeInBytes) {} public: /// \name Construction /// @{ static const MCConstantExpr *create(int64_t Value, MCContext &Ctx, bool PrintInHex = false, unsigned SizeInBytes = 0); /// @} /// \name Accessors /// @{ int64_t getValue() const { return Value; } unsigned getSizeInBytes() const { return SizeInBytes; } bool useHexFormat() const { return PrintInHex; } /// @} static bool classof(const MCExpr *E) { return E->getKind() == MCExpr::Constant; } }; /// Represent a reference to a symbol from inside an expression. /// /// A symbol reference in an expression may be a use of a label, a use of an /// assembler variable (defined constant), or constitute an implicit definition /// of the symbol as external. class MCSymbolRefExpr : public MCExpr { public: enum VariantKind : uint16_t { VK_None, VK_Invalid, VK_GOT, VK_GOTOFF, VK_GOTREL, VK_PCREL, VK_GOTPCREL, VK_GOTTPOFF, VK_INDNTPOFF, VK_NTPOFF, VK_GOTNTPOFF, VK_PLT, VK_TLSGD, VK_TLSLD, VK_TLSLDM, VK_TPOFF, VK_DTPOFF, VK_TLSCALL, // symbol(tlscall) VK_TLSDESC, // symbol(tlsdesc) VK_TLVP, // Mach-O thread local variable relocations VK_TLVPPAGE, VK_TLVPPAGEOFF, VK_PAGE, VK_PAGEOFF, VK_GOTPAGE, VK_GOTPAGEOFF, VK_SECREL, VK_SIZE, // symbol@SIZE VK_WEAKREF, // The link between the symbols in .weakref foo, bar VK_X86_ABS8, VK_ARM_NONE, VK_ARM_GOT_PREL, VK_ARM_TARGET1, VK_ARM_TARGET2, VK_ARM_PREL31, VK_ARM_SBREL, // symbol(sbrel) VK_ARM_TLSLDO, // symbol(tlsldo) VK_ARM_TLSDESCSEQ, VK_AVR_NONE, VK_AVR_LO8, VK_AVR_HI8, VK_AVR_HLO8, VK_AVR_DIFF8, VK_AVR_DIFF16, VK_AVR_DIFF32, VK_PPC_LO, // symbol@l VK_PPC_HI, // symbol@h VK_PPC_HA, // symbol@ha VK_PPC_HIGH, // symbol@high VK_PPC_HIGHA, // symbol@higha VK_PPC_HIGHER, // symbol@higher VK_PPC_HIGHERA, // symbol@highera VK_PPC_HIGHEST, // symbol@highest VK_PPC_HIGHESTA, // symbol@highesta VK_PPC_GOT_LO, // symbol@got@l VK_PPC_GOT_HI, // symbol@got@h VK_PPC_GOT_HA, // symbol@got@ha VK_PPC_TOCBASE, // symbol@tocbase VK_PPC_TOC, // symbol@toc VK_PPC_TOC_LO, // symbol@toc@l VK_PPC_TOC_HI, // symbol@toc@h VK_PPC_TOC_HA, // symbol@toc@ha VK_PPC_U, // symbol@u VK_PPC_L, // symbol@l VK_PPC_DTPMOD, // symbol@dtpmod VK_PPC_TPREL_LO, // symbol@tprel@l VK_PPC_TPREL_HI, // symbol@tprel@h VK_PPC_TPREL_HA, // symbol@tprel@ha VK_PPC_TPREL_HIGH, // symbol@tprel@high VK_PPC_TPREL_HIGHA, // symbol@tprel@higha VK_PPC_TPREL_HIGHER, // symbol@tprel@higher VK_PPC_TPREL_HIGHERA, // symbol@tprel@highera VK_PPC_TPREL_HIGHEST, // symbol@tprel@highest VK_PPC_TPREL_HIGHESTA, // symbol@tprel@highesta VK_PPC_DTPREL_LO, // symbol@dtprel@l VK_PPC_DTPREL_HI, // symbol@dtprel@h VK_PPC_DTPREL_HA, // symbol@dtprel@ha VK_PPC_DTPREL_HIGH, // symbol@dtprel@high VK_PPC_DTPREL_HIGHA, // symbol@dtprel@higha VK_PPC_DTPREL_HIGHER, // symbol@dtprel@higher VK_PPC_DTPREL_HIGHERA, // symbol@dtprel@highera VK_PPC_DTPREL_HIGHEST, // symbol@dtprel@highest VK_PPC_DTPREL_HIGHESTA,// symbol@dtprel@highesta VK_PPC_GOT_TPREL, // symbol@got@tprel VK_PPC_GOT_TPREL_LO, // symbol@got@tprel@l VK_PPC_GOT_TPREL_HI, // symbol@got@tprel@h VK_PPC_GOT_TPREL_HA, // symbol@got@tprel@ha VK_PPC_GOT_DTPREL, // symbol@got@dtprel VK_PPC_GOT_DTPREL_LO, // symbol@got@dtprel@l VK_PPC_GOT_DTPREL_HI, // symbol@got@dtprel@h VK_PPC_GOT_DTPREL_HA, // symbol@got@dtprel@ha VK_PPC_TLS, // symbol@tls VK_PPC_GOT_TLSGD, // symbol@got@tlsgd VK_PPC_GOT_TLSGD_LO, // symbol@got@tlsgd@l VK_PPC_GOT_TLSGD_HI, // symbol@got@tlsgd@h VK_PPC_GOT_TLSGD_HA, // symbol@got@tlsgd@ha VK_PPC_TLSGD, // symbol@tlsgd VK_PPC_GOT_TLSLD, // symbol@got@tlsld VK_PPC_GOT_TLSLD_LO, // symbol@got@tlsld@l VK_PPC_GOT_TLSLD_HI, // symbol@got@tlsld@h VK_PPC_GOT_TLSLD_HA, // symbol@got@tlsld@ha VK_PPC_TLSLD, // symbol@tlsld VK_PPC_LOCAL, // symbol@local + VK_PPC_NOTOC, // symbol@notoc VK_COFF_IMGREL32, // symbol@imgrel (image-relative) VK_Hexagon_LO16, VK_Hexagon_HI16, VK_Hexagon_GPREL, VK_Hexagon_GD_GOT, VK_Hexagon_LD_GOT, VK_Hexagon_GD_PLT, VK_Hexagon_LD_PLT, VK_Hexagon_IE, VK_Hexagon_IE_GOT, VK_WASM_TYPEINDEX, // Reference to a symbol's type (signature) VK_WASM_MBREL, // Memory address relative to memory base VK_WASM_TBREL, // Table index relative to table bare VK_AMDGPU_GOTPCREL32_LO, // symbol@gotpcrel32@lo VK_AMDGPU_GOTPCREL32_HI, // symbol@gotpcrel32@hi VK_AMDGPU_REL32_LO, // symbol@rel32@lo VK_AMDGPU_REL32_HI, // symbol@rel32@hi VK_AMDGPU_REL64, // symbol@rel64 VK_AMDGPU_ABS32_LO, // symbol@abs32@lo VK_AMDGPU_ABS32_HI, // symbol@abs32@hi VK_TPREL, VK_DTPREL }; private: /// The symbol reference modifier. const VariantKind Kind; /// Specifies how the variant kind should be printed. const unsigned UseParensForSymbolVariant : 1; // FIXME: Remove this bit. const unsigned HasSubsectionsViaSymbols : 1; /// The symbol being referenced. const MCSymbol *Symbol; explicit MCSymbolRefExpr(const MCSymbol *Symbol, VariantKind Kind, const MCAsmInfo *MAI, SMLoc Loc = SMLoc()); public: /// \name Construction /// @{ static const MCSymbolRefExpr *create(const MCSymbol *Symbol, MCContext &Ctx) { return MCSymbolRefExpr::create(Symbol, VK_None, Ctx); } static const MCSymbolRefExpr *create(const MCSymbol *Symbol, VariantKind Kind, MCContext &Ctx, SMLoc Loc = SMLoc()); static const MCSymbolRefExpr *create(StringRef Name, VariantKind Kind, MCContext &Ctx); /// @} /// \name Accessors /// @{ const MCSymbol &getSymbol() const { return *Symbol; } VariantKind getKind() const { return Kind; } void printVariantKind(raw_ostream &OS) const; bool hasSubsectionsViaSymbols() const { return HasSubsectionsViaSymbols; } /// @} /// \name Static Utility Functions /// @{ static StringRef getVariantKindName(VariantKind Kind); static VariantKind getVariantKindForName(StringRef Name); /// @} static bool classof(const MCExpr *E) { return E->getKind() == MCExpr::SymbolRef; } }; /// Unary assembler expressions. class MCUnaryExpr : public MCExpr { public: enum Opcode { LNot, ///< Logical negation. Minus, ///< Unary minus. Not, ///< Bitwise negation. Plus ///< Unary plus. }; private: Opcode Op; const MCExpr *Expr; MCUnaryExpr(Opcode Op, const MCExpr *Expr, SMLoc Loc) : MCExpr(MCExpr::Unary, Loc), Op(Op), Expr(Expr) {} public: /// \name Construction /// @{ static const MCUnaryExpr *create(Opcode Op, const MCExpr *Expr, MCContext &Ctx, SMLoc Loc = SMLoc()); static const MCUnaryExpr *createLNot(const MCExpr *Expr, MCContext &Ctx, SMLoc Loc = SMLoc()) { return create(LNot, Expr, Ctx, Loc); } static const MCUnaryExpr *createMinus(const MCExpr *Expr, MCContext &Ctx, SMLoc Loc = SMLoc()) { return create(Minus, Expr, Ctx, Loc); } static const MCUnaryExpr *createNot(const MCExpr *Expr, MCContext &Ctx, SMLoc Loc = SMLoc()) { return create(Not, Expr, Ctx, Loc); } static const MCUnaryExpr *createPlus(const MCExpr *Expr, MCContext &Ctx, SMLoc Loc = SMLoc()) { return create(Plus, Expr, Ctx, Loc); } /// @} /// \name Accessors /// @{ /// Get the kind of this unary expression. Opcode getOpcode() const { return Op; } /// Get the child of this unary expression. const MCExpr *getSubExpr() const { return Expr; } /// @} static bool classof(const MCExpr *E) { return E->getKind() == MCExpr::Unary; } }; /// Binary assembler expressions. class MCBinaryExpr : public MCExpr { public: enum Opcode { Add, ///< Addition. And, ///< Bitwise and. Div, ///< Signed division. EQ, ///< Equality comparison. GT, ///< Signed greater than comparison (result is either 0 or some ///< target-specific non-zero value) GTE, ///< Signed greater than or equal comparison (result is either 0 or ///< some target-specific non-zero value). LAnd, ///< Logical and. LOr, ///< Logical or. LT, ///< Signed less than comparison (result is either 0 or ///< some target-specific non-zero value). LTE, ///< Signed less than or equal comparison (result is either 0 or ///< some target-specific non-zero value). Mod, ///< Signed remainder. Mul, ///< Multiplication. NE, ///< Inequality comparison. Or, ///< Bitwise or. Shl, ///< Shift left. AShr, ///< Arithmetic shift right. LShr, ///< Logical shift right. Sub, ///< Subtraction. Xor ///< Bitwise exclusive or. }; private: Opcode Op; const MCExpr *LHS, *RHS; MCBinaryExpr(Opcode Op, const MCExpr *LHS, const MCExpr *RHS, SMLoc Loc = SMLoc()) : MCExpr(MCExpr::Binary, Loc), Op(Op), LHS(LHS), RHS(RHS) {} public: /// \name Construction /// @{ static const MCBinaryExpr *create(Opcode Op, const MCExpr *LHS, const MCExpr *RHS, MCContext &Ctx, SMLoc Loc = SMLoc()); static const MCBinaryExpr *createAdd(const MCExpr *LHS, const MCExpr *RHS, MCContext &Ctx) { return create(Add, LHS, RHS, Ctx); } static const MCBinaryExpr *createAnd(const MCExpr *LHS, const MCExpr *RHS, MCContext &Ctx) { return create(And, LHS, RHS, Ctx); } static const MCBinaryExpr *createDiv(const MCExpr *LHS, const MCExpr *RHS, MCContext &Ctx) { return create(Div, LHS, RHS, Ctx); } static const MCBinaryExpr *createEQ(const MCExpr *LHS, const MCExpr *RHS, MCContext &Ctx) { return create(EQ, LHS, RHS, Ctx); } static const MCBinaryExpr *createGT(const MCExpr *LHS, const MCExpr *RHS, MCContext &Ctx) { return create(GT, LHS, RHS, Ctx); } static const MCBinaryExpr *createGTE(const MCExpr *LHS, const MCExpr *RHS, MCContext &Ctx) { return create(GTE, LHS, RHS, Ctx); } static const MCBinaryExpr *createLAnd(const MCExpr *LHS, const MCExpr *RHS, MCContext &Ctx) { return create(LAnd, LHS, RHS, Ctx); } static const MCBinaryExpr *createLOr(const MCExpr *LHS, const MCExpr *RHS, MCContext &Ctx) { return create(LOr, LHS, RHS, Ctx); } static const MCBinaryExpr *createLT(const MCExpr *LHS, const MCExpr *RHS, MCContext &Ctx) { return create(LT, LHS, RHS, Ctx); } static const MCBinaryExpr *createLTE(const MCExpr *LHS, const MCExpr *RHS, MCContext &Ctx) { return create(LTE, LHS, RHS, Ctx); } static const MCBinaryExpr *createMod(const MCExpr *LHS, const MCExpr *RHS, MCContext &Ctx) { return create(Mod, LHS, RHS, Ctx); } static const MCBinaryExpr *createMul(const MCExpr *LHS, const MCExpr *RHS, MCContext &Ctx) { return create(Mul, LHS, RHS, Ctx); } static const MCBinaryExpr *createNE(const MCExpr *LHS, const MCExpr *RHS, MCContext &Ctx) { return create(NE, LHS, RHS, Ctx); } static const MCBinaryExpr *createOr(const MCExpr *LHS, const MCExpr *RHS, MCContext &Ctx) { return create(Or, LHS, RHS, Ctx); } static const MCBinaryExpr *createShl(const MCExpr *LHS, const MCExpr *RHS, MCContext &Ctx) { return create(Shl, LHS, RHS, Ctx); } static const MCBinaryExpr *createAShr(const MCExpr *LHS, const MCExpr *RHS, MCContext &Ctx) { return create(AShr, LHS, RHS, Ctx); } static const MCBinaryExpr *createLShr(const MCExpr *LHS, const MCExpr *RHS, MCContext &Ctx) { return create(LShr, LHS, RHS, Ctx); } static const MCBinaryExpr *createSub(const MCExpr *LHS, const MCExpr *RHS, MCContext &Ctx) { return create(Sub, LHS, RHS, Ctx); } static const MCBinaryExpr *createXor(const MCExpr *LHS, const MCExpr *RHS, MCContext &Ctx) { return create(Xor, LHS, RHS, Ctx); } /// @} /// \name Accessors /// @{ /// Get the kind of this binary expression. Opcode getOpcode() const { return Op; } /// Get the left-hand side expression of the binary operator. const MCExpr *getLHS() const { return LHS; } /// Get the right-hand side expression of the binary operator. const MCExpr *getRHS() const { return RHS; } /// @} static bool classof(const MCExpr *E) { return E->getKind() == MCExpr::Binary; } }; /// This is an extension point for target-specific MCExpr subclasses to /// implement. /// /// NOTE: All subclasses are required to have trivial destructors because /// MCExprs are bump pointer allocated and not destructed. class MCTargetExpr : public MCExpr { virtual void anchor(); protected: MCTargetExpr() : MCExpr(Target, SMLoc()) {} virtual ~MCTargetExpr() = default; public: virtual void printImpl(raw_ostream &OS, const MCAsmInfo *MAI) const = 0; virtual bool evaluateAsRelocatableImpl(MCValue &Res, const MCAsmLayout *Layout, const MCFixup *Fixup) const = 0; // allow Target Expressions to be checked for equality virtual bool isEqualTo(const MCExpr *x) const { return false; } // This should be set when assigned expressions are not valid ".set" // expressions, e.g. registers, and must be inlined. virtual bool inlineAssignedExpr() const { return false; } virtual void visitUsedExpr(MCStreamer& Streamer) const = 0; virtual MCFragment *findAssociatedFragment() const = 0; virtual void fixELFSymbolsInTLSFixups(MCAssembler &) const = 0; static bool classof(const MCExpr *E) { return E->getKind() == MCExpr::Target; } }; } // end namespace llvm #endif // LLVM_MC_MCEXPR_H diff --git a/llvm/lib/MC/MCExpr.cpp b/llvm/lib/MC/MCExpr.cpp index 2ddbf944f8ed..091880250ec5 100644 --- a/llvm/lib/MC/MCExpr.cpp +++ b/llvm/lib/MC/MCExpr.cpp @@ -1,946 +1,948 @@ //===- MCExpr.cpp - Assembly Level Expression Implementation --------------===// // // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. // See https://llvm.org/LICENSE.txt for license information. // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// #include "llvm/MC/MCExpr.h" #include "llvm/ADT/Statistic.h" #include "llvm/ADT/StringExtras.h" #include "llvm/ADT/StringSwitch.h" #include "llvm/Config/llvm-config.h" #include "llvm/MC/MCAsmBackend.h" #include "llvm/MC/MCAsmInfo.h" #include "llvm/MC/MCAsmLayout.h" #include "llvm/MC/MCAssembler.h" #include "llvm/MC/MCContext.h" #include "llvm/MC/MCObjectWriter.h" #include "llvm/MC/MCSymbol.h" #include "llvm/MC/MCValue.h" #include "llvm/Support/Casting.h" #include "llvm/Support/Compiler.h" #include "llvm/Support/Debug.h" #include "llvm/Support/ErrorHandling.h" #include "llvm/Support/raw_ostream.h" #include #include using namespace llvm; #define DEBUG_TYPE "mcexpr" namespace { namespace stats { STATISTIC(MCExprEvaluate, "Number of MCExpr evaluations"); } // end namespace stats } // end anonymous namespace void MCExpr::print(raw_ostream &OS, const MCAsmInfo *MAI, bool InParens) const { switch (getKind()) { case MCExpr::Target: return cast(this)->printImpl(OS, MAI); case MCExpr::Constant: { auto Value = cast(*this).getValue(); auto PrintInHex = cast(*this).useHexFormat(); auto SizeInBytes = cast(*this).getSizeInBytes(); if (PrintInHex) switch (SizeInBytes) { default: OS << "0x" << Twine::utohexstr(Value); break; case 1: OS << format("0x%02" PRIx64, Value); break; case 2: OS << format("0x%04" PRIx64, Value); break; case 4: OS << format("0x%08" PRIx64, Value); break; case 8: OS << format("0x%016" PRIx64, Value); break; } else OS << Value; return; } case MCExpr::SymbolRef: { const MCSymbolRefExpr &SRE = cast(*this); const MCSymbol &Sym = SRE.getSymbol(); // Parenthesize names that start with $ so that they don't look like // absolute names. bool UseParens = !InParens && !Sym.getName().empty() && Sym.getName()[0] == '$'; if (UseParens) { OS << '('; Sym.print(OS, MAI); OS << ')'; } else Sym.print(OS, MAI); if (SRE.getKind() != MCSymbolRefExpr::VK_None) SRE.printVariantKind(OS); return; } case MCExpr::Unary: { const MCUnaryExpr &UE = cast(*this); switch (UE.getOpcode()) { case MCUnaryExpr::LNot: OS << '!'; break; case MCUnaryExpr::Minus: OS << '-'; break; case MCUnaryExpr::Not: OS << '~'; break; case MCUnaryExpr::Plus: OS << '+'; break; } bool Binary = UE.getSubExpr()->getKind() == MCExpr::Binary; if (Binary) OS << "("; UE.getSubExpr()->print(OS, MAI); if (Binary) OS << ")"; return; } case MCExpr::Binary: { const MCBinaryExpr &BE = cast(*this); // Only print parens around the LHS if it is non-trivial. if (isa(BE.getLHS()) || isa(BE.getLHS())) { BE.getLHS()->print(OS, MAI); } else { OS << '('; BE.getLHS()->print(OS, MAI); OS << ')'; } switch (BE.getOpcode()) { case MCBinaryExpr::Add: // Print "X-42" instead of "X+-42". if (const MCConstantExpr *RHSC = dyn_cast(BE.getRHS())) { if (RHSC->getValue() < 0) { OS << RHSC->getValue(); return; } } OS << '+'; break; case MCBinaryExpr::AShr: OS << ">>"; break; case MCBinaryExpr::And: OS << '&'; break; case MCBinaryExpr::Div: OS << '/'; break; case MCBinaryExpr::EQ: OS << "=="; break; case MCBinaryExpr::GT: OS << '>'; break; case MCBinaryExpr::GTE: OS << ">="; break; case MCBinaryExpr::LAnd: OS << "&&"; break; case MCBinaryExpr::LOr: OS << "||"; break; case MCBinaryExpr::LShr: OS << ">>"; break; case MCBinaryExpr::LT: OS << '<'; break; case MCBinaryExpr::LTE: OS << "<="; break; case MCBinaryExpr::Mod: OS << '%'; break; case MCBinaryExpr::Mul: OS << '*'; break; case MCBinaryExpr::NE: OS << "!="; break; case MCBinaryExpr::Or: OS << '|'; break; case MCBinaryExpr::Shl: OS << "<<"; break; case MCBinaryExpr::Sub: OS << '-'; break; case MCBinaryExpr::Xor: OS << '^'; break; } // Only print parens around the LHS if it is non-trivial. if (isa(BE.getRHS()) || isa(BE.getRHS())) { BE.getRHS()->print(OS, MAI); } else { OS << '('; BE.getRHS()->print(OS, MAI); OS << ')'; } return; } } llvm_unreachable("Invalid expression kind!"); } #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) LLVM_DUMP_METHOD void MCExpr::dump() const { dbgs() << *this; dbgs() << '\n'; } #endif /* *** */ const MCBinaryExpr *MCBinaryExpr::create(Opcode Opc, const MCExpr *LHS, const MCExpr *RHS, MCContext &Ctx, SMLoc Loc) { return new (Ctx) MCBinaryExpr(Opc, LHS, RHS, Loc); } const MCUnaryExpr *MCUnaryExpr::create(Opcode Opc, const MCExpr *Expr, MCContext &Ctx, SMLoc Loc) { return new (Ctx) MCUnaryExpr(Opc, Expr, Loc); } const MCConstantExpr *MCConstantExpr::create(int64_t Value, MCContext &Ctx, bool PrintInHex, unsigned SizeInBytes) { return new (Ctx) MCConstantExpr(Value, PrintInHex, SizeInBytes); } /* *** */ MCSymbolRefExpr::MCSymbolRefExpr(const MCSymbol *Symbol, VariantKind Kind, const MCAsmInfo *MAI, SMLoc Loc) : MCExpr(MCExpr::SymbolRef, Loc), Kind(Kind), UseParensForSymbolVariant(MAI->useParensForSymbolVariant()), HasSubsectionsViaSymbols(MAI->hasSubsectionsViaSymbols()), Symbol(Symbol) { assert(Symbol); } const MCSymbolRefExpr *MCSymbolRefExpr::create(const MCSymbol *Sym, VariantKind Kind, MCContext &Ctx, SMLoc Loc) { return new (Ctx) MCSymbolRefExpr(Sym, Kind, Ctx.getAsmInfo(), Loc); } const MCSymbolRefExpr *MCSymbolRefExpr::create(StringRef Name, VariantKind Kind, MCContext &Ctx) { return create(Ctx.getOrCreateSymbol(Name), Kind, Ctx); } StringRef MCSymbolRefExpr::getVariantKindName(VariantKind Kind) { switch (Kind) { case VK_Invalid: return "<>"; case VK_None: return "<>"; case VK_DTPOFF: return "DTPOFF"; case VK_DTPREL: return "DTPREL"; case VK_GOT: return "GOT"; case VK_GOTOFF: return "GOTOFF"; case VK_GOTREL: return "GOTREL"; case VK_PCREL: return "PCREL"; case VK_GOTPCREL: return "GOTPCREL"; case VK_GOTTPOFF: return "GOTTPOFF"; case VK_INDNTPOFF: return "INDNTPOFF"; case VK_NTPOFF: return "NTPOFF"; case VK_GOTNTPOFF: return "GOTNTPOFF"; case VK_PLT: return "PLT"; case VK_TLSGD: return "TLSGD"; case VK_TLSLD: return "TLSLD"; case VK_TLSLDM: return "TLSLDM"; case VK_TPOFF: return "TPOFF"; case VK_TPREL: return "TPREL"; case VK_TLSCALL: return "tlscall"; case VK_TLSDESC: return "tlsdesc"; case VK_TLVP: return "TLVP"; case VK_TLVPPAGE: return "TLVPPAGE"; case VK_TLVPPAGEOFF: return "TLVPPAGEOFF"; case VK_PAGE: return "PAGE"; case VK_PAGEOFF: return "PAGEOFF"; case VK_GOTPAGE: return "GOTPAGE"; case VK_GOTPAGEOFF: return "GOTPAGEOFF"; case VK_SECREL: return "SECREL32"; case VK_SIZE: return "SIZE"; case VK_WEAKREF: return "WEAKREF"; case VK_X86_ABS8: return "ABS8"; case VK_ARM_NONE: return "none"; case VK_ARM_GOT_PREL: return "GOT_PREL"; case VK_ARM_TARGET1: return "target1"; case VK_ARM_TARGET2: return "target2"; case VK_ARM_PREL31: return "prel31"; case VK_ARM_SBREL: return "sbrel"; case VK_ARM_TLSLDO: return "tlsldo"; case VK_ARM_TLSDESCSEQ: return "tlsdescseq"; case VK_AVR_NONE: return "none"; case VK_AVR_LO8: return "lo8"; case VK_AVR_HI8: return "hi8"; case VK_AVR_HLO8: return "hlo8"; case VK_AVR_DIFF8: return "diff8"; case VK_AVR_DIFF16: return "diff16"; case VK_AVR_DIFF32: return "diff32"; case VK_PPC_LO: return "l"; case VK_PPC_HI: return "h"; case VK_PPC_HA: return "ha"; case VK_PPC_HIGH: return "high"; case VK_PPC_HIGHA: return "higha"; case VK_PPC_HIGHER: return "higher"; case VK_PPC_HIGHERA: return "highera"; case VK_PPC_HIGHEST: return "highest"; case VK_PPC_HIGHESTA: return "highesta"; case VK_PPC_GOT_LO: return "got@l"; case VK_PPC_GOT_HI: return "got@h"; case VK_PPC_GOT_HA: return "got@ha"; case VK_PPC_TOCBASE: return "tocbase"; case VK_PPC_TOC: return "toc"; case VK_PPC_TOC_LO: return "toc@l"; case VK_PPC_TOC_HI: return "toc@h"; case VK_PPC_TOC_HA: return "toc@ha"; case VK_PPC_U: return "u"; case VK_PPC_L: return "l"; case VK_PPC_DTPMOD: return "dtpmod"; case VK_PPC_TPREL_LO: return "tprel@l"; case VK_PPC_TPREL_HI: return "tprel@h"; case VK_PPC_TPREL_HA: return "tprel@ha"; case VK_PPC_TPREL_HIGH: return "tprel@high"; case VK_PPC_TPREL_HIGHA: return "tprel@higha"; case VK_PPC_TPREL_HIGHER: return "tprel@higher"; case VK_PPC_TPREL_HIGHERA: return "tprel@highera"; case VK_PPC_TPREL_HIGHEST: return "tprel@highest"; case VK_PPC_TPREL_HIGHESTA: return "tprel@highesta"; case VK_PPC_DTPREL_LO: return "dtprel@l"; case VK_PPC_DTPREL_HI: return "dtprel@h"; case VK_PPC_DTPREL_HA: return "dtprel@ha"; case VK_PPC_DTPREL_HIGH: return "dtprel@high"; case VK_PPC_DTPREL_HIGHA: return "dtprel@higha"; case VK_PPC_DTPREL_HIGHER: return "dtprel@higher"; case VK_PPC_DTPREL_HIGHERA: return "dtprel@highera"; case VK_PPC_DTPREL_HIGHEST: return "dtprel@highest"; case VK_PPC_DTPREL_HIGHESTA: return "dtprel@highesta"; case VK_PPC_GOT_TPREL: return "got@tprel"; case VK_PPC_GOT_TPREL_LO: return "got@tprel@l"; case VK_PPC_GOT_TPREL_HI: return "got@tprel@h"; case VK_PPC_GOT_TPREL_HA: return "got@tprel@ha"; case VK_PPC_GOT_DTPREL: return "got@dtprel"; case VK_PPC_GOT_DTPREL_LO: return "got@dtprel@l"; case VK_PPC_GOT_DTPREL_HI: return "got@dtprel@h"; case VK_PPC_GOT_DTPREL_HA: return "got@dtprel@ha"; case VK_PPC_TLS: return "tls"; case VK_PPC_GOT_TLSGD: return "got@tlsgd"; case VK_PPC_GOT_TLSGD_LO: return "got@tlsgd@l"; case VK_PPC_GOT_TLSGD_HI: return "got@tlsgd@h"; case VK_PPC_GOT_TLSGD_HA: return "got@tlsgd@ha"; case VK_PPC_TLSGD: return "tlsgd"; case VK_PPC_GOT_TLSLD: return "got@tlsld"; case VK_PPC_GOT_TLSLD_LO: return "got@tlsld@l"; case VK_PPC_GOT_TLSLD_HI: return "got@tlsld@h"; case VK_PPC_GOT_TLSLD_HA: return "got@tlsld@ha"; case VK_PPC_TLSLD: return "tlsld"; case VK_PPC_LOCAL: return "local"; + case VK_PPC_NOTOC: return "notoc"; case VK_COFF_IMGREL32: return "IMGREL"; case VK_Hexagon_LO16: return "LO16"; case VK_Hexagon_HI16: return "HI16"; case VK_Hexagon_GPREL: return "GPREL"; case VK_Hexagon_GD_GOT: return "GDGOT"; case VK_Hexagon_LD_GOT: return "LDGOT"; case VK_Hexagon_GD_PLT: return "GDPLT"; case VK_Hexagon_LD_PLT: return "LDPLT"; case VK_Hexagon_IE: return "IE"; case VK_Hexagon_IE_GOT: return "IEGOT"; case VK_WASM_TYPEINDEX: return "TYPEINDEX"; case VK_WASM_MBREL: return "MBREL"; case VK_WASM_TBREL: return "TBREL"; case VK_AMDGPU_GOTPCREL32_LO: return "gotpcrel32@lo"; case VK_AMDGPU_GOTPCREL32_HI: return "gotpcrel32@hi"; case VK_AMDGPU_REL32_LO: return "rel32@lo"; case VK_AMDGPU_REL32_HI: return "rel32@hi"; case VK_AMDGPU_REL64: return "rel64"; case VK_AMDGPU_ABS32_LO: return "abs32@lo"; case VK_AMDGPU_ABS32_HI: return "abs32@hi"; } llvm_unreachable("Invalid variant kind"); } MCSymbolRefExpr::VariantKind MCSymbolRefExpr::getVariantKindForName(StringRef Name) { return StringSwitch(Name.lower()) .Case("dtprel", VK_DTPREL) .Case("dtpoff", VK_DTPOFF) .Case("got", VK_GOT) .Case("gotoff", VK_GOTOFF) .Case("gotrel", VK_GOTREL) .Case("pcrel", VK_PCREL) .Case("gotpcrel", VK_GOTPCREL) .Case("gottpoff", VK_GOTTPOFF) .Case("indntpoff", VK_INDNTPOFF) .Case("ntpoff", VK_NTPOFF) .Case("gotntpoff", VK_GOTNTPOFF) .Case("plt", VK_PLT) .Case("tlscall", VK_TLSCALL) .Case("tlsdesc", VK_TLSDESC) .Case("tlsgd", VK_TLSGD) .Case("tlsld", VK_TLSLD) .Case("tlsldm", VK_TLSLDM) .Case("tpoff", VK_TPOFF) .Case("tprel", VK_TPREL) .Case("tlvp", VK_TLVP) .Case("tlvppage", VK_TLVPPAGE) .Case("tlvppageoff", VK_TLVPPAGEOFF) .Case("page", VK_PAGE) .Case("pageoff", VK_PAGEOFF) .Case("gotpage", VK_GOTPAGE) .Case("gotpageoff", VK_GOTPAGEOFF) .Case("imgrel", VK_COFF_IMGREL32) .Case("secrel32", VK_SECREL) .Case("size", VK_SIZE) .Case("abs8", VK_X86_ABS8) .Case("l", VK_PPC_LO) .Case("h", VK_PPC_HI) .Case("ha", VK_PPC_HA) .Case("high", VK_PPC_HIGH) .Case("higha", VK_PPC_HIGHA) .Case("higher", VK_PPC_HIGHER) .Case("highera", VK_PPC_HIGHERA) .Case("highest", VK_PPC_HIGHEST) .Case("highesta", VK_PPC_HIGHESTA) .Case("got@l", VK_PPC_GOT_LO) .Case("got@h", VK_PPC_GOT_HI) .Case("got@ha", VK_PPC_GOT_HA) .Case("local", VK_PPC_LOCAL) .Case("tocbase", VK_PPC_TOCBASE) .Case("toc", VK_PPC_TOC) .Case("toc@l", VK_PPC_TOC_LO) .Case("toc@h", VK_PPC_TOC_HI) .Case("toc@ha", VK_PPC_TOC_HA) .Case("u", VK_PPC_U) .Case("l", VK_PPC_L) .Case("tls", VK_PPC_TLS) .Case("dtpmod", VK_PPC_DTPMOD) .Case("tprel@l", VK_PPC_TPREL_LO) .Case("tprel@h", VK_PPC_TPREL_HI) .Case("tprel@ha", VK_PPC_TPREL_HA) .Case("tprel@high", VK_PPC_TPREL_HIGH) .Case("tprel@higha", VK_PPC_TPREL_HIGHA) .Case("tprel@higher", VK_PPC_TPREL_HIGHER) .Case("tprel@highera", VK_PPC_TPREL_HIGHERA) .Case("tprel@highest", VK_PPC_TPREL_HIGHEST) .Case("tprel@highesta", VK_PPC_TPREL_HIGHESTA) .Case("dtprel@l", VK_PPC_DTPREL_LO) .Case("dtprel@h", VK_PPC_DTPREL_HI) .Case("dtprel@ha", VK_PPC_DTPREL_HA) .Case("dtprel@high", VK_PPC_DTPREL_HIGH) .Case("dtprel@higha", VK_PPC_DTPREL_HIGHA) .Case("dtprel@higher", VK_PPC_DTPREL_HIGHER) .Case("dtprel@highera", VK_PPC_DTPREL_HIGHERA) .Case("dtprel@highest", VK_PPC_DTPREL_HIGHEST) .Case("dtprel@highesta", VK_PPC_DTPREL_HIGHESTA) .Case("got@tprel", VK_PPC_GOT_TPREL) .Case("got@tprel@l", VK_PPC_GOT_TPREL_LO) .Case("got@tprel@h", VK_PPC_GOT_TPREL_HI) .Case("got@tprel@ha", VK_PPC_GOT_TPREL_HA) .Case("got@dtprel", VK_PPC_GOT_DTPREL) .Case("got@dtprel@l", VK_PPC_GOT_DTPREL_LO) .Case("got@dtprel@h", VK_PPC_GOT_DTPREL_HI) .Case("got@dtprel@ha", VK_PPC_GOT_DTPREL_HA) .Case("got@tlsgd", VK_PPC_GOT_TLSGD) .Case("got@tlsgd@l", VK_PPC_GOT_TLSGD_LO) .Case("got@tlsgd@h", VK_PPC_GOT_TLSGD_HI) .Case("got@tlsgd@ha", VK_PPC_GOT_TLSGD_HA) .Case("got@tlsld", VK_PPC_GOT_TLSLD) .Case("got@tlsld@l", VK_PPC_GOT_TLSLD_LO) .Case("got@tlsld@h", VK_PPC_GOT_TLSLD_HI) .Case("got@tlsld@ha", VK_PPC_GOT_TLSLD_HA) + .Case("notoc", VK_PPC_NOTOC) .Case("gdgot", VK_Hexagon_GD_GOT) .Case("gdplt", VK_Hexagon_GD_PLT) .Case("iegot", VK_Hexagon_IE_GOT) .Case("ie", VK_Hexagon_IE) .Case("ldgot", VK_Hexagon_LD_GOT) .Case("ldplt", VK_Hexagon_LD_PLT) .Case("none", VK_ARM_NONE) .Case("got_prel", VK_ARM_GOT_PREL) .Case("target1", VK_ARM_TARGET1) .Case("target2", VK_ARM_TARGET2) .Case("prel31", VK_ARM_PREL31) .Case("sbrel", VK_ARM_SBREL) .Case("tlsldo", VK_ARM_TLSLDO) .Case("lo8", VK_AVR_LO8) .Case("hi8", VK_AVR_HI8) .Case("hlo8", VK_AVR_HLO8) .Case("typeindex", VK_WASM_TYPEINDEX) .Case("tbrel", VK_WASM_TBREL) .Case("mbrel", VK_WASM_MBREL) .Case("gotpcrel32@lo", VK_AMDGPU_GOTPCREL32_LO) .Case("gotpcrel32@hi", VK_AMDGPU_GOTPCREL32_HI) .Case("rel32@lo", VK_AMDGPU_REL32_LO) .Case("rel32@hi", VK_AMDGPU_REL32_HI) .Case("rel64", VK_AMDGPU_REL64) .Case("abs32@lo", VK_AMDGPU_ABS32_LO) .Case("abs32@hi", VK_AMDGPU_ABS32_HI) .Default(VK_Invalid); } void MCSymbolRefExpr::printVariantKind(raw_ostream &OS) const { if (UseParensForSymbolVariant) OS << '(' << MCSymbolRefExpr::getVariantKindName(getKind()) << ')'; else OS << '@' << MCSymbolRefExpr::getVariantKindName(getKind()); } /* *** */ void MCTargetExpr::anchor() {} /* *** */ bool MCExpr::evaluateAsAbsolute(int64_t &Res) const { return evaluateAsAbsolute(Res, nullptr, nullptr, nullptr, false); } bool MCExpr::evaluateAsAbsolute(int64_t &Res, const MCAsmLayout &Layout) const { return evaluateAsAbsolute(Res, &Layout.getAssembler(), &Layout, nullptr, false); } bool MCExpr::evaluateAsAbsolute(int64_t &Res, const MCAsmLayout &Layout, const SectionAddrMap &Addrs) const { // Setting InSet causes us to absolutize differences across sections and that // is what the MachO writer uses Addrs for. return evaluateAsAbsolute(Res, &Layout.getAssembler(), &Layout, &Addrs, true); } bool MCExpr::evaluateAsAbsolute(int64_t &Res, const MCAssembler &Asm) const { return evaluateAsAbsolute(Res, &Asm, nullptr, nullptr, false); } bool MCExpr::evaluateAsAbsolute(int64_t &Res, const MCAssembler *Asm) const { return evaluateAsAbsolute(Res, Asm, nullptr, nullptr, false); } bool MCExpr::evaluateKnownAbsolute(int64_t &Res, const MCAsmLayout &Layout) const { return evaluateAsAbsolute(Res, &Layout.getAssembler(), &Layout, nullptr, true); } bool MCExpr::evaluateAsAbsolute(int64_t &Res, const MCAssembler *Asm, const MCAsmLayout *Layout, const SectionAddrMap *Addrs, bool InSet) const { MCValue Value; // Fast path constants. if (const MCConstantExpr *CE = dyn_cast(this)) { Res = CE->getValue(); return true; } bool IsRelocatable = evaluateAsRelocatableImpl(Value, Asm, Layout, nullptr, Addrs, InSet); // Record the current value. Res = Value.getConstant(); return IsRelocatable && Value.isAbsolute(); } /// Helper method for \see EvaluateSymbolAdd(). static void AttemptToFoldSymbolOffsetDifference( const MCAssembler *Asm, const MCAsmLayout *Layout, const SectionAddrMap *Addrs, bool InSet, const MCSymbolRefExpr *&A, const MCSymbolRefExpr *&B, int64_t &Addend) { if (!A || !B) return; const MCSymbol &SA = A->getSymbol(); const MCSymbol &SB = B->getSymbol(); if (SA.isUndefined() || SB.isUndefined()) return; if (!Asm->getWriter().isSymbolRefDifferenceFullyResolved(*Asm, A, B, InSet)) return; if (SA.getFragment() == SB.getFragment() && !SA.isVariable() && !SA.isUnset() && !SB.isVariable() && !SB.isUnset()) { Addend += (SA.getOffset() - SB.getOffset()); // Pointers to Thumb symbols need to have their low-bit set to allow // for interworking. if (Asm->isThumbFunc(&SA)) Addend |= 1; // If symbol is labeled as micromips, we set low-bit to ensure // correct offset in .gcc_except_table if (Asm->getBackend().isMicroMips(&SA)) Addend |= 1; // Clear the symbol expr pointers to indicate we have folded these // operands. A = B = nullptr; return; } if (!Layout) return; const MCSection &SecA = *SA.getFragment()->getParent(); const MCSection &SecB = *SB.getFragment()->getParent(); if ((&SecA != &SecB) && !Addrs) return; // Eagerly evaluate. Addend += Layout->getSymbolOffset(A->getSymbol()) - Layout->getSymbolOffset(B->getSymbol()); if (Addrs && (&SecA != &SecB)) Addend += (Addrs->lookup(&SecA) - Addrs->lookup(&SecB)); // Pointers to Thumb symbols need to have their low-bit set to allow // for interworking. if (Asm->isThumbFunc(&SA)) Addend |= 1; // If symbol is labeled as micromips, we set low-bit to ensure // correct offset in .gcc_except_table if (Asm->getBackend().isMicroMips(&SA)) Addend |= 1; // Clear the symbol expr pointers to indicate we have folded these // operands. A = B = nullptr; } static bool canFold(const MCAssembler *Asm, const MCSymbolRefExpr *A, const MCSymbolRefExpr *B, bool InSet) { if (InSet) return true; if (!Asm->getBackend().requiresDiffExpressionRelocations()) return true; const MCSymbol &CheckSym = A ? A->getSymbol() : B->getSymbol(); if (!CheckSym.isInSection()) return true; if (!CheckSym.getSection().hasInstructions()) return true; return false; } /// Evaluate the result of an add between (conceptually) two MCValues. /// /// This routine conceptually attempts to construct an MCValue: /// Result = (Result_A - Result_B + Result_Cst) /// from two MCValue's LHS and RHS where /// Result = LHS + RHS /// and /// Result = (LHS_A - LHS_B + LHS_Cst) + (RHS_A - RHS_B + RHS_Cst). /// /// This routine attempts to aggressively fold the operands such that the result /// is representable in an MCValue, but may not always succeed. /// /// \returns True on success, false if the result is not representable in an /// MCValue. /// NOTE: It is really important to have both the Asm and Layout arguments. /// They might look redundant, but this function can be used before layout /// is done (see the object streamer for example) and having the Asm argument /// lets us avoid relaxations early. static bool EvaluateSymbolicAdd(const MCAssembler *Asm, const MCAsmLayout *Layout, const SectionAddrMap *Addrs, bool InSet, const MCValue &LHS, const MCSymbolRefExpr *RHS_A, const MCSymbolRefExpr *RHS_B, int64_t RHS_Cst, MCValue &Res) { // FIXME: This routine (and other evaluation parts) are *incredibly* sloppy // about dealing with modifiers. This will ultimately bite us, one day. const MCSymbolRefExpr *LHS_A = LHS.getSymA(); const MCSymbolRefExpr *LHS_B = LHS.getSymB(); int64_t LHS_Cst = LHS.getConstant(); // Fold the result constant immediately. int64_t Result_Cst = LHS_Cst + RHS_Cst; assert((!Layout || Asm) && "Must have an assembler object if layout is given!"); // If we have a layout, we can fold resolved differences. Do not do this if // the backend requires this to be emitted as individual relocations, unless // the InSet flag is set to get the current difference anyway (used for // example to calculate symbol sizes). if (Asm && canFold(Asm, LHS_A, LHS_B, InSet)) { // First, fold out any differences which are fully resolved. By // reassociating terms in // Result = (LHS_A - LHS_B + LHS_Cst) + (RHS_A - RHS_B + RHS_Cst). // we have the four possible differences: // (LHS_A - LHS_B), // (LHS_A - RHS_B), // (RHS_A - LHS_B), // (RHS_A - RHS_B). // Since we are attempting to be as aggressive as possible about folding, we // attempt to evaluate each possible alternative. AttemptToFoldSymbolOffsetDifference(Asm, Layout, Addrs, InSet, LHS_A, LHS_B, Result_Cst); AttemptToFoldSymbolOffsetDifference(Asm, Layout, Addrs, InSet, LHS_A, RHS_B, Result_Cst); AttemptToFoldSymbolOffsetDifference(Asm, Layout, Addrs, InSet, RHS_A, LHS_B, Result_Cst); AttemptToFoldSymbolOffsetDifference(Asm, Layout, Addrs, InSet, RHS_A, RHS_B, Result_Cst); } // We can't represent the addition or subtraction of two symbols. if ((LHS_A && RHS_A) || (LHS_B && RHS_B)) return false; // At this point, we have at most one additive symbol and one subtractive // symbol -- find them. const MCSymbolRefExpr *A = LHS_A ? LHS_A : RHS_A; const MCSymbolRefExpr *B = LHS_B ? LHS_B : RHS_B; Res = MCValue::get(A, B, Result_Cst); return true; } bool MCExpr::evaluateAsRelocatable(MCValue &Res, const MCAsmLayout *Layout, const MCFixup *Fixup) const { MCAssembler *Assembler = Layout ? &Layout->getAssembler() : nullptr; return evaluateAsRelocatableImpl(Res, Assembler, Layout, Fixup, nullptr, false); } bool MCExpr::evaluateAsValue(MCValue &Res, const MCAsmLayout &Layout) const { MCAssembler *Assembler = &Layout.getAssembler(); return evaluateAsRelocatableImpl(Res, Assembler, &Layout, nullptr, nullptr, true); } static bool canExpand(const MCSymbol &Sym, bool InSet) { const MCExpr *Expr = Sym.getVariableValue(); const auto *Inner = dyn_cast(Expr); if (Inner) { if (Inner->getKind() == MCSymbolRefExpr::VK_WEAKREF) return false; } if (InSet) return true; return !Sym.isInSection(); } bool MCExpr::evaluateAsRelocatableImpl(MCValue &Res, const MCAssembler *Asm, const MCAsmLayout *Layout, const MCFixup *Fixup, const SectionAddrMap *Addrs, bool InSet) const { ++stats::MCExprEvaluate; switch (getKind()) { case Target: return cast(this)->evaluateAsRelocatableImpl(Res, Layout, Fixup); case Constant: Res = MCValue::get(cast(this)->getValue()); return true; case SymbolRef: { const MCSymbolRefExpr *SRE = cast(this); const MCSymbol &Sym = SRE->getSymbol(); // Evaluate recursively if this is a variable. if (Sym.isVariable() && SRE->getKind() == MCSymbolRefExpr::VK_None && canExpand(Sym, InSet)) { bool IsMachO = SRE->hasSubsectionsViaSymbols(); if (Sym.getVariableValue()->evaluateAsRelocatableImpl( Res, Asm, Layout, Fixup, Addrs, InSet || IsMachO)) { if (!IsMachO) return true; const MCSymbolRefExpr *A = Res.getSymA(); const MCSymbolRefExpr *B = Res.getSymB(); // FIXME: This is small hack. Given // a = b + 4 // .long a // the OS X assembler will completely drop the 4. We should probably // include it in the relocation or produce an error if that is not // possible. // Allow constant expressions. if (!A && !B) return true; // Allows aliases with zero offset. if (Res.getConstant() == 0 && (!A || !B)) return true; } } Res = MCValue::get(SRE, nullptr, 0); return true; } case Unary: { const MCUnaryExpr *AUE = cast(this); MCValue Value; if (!AUE->getSubExpr()->evaluateAsRelocatableImpl(Value, Asm, Layout, Fixup, Addrs, InSet)) return false; switch (AUE->getOpcode()) { case MCUnaryExpr::LNot: if (!Value.isAbsolute()) return false; Res = MCValue::get(!Value.getConstant()); break; case MCUnaryExpr::Minus: /// -(a - b + const) ==> (b - a - const) if (Value.getSymA() && !Value.getSymB()) return false; // The cast avoids undefined behavior if the constant is INT64_MIN. Res = MCValue::get(Value.getSymB(), Value.getSymA(), -(uint64_t)Value.getConstant()); break; case MCUnaryExpr::Not: if (!Value.isAbsolute()) return false; Res = MCValue::get(~Value.getConstant()); break; case MCUnaryExpr::Plus: Res = Value; break; } return true; } case Binary: { const MCBinaryExpr *ABE = cast(this); MCValue LHSValue, RHSValue; if (!ABE->getLHS()->evaluateAsRelocatableImpl(LHSValue, Asm, Layout, Fixup, Addrs, InSet) || !ABE->getRHS()->evaluateAsRelocatableImpl(RHSValue, Asm, Layout, Fixup, Addrs, InSet)) { // Check if both are Target Expressions, see if we can compare them. if (const MCTargetExpr *L = dyn_cast(ABE->getLHS())) if (const MCTargetExpr *R = cast(ABE->getRHS())) { switch (ABE->getOpcode()) { case MCBinaryExpr::EQ: Res = MCValue::get((L->isEqualTo(R)) ? -1 : 0); return true; case MCBinaryExpr::NE: Res = MCValue::get((R->isEqualTo(R)) ? 0 : -1); return true; default: break; } } return false; } // We only support a few operations on non-constant expressions, handle // those first. if (!LHSValue.isAbsolute() || !RHSValue.isAbsolute()) { switch (ABE->getOpcode()) { default: return false; case MCBinaryExpr::Sub: // Negate RHS and add. // The cast avoids undefined behavior if the constant is INT64_MIN. return EvaluateSymbolicAdd(Asm, Layout, Addrs, InSet, LHSValue, RHSValue.getSymB(), RHSValue.getSymA(), -(uint64_t)RHSValue.getConstant(), Res); case MCBinaryExpr::Add: return EvaluateSymbolicAdd(Asm, Layout, Addrs, InSet, LHSValue, RHSValue.getSymA(), RHSValue.getSymB(), RHSValue.getConstant(), Res); } } // FIXME: We need target hooks for the evaluation. It may be limited in // width, and gas defines the result of comparisons differently from // Apple as. int64_t LHS = LHSValue.getConstant(), RHS = RHSValue.getConstant(); int64_t Result = 0; auto Op = ABE->getOpcode(); switch (Op) { case MCBinaryExpr::AShr: Result = LHS >> RHS; break; case MCBinaryExpr::Add: Result = LHS + RHS; break; case MCBinaryExpr::And: Result = LHS & RHS; break; case MCBinaryExpr::Div: case MCBinaryExpr::Mod: // Handle division by zero. gas just emits a warning and keeps going, // we try to be stricter. // FIXME: Currently the caller of this function has no way to understand // we're bailing out because of 'division by zero'. Therefore, it will // emit a 'expected relocatable expression' error. It would be nice to // change this code to emit a better diagnostic. if (RHS == 0) return false; if (ABE->getOpcode() == MCBinaryExpr::Div) Result = LHS / RHS; else Result = LHS % RHS; break; case MCBinaryExpr::EQ: Result = LHS == RHS; break; case MCBinaryExpr::GT: Result = LHS > RHS; break; case MCBinaryExpr::GTE: Result = LHS >= RHS; break; case MCBinaryExpr::LAnd: Result = LHS && RHS; break; case MCBinaryExpr::LOr: Result = LHS || RHS; break; case MCBinaryExpr::LShr: Result = uint64_t(LHS) >> uint64_t(RHS); break; case MCBinaryExpr::LT: Result = LHS < RHS; break; case MCBinaryExpr::LTE: Result = LHS <= RHS; break; case MCBinaryExpr::Mul: Result = LHS * RHS; break; case MCBinaryExpr::NE: Result = LHS != RHS; break; case MCBinaryExpr::Or: Result = LHS | RHS; break; case MCBinaryExpr::Shl: Result = uint64_t(LHS) << uint64_t(RHS); break; case MCBinaryExpr::Sub: Result = LHS - RHS; break; case MCBinaryExpr::Xor: Result = LHS ^ RHS; break; } switch (Op) { default: Res = MCValue::get(Result); break; case MCBinaryExpr::EQ: case MCBinaryExpr::GT: case MCBinaryExpr::GTE: case MCBinaryExpr::LT: case MCBinaryExpr::LTE: case MCBinaryExpr::NE: // A comparison operator returns a -1 if true and 0 if false. Res = MCValue::get(Result ? -1 : 0); break; } return true; } } llvm_unreachable("Invalid assembly expression kind!"); } MCFragment *MCExpr::findAssociatedFragment() const { switch (getKind()) { case Target: // We never look through target specific expressions. return cast(this)->findAssociatedFragment(); case Constant: return MCSymbol::AbsolutePseudoFragment; case SymbolRef: { const MCSymbolRefExpr *SRE = cast(this); const MCSymbol &Sym = SRE->getSymbol(); return Sym.getFragment(); } case Unary: return cast(this)->getSubExpr()->findAssociatedFragment(); case Binary: { const MCBinaryExpr *BE = cast(this); MCFragment *LHS_F = BE->getLHS()->findAssociatedFragment(); MCFragment *RHS_F = BE->getRHS()->findAssociatedFragment(); // If either is absolute, return the other. if (LHS_F == MCSymbol::AbsolutePseudoFragment) return RHS_F; if (RHS_F == MCSymbol::AbsolutePseudoFragment) return LHS_F; // Not always correct, but probably the best we can do without more context. if (BE->getOpcode() == MCBinaryExpr::Sub) return MCSymbol::AbsolutePseudoFragment; // Otherwise, return the first non-null fragment. return LHS_F ? LHS_F : RHS_F; } } llvm_unreachable("Invalid assembly expression kind!"); } diff --git a/llvm/lib/Target/PowerPC/MCTargetDesc/PPCAsmBackend.cpp b/llvm/lib/Target/PowerPC/MCTargetDesc/PPCAsmBackend.cpp index 7320c1e6f51d..cb7d4293e0f1 100644 --- a/llvm/lib/Target/PowerPC/MCTargetDesc/PPCAsmBackend.cpp +++ b/llvm/lib/Target/PowerPC/MCTargetDesc/PPCAsmBackend.cpp @@ -1,265 +1,270 @@ //===-- PPCAsmBackend.cpp - PPC Assembler Backend -------------------------===// // // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. // See https://llvm.org/LICENSE.txt for license information. // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// #include "MCTargetDesc/PPCFixupKinds.h" #include "MCTargetDesc/PPCMCTargetDesc.h" #include "llvm/BinaryFormat/ELF.h" #include "llvm/BinaryFormat/MachO.h" #include "llvm/MC/MCAsmBackend.h" #include "llvm/MC/MCAssembler.h" #include "llvm/MC/MCELFObjectWriter.h" #include "llvm/MC/MCFixupKindInfo.h" #include "llvm/MC/MCMachObjectWriter.h" #include "llvm/MC/MCObjectWriter.h" #include "llvm/MC/MCSectionMachO.h" #include "llvm/MC/MCSubtargetInfo.h" #include "llvm/MC/MCSymbolELF.h" #include "llvm/MC/MCValue.h" #include "llvm/Support/ErrorHandling.h" #include "llvm/Support/TargetRegistry.h" using namespace llvm; static uint64_t adjustFixupValue(unsigned Kind, uint64_t Value) { switch (Kind) { default: llvm_unreachable("Unknown fixup kind!"); case FK_Data_1: case FK_Data_2: case FK_Data_4: case FK_Data_8: case PPC::fixup_ppc_nofixup: return Value; case PPC::fixup_ppc_brcond14: case PPC::fixup_ppc_brcond14abs: return Value & 0xfffc; case PPC::fixup_ppc_br24: case PPC::fixup_ppc_br24abs: + case PPC::fixup_ppc_br24_notoc: return Value & 0x3fffffc; case PPC::fixup_ppc_half16: return Value & 0xffff; case PPC::fixup_ppc_half16ds: return Value & 0xfffc; } } static unsigned getFixupKindNumBytes(unsigned Kind) { switch (Kind) { default: llvm_unreachable("Unknown fixup kind!"); case FK_Data_1: return 1; case FK_Data_2: case PPC::fixup_ppc_half16: case PPC::fixup_ppc_half16ds: return 2; case FK_Data_4: case PPC::fixup_ppc_brcond14: case PPC::fixup_ppc_brcond14abs: case PPC::fixup_ppc_br24: case PPC::fixup_ppc_br24abs: + case PPC::fixup_ppc_br24_notoc: return 4; case FK_Data_8: return 8; case PPC::fixup_ppc_nofixup: return 0; } } namespace { class PPCAsmBackend : public MCAsmBackend { protected: Triple TT; public: PPCAsmBackend(const Target &T, const Triple &TT) : MCAsmBackend(TT.isLittleEndian() ? support::little : support::big), TT(TT) {} unsigned getNumFixupKinds() const override { return PPC::NumTargetFixupKinds; } const MCFixupKindInfo &getFixupKindInfo(MCFixupKind Kind) const override { const static MCFixupKindInfo InfosBE[PPC::NumTargetFixupKinds] = { // name offset bits flags { "fixup_ppc_br24", 6, 24, MCFixupKindInfo::FKF_IsPCRel }, + { "fixup_ppc_br24_notoc", 6, 24, MCFixupKindInfo::FKF_IsPCRel }, { "fixup_ppc_brcond14", 16, 14, MCFixupKindInfo::FKF_IsPCRel }, { "fixup_ppc_br24abs", 6, 24, 0 }, { "fixup_ppc_brcond14abs", 16, 14, 0 }, { "fixup_ppc_half16", 0, 16, 0 }, { "fixup_ppc_half16ds", 0, 14, 0 }, { "fixup_ppc_nofixup", 0, 0, 0 } }; const static MCFixupKindInfo InfosLE[PPC::NumTargetFixupKinds] = { // name offset bits flags { "fixup_ppc_br24", 2, 24, MCFixupKindInfo::FKF_IsPCRel }, + { "fixup_ppc_br24_notoc", 2, 24, MCFixupKindInfo::FKF_IsPCRel }, { "fixup_ppc_brcond14", 2, 14, MCFixupKindInfo::FKF_IsPCRel }, { "fixup_ppc_br24abs", 2, 24, 0 }, { "fixup_ppc_brcond14abs", 2, 14, 0 }, { "fixup_ppc_half16", 0, 16, 0 }, { "fixup_ppc_half16ds", 2, 14, 0 }, { "fixup_ppc_nofixup", 0, 0, 0 } }; // Fixup kinds from .reloc directive are like R_PPC_NONE/R_PPC64_NONE. They // do not require any extra processing. if (Kind >= FirstLiteralRelocationKind) return MCAsmBackend::getFixupKindInfo(FK_NONE); if (Kind < FirstTargetFixupKind) return MCAsmBackend::getFixupKindInfo(Kind); assert(unsigned(Kind - FirstTargetFixupKind) < getNumFixupKinds() && "Invalid kind!"); return (Endian == support::little ? InfosLE : InfosBE)[Kind - FirstTargetFixupKind]; } void applyFixup(const MCAssembler &Asm, const MCFixup &Fixup, const MCValue &Target, MutableArrayRef Data, uint64_t Value, bool IsResolved, const MCSubtargetInfo *STI) const override { MCFixupKind Kind = Fixup.getKind(); if (Kind >= FirstLiteralRelocationKind) return; Value = adjustFixupValue(Kind, Value); if (!Value) return; // Doesn't change encoding. unsigned Offset = Fixup.getOffset(); unsigned NumBytes = getFixupKindNumBytes(Kind); // For each byte of the fragment that the fixup touches, mask in the bits // from the fixup value. The Value has been "split up" into the appropriate // bitfields above. for (unsigned i = 0; i != NumBytes; ++i) { unsigned Idx = Endian == support::little ? i : (NumBytes - 1 - i); Data[Offset + i] |= uint8_t((Value >> (Idx * 8)) & 0xff); } } bool shouldForceRelocation(const MCAssembler &Asm, const MCFixup &Fixup, const MCValue &Target) override { MCFixupKind Kind = Fixup.getKind(); switch ((unsigned)Kind) { default: return Kind >= FirstLiteralRelocationKind; case PPC::fixup_ppc_br24: case PPC::fixup_ppc_br24abs: + case PPC::fixup_ppc_br24_notoc: // If the target symbol has a local entry point we must not attempt // to resolve the fixup directly. Emit a relocation and leave // resolution of the final target address to the linker. if (const MCSymbolRefExpr *A = Target.getSymA()) { if (const auto *S = dyn_cast(&A->getSymbol())) { // The "other" values are stored in the last 6 bits of the second // byte. The traditional defines for STO values assume the full byte // and thus the shift to pack it. unsigned Other = S->getOther() << 2; if ((Other & ELF::STO_PPC64_LOCAL_MASK) != 0) return true; } } return false; } } bool mayNeedRelaxation(const MCInst &Inst, const MCSubtargetInfo &STI) const override { // FIXME. return false; } bool fixupNeedsRelaxation(const MCFixup &Fixup, uint64_t Value, const MCRelaxableFragment *DF, const MCAsmLayout &Layout) const override { // FIXME. llvm_unreachable("relaxInstruction() unimplemented"); } void relaxInstruction(const MCInst &Inst, const MCSubtargetInfo &STI, MCInst &Res) const override { // FIXME. llvm_unreachable("relaxInstruction() unimplemented"); } bool writeNopData(raw_ostream &OS, uint64_t Count) const override { uint64_t NumNops = Count / 4; for (uint64_t i = 0; i != NumNops; ++i) support::endian::write(OS, 0x60000000, Endian); OS.write_zeros(Count % 4); return true; } }; } // end anonymous namespace // FIXME: This should be in a separate file. namespace { class ELFPPCAsmBackend : public PPCAsmBackend { public: ELFPPCAsmBackend(const Target &T, const Triple &TT) : PPCAsmBackend(T, TT) {} std::unique_ptr createObjectTargetWriter() const override { uint8_t OSABI = MCELFObjectTargetWriter::getOSABI(TT.getOS()); bool Is64 = TT.isPPC64(); return createPPCELFObjectWriter(Is64, OSABI); } Optional getFixupKind(StringRef Name) const override; }; class XCOFFPPCAsmBackend : public PPCAsmBackend { public: XCOFFPPCAsmBackend(const Target &T, const Triple &TT) : PPCAsmBackend(T, TT) {} std::unique_ptr createObjectTargetWriter() const override { return createPPCXCOFFObjectWriter(TT.isArch64Bit()); } }; } // end anonymous namespace Optional ELFPPCAsmBackend::getFixupKind(StringRef Name) const { if (TT.isOSBinFormatELF()) { unsigned Type; if (TT.isPPC64()) { Type = llvm::StringSwitch(Name) #define ELF_RELOC(X, Y) .Case(#X, Y) #include "llvm/BinaryFormat/ELFRelocs/PowerPC64.def" #undef ELF_RELOC .Default(-1u); } else { Type = llvm::StringSwitch(Name) #define ELF_RELOC(X, Y) .Case(#X, Y) #include "llvm/BinaryFormat/ELFRelocs/PowerPC.def" #undef ELF_RELOC .Default(-1u); } if (Type != -1u) return static_cast(FirstLiteralRelocationKind + Type); } return None; } MCAsmBackend *llvm::createPPCAsmBackend(const Target &T, const MCSubtargetInfo &STI, const MCRegisterInfo &MRI, const MCTargetOptions &Options) { const Triple &TT = STI.getTargetTriple(); if (TT.isOSBinFormatXCOFF()) return new XCOFFPPCAsmBackend(T, TT); return new ELFPPCAsmBackend(T, TT); } diff --git a/llvm/lib/Target/PowerPC/MCTargetDesc/PPCELFObjectWriter.cpp b/llvm/lib/Target/PowerPC/MCTargetDesc/PPCELFObjectWriter.cpp index c0379620a197..99a8207832b0 100644 --- a/llvm/lib/Target/PowerPC/MCTargetDesc/PPCELFObjectWriter.cpp +++ b/llvm/lib/Target/PowerPC/MCTargetDesc/PPCELFObjectWriter.cpp @@ -1,447 +1,452 @@ //===-- PPCELFObjectWriter.cpp - PPC ELF Writer ---------------------------===// // // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. // See https://llvm.org/LICENSE.txt for license information. // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// #include "MCTargetDesc/PPCFixupKinds.h" #include "MCTargetDesc/PPCMCExpr.h" #include "MCTargetDesc/PPCMCTargetDesc.h" #include "llvm/ADT/STLExtras.h" #include "llvm/MC/MCELFObjectWriter.h" #include "llvm/MC/MCExpr.h" #include "llvm/MC/MCObjectWriter.h" #include "llvm/MC/MCSymbolELF.h" #include "llvm/MC/MCValue.h" #include "llvm/Support/ErrorHandling.h" using namespace llvm; namespace { class PPCELFObjectWriter : public MCELFObjectTargetWriter { public: PPCELFObjectWriter(bool Is64Bit, uint8_t OSABI); protected: unsigned getRelocType(MCContext &Ctx, const MCValue &Target, const MCFixup &Fixup, bool IsPCRel) const override; bool needsRelocateWithSymbol(const MCSymbol &Sym, unsigned Type) const override; }; } PPCELFObjectWriter::PPCELFObjectWriter(bool Is64Bit, uint8_t OSABI) : MCELFObjectTargetWriter(Is64Bit, OSABI, Is64Bit ? ELF::EM_PPC64 : ELF::EM_PPC, /*HasRelocationAddend*/ true) {} static MCSymbolRefExpr::VariantKind getAccessVariant(const MCValue &Target, const MCFixup &Fixup) { const MCExpr *Expr = Fixup.getValue(); if (Expr->getKind() != MCExpr::Target) return Target.getAccessVariant(); switch (cast(Expr)->getKind()) { case PPCMCExpr::VK_PPC_None: return MCSymbolRefExpr::VK_None; case PPCMCExpr::VK_PPC_LO: return MCSymbolRefExpr::VK_PPC_LO; case PPCMCExpr::VK_PPC_HI: return MCSymbolRefExpr::VK_PPC_HI; case PPCMCExpr::VK_PPC_HA: return MCSymbolRefExpr::VK_PPC_HA; case PPCMCExpr::VK_PPC_HIGH: return MCSymbolRefExpr::VK_PPC_HIGH; case PPCMCExpr::VK_PPC_HIGHA: return MCSymbolRefExpr::VK_PPC_HIGHA; case PPCMCExpr::VK_PPC_HIGHERA: return MCSymbolRefExpr::VK_PPC_HIGHERA; case PPCMCExpr::VK_PPC_HIGHER: return MCSymbolRefExpr::VK_PPC_HIGHER; case PPCMCExpr::VK_PPC_HIGHEST: return MCSymbolRefExpr::VK_PPC_HIGHEST; case PPCMCExpr::VK_PPC_HIGHESTA: return MCSymbolRefExpr::VK_PPC_HIGHESTA; } llvm_unreachable("unknown PPCMCExpr kind"); } unsigned PPCELFObjectWriter::getRelocType(MCContext &Ctx, const MCValue &Target, const MCFixup &Fixup, bool IsPCRel) const { MCFixupKind Kind = Fixup.getKind(); if (Kind >= FirstLiteralRelocationKind) return Kind - FirstLiteralRelocationKind; MCSymbolRefExpr::VariantKind Modifier = getAccessVariant(Target, Fixup); // determine the type of the relocation unsigned Type; if (IsPCRel) { switch (Fixup.getTargetKind()) { default: llvm_unreachable("Unimplemented"); case PPC::fixup_ppc_br24: case PPC::fixup_ppc_br24abs: + case PPC::fixup_ppc_br24_notoc: switch (Modifier) { default: llvm_unreachable("Unsupported Modifier"); case MCSymbolRefExpr::VK_None: Type = ELF::R_PPC_REL24; break; case MCSymbolRefExpr::VK_PLT: Type = ELF::R_PPC_PLTREL24; break; case MCSymbolRefExpr::VK_PPC_LOCAL: Type = ELF::R_PPC_LOCAL24PC; break; + case MCSymbolRefExpr::VK_PPC_NOTOC: + Type = ELF::R_PPC64_REL24_NOTOC; + break; } break; case PPC::fixup_ppc_brcond14: case PPC::fixup_ppc_brcond14abs: Type = ELF::R_PPC_REL14; break; case PPC::fixup_ppc_half16: switch (Modifier) { default: llvm_unreachable("Unsupported Modifier"); case MCSymbolRefExpr::VK_None: Type = ELF::R_PPC_REL16; break; case MCSymbolRefExpr::VK_PPC_LO: Type = ELF::R_PPC_REL16_LO; break; case MCSymbolRefExpr::VK_PPC_HI: Type = ELF::R_PPC_REL16_HI; break; case MCSymbolRefExpr::VK_PPC_HA: Type = ELF::R_PPC_REL16_HA; break; } break; case PPC::fixup_ppc_half16ds: Target.print(errs()); errs() << '\n'; report_fatal_error("Invalid PC-relative half16ds relocation"); case FK_Data_4: case FK_PCRel_4: Type = ELF::R_PPC_REL32; break; case FK_Data_8: case FK_PCRel_8: Type = ELF::R_PPC64_REL64; break; } } else { switch (Fixup.getTargetKind()) { default: llvm_unreachable("invalid fixup kind!"); case PPC::fixup_ppc_br24abs: Type = ELF::R_PPC_ADDR24; break; case PPC::fixup_ppc_brcond14abs: Type = ELF::R_PPC_ADDR14; // XXX: or BRNTAKEN?_ break; case PPC::fixup_ppc_half16: switch (Modifier) { default: llvm_unreachable("Unsupported Modifier"); case MCSymbolRefExpr::VK_None: Type = ELF::R_PPC_ADDR16; break; case MCSymbolRefExpr::VK_PPC_LO: Type = ELF::R_PPC_ADDR16_LO; break; case MCSymbolRefExpr::VK_PPC_HI: Type = ELF::R_PPC_ADDR16_HI; break; case MCSymbolRefExpr::VK_PPC_HA: Type = ELF::R_PPC_ADDR16_HA; break; case MCSymbolRefExpr::VK_PPC_HIGH: Type = ELF::R_PPC64_ADDR16_HIGH; break; case MCSymbolRefExpr::VK_PPC_HIGHA: Type = ELF::R_PPC64_ADDR16_HIGHA; break; case MCSymbolRefExpr::VK_PPC_HIGHER: Type = ELF::R_PPC64_ADDR16_HIGHER; break; case MCSymbolRefExpr::VK_PPC_HIGHERA: Type = ELF::R_PPC64_ADDR16_HIGHERA; break; case MCSymbolRefExpr::VK_PPC_HIGHEST: Type = ELF::R_PPC64_ADDR16_HIGHEST; break; case MCSymbolRefExpr::VK_PPC_HIGHESTA: Type = ELF::R_PPC64_ADDR16_HIGHESTA; break; case MCSymbolRefExpr::VK_GOT: Type = ELF::R_PPC_GOT16; break; case MCSymbolRefExpr::VK_PPC_GOT_LO: Type = ELF::R_PPC_GOT16_LO; break; case MCSymbolRefExpr::VK_PPC_GOT_HI: Type = ELF::R_PPC_GOT16_HI; break; case MCSymbolRefExpr::VK_PPC_GOT_HA: Type = ELF::R_PPC_GOT16_HA; break; case MCSymbolRefExpr::VK_PPC_TOC: Type = ELF::R_PPC64_TOC16; break; case MCSymbolRefExpr::VK_PPC_TOC_LO: Type = ELF::R_PPC64_TOC16_LO; break; case MCSymbolRefExpr::VK_PPC_TOC_HI: Type = ELF::R_PPC64_TOC16_HI; break; case MCSymbolRefExpr::VK_PPC_TOC_HA: Type = ELF::R_PPC64_TOC16_HA; break; case MCSymbolRefExpr::VK_TPREL: Type = ELF::R_PPC_TPREL16; break; case MCSymbolRefExpr::VK_PPC_TPREL_LO: Type = ELF::R_PPC_TPREL16_LO; break; case MCSymbolRefExpr::VK_PPC_TPREL_HI: Type = ELF::R_PPC_TPREL16_HI; break; case MCSymbolRefExpr::VK_PPC_TPREL_HA: Type = ELF::R_PPC_TPREL16_HA; break; case MCSymbolRefExpr::VK_PPC_TPREL_HIGH: Type = ELF::R_PPC64_TPREL16_HIGH; break; case MCSymbolRefExpr::VK_PPC_TPREL_HIGHA: Type = ELF::R_PPC64_TPREL16_HIGHA; break; case MCSymbolRefExpr::VK_PPC_TPREL_HIGHER: Type = ELF::R_PPC64_TPREL16_HIGHER; break; case MCSymbolRefExpr::VK_PPC_TPREL_HIGHERA: Type = ELF::R_PPC64_TPREL16_HIGHERA; break; case MCSymbolRefExpr::VK_PPC_TPREL_HIGHEST: Type = ELF::R_PPC64_TPREL16_HIGHEST; break; case MCSymbolRefExpr::VK_PPC_TPREL_HIGHESTA: Type = ELF::R_PPC64_TPREL16_HIGHESTA; break; case MCSymbolRefExpr::VK_DTPREL: Type = ELF::R_PPC64_DTPREL16; break; case MCSymbolRefExpr::VK_PPC_DTPREL_LO: Type = ELF::R_PPC64_DTPREL16_LO; break; case MCSymbolRefExpr::VK_PPC_DTPREL_HI: Type = ELF::R_PPC64_DTPREL16_HI; break; case MCSymbolRefExpr::VK_PPC_DTPREL_HA: Type = ELF::R_PPC64_DTPREL16_HA; break; case MCSymbolRefExpr::VK_PPC_DTPREL_HIGH: Type = ELF::R_PPC64_DTPREL16_HIGH; break; case MCSymbolRefExpr::VK_PPC_DTPREL_HIGHA: Type = ELF::R_PPC64_DTPREL16_HIGHA; break; case MCSymbolRefExpr::VK_PPC_DTPREL_HIGHER: Type = ELF::R_PPC64_DTPREL16_HIGHER; break; case MCSymbolRefExpr::VK_PPC_DTPREL_HIGHERA: Type = ELF::R_PPC64_DTPREL16_HIGHERA; break; case MCSymbolRefExpr::VK_PPC_DTPREL_HIGHEST: Type = ELF::R_PPC64_DTPREL16_HIGHEST; break; case MCSymbolRefExpr::VK_PPC_DTPREL_HIGHESTA: Type = ELF::R_PPC64_DTPREL16_HIGHESTA; break; case MCSymbolRefExpr::VK_PPC_GOT_TLSGD: if (is64Bit()) Type = ELF::R_PPC64_GOT_TLSGD16; else Type = ELF::R_PPC_GOT_TLSGD16; break; case MCSymbolRefExpr::VK_PPC_GOT_TLSGD_LO: Type = ELF::R_PPC64_GOT_TLSGD16_LO; break; case MCSymbolRefExpr::VK_PPC_GOT_TLSGD_HI: Type = ELF::R_PPC64_GOT_TLSGD16_HI; break; case MCSymbolRefExpr::VK_PPC_GOT_TLSGD_HA: Type = ELF::R_PPC64_GOT_TLSGD16_HA; break; case MCSymbolRefExpr::VK_PPC_GOT_TLSLD: if (is64Bit()) Type = ELF::R_PPC64_GOT_TLSLD16; else Type = ELF::R_PPC_GOT_TLSLD16; break; case MCSymbolRefExpr::VK_PPC_GOT_TLSLD_LO: Type = ELF::R_PPC64_GOT_TLSLD16_LO; break; case MCSymbolRefExpr::VK_PPC_GOT_TLSLD_HI: Type = ELF::R_PPC64_GOT_TLSLD16_HI; break; case MCSymbolRefExpr::VK_PPC_GOT_TLSLD_HA: Type = ELF::R_PPC64_GOT_TLSLD16_HA; break; case MCSymbolRefExpr::VK_PPC_GOT_TPREL: /* We don't have R_PPC64_GOT_TPREL16, but since GOT offsets are always 4-aligned, we can use R_PPC64_GOT_TPREL16_DS. */ Type = ELF::R_PPC64_GOT_TPREL16_DS; break; case MCSymbolRefExpr::VK_PPC_GOT_TPREL_LO: /* We don't have R_PPC64_GOT_TPREL16_LO, but since GOT offsets are always 4-aligned, we can use R_PPC64_GOT_TPREL16_LO_DS. */ Type = ELF::R_PPC64_GOT_TPREL16_LO_DS; break; case MCSymbolRefExpr::VK_PPC_GOT_TPREL_HI: Type = ELF::R_PPC64_GOT_TPREL16_HI; break; case MCSymbolRefExpr::VK_PPC_GOT_DTPREL: /* We don't have R_PPC64_GOT_DTPREL16, but since GOT offsets are always 4-aligned, we can use R_PPC64_GOT_DTPREL16_DS. */ Type = ELF::R_PPC64_GOT_DTPREL16_DS; break; case MCSymbolRefExpr::VK_PPC_GOT_DTPREL_LO: /* We don't have R_PPC64_GOT_DTPREL16_LO, but since GOT offsets are always 4-aligned, we can use R_PPC64_GOT_DTPREL16_LO_DS. */ Type = ELF::R_PPC64_GOT_DTPREL16_LO_DS; break; case MCSymbolRefExpr::VK_PPC_GOT_TPREL_HA: Type = ELF::R_PPC64_GOT_TPREL16_HA; break; case MCSymbolRefExpr::VK_PPC_GOT_DTPREL_HI: Type = ELF::R_PPC64_GOT_DTPREL16_HI; break; case MCSymbolRefExpr::VK_PPC_GOT_DTPREL_HA: Type = ELF::R_PPC64_GOT_DTPREL16_HA; break; } break; case PPC::fixup_ppc_half16ds: switch (Modifier) { default: llvm_unreachable("Unsupported Modifier"); case MCSymbolRefExpr::VK_None: Type = ELF::R_PPC64_ADDR16_DS; break; case MCSymbolRefExpr::VK_PPC_LO: Type = ELF::R_PPC64_ADDR16_LO_DS; break; case MCSymbolRefExpr::VK_GOT: Type = ELF::R_PPC64_GOT16_DS; break; case MCSymbolRefExpr::VK_PPC_GOT_LO: Type = ELF::R_PPC64_GOT16_LO_DS; break; case MCSymbolRefExpr::VK_PPC_TOC: Type = ELF::R_PPC64_TOC16_DS; break; case MCSymbolRefExpr::VK_PPC_TOC_LO: Type = ELF::R_PPC64_TOC16_LO_DS; break; case MCSymbolRefExpr::VK_TPREL: Type = ELF::R_PPC64_TPREL16_DS; break; case MCSymbolRefExpr::VK_PPC_TPREL_LO: Type = ELF::R_PPC64_TPREL16_LO_DS; break; case MCSymbolRefExpr::VK_DTPREL: Type = ELF::R_PPC64_DTPREL16_DS; break; case MCSymbolRefExpr::VK_PPC_DTPREL_LO: Type = ELF::R_PPC64_DTPREL16_LO_DS; break; case MCSymbolRefExpr::VK_PPC_GOT_TPREL: Type = ELF::R_PPC64_GOT_TPREL16_DS; break; case MCSymbolRefExpr::VK_PPC_GOT_TPREL_LO: Type = ELF::R_PPC64_GOT_TPREL16_LO_DS; break; case MCSymbolRefExpr::VK_PPC_GOT_DTPREL: Type = ELF::R_PPC64_GOT_DTPREL16_DS; break; case MCSymbolRefExpr::VK_PPC_GOT_DTPREL_LO: Type = ELF::R_PPC64_GOT_DTPREL16_LO_DS; break; } break; case PPC::fixup_ppc_nofixup: switch (Modifier) { default: llvm_unreachable("Unsupported Modifier"); case MCSymbolRefExpr::VK_PPC_TLSGD: if (is64Bit()) Type = ELF::R_PPC64_TLSGD; else Type = ELF::R_PPC_TLSGD; break; case MCSymbolRefExpr::VK_PPC_TLSLD: if (is64Bit()) Type = ELF::R_PPC64_TLSLD; else Type = ELF::R_PPC_TLSLD; break; case MCSymbolRefExpr::VK_PPC_TLS: if (is64Bit()) Type = ELF::R_PPC64_TLS; else Type = ELF::R_PPC_TLS; break; } break; case FK_Data_8: switch (Modifier) { default: llvm_unreachable("Unsupported Modifier"); case MCSymbolRefExpr::VK_PPC_TOCBASE: Type = ELF::R_PPC64_TOC; break; case MCSymbolRefExpr::VK_None: Type = ELF::R_PPC64_ADDR64; break; case MCSymbolRefExpr::VK_PPC_DTPMOD: Type = ELF::R_PPC64_DTPMOD64; break; case MCSymbolRefExpr::VK_TPREL: Type = ELF::R_PPC64_TPREL64; break; case MCSymbolRefExpr::VK_DTPREL: Type = ELF::R_PPC64_DTPREL64; break; } break; case FK_Data_4: Type = ELF::R_PPC_ADDR32; break; case FK_Data_2: Type = ELF::R_PPC_ADDR16; break; } } return Type; } bool PPCELFObjectWriter::needsRelocateWithSymbol(const MCSymbol &Sym, unsigned Type) const { switch (Type) { default: return false; case ELF::R_PPC_REL24: + case ELF::R_PPC64_REL24_NOTOC: // If the target symbol has a local entry point, we must keep the // target symbol to preserve that information for the linker. // The "other" values are stored in the last 6 bits of the second byte. // The traditional defines for STO values assume the full byte and thus // the shift to pack it. unsigned Other = cast(Sym).getOther() << 2; return (Other & ELF::STO_PPC64_LOCAL_MASK) != 0; } } std::unique_ptr llvm::createPPCELFObjectWriter(bool Is64Bit, uint8_t OSABI) { return std::make_unique(Is64Bit, OSABI); } diff --git a/llvm/lib/Target/PowerPC/MCTargetDesc/PPCFixupKinds.h b/llvm/lib/Target/PowerPC/MCTargetDesc/PPCFixupKinds.h index 845489788c86..efa7f0ad86d9 100644 --- a/llvm/lib/Target/PowerPC/MCTargetDesc/PPCFixupKinds.h +++ b/llvm/lib/Target/PowerPC/MCTargetDesc/PPCFixupKinds.h @@ -1,51 +1,55 @@ //===-- PPCFixupKinds.h - PPC Specific Fixup Entries ------------*- C++ -*-===// // // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. // See https://llvm.org/LICENSE.txt for license information. // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// #ifndef LLVM_LIB_TARGET_POWERPC_MCTARGETDESC_PPCFIXUPKINDS_H #define LLVM_LIB_TARGET_POWERPC_MCTARGETDESC_PPCFIXUPKINDS_H #include "llvm/MC/MCFixup.h" #undef PPC namespace llvm { namespace PPC { enum Fixups { // 24-bit PC relative relocation for direct branches like 'b' and 'bl'. fixup_ppc_br24 = FirstTargetFixupKind, + // 24-bit PC relative relocation for direct branches like 'b' and 'bl' where + // the caller does not use the TOC. + fixup_ppc_br24_notoc, + /// 14-bit PC relative relocation for conditional branches. fixup_ppc_brcond14, /// 24-bit absolute relocation for direct branches like 'ba' and 'bla'. fixup_ppc_br24abs, /// 14-bit absolute relocation for conditional branches. fixup_ppc_brcond14abs, /// A 16-bit fixup corresponding to lo16(_foo) or ha16(_foo) for instrs like /// 'li' or 'addis'. fixup_ppc_half16, /// A 14-bit fixup corresponding to lo16(_foo) with implied 2 zero bits for /// instrs like 'std'. fixup_ppc_half16ds, /// Not a true fixup, but ties a symbol to a call to __tls_get_addr for the /// TLS general and local dynamic models, or inserts the thread-pointer /// register number. fixup_ppc_nofixup, // Marker LastTargetFixupKind, NumTargetFixupKinds = LastTargetFixupKind - FirstTargetFixupKind }; } } #endif diff --git a/llvm/lib/Target/PowerPC/MCTargetDesc/PPCMCCodeEmitter.cpp b/llvm/lib/Target/PowerPC/MCTargetDesc/PPCMCCodeEmitter.cpp index 672f6e02ce93..06df3bde48f6 100644 --- a/llvm/lib/Target/PowerPC/MCTargetDesc/PPCMCCodeEmitter.cpp +++ b/llvm/lib/Target/PowerPC/MCTargetDesc/PPCMCCodeEmitter.cpp @@ -1,356 +1,358 @@ //===-- PPCMCCodeEmitter.cpp - Convert PPC code to machine code -----------===// // // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. // See https://llvm.org/LICENSE.txt for license information. // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // // This file implements the PPCMCCodeEmitter class. // //===----------------------------------------------------------------------===// #include "MCTargetDesc/PPCFixupKinds.h" #include "PPCInstrInfo.h" #include "PPCMCCodeEmitter.h" #include "llvm/ADT/SmallVector.h" #include "llvm/ADT/Statistic.h" #include "llvm/ADT/Triple.h" #include "llvm/MC/MCFixup.h" #include "llvm/MC/MCInstrDesc.h" #include "llvm/MC/MCRegisterInfo.h" #include "llvm/Support/Endian.h" #include "llvm/Support/EndianStream.h" #include "llvm/Support/ErrorHandling.h" #include "llvm/Support/MathExtras.h" #include "llvm/Support/raw_ostream.h" #include #include using namespace llvm; #define DEBUG_TYPE "mccodeemitter" STATISTIC(MCNumEmitted, "Number of MC instructions emitted"); MCCodeEmitter *llvm::createPPCMCCodeEmitter(const MCInstrInfo &MCII, const MCRegisterInfo &MRI, MCContext &Ctx) { return new PPCMCCodeEmitter(MCII, Ctx); } unsigned PPCMCCodeEmitter:: getDirectBrEncoding(const MCInst &MI, unsigned OpNo, SmallVectorImpl &Fixups, const MCSubtargetInfo &STI) const { const MCOperand &MO = MI.getOperand(OpNo); if (MO.isReg() || MO.isImm()) return getMachineOpValue(MI, MO, Fixups, STI); // Add a fixup for the branch target. Fixups.push_back(MCFixup::create(0, MO.getExpr(), - (MCFixupKind)PPC::fixup_ppc_br24)); + ((MI.getOpcode() == PPC::BL8_NOTOC) + ? (MCFixupKind)PPC::fixup_ppc_br24_notoc + : (MCFixupKind)PPC::fixup_ppc_br24))); return 0; } unsigned PPCMCCodeEmitter::getCondBrEncoding(const MCInst &MI, unsigned OpNo, SmallVectorImpl &Fixups, const MCSubtargetInfo &STI) const { const MCOperand &MO = MI.getOperand(OpNo); if (MO.isReg() || MO.isImm()) return getMachineOpValue(MI, MO, Fixups, STI); // Add a fixup for the branch target. Fixups.push_back(MCFixup::create(0, MO.getExpr(), (MCFixupKind)PPC::fixup_ppc_brcond14)); return 0; } unsigned PPCMCCodeEmitter:: getAbsDirectBrEncoding(const MCInst &MI, unsigned OpNo, SmallVectorImpl &Fixups, const MCSubtargetInfo &STI) const { const MCOperand &MO = MI.getOperand(OpNo); if (MO.isReg() || MO.isImm()) return getMachineOpValue(MI, MO, Fixups, STI); // Add a fixup for the branch target. Fixups.push_back(MCFixup::create(0, MO.getExpr(), (MCFixupKind)PPC::fixup_ppc_br24abs)); return 0; } unsigned PPCMCCodeEmitter:: getAbsCondBrEncoding(const MCInst &MI, unsigned OpNo, SmallVectorImpl &Fixups, const MCSubtargetInfo &STI) const { const MCOperand &MO = MI.getOperand(OpNo); if (MO.isReg() || MO.isImm()) return getMachineOpValue(MI, MO, Fixups, STI); // Add a fixup for the branch target. Fixups.push_back(MCFixup::create(0, MO.getExpr(), (MCFixupKind)PPC::fixup_ppc_brcond14abs)); return 0; } unsigned PPCMCCodeEmitter::getImm16Encoding(const MCInst &MI, unsigned OpNo, SmallVectorImpl &Fixups, const MCSubtargetInfo &STI) const { const MCOperand &MO = MI.getOperand(OpNo); if (MO.isReg() || MO.isImm()) return getMachineOpValue(MI, MO, Fixups, STI); // Add a fixup for the immediate field. Fixups.push_back(MCFixup::create(IsLittleEndian? 0 : 2, MO.getExpr(), (MCFixupKind)PPC::fixup_ppc_half16)); return 0; } unsigned PPCMCCodeEmitter::getMemRIEncoding(const MCInst &MI, unsigned OpNo, SmallVectorImpl &Fixups, const MCSubtargetInfo &STI) const { // Encode (imm, reg) as a memri, which has the low 16-bits as the // displacement and the next 5 bits as the register #. assert(MI.getOperand(OpNo+1).isReg()); unsigned RegBits = getMachineOpValue(MI, MI.getOperand(OpNo+1), Fixups, STI) << 16; const MCOperand &MO = MI.getOperand(OpNo); if (MO.isImm()) return (getMachineOpValue(MI, MO, Fixups, STI) & 0xFFFF) | RegBits; // Add a fixup for the displacement field. Fixups.push_back(MCFixup::create(IsLittleEndian? 0 : 2, MO.getExpr(), (MCFixupKind)PPC::fixup_ppc_half16)); return RegBits; } unsigned PPCMCCodeEmitter::getMemRIXEncoding(const MCInst &MI, unsigned OpNo, SmallVectorImpl &Fixups, const MCSubtargetInfo &STI) const { // Encode (imm, reg) as a memrix, which has the low 14-bits as the // displacement and the next 5 bits as the register #. assert(MI.getOperand(OpNo+1).isReg()); unsigned RegBits = getMachineOpValue(MI, MI.getOperand(OpNo+1), Fixups, STI) << 14; const MCOperand &MO = MI.getOperand(OpNo); if (MO.isImm()) return ((getMachineOpValue(MI, MO, Fixups, STI) >> 2) & 0x3FFF) | RegBits; // Add a fixup for the displacement field. Fixups.push_back(MCFixup::create(IsLittleEndian? 0 : 2, MO.getExpr(), (MCFixupKind)PPC::fixup_ppc_half16ds)); return RegBits; } unsigned PPCMCCodeEmitter::getMemRIX16Encoding(const MCInst &MI, unsigned OpNo, SmallVectorImpl &Fixups, const MCSubtargetInfo &STI) const { // Encode (imm, reg) as a memrix16, which has the low 12-bits as the // displacement and the next 5 bits as the register #. assert(MI.getOperand(OpNo+1).isReg()); unsigned RegBits = getMachineOpValue(MI, MI.getOperand(OpNo+1), Fixups, STI) << 12; const MCOperand &MO = MI.getOperand(OpNo); if (MO.isImm()) { assert(!(MO.getImm() % 16) && "Expecting an immediate that is a multiple of 16"); return ((getMachineOpValue(MI, MO, Fixups, STI) >> 4) & 0xFFF) | RegBits; } // Otherwise add a fixup for the displacement field. Fixups.push_back(MCFixup::create(IsLittleEndian? 0 : 2, MO.getExpr(), (MCFixupKind)PPC::fixup_ppc_half16ds)); return RegBits; } uint64_t PPCMCCodeEmitter::getMemRI34PCRelEncoding(const MCInst &MI, unsigned OpNo, SmallVectorImpl &Fixups, const MCSubtargetInfo &STI) const { // Encode (imm, reg) as a memri34, which has the low 34-bits as the // displacement and the next 5 bits as an immediate 0. assert(MI.getOperand(OpNo + 1).isImm() && "Expecting an immediate."); uint64_t RegBits = getMachineOpValue(MI, MI.getOperand(OpNo + 1), Fixups, STI) << 34; if (RegBits != 0) report_fatal_error("Operand must be 0"); const MCOperand &MO = MI.getOperand(OpNo); return ((getMachineOpValue(MI, MO, Fixups, STI)) & 0x3FFFFFFFFUL) | RegBits; } uint64_t PPCMCCodeEmitter::getMemRI34Encoding(const MCInst &MI, unsigned OpNo, SmallVectorImpl &Fixups, const MCSubtargetInfo &STI) const { // Encode (imm, reg) as a memri34, which has the low 34-bits as the // displacement and the next 5 bits as the register #. assert(MI.getOperand(OpNo + 1).isReg() && "Expecting a register."); uint64_t RegBits = getMachineOpValue(MI, MI.getOperand(OpNo + 1), Fixups, STI) << 34; const MCOperand &MO = MI.getOperand(OpNo); return ((getMachineOpValue(MI, MO, Fixups, STI)) & 0x3FFFFFFFFUL) | RegBits; } unsigned PPCMCCodeEmitter::getSPE8DisEncoding(const MCInst &MI, unsigned OpNo, SmallVectorImpl &Fixups, const MCSubtargetInfo &STI) const { // Encode (imm, reg) as a spe8dis, which has the low 5-bits of (imm / 8) // as the displacement and the next 5 bits as the register #. assert(MI.getOperand(OpNo+1).isReg()); uint32_t RegBits = getMachineOpValue(MI, MI.getOperand(OpNo+1), Fixups, STI) << 5; const MCOperand &MO = MI.getOperand(OpNo); assert(MO.isImm()); uint32_t Imm = getMachineOpValue(MI, MO, Fixups, STI) >> 3; return reverseBits(Imm | RegBits) >> 22; } unsigned PPCMCCodeEmitter::getSPE4DisEncoding(const MCInst &MI, unsigned OpNo, SmallVectorImpl &Fixups, const MCSubtargetInfo &STI) const { // Encode (imm, reg) as a spe4dis, which has the low 5-bits of (imm / 4) // as the displacement and the next 5 bits as the register #. assert(MI.getOperand(OpNo+1).isReg()); uint32_t RegBits = getMachineOpValue(MI, MI.getOperand(OpNo+1), Fixups, STI) << 5; const MCOperand &MO = MI.getOperand(OpNo); assert(MO.isImm()); uint32_t Imm = getMachineOpValue(MI, MO, Fixups, STI) >> 2; return reverseBits(Imm | RegBits) >> 22; } unsigned PPCMCCodeEmitter::getSPE2DisEncoding(const MCInst &MI, unsigned OpNo, SmallVectorImpl &Fixups, const MCSubtargetInfo &STI) const { // Encode (imm, reg) as a spe2dis, which has the low 5-bits of (imm / 2) // as the displacement and the next 5 bits as the register #. assert(MI.getOperand(OpNo+1).isReg()); uint32_t RegBits = getMachineOpValue(MI, MI.getOperand(OpNo+1), Fixups, STI) << 5; const MCOperand &MO = MI.getOperand(OpNo); assert(MO.isImm()); uint32_t Imm = getMachineOpValue(MI, MO, Fixups, STI) >> 1; return reverseBits(Imm | RegBits) >> 22; } unsigned PPCMCCodeEmitter::getTLSRegEncoding(const MCInst &MI, unsigned OpNo, SmallVectorImpl &Fixups, const MCSubtargetInfo &STI) const { const MCOperand &MO = MI.getOperand(OpNo); if (MO.isReg()) return getMachineOpValue(MI, MO, Fixups, STI); // Add a fixup for the TLS register, which simply provides a relocation // hint to the linker that this statement is part of a relocation sequence. // Return the thread-pointer register's encoding. Fixups.push_back(MCFixup::create(0, MO.getExpr(), (MCFixupKind)PPC::fixup_ppc_nofixup)); const Triple &TT = STI.getTargetTriple(); bool isPPC64 = TT.isPPC64(); return CTX.getRegisterInfo()->getEncodingValue(isPPC64 ? PPC::X13 : PPC::R2); } unsigned PPCMCCodeEmitter::getTLSCallEncoding(const MCInst &MI, unsigned OpNo, SmallVectorImpl &Fixups, const MCSubtargetInfo &STI) const { // For special TLS calls, we need two fixups; one for the branch target // (__tls_get_addr), which we create via getDirectBrEncoding as usual, // and one for the TLSGD or TLSLD symbol, which is emitted here. const MCOperand &MO = MI.getOperand(OpNo+1); Fixups.push_back(MCFixup::create(0, MO.getExpr(), (MCFixupKind)PPC::fixup_ppc_nofixup)); return getDirectBrEncoding(MI, OpNo, Fixups, STI); } unsigned PPCMCCodeEmitter:: get_crbitm_encoding(const MCInst &MI, unsigned OpNo, SmallVectorImpl &Fixups, const MCSubtargetInfo &STI) const { const MCOperand &MO = MI.getOperand(OpNo); assert((MI.getOpcode() == PPC::MTOCRF || MI.getOpcode() == PPC::MTOCRF8 || MI.getOpcode() == PPC::MFOCRF || MI.getOpcode() == PPC::MFOCRF8) && (MO.getReg() >= PPC::CR0 && MO.getReg() <= PPC::CR7)); return 0x80 >> CTX.getRegisterInfo()->getEncodingValue(MO.getReg()); } // Get the index for this operand in this instruction. This is needed for // computing the register number in PPCInstrInfo::getRegNumForOperand() for // any instructions that use a different numbering scheme for registers in // different operands. static unsigned getOpIdxForMO(const MCInst &MI, const MCOperand &MO) { for (unsigned i = 0; i < MI.getNumOperands(); i++) { const MCOperand &Op = MI.getOperand(i); if (&Op == &MO) return i; } llvm_unreachable("This operand is not part of this instruction"); return ~0U; // Silence any warnings about no return. } uint64_t PPCMCCodeEmitter:: getMachineOpValue(const MCInst &MI, const MCOperand &MO, SmallVectorImpl &Fixups, const MCSubtargetInfo &STI) const { if (MO.isReg()) { // MTOCRF/MFOCRF should go through get_crbitm_encoding for the CR operand. // The GPR operand should come through here though. assert((MI.getOpcode() != PPC::MTOCRF && MI.getOpcode() != PPC::MTOCRF8 && MI.getOpcode() != PPC::MFOCRF && MI.getOpcode() != PPC::MFOCRF8) || MO.getReg() < PPC::CR0 || MO.getReg() > PPC::CR7); unsigned OpNo = getOpIdxForMO(MI, MO); unsigned Reg = PPCInstrInfo::getRegNumForOperand(MCII.get(MI.getOpcode()), MO.getReg(), OpNo); return CTX.getRegisterInfo()->getEncodingValue(Reg); } assert(MO.isImm() && "Relocation required in an instruction that we cannot encode!"); return MO.getImm(); } void PPCMCCodeEmitter::encodeInstruction( const MCInst &MI, raw_ostream &OS, SmallVectorImpl &Fixups, const MCSubtargetInfo &STI) const { verifyInstructionPredicates(MI, computeAvailableFeatures(STI.getFeatureBits())); uint64_t Bits = getBinaryCodeForInstr(MI, Fixups, STI); // Output the constant in big/little endian byte order. unsigned Size = getInstSizeInBytes(MI); support::endianness E = IsLittleEndian ? support::little : support::big; switch (Size) { case 0: break; case 4: support::endian::write(OS, Bits, E); break; case 8: // If we emit a pair of instructions, the first one is // always in the top 32 bits, even on little-endian. support::endian::write(OS, Bits >> 32, E); support::endian::write(OS, Bits, E); break; default: llvm_unreachable("Invalid instruction size"); } ++MCNumEmitted; // Keep track of the # of mi's emitted. } // Get the number of bytes used to encode the given MCInst. unsigned PPCMCCodeEmitter::getInstSizeInBytes(const MCInst &MI) const { unsigned Opcode = MI.getOpcode(); const MCInstrDesc &Desc = MCII.get(Opcode); return Desc.getSize(); } bool PPCMCCodeEmitter::isPrefixedInstruction(const MCInst &MI) const { unsigned Opcode = MI.getOpcode(); const PPCInstrInfo *InstrInfo = static_cast(&MCII); return InstrInfo->isPrefixed(Opcode); } #define ENABLE_INSTR_PREDICATE_VERIFIER #include "PPCGenMCCodeEmitter.inc" diff --git a/llvm/lib/Target/PowerPC/MCTargetDesc/PPCMCTargetDesc.cpp b/llvm/lib/Target/PowerPC/MCTargetDesc/PPCMCTargetDesc.cpp index 9d3ecc07b3b2..c85b08ad77e0 100644 --- a/llvm/lib/Target/PowerPC/MCTargetDesc/PPCMCTargetDesc.cpp +++ b/llvm/lib/Target/PowerPC/MCTargetDesc/PPCMCTargetDesc.cpp @@ -1,342 +1,363 @@ //===-- PPCMCTargetDesc.cpp - PowerPC Target Descriptions -----------------===// // // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. // See https://llvm.org/LICENSE.txt for license information. // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // // This file provides PowerPC specific target descriptions. // //===----------------------------------------------------------------------===// #include "MCTargetDesc/PPCMCTargetDesc.h" #include "MCTargetDesc/PPCInstPrinter.h" #include "MCTargetDesc/PPCMCAsmInfo.h" #include "PPCELFStreamer.h" #include "PPCTargetStreamer.h" #include "TargetInfo/PowerPCTargetInfo.h" #include "llvm/ADT/SmallPtrSet.h" #include "llvm/ADT/StringRef.h" #include "llvm/ADT/Triple.h" #include "llvm/BinaryFormat/ELF.h" #include "llvm/MC/MCAssembler.h" #include "llvm/MC/MCAsmBackend.h" #include "llvm/MC/MCCodeEmitter.h" #include "llvm/MC/MCContext.h" #include "llvm/MC/MCDwarf.h" #include "llvm/MC/MCELFStreamer.h" #include "llvm/MC/MCExpr.h" #include "llvm/MC/MCInstrInfo.h" #include "llvm/MC/MCObjectWriter.h" #include "llvm/MC/MCRegisterInfo.h" #include "llvm/MC/MCStreamer.h" #include "llvm/MC/MCSubtargetInfo.h" #include "llvm/MC/MCSymbol.h" #include "llvm/MC/MCSymbolELF.h" #include "llvm/MC/MCSymbolXCOFF.h" #include "llvm/Support/Casting.h" #include "llvm/Support/CodeGen.h" #include "llvm/Support/ErrorHandling.h" #include "llvm/Support/FormattedStream.h" #include "llvm/Support/TargetRegistry.h" #include "llvm/Support/raw_ostream.h" using namespace llvm; #define GET_INSTRINFO_MC_DESC #include "PPCGenInstrInfo.inc" #define GET_SUBTARGETINFO_MC_DESC #include "PPCGenSubtargetInfo.inc" #define GET_REGINFO_MC_DESC #include "PPCGenRegisterInfo.inc" PPCTargetStreamer::PPCTargetStreamer(MCStreamer &S) : MCTargetStreamer(S) {} // Pin the vtable to this file. PPCTargetStreamer::~PPCTargetStreamer() = default; static MCInstrInfo *createPPCMCInstrInfo() { MCInstrInfo *X = new MCInstrInfo(); InitPPCMCInstrInfo(X); return X; } static MCRegisterInfo *createPPCMCRegisterInfo(const Triple &TT) { bool isPPC64 = (TT.getArch() == Triple::ppc64 || TT.getArch() == Triple::ppc64le); unsigned Flavour = isPPC64 ? 0 : 1; unsigned RA = isPPC64 ? PPC::LR8 : PPC::LR; MCRegisterInfo *X = new MCRegisterInfo(); InitPPCMCRegisterInfo(X, RA, Flavour, Flavour); return X; } static MCSubtargetInfo *createPPCMCSubtargetInfo(const Triple &TT, StringRef CPU, StringRef FS) { return createPPCMCSubtargetInfoImpl(TT, CPU, FS); } static MCAsmInfo *createPPCMCAsmInfo(const MCRegisterInfo &MRI, const Triple &TheTriple, const MCTargetOptions &Options) { bool isPPC64 = (TheTriple.getArch() == Triple::ppc64 || TheTriple.getArch() == Triple::ppc64le); MCAsmInfo *MAI; if (TheTriple.isOSBinFormatXCOFF()) MAI = new PPCXCOFFMCAsmInfo(isPPC64, TheTriple); else MAI = new PPCELFMCAsmInfo(isPPC64, TheTriple); // Initial state of the frame pointer is R1. unsigned Reg = isPPC64 ? PPC::X1 : PPC::R1; MCCFIInstruction Inst = MCCFIInstruction::createDefCfa(nullptr, MRI.getDwarfRegNum(Reg, true), 0); MAI->addInitialFrameState(Inst); return MAI; } static MCStreamer *createPPCMCStreamer(const Triple &T, MCContext &Context, std::unique_ptr &&MAB, std::unique_ptr &&OW, std::unique_ptr &&Emitter, bool RelaxAll) { return createPPCELFStreamer(Context, std::move(MAB), std::move(OW), std::move(Emitter)); } namespace { class PPCTargetAsmStreamer : public PPCTargetStreamer { formatted_raw_ostream &OS; public: PPCTargetAsmStreamer(MCStreamer &S, formatted_raw_ostream &OS) : PPCTargetStreamer(S), OS(OS) {} void emitTCEntry(const MCSymbol &S) override { const MCAsmInfo *MAI = Streamer.getContext().getAsmInfo(); OS << "\t.tc "; OS << (MAI->getSymbolsHaveSMC() ? cast(S).getUnqualifiedName() : S.getName()); OS << "[TC],"; OS << S.getName(); OS << '\n'; } void emitMachine(StringRef CPU) override { OS << "\t.machine " << CPU << '\n'; } void emitAbiVersion(int AbiVersion) override { OS << "\t.abiversion " << AbiVersion << '\n'; } void emitLocalEntry(MCSymbolELF *S, const MCExpr *LocalOffset) override { const MCAsmInfo *MAI = Streamer.getContext().getAsmInfo(); OS << "\t.localentry\t"; S->print(OS, MAI); OS << ", "; LocalOffset->print(OS, MAI); OS << '\n'; } }; class PPCTargetELFStreamer : public PPCTargetStreamer { public: PPCTargetELFStreamer(MCStreamer &S) : PPCTargetStreamer(S) {} MCELFStreamer &getStreamer() { return static_cast(Streamer); } void emitTCEntry(const MCSymbol &S) override { // Creates a R_PPC64_TOC relocation Streamer.emitValueToAlignment(8); Streamer.emitSymbolValue(&S, 8); } void emitMachine(StringRef CPU) override { // FIXME: Is there anything to do in here or does this directive only // limit the parser? } void emitAbiVersion(int AbiVersion) override { MCAssembler &MCA = getStreamer().getAssembler(); unsigned Flags = MCA.getELFHeaderEFlags(); Flags &= ~ELF::EF_PPC64_ABI; Flags |= (AbiVersion & ELF::EF_PPC64_ABI); MCA.setELFHeaderEFlags(Flags); } void emitLocalEntry(MCSymbolELF *S, const MCExpr *LocalOffset) override { MCAssembler &MCA = getStreamer().getAssembler(); - int64_t Res; - if (!LocalOffset->evaluateAsAbsolute(Res, MCA)) - report_fatal_error(".localentry expression must be absolute."); - - unsigned Encoded = ELF::encodePPC64LocalEntryOffset(Res); - if (Res != ELF::decodePPC64LocalEntryOffset(Encoded)) - report_fatal_error(".localentry expression cannot be encoded."); + // encodePPC64LocalEntryOffset will report an error if it cannot + // encode LocalOffset. + unsigned Encoded = encodePPC64LocalEntryOffset(LocalOffset); unsigned Other = S->getOther(); Other &= ~ELF::STO_PPC64_LOCAL_MASK; Other |= Encoded; S->setOther(Other); // For GAS compatibility, unless we already saw a .abiversion directive, // set e_flags to indicate ELFv2 ABI. unsigned Flags = MCA.getELFHeaderEFlags(); if ((Flags & ELF::EF_PPC64_ABI) == 0) MCA.setELFHeaderEFlags(Flags | 2); } void emitAssignment(MCSymbol *S, const MCExpr *Value) override { auto *Symbol = cast(S); // When encoding an assignment to set symbol A to symbol B, also copy // the st_other bits encoding the local entry point offset. if (copyLocalEntry(Symbol, Value)) UpdateOther.insert(Symbol); else UpdateOther.erase(Symbol); } void finish() override { for (auto *Sym : UpdateOther) if (Sym->isVariable()) copyLocalEntry(Sym, Sym->getVariableValue()); } private: SmallPtrSet UpdateOther; bool copyLocalEntry(MCSymbolELF *D, const MCExpr *S) { auto *Ref = dyn_cast(S); if (!Ref) return false; const auto &RhsSym = cast(Ref->getSymbol()); unsigned Other = D->getOther(); Other &= ~ELF::STO_PPC64_LOCAL_MASK; Other |= RhsSym.getOther() & ELF::STO_PPC64_LOCAL_MASK; D->setOther(Other); return true; } + + unsigned encodePPC64LocalEntryOffset(const MCExpr *LocalOffset) { + MCAssembler &MCA = getStreamer().getAssembler(); + int64_t Offset; + if (!LocalOffset->evaluateAsAbsolute(Offset, MCA)) + MCA.getContext().reportFatalError( + LocalOffset->getLoc(), ".localentry expression must be absolute."); + + switch (Offset) { + default: + MCA.getContext().reportFatalError( + LocalOffset->getLoc(), + ".localentry expression is not a valid power of 2."); + case 0: + return 0; + case 1: + return 1 << ELF::STO_PPC64_LOCAL_BIT; + case 4: + case 8: + case 16: + case 32: + case 64: + return (int)Log2(Offset) << (int)ELF::STO_PPC64_LOCAL_BIT; + } + } }; class PPCTargetMachOStreamer : public PPCTargetStreamer { public: PPCTargetMachOStreamer(MCStreamer &S) : PPCTargetStreamer(S) {} void emitTCEntry(const MCSymbol &S) override { llvm_unreachable("Unknown pseudo-op: .tc"); } void emitMachine(StringRef CPU) override { // FIXME: We should update the CPUType, CPUSubType in the Object file if // the new values are different from the defaults. } void emitAbiVersion(int AbiVersion) override { llvm_unreachable("Unknown pseudo-op: .abiversion"); } void emitLocalEntry(MCSymbolELF *S, const MCExpr *LocalOffset) override { llvm_unreachable("Unknown pseudo-op: .localentry"); } }; class PPCTargetXCOFFStreamer : public PPCTargetStreamer { public: PPCTargetXCOFFStreamer(MCStreamer &S) : PPCTargetStreamer(S) {} void emitTCEntry(const MCSymbol &S) override { const MCAsmInfo *MAI = Streamer.getContext().getAsmInfo(); const unsigned PointerSize = MAI->getCodePointerSize(); Streamer.emitValueToAlignment(PointerSize); Streamer.emitSymbolValue(&S, PointerSize); } void emitMachine(StringRef CPU) override { llvm_unreachable("Machine pseudo-ops are invalid for XCOFF."); } void emitAbiVersion(int AbiVersion) override { llvm_unreachable("ABI-version pseudo-ops are invalid for XCOFF."); } void emitLocalEntry(MCSymbolELF *S, const MCExpr *LocalOffset) override { llvm_unreachable("Local-entry pseudo-ops are invalid for XCOFF."); } }; } // end anonymous namespace static MCTargetStreamer *createAsmTargetStreamer(MCStreamer &S, formatted_raw_ostream &OS, MCInstPrinter *InstPrint, bool isVerboseAsm) { return new PPCTargetAsmStreamer(S, OS); } static MCTargetStreamer * createObjectTargetStreamer(MCStreamer &S, const MCSubtargetInfo &STI) { const Triple &TT = STI.getTargetTriple(); if (TT.isOSBinFormatELF()) return new PPCTargetELFStreamer(S); if (TT.isOSBinFormatXCOFF()) return new PPCTargetXCOFFStreamer(S); return new PPCTargetMachOStreamer(S); } static MCInstPrinter *createPPCMCInstPrinter(const Triple &T, unsigned SyntaxVariant, const MCAsmInfo &MAI, const MCInstrInfo &MII, const MCRegisterInfo &MRI) { return new PPCInstPrinter(MAI, MII, MRI, T); } extern "C" LLVM_EXTERNAL_VISIBILITY void LLVMInitializePowerPCTargetMC() { for (Target *T : {&getThePPC32Target(), &getThePPC64Target(), &getThePPC64LETarget()}) { // Register the MC asm info. RegisterMCAsmInfoFn C(*T, createPPCMCAsmInfo); // Register the MC instruction info. TargetRegistry::RegisterMCInstrInfo(*T, createPPCMCInstrInfo); // Register the MC register info. TargetRegistry::RegisterMCRegInfo(*T, createPPCMCRegisterInfo); // Register the MC subtarget info. TargetRegistry::RegisterMCSubtargetInfo(*T, createPPCMCSubtargetInfo); // Register the MC Code Emitter TargetRegistry::RegisterMCCodeEmitter(*T, createPPCMCCodeEmitter); // Register the asm backend. TargetRegistry::RegisterMCAsmBackend(*T, createPPCAsmBackend); // Register the elf streamer. TargetRegistry::RegisterELFStreamer(*T, createPPCMCStreamer); // Register the object target streamer. TargetRegistry::RegisterObjectTargetStreamer(*T, createObjectTargetStreamer); // Register the asm target streamer. TargetRegistry::RegisterAsmTargetStreamer(*T, createAsmTargetStreamer); // Register the MCInstPrinter. TargetRegistry::RegisterMCInstPrinter(*T, createPPCMCInstPrinter); } } diff --git a/llvm/lib/Target/PowerPC/PPCAsmPrinter.cpp b/llvm/lib/Target/PowerPC/PPCAsmPrinter.cpp index 0a256cb5876b..e0bbb8f8dd3e 100644 --- a/llvm/lib/Target/PowerPC/PPCAsmPrinter.cpp +++ b/llvm/lib/Target/PowerPC/PPCAsmPrinter.cpp @@ -1,1770 +1,1801 @@ //===-- PPCAsmPrinter.cpp - Print machine instrs to PowerPC assembly ------===// // // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. // See https://llvm.org/LICENSE.txt for license information. // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // // This file contains a printer that converts from our internal representation // of machine-dependent LLVM code to PowerPC assembly language. This printer is // the output mechanism used by `llc'. // // Documentation at http://developer.apple.com/documentation/DeveloperTools/ // Reference/Assembler/ASMIntroduction/chapter_1_section_1.html // //===----------------------------------------------------------------------===// #include "MCTargetDesc/PPCInstPrinter.h" #include "MCTargetDesc/PPCMCExpr.h" #include "MCTargetDesc/PPCMCTargetDesc.h" #include "MCTargetDesc/PPCPredicates.h" #include "PPC.h" #include "PPCInstrInfo.h" #include "PPCMachineFunctionInfo.h" #include "PPCSubtarget.h" #include "PPCTargetMachine.h" #include "PPCTargetStreamer.h" #include "TargetInfo/PowerPCTargetInfo.h" #include "llvm/ADT/MapVector.h" #include "llvm/ADT/StringRef.h" #include "llvm/ADT/Triple.h" #include "llvm/ADT/Twine.h" #include "llvm/BinaryFormat/ELF.h" #include "llvm/BinaryFormat/MachO.h" #include "llvm/CodeGen/AsmPrinter.h" #include "llvm/CodeGen/MachineBasicBlock.h" #include "llvm/CodeGen/MachineFunction.h" #include "llvm/CodeGen/MachineInstr.h" #include "llvm/CodeGen/MachineModuleInfoImpls.h" #include "llvm/CodeGen/MachineOperand.h" #include "llvm/CodeGen/MachineRegisterInfo.h" #include "llvm/CodeGen/StackMaps.h" #include "llvm/CodeGen/TargetLoweringObjectFileImpl.h" #include "llvm/IR/DataLayout.h" #include "llvm/IR/GlobalValue.h" #include "llvm/IR/GlobalVariable.h" #include "llvm/IR/Module.h" #include "llvm/MC/MCAsmInfo.h" #include "llvm/MC/MCContext.h" #include "llvm/MC/MCExpr.h" #include "llvm/MC/MCInst.h" #include "llvm/MC/MCInstBuilder.h" #include "llvm/MC/MCSectionELF.h" #include "llvm/MC/MCSectionMachO.h" #include "llvm/MC/MCSectionXCOFF.h" #include "llvm/MC/MCStreamer.h" #include "llvm/MC/MCSymbol.h" #include "llvm/MC/MCSymbolELF.h" #include "llvm/MC/MCSymbolXCOFF.h" #include "llvm/MC/SectionKind.h" #include "llvm/Support/Casting.h" #include "llvm/Support/CodeGen.h" #include "llvm/Support/Debug.h" #include "llvm/Support/ErrorHandling.h" #include "llvm/Support/TargetRegistry.h" #include "llvm/Support/raw_ostream.h" #include "llvm/Target/TargetMachine.h" #include #include #include #include #include using namespace llvm; #define DEBUG_TYPE "asmprinter" namespace { class PPCAsmPrinter : public AsmPrinter { protected: MapVector TOC; const PPCSubtarget *Subtarget = nullptr; StackMaps SM; virtual MCSymbol *getMCSymbolForTOCPseudoMO(const MachineOperand &MO); public: explicit PPCAsmPrinter(TargetMachine &TM, std::unique_ptr Streamer) : AsmPrinter(TM, std::move(Streamer)), SM(*this) {} StringRef getPassName() const override { return "PowerPC Assembly Printer"; } MCSymbol *lookUpOrCreateTOCEntry(const MCSymbol *Sym); bool doInitialization(Module &M) override { if (!TOC.empty()) TOC.clear(); return AsmPrinter::doInitialization(M); } void emitInstruction(const MachineInstr *MI) override; /// This function is for PrintAsmOperand and PrintAsmMemoryOperand, /// invoked by EmitMSInlineAsmStr and EmitGCCInlineAsmStr only. /// The \p MI would be INLINEASM ONLY. void printOperand(const MachineInstr *MI, unsigned OpNo, raw_ostream &O); void PrintSymbolOperand(const MachineOperand &MO, raw_ostream &O) override; bool PrintAsmOperand(const MachineInstr *MI, unsigned OpNo, const char *ExtraCode, raw_ostream &O) override; bool PrintAsmMemoryOperand(const MachineInstr *MI, unsigned OpNo, const char *ExtraCode, raw_ostream &O) override; void emitEndOfAsmFile(Module &M) override; void LowerSTACKMAP(StackMaps &SM, const MachineInstr &MI); void LowerPATCHPOINT(StackMaps &SM, const MachineInstr &MI); void EmitTlsCall(const MachineInstr *MI, MCSymbolRefExpr::VariantKind VK); bool runOnMachineFunction(MachineFunction &MF) override { Subtarget = &MF.getSubtarget(); bool Changed = AsmPrinter::runOnMachineFunction(MF); emitXRayTable(); return Changed; } }; /// PPCLinuxAsmPrinter - PowerPC assembly printer, customized for Linux class PPCLinuxAsmPrinter : public PPCAsmPrinter { public: explicit PPCLinuxAsmPrinter(TargetMachine &TM, std::unique_ptr Streamer) : PPCAsmPrinter(TM, std::move(Streamer)) {} StringRef getPassName() const override { return "Linux PPC Assembly Printer"; } void emitStartOfAsmFile(Module &M) override; void emitEndOfAsmFile(Module &) override; void emitFunctionEntryLabel() override; void emitFunctionBodyStart() override; void emitFunctionBodyEnd() override; void emitInstruction(const MachineInstr *MI) override; }; class PPCAIXAsmPrinter : public PPCAsmPrinter { private: static void ValidateGV(const GlobalVariable *GV); protected: MCSymbol *getMCSymbolForTOCPseudoMO(const MachineOperand &MO) override; public: PPCAIXAsmPrinter(TargetMachine &TM, std::unique_ptr Streamer) : PPCAsmPrinter(TM, std::move(Streamer)) {} StringRef getPassName() const override { return "AIX PPC Assembly Printer"; } void SetupMachineFunction(MachineFunction &MF) override; const MCExpr *lowerConstant(const Constant *CV) override; void emitGlobalVariable(const GlobalVariable *GV) override; void emitFunctionDescriptor() override; void emitEndOfAsmFile(Module &) override; }; } // end anonymous namespace void PPCAsmPrinter::PrintSymbolOperand(const MachineOperand &MO, raw_ostream &O) { // Computing the address of a global symbol, not calling it. const GlobalValue *GV = MO.getGlobal(); getSymbol(GV)->print(O, MAI); printOffset(MO.getOffset(), O); } void PPCAsmPrinter::printOperand(const MachineInstr *MI, unsigned OpNo, raw_ostream &O) { const DataLayout &DL = getDataLayout(); const MachineOperand &MO = MI->getOperand(OpNo); switch (MO.getType()) { case MachineOperand::MO_Register: { // The MI is INLINEASM ONLY and UseVSXReg is always false. const char *RegName = PPCInstPrinter::getRegisterName(MO.getReg()); // Linux assembler (Others?) does not take register mnemonics. // FIXME - What about special registers used in mfspr/mtspr? O << PPCRegisterInfo::stripRegisterPrefix(RegName); return; } case MachineOperand::MO_Immediate: O << MO.getImm(); return; case MachineOperand::MO_MachineBasicBlock: MO.getMBB()->getSymbol()->print(O, MAI); return; case MachineOperand::MO_ConstantPoolIndex: O << DL.getPrivateGlobalPrefix() << "CPI" << getFunctionNumber() << '_' << MO.getIndex(); return; case MachineOperand::MO_BlockAddress: GetBlockAddressSymbol(MO.getBlockAddress())->print(O, MAI); return; case MachineOperand::MO_GlobalAddress: { PrintSymbolOperand(MO, O); return; } default: O << ""; return; } } /// PrintAsmOperand - Print out an operand for an inline asm expression. /// bool PPCAsmPrinter::PrintAsmOperand(const MachineInstr *MI, unsigned OpNo, const char *ExtraCode, raw_ostream &O) { // Does this asm operand have a single letter operand modifier? if (ExtraCode && ExtraCode[0]) { if (ExtraCode[1] != 0) return true; // Unknown modifier. switch (ExtraCode[0]) { default: // See if this is a generic print operand return AsmPrinter::PrintAsmOperand(MI, OpNo, ExtraCode, O); case 'L': // Write second word of DImode reference. // Verify that this operand has two consecutive registers. if (!MI->getOperand(OpNo).isReg() || OpNo+1 == MI->getNumOperands() || !MI->getOperand(OpNo+1).isReg()) return true; ++OpNo; // Return the high-part. break; case 'I': // Write 'i' if an integer constant, otherwise nothing. Used to print // addi vs add, etc. if (MI->getOperand(OpNo).isImm()) O << "i"; return false; case 'x': if(!MI->getOperand(OpNo).isReg()) return true; // This operand uses VSX numbering. // If the operand is a VMX register, convert it to a VSX register. Register Reg = MI->getOperand(OpNo).getReg(); if (PPCInstrInfo::isVRRegister(Reg)) Reg = PPC::VSX32 + (Reg - PPC::V0); else if (PPCInstrInfo::isVFRegister(Reg)) Reg = PPC::VSX32 + (Reg - PPC::VF0); const char *RegName; RegName = PPCInstPrinter::getRegisterName(Reg); RegName = PPCRegisterInfo::stripRegisterPrefix(RegName); O << RegName; return false; } } printOperand(MI, OpNo, O); return false; } // At the moment, all inline asm memory operands are a single register. // In any case, the output of this routine should always be just one // assembler operand. bool PPCAsmPrinter::PrintAsmMemoryOperand(const MachineInstr *MI, unsigned OpNo, const char *ExtraCode, raw_ostream &O) { if (ExtraCode && ExtraCode[0]) { if (ExtraCode[1] != 0) return true; // Unknown modifier. switch (ExtraCode[0]) { default: return true; // Unknown modifier. case 'y': { // A memory reference for an X-form instruction O << "0, "; printOperand(MI, OpNo, O); return false; } case 'U': // Print 'u' for update form. case 'X': // Print 'x' for indexed form. { // FIXME: Currently for PowerPC memory operands are always loaded // into a register, so we never get an update or indexed form. // This is bad even for offset forms, since even if we know we // have a value in -16(r1), we will generate a load into r // and then load from 0(r). Until that issue is fixed, // tolerate 'U' and 'X' but don't output anything. assert(MI->getOperand(OpNo).isReg()); return false; } } } assert(MI->getOperand(OpNo).isReg()); O << "0("; printOperand(MI, OpNo, O); O << ")"; return false; } /// lookUpOrCreateTOCEntry -- Given a symbol, look up whether a TOC entry /// exists for it. If not, create one. Then return a symbol that references /// the TOC entry. MCSymbol *PPCAsmPrinter::lookUpOrCreateTOCEntry(const MCSymbol *Sym) { MCSymbol *&TOCEntry = TOC[Sym]; if (!TOCEntry) TOCEntry = createTempSymbol("C"); return TOCEntry; } void PPCAsmPrinter::emitEndOfAsmFile(Module &M) { emitStackMaps(SM); } void PPCAsmPrinter::LowerSTACKMAP(StackMaps &SM, const MachineInstr &MI) { unsigned NumNOPBytes = MI.getOperand(1).getImm(); auto &Ctx = OutStreamer->getContext(); MCSymbol *MILabel = Ctx.createTempSymbol(); OutStreamer->emitLabel(MILabel); SM.recordStackMap(*MILabel, MI); assert(NumNOPBytes % 4 == 0 && "Invalid number of NOP bytes requested!"); // Scan ahead to trim the shadow. const MachineBasicBlock &MBB = *MI.getParent(); MachineBasicBlock::const_iterator MII(MI); ++MII; while (NumNOPBytes > 0) { if (MII == MBB.end() || MII->isCall() || MII->getOpcode() == PPC::DBG_VALUE || MII->getOpcode() == TargetOpcode::PATCHPOINT || MII->getOpcode() == TargetOpcode::STACKMAP) break; ++MII; NumNOPBytes -= 4; } // Emit nops. for (unsigned i = 0; i < NumNOPBytes; i += 4) EmitToStreamer(*OutStreamer, MCInstBuilder(PPC::NOP)); } // Lower a patchpoint of the form: // [], , , , void PPCAsmPrinter::LowerPATCHPOINT(StackMaps &SM, const MachineInstr &MI) { auto &Ctx = OutStreamer->getContext(); MCSymbol *MILabel = Ctx.createTempSymbol(); OutStreamer->emitLabel(MILabel); SM.recordPatchPoint(*MILabel, MI); PatchPointOpers Opers(&MI); unsigned EncodedBytes = 0; const MachineOperand &CalleeMO = Opers.getCallTarget(); if (CalleeMO.isImm()) { int64_t CallTarget = CalleeMO.getImm(); if (CallTarget) { assert((CallTarget & 0xFFFFFFFFFFFF) == CallTarget && "High 16 bits of call target should be zero."); Register ScratchReg = MI.getOperand(Opers.getNextScratchIdx()).getReg(); EncodedBytes = 0; // Materialize the jump address: EmitToStreamer(*OutStreamer, MCInstBuilder(PPC::LI8) .addReg(ScratchReg) .addImm((CallTarget >> 32) & 0xFFFF)); ++EncodedBytes; EmitToStreamer(*OutStreamer, MCInstBuilder(PPC::RLDIC) .addReg(ScratchReg) .addReg(ScratchReg) .addImm(32).addImm(16)); ++EncodedBytes; EmitToStreamer(*OutStreamer, MCInstBuilder(PPC::ORIS8) .addReg(ScratchReg) .addReg(ScratchReg) .addImm((CallTarget >> 16) & 0xFFFF)); ++EncodedBytes; EmitToStreamer(*OutStreamer, MCInstBuilder(PPC::ORI8) .addReg(ScratchReg) .addReg(ScratchReg) .addImm(CallTarget & 0xFFFF)); // Save the current TOC pointer before the remote call. int TOCSaveOffset = Subtarget->getFrameLowering()->getTOCSaveOffset(); EmitToStreamer(*OutStreamer, MCInstBuilder(PPC::STD) .addReg(PPC::X2) .addImm(TOCSaveOffset) .addReg(PPC::X1)); ++EncodedBytes; // If we're on ELFv1, then we need to load the actual function pointer // from the function descriptor. if (!Subtarget->isELFv2ABI()) { // Load the new TOC pointer and the function address, but not r11 // (needing this is rare, and loading it here would prevent passing it // via a 'nest' parameter. EmitToStreamer(*OutStreamer, MCInstBuilder(PPC::LD) .addReg(PPC::X2) .addImm(8) .addReg(ScratchReg)); ++EncodedBytes; EmitToStreamer(*OutStreamer, MCInstBuilder(PPC::LD) .addReg(ScratchReg) .addImm(0) .addReg(ScratchReg)); ++EncodedBytes; } EmitToStreamer(*OutStreamer, MCInstBuilder(PPC::MTCTR8) .addReg(ScratchReg)); ++EncodedBytes; EmitToStreamer(*OutStreamer, MCInstBuilder(PPC::BCTRL8)); ++EncodedBytes; // Restore the TOC pointer after the call. EmitToStreamer(*OutStreamer, MCInstBuilder(PPC::LD) .addReg(PPC::X2) .addImm(TOCSaveOffset) .addReg(PPC::X1)); ++EncodedBytes; } } else if (CalleeMO.isGlobal()) { const GlobalValue *GValue = CalleeMO.getGlobal(); MCSymbol *MOSymbol = getSymbol(GValue); const MCExpr *SymVar = MCSymbolRefExpr::create(MOSymbol, OutContext); EmitToStreamer(*OutStreamer, MCInstBuilder(PPC::BL8_NOP) .addExpr(SymVar)); EncodedBytes += 2; } // Each instruction is 4 bytes. EncodedBytes *= 4; // Emit padding. unsigned NumBytes = Opers.getNumPatchBytes(); assert(NumBytes >= EncodedBytes && "Patchpoint can't request size less than the length of a call."); assert((NumBytes - EncodedBytes) % 4 == 0 && "Invalid number of NOP bytes requested!"); for (unsigned i = EncodedBytes; i < NumBytes; i += 4) EmitToStreamer(*OutStreamer, MCInstBuilder(PPC::NOP)); } /// EmitTlsCall -- Given a GETtls[ld]ADDR[32] instruction, print a /// call to __tls_get_addr to the current output stream. void PPCAsmPrinter::EmitTlsCall(const MachineInstr *MI, MCSymbolRefExpr::VariantKind VK) { StringRef Name = "__tls_get_addr"; MCSymbol *TlsGetAddr = OutContext.getOrCreateSymbol(Name); MCSymbolRefExpr::VariantKind Kind = MCSymbolRefExpr::VK_None; const Module *M = MF->getFunction().getParent(); assert(MI->getOperand(0).isReg() && ((Subtarget->isPPC64() && MI->getOperand(0).getReg() == PPC::X3) || (!Subtarget->isPPC64() && MI->getOperand(0).getReg() == PPC::R3)) && "GETtls[ld]ADDR[32] must define GPR3"); assert(MI->getOperand(1).isReg() && ((Subtarget->isPPC64() && MI->getOperand(1).getReg() == PPC::X3) || (!Subtarget->isPPC64() && MI->getOperand(1).getReg() == PPC::R3)) && "GETtls[ld]ADDR[32] must read GPR3"); if (Subtarget->is32BitELFABI() && isPositionIndependent()) Kind = MCSymbolRefExpr::VK_PLT; const MCExpr *TlsRef = MCSymbolRefExpr::create(TlsGetAddr, Kind, OutContext); // Add 32768 offset to the symbol so we follow up the latest GOT/PLT ABI. if (Kind == MCSymbolRefExpr::VK_PLT && Subtarget->isSecurePlt() && M->getPICLevel() == PICLevel::BigPIC) TlsRef = MCBinaryExpr::createAdd( TlsRef, MCConstantExpr::create(32768, OutContext), OutContext); const MachineOperand &MO = MI->getOperand(2); const GlobalValue *GValue = MO.getGlobal(); MCSymbol *MOSymbol = getSymbol(GValue); const MCExpr *SymVar = MCSymbolRefExpr::create(MOSymbol, VK, OutContext); EmitToStreamer(*OutStreamer, MCInstBuilder(Subtarget->isPPC64() ? PPC::BL8_NOP_TLS : PPC::BL_TLS) .addExpr(TlsRef) .addExpr(SymVar)); } /// Map a machine operand for a TOC pseudo-machine instruction to its /// corresponding MCSymbol. MCSymbol *PPCAsmPrinter::getMCSymbolForTOCPseudoMO(const MachineOperand &MO) { switch (MO.getType()) { case MachineOperand::MO_GlobalAddress: return getSymbol(MO.getGlobal()); case MachineOperand::MO_ConstantPoolIndex: return GetCPISymbol(MO.getIndex()); case MachineOperand::MO_JumpTableIndex: return GetJTISymbol(MO.getIndex()); case MachineOperand::MO_BlockAddress: return GetBlockAddressSymbol(MO.getBlockAddress()); default: llvm_unreachable("Unexpected operand type to get symbol."); } } /// EmitInstruction -- Print out a single PowerPC MI in Darwin syntax to /// the current output stream. /// void PPCAsmPrinter::emitInstruction(const MachineInstr *MI) { MCInst TmpInst; const bool IsPPC64 = Subtarget->isPPC64(); const bool IsAIX = Subtarget->isAIXABI(); const Module *M = MF->getFunction().getParent(); PICLevel::Level PL = M->getPICLevel(); #ifndef NDEBUG // Validate that SPE and FPU are mutually exclusive in codegen if (!MI->isInlineAsm()) { for (const MachineOperand &MO: MI->operands()) { if (MO.isReg()) { Register Reg = MO.getReg(); if (Subtarget->hasSPE()) { if (PPC::F4RCRegClass.contains(Reg) || PPC::F8RCRegClass.contains(Reg) || PPC::QBRCRegClass.contains(Reg) || PPC::QFRCRegClass.contains(Reg) || PPC::QSRCRegClass.contains(Reg) || PPC::VFRCRegClass.contains(Reg) || PPC::VRRCRegClass.contains(Reg) || PPC::VSFRCRegClass.contains(Reg) || PPC::VSSRCRegClass.contains(Reg) ) llvm_unreachable("SPE targets cannot have FPRegs!"); } else { if (PPC::SPERCRegClass.contains(Reg)) llvm_unreachable("SPE register found in FPU-targeted code!"); } } } } #endif // Lower multi-instruction pseudo operations. switch (MI->getOpcode()) { default: break; case TargetOpcode::DBG_VALUE: llvm_unreachable("Should be handled target independently"); case TargetOpcode::STACKMAP: return LowerSTACKMAP(SM, *MI); case TargetOpcode::PATCHPOINT: return LowerPATCHPOINT(SM, *MI); case PPC::MoveGOTtoLR: { // Transform %lr = MoveGOTtoLR // Into this: bl _GLOBAL_OFFSET_TABLE_@local-4 // _GLOBAL_OFFSET_TABLE_@local-4 (instruction preceding // _GLOBAL_OFFSET_TABLE_) has exactly one instruction: // blrl // This will return the pointer to _GLOBAL_OFFSET_TABLE_@local MCSymbol *GOTSymbol = OutContext.getOrCreateSymbol(StringRef("_GLOBAL_OFFSET_TABLE_")); const MCExpr *OffsExpr = MCBinaryExpr::createSub(MCSymbolRefExpr::create(GOTSymbol, MCSymbolRefExpr::VK_PPC_LOCAL, OutContext), MCConstantExpr::create(4, OutContext), OutContext); // Emit the 'bl'. EmitToStreamer(*OutStreamer, MCInstBuilder(PPC::BL).addExpr(OffsExpr)); return; } case PPC::MovePCtoLR: case PPC::MovePCtoLR8: { // Transform %lr = MovePCtoLR // Into this, where the label is the PIC base: // bl L1$pb // L1$pb: MCSymbol *PICBase = MF->getPICBaseSymbol(); // Emit the 'bl'. EmitToStreamer(*OutStreamer, MCInstBuilder(PPC::BL) // FIXME: We would like an efficient form for this, so we // don't have to do a lot of extra uniquing. .addExpr(MCSymbolRefExpr::create(PICBase, OutContext))); // Emit the label. OutStreamer->emitLabel(PICBase); return; } case PPC::UpdateGBR: { // Transform %rd = UpdateGBR(%rt, %ri) // Into: lwz %rt, .L0$poff - .L0$pb(%ri) // add %rd, %rt, %ri // or into (if secure plt mode is on): // addis r30, r30, {.LTOC,_GLOBAL_OFFSET_TABLE} - .L0$pb@ha // addi r30, r30, {.LTOC,_GLOBAL_OFFSET_TABLE} - .L0$pb@l // Get the offset from the GOT Base Register to the GOT LowerPPCMachineInstrToMCInst(MI, TmpInst, *this); if (Subtarget->isSecurePlt() && isPositionIndependent() ) { unsigned PICR = TmpInst.getOperand(0).getReg(); MCSymbol *BaseSymbol = OutContext.getOrCreateSymbol( M->getPICLevel() == PICLevel::SmallPIC ? "_GLOBAL_OFFSET_TABLE_" : ".LTOC"); const MCExpr *PB = MCSymbolRefExpr::create(MF->getPICBaseSymbol(), OutContext); const MCExpr *DeltaExpr = MCBinaryExpr::createSub( MCSymbolRefExpr::create(BaseSymbol, OutContext), PB, OutContext); const MCExpr *DeltaHi = PPCMCExpr::createHa(DeltaExpr, OutContext); EmitToStreamer( *OutStreamer, MCInstBuilder(PPC::ADDIS).addReg(PICR).addReg(PICR).addExpr(DeltaHi)); const MCExpr *DeltaLo = PPCMCExpr::createLo(DeltaExpr, OutContext); EmitToStreamer( *OutStreamer, MCInstBuilder(PPC::ADDI).addReg(PICR).addReg(PICR).addExpr(DeltaLo)); return; } else { MCSymbol *PICOffset = MF->getInfo()->getPICOffsetSymbol(); TmpInst.setOpcode(PPC::LWZ); const MCExpr *Exp = MCSymbolRefExpr::create(PICOffset, MCSymbolRefExpr::VK_None, OutContext); const MCExpr *PB = MCSymbolRefExpr::create(MF->getPICBaseSymbol(), MCSymbolRefExpr::VK_None, OutContext); const MCOperand TR = TmpInst.getOperand(1); const MCOperand PICR = TmpInst.getOperand(0); // Step 1: lwz %rt, .L$poff - .L$pb(%ri) TmpInst.getOperand(1) = MCOperand::createExpr(MCBinaryExpr::createSub(Exp, PB, OutContext)); TmpInst.getOperand(0) = TR; TmpInst.getOperand(2) = PICR; EmitToStreamer(*OutStreamer, TmpInst); TmpInst.setOpcode(PPC::ADD4); TmpInst.getOperand(0) = PICR; TmpInst.getOperand(1) = TR; TmpInst.getOperand(2) = PICR; EmitToStreamer(*OutStreamer, TmpInst); return; } } case PPC::LWZtoc: { // Transform %rN = LWZtoc @op1, %r2 LowerPPCMachineInstrToMCInst(MI, TmpInst, *this); // Change the opcode to LWZ. TmpInst.setOpcode(PPC::LWZ); const MachineOperand &MO = MI->getOperand(1); assert((MO.isGlobal() || MO.isCPI() || MO.isJTI() || MO.isBlockAddress()) && "Invalid operand for LWZtoc."); // Map the operand to its corresponding MCSymbol. const MCSymbol *const MOSymbol = getMCSymbolForTOCPseudoMO(MO); // Create a reference to the GOT entry for the symbol. The GOT entry will be // synthesized later. if (PL == PICLevel::SmallPIC && !IsAIX) { const MCExpr *Exp = MCSymbolRefExpr::create(MOSymbol, MCSymbolRefExpr::VK_GOT, OutContext); TmpInst.getOperand(1) = MCOperand::createExpr(Exp); EmitToStreamer(*OutStreamer, TmpInst); return; } // Otherwise, use the TOC. 'TOCEntry' is a label used to reference the // storage allocated in the TOC which contains the address of // 'MOSymbol'. Said TOC entry will be synthesized later. MCSymbol *TOCEntry = lookUpOrCreateTOCEntry(MOSymbol); const MCExpr *Exp = MCSymbolRefExpr::create(TOCEntry, MCSymbolRefExpr::VK_None, OutContext); // AIX uses the label directly as the lwz displacement operand for // references into the toc section. The displacement value will be generated // relative to the toc-base. if (IsAIX) { assert( TM.getCodeModel() == CodeModel::Small && "This pseudo should only be selected for 32-bit small code model."); TmpInst.getOperand(1) = MCOperand::createExpr(Exp); EmitToStreamer(*OutStreamer, TmpInst); return; } // Create an explicit subtract expression between the local symbol and // '.LTOC' to manifest the toc-relative offset. const MCExpr *PB = MCSymbolRefExpr::create( OutContext.getOrCreateSymbol(Twine(".LTOC")), OutContext); Exp = MCBinaryExpr::createSub(Exp, PB, OutContext); TmpInst.getOperand(1) = MCOperand::createExpr(Exp); EmitToStreamer(*OutStreamer, TmpInst); return; } case PPC::LDtocJTI: case PPC::LDtocCPT: case PPC::LDtocBA: case PPC::LDtoc: { // Transform %x3 = LDtoc @min1, %x2 LowerPPCMachineInstrToMCInst(MI, TmpInst, *this); // Change the opcode to LD. TmpInst.setOpcode(PPC::LD); const MachineOperand &MO = MI->getOperand(1); assert((MO.isGlobal() || MO.isCPI() || MO.isJTI() || MO.isBlockAddress()) && "Invalid operand!"); // Map the machine operand to its corresponding MCSymbol, then map the // global address operand to be a reference to the TOC entry we will // synthesize later. MCSymbol *TOCEntry = lookUpOrCreateTOCEntry(getMCSymbolForTOCPseudoMO(MO)); const MCSymbolRefExpr::VariantKind VK = IsAIX ? MCSymbolRefExpr::VK_None : MCSymbolRefExpr::VK_PPC_TOC; const MCExpr *Exp = MCSymbolRefExpr::create(TOCEntry, VK, OutContext); TmpInst.getOperand(1) = MCOperand::createExpr(Exp); EmitToStreamer(*OutStreamer, TmpInst); return; } case PPC::ADDIStocHA: { assert((IsAIX && !IsPPC64 && TM.getCodeModel() == CodeModel::Large) && "This pseudo should only be selected for 32-bit large code model on" " AIX."); // Transform %rd = ADDIStocHA %rA, @sym(%r2) LowerPPCMachineInstrToMCInst(MI, TmpInst, *this); // Change the opcode to ADDIS. TmpInst.setOpcode(PPC::ADDIS); const MachineOperand &MO = MI->getOperand(2); assert((MO.isGlobal() || MO.isCPI() || MO.isJTI() || MO.isBlockAddress()) && "Invalid operand for ADDIStocHA."); // Map the machine operand to its corresponding MCSymbol. MCSymbol *MOSymbol = getMCSymbolForTOCPseudoMO(MO); // Always use TOC on AIX. Map the global address operand to be a reference // to the TOC entry we will synthesize later. 'TOCEntry' is a label used to // reference the storage allocated in the TOC which contains the address of // 'MOSymbol'. MCSymbol *TOCEntry = lookUpOrCreateTOCEntry(MOSymbol); const MCExpr *Exp = MCSymbolRefExpr::create(TOCEntry, MCSymbolRefExpr::VK_PPC_U, OutContext); TmpInst.getOperand(2) = MCOperand::createExpr(Exp); EmitToStreamer(*OutStreamer, TmpInst); return; } case PPC::LWZtocL: { assert(IsAIX && !IsPPC64 && TM.getCodeModel() == CodeModel::Large && "This pseudo should only be selected for 32-bit large code model on" " AIX."); // Transform %rd = LWZtocL @sym, %rs. LowerPPCMachineInstrToMCInst(MI, TmpInst, *this); // Change the opcode to lwz. TmpInst.setOpcode(PPC::LWZ); const MachineOperand &MO = MI->getOperand(1); assert((MO.isGlobal() || MO.isCPI() || MO.isJTI() || MO.isBlockAddress()) && "Invalid operand for LWZtocL."); // Map the machine operand to its corresponding MCSymbol. MCSymbol *MOSymbol = getMCSymbolForTOCPseudoMO(MO); // Always use TOC on AIX. Map the global address operand to be a reference // to the TOC entry we will synthesize later. 'TOCEntry' is a label used to // reference the storage allocated in the TOC which contains the address of // 'MOSymbol'. MCSymbol *TOCEntry = lookUpOrCreateTOCEntry(MOSymbol); const MCExpr *Exp = MCSymbolRefExpr::create(TOCEntry, MCSymbolRefExpr::VK_PPC_L, OutContext); TmpInst.getOperand(1) = MCOperand::createExpr(Exp); EmitToStreamer(*OutStreamer, TmpInst); return; } case PPC::ADDIStocHA8: { // Transform %xd = ADDIStocHA8 %x2, @sym LowerPPCMachineInstrToMCInst(MI, TmpInst, *this); // Change the opcode to ADDIS8. If the global address is the address of // an external symbol, is a jump table address, is a block address, or is a // constant pool index with large code model enabled, then generate a TOC // entry and reference that. Otherwise, reference the symbol directly. TmpInst.setOpcode(PPC::ADDIS8); const MachineOperand &MO = MI->getOperand(2); assert((MO.isGlobal() || MO.isCPI() || MO.isJTI() || MO.isBlockAddress()) && "Invalid operand for ADDIStocHA8!"); const MCSymbol *MOSymbol = getMCSymbolForTOCPseudoMO(MO); const bool GlobalToc = MO.isGlobal() && Subtarget->isGVIndirectSymbol(MO.getGlobal()); if (GlobalToc || MO.isJTI() || MO.isBlockAddress() || (MO.isCPI() && TM.getCodeModel() == CodeModel::Large)) MOSymbol = lookUpOrCreateTOCEntry(MOSymbol); const MCSymbolRefExpr::VariantKind VK = IsAIX ? MCSymbolRefExpr::VK_PPC_U : MCSymbolRefExpr::VK_PPC_TOC_HA; const MCExpr *Exp = MCSymbolRefExpr::create(MOSymbol, VK, OutContext); if (!MO.isJTI() && MO.getOffset()) Exp = MCBinaryExpr::createAdd(Exp, MCConstantExpr::create(MO.getOffset(), OutContext), OutContext); TmpInst.getOperand(2) = MCOperand::createExpr(Exp); EmitToStreamer(*OutStreamer, TmpInst); return; } case PPC::LDtocL: { // Transform %xd = LDtocL @sym, %xs LowerPPCMachineInstrToMCInst(MI, TmpInst, *this); // Change the opcode to LD. If the global address is the address of // an external symbol, is a jump table address, is a block address, or is // a constant pool index with large code model enabled, then generate a // TOC entry and reference that. Otherwise, reference the symbol directly. TmpInst.setOpcode(PPC::LD); const MachineOperand &MO = MI->getOperand(1); assert((MO.isGlobal() || MO.isCPI() || MO.isJTI() || MO.isBlockAddress()) && "Invalid operand for LDtocL!"); LLVM_DEBUG(assert( (!MO.isGlobal() || Subtarget->isGVIndirectSymbol(MO.getGlobal())) && "LDtocL used on symbol that could be accessed directly is " "invalid. Must match ADDIStocHA8.")); const MCSymbol *MOSymbol = getMCSymbolForTOCPseudoMO(MO); if (!MO.isCPI() || TM.getCodeModel() == CodeModel::Large) MOSymbol = lookUpOrCreateTOCEntry(MOSymbol); const MCSymbolRefExpr::VariantKind VK = IsAIX ? MCSymbolRefExpr::VK_PPC_L : MCSymbolRefExpr::VK_PPC_TOC_LO; const MCExpr *Exp = MCSymbolRefExpr::create(MOSymbol, VK, OutContext); TmpInst.getOperand(1) = MCOperand::createExpr(Exp); EmitToStreamer(*OutStreamer, TmpInst); return; } case PPC::ADDItocL: { // Transform %xd = ADDItocL %xs, @sym LowerPPCMachineInstrToMCInst(MI, TmpInst, *this); // Change the opcode to ADDI8. If the global address is external, then // generate a TOC entry and reference that. Otherwise, reference the // symbol directly. TmpInst.setOpcode(PPC::ADDI8); const MachineOperand &MO = MI->getOperand(2); assert((MO.isGlobal() || MO.isCPI()) && "Invalid operand for ADDItocL."); LLVM_DEBUG(assert( !(MO.isGlobal() && Subtarget->isGVIndirectSymbol(MO.getGlobal())) && "Interposable definitions must use indirect access.")); const MCExpr *Exp = MCSymbolRefExpr::create(getMCSymbolForTOCPseudoMO(MO), MCSymbolRefExpr::VK_PPC_TOC_LO, OutContext); TmpInst.getOperand(2) = MCOperand::createExpr(Exp); EmitToStreamer(*OutStreamer, TmpInst); return; } case PPC::ADDISgotTprelHA: { // Transform: %xd = ADDISgotTprelHA %x2, @sym // Into: %xd = ADDIS8 %x2, sym@got@tlsgd@ha assert(IsPPC64 && "Not supported for 32-bit PowerPC"); const MachineOperand &MO = MI->getOperand(2); const GlobalValue *GValue = MO.getGlobal(); MCSymbol *MOSymbol = getSymbol(GValue); const MCExpr *SymGotTprel = MCSymbolRefExpr::create(MOSymbol, MCSymbolRefExpr::VK_PPC_GOT_TPREL_HA, OutContext); EmitToStreamer(*OutStreamer, MCInstBuilder(PPC::ADDIS8) .addReg(MI->getOperand(0).getReg()) .addReg(MI->getOperand(1).getReg()) .addExpr(SymGotTprel)); return; } case PPC::LDgotTprelL: case PPC::LDgotTprelL32: { // Transform %xd = LDgotTprelL @sym, %xs LowerPPCMachineInstrToMCInst(MI, TmpInst, *this); // Change the opcode to LD. TmpInst.setOpcode(IsPPC64 ? PPC::LD : PPC::LWZ); const MachineOperand &MO = MI->getOperand(1); const GlobalValue *GValue = MO.getGlobal(); MCSymbol *MOSymbol = getSymbol(GValue); const MCExpr *Exp = MCSymbolRefExpr::create( MOSymbol, IsPPC64 ? MCSymbolRefExpr::VK_PPC_GOT_TPREL_LO : MCSymbolRefExpr::VK_PPC_GOT_TPREL, OutContext); TmpInst.getOperand(1) = MCOperand::createExpr(Exp); EmitToStreamer(*OutStreamer, TmpInst); return; } case PPC::PPC32PICGOT: { MCSymbol *GOTSymbol = OutContext.getOrCreateSymbol(StringRef("_GLOBAL_OFFSET_TABLE_")); MCSymbol *GOTRef = OutContext.createTempSymbol(); MCSymbol *NextInstr = OutContext.createTempSymbol(); EmitToStreamer(*OutStreamer, MCInstBuilder(PPC::BL) // FIXME: We would like an efficient form for this, so we don't have to do // a lot of extra uniquing. .addExpr(MCSymbolRefExpr::create(NextInstr, OutContext))); const MCExpr *OffsExpr = MCBinaryExpr::createSub(MCSymbolRefExpr::create(GOTSymbol, OutContext), MCSymbolRefExpr::create(GOTRef, OutContext), OutContext); OutStreamer->emitLabel(GOTRef); OutStreamer->emitValue(OffsExpr, 4); OutStreamer->emitLabel(NextInstr); EmitToStreamer(*OutStreamer, MCInstBuilder(PPC::MFLR) .addReg(MI->getOperand(0).getReg())); EmitToStreamer(*OutStreamer, MCInstBuilder(PPC::LWZ) .addReg(MI->getOperand(1).getReg()) .addImm(0) .addReg(MI->getOperand(0).getReg())); EmitToStreamer(*OutStreamer, MCInstBuilder(PPC::ADD4) .addReg(MI->getOperand(0).getReg()) .addReg(MI->getOperand(1).getReg()) .addReg(MI->getOperand(0).getReg())); return; } case PPC::PPC32GOT: { MCSymbol *GOTSymbol = OutContext.getOrCreateSymbol(StringRef("_GLOBAL_OFFSET_TABLE_")); const MCExpr *SymGotTlsL = MCSymbolRefExpr::create( GOTSymbol, MCSymbolRefExpr::VK_PPC_LO, OutContext); const MCExpr *SymGotTlsHA = MCSymbolRefExpr::create( GOTSymbol, MCSymbolRefExpr::VK_PPC_HA, OutContext); EmitToStreamer(*OutStreamer, MCInstBuilder(PPC::LI) .addReg(MI->getOperand(0).getReg()) .addExpr(SymGotTlsL)); EmitToStreamer(*OutStreamer, MCInstBuilder(PPC::ADDIS) .addReg(MI->getOperand(0).getReg()) .addReg(MI->getOperand(0).getReg()) .addExpr(SymGotTlsHA)); return; } case PPC::ADDIStlsgdHA: { // Transform: %xd = ADDIStlsgdHA %x2, @sym // Into: %xd = ADDIS8 %x2, sym@got@tlsgd@ha assert(IsPPC64 && "Not supported for 32-bit PowerPC"); const MachineOperand &MO = MI->getOperand(2); const GlobalValue *GValue = MO.getGlobal(); MCSymbol *MOSymbol = getSymbol(GValue); const MCExpr *SymGotTlsGD = MCSymbolRefExpr::create(MOSymbol, MCSymbolRefExpr::VK_PPC_GOT_TLSGD_HA, OutContext); EmitToStreamer(*OutStreamer, MCInstBuilder(PPC::ADDIS8) .addReg(MI->getOperand(0).getReg()) .addReg(MI->getOperand(1).getReg()) .addExpr(SymGotTlsGD)); return; } case PPC::ADDItlsgdL: // Transform: %xd = ADDItlsgdL %xs, @sym // Into: %xd = ADDI8 %xs, sym@got@tlsgd@l case PPC::ADDItlsgdL32: { // Transform: %rd = ADDItlsgdL32 %rs, @sym // Into: %rd = ADDI %rs, sym@got@tlsgd const MachineOperand &MO = MI->getOperand(2); const GlobalValue *GValue = MO.getGlobal(); MCSymbol *MOSymbol = getSymbol(GValue); const MCExpr *SymGotTlsGD = MCSymbolRefExpr::create( MOSymbol, IsPPC64 ? MCSymbolRefExpr::VK_PPC_GOT_TLSGD_LO : MCSymbolRefExpr::VK_PPC_GOT_TLSGD, OutContext); EmitToStreamer(*OutStreamer, MCInstBuilder(IsPPC64 ? PPC::ADDI8 : PPC::ADDI) .addReg(MI->getOperand(0).getReg()) .addReg(MI->getOperand(1).getReg()) .addExpr(SymGotTlsGD)); return; } case PPC::GETtlsADDR: // Transform: %x3 = GETtlsADDR %x3, @sym // Into: BL8_NOP_TLS __tls_get_addr(sym at tlsgd) case PPC::GETtlsADDR32: { // Transform: %r3 = GETtlsADDR32 %r3, @sym // Into: BL_TLS __tls_get_addr(sym at tlsgd)@PLT EmitTlsCall(MI, MCSymbolRefExpr::VK_PPC_TLSGD); return; } case PPC::ADDIStlsldHA: { // Transform: %xd = ADDIStlsldHA %x2, @sym // Into: %xd = ADDIS8 %x2, sym@got@tlsld@ha assert(IsPPC64 && "Not supported for 32-bit PowerPC"); const MachineOperand &MO = MI->getOperand(2); const GlobalValue *GValue = MO.getGlobal(); MCSymbol *MOSymbol = getSymbol(GValue); const MCExpr *SymGotTlsLD = MCSymbolRefExpr::create(MOSymbol, MCSymbolRefExpr::VK_PPC_GOT_TLSLD_HA, OutContext); EmitToStreamer(*OutStreamer, MCInstBuilder(PPC::ADDIS8) .addReg(MI->getOperand(0).getReg()) .addReg(MI->getOperand(1).getReg()) .addExpr(SymGotTlsLD)); return; } case PPC::ADDItlsldL: // Transform: %xd = ADDItlsldL %xs, @sym // Into: %xd = ADDI8 %xs, sym@got@tlsld@l case PPC::ADDItlsldL32: { // Transform: %rd = ADDItlsldL32 %rs, @sym // Into: %rd = ADDI %rs, sym@got@tlsld const MachineOperand &MO = MI->getOperand(2); const GlobalValue *GValue = MO.getGlobal(); MCSymbol *MOSymbol = getSymbol(GValue); const MCExpr *SymGotTlsLD = MCSymbolRefExpr::create( MOSymbol, IsPPC64 ? MCSymbolRefExpr::VK_PPC_GOT_TLSLD_LO : MCSymbolRefExpr::VK_PPC_GOT_TLSLD, OutContext); EmitToStreamer(*OutStreamer, MCInstBuilder(IsPPC64 ? PPC::ADDI8 : PPC::ADDI) .addReg(MI->getOperand(0).getReg()) .addReg(MI->getOperand(1).getReg()) .addExpr(SymGotTlsLD)); return; } case PPC::GETtlsldADDR: // Transform: %x3 = GETtlsldADDR %x3, @sym // Into: BL8_NOP_TLS __tls_get_addr(sym at tlsld) case PPC::GETtlsldADDR32: { // Transform: %r3 = GETtlsldADDR32 %r3, @sym // Into: BL_TLS __tls_get_addr(sym at tlsld)@PLT EmitTlsCall(MI, MCSymbolRefExpr::VK_PPC_TLSLD); return; } case PPC::ADDISdtprelHA: // Transform: %xd = ADDISdtprelHA %xs, @sym // Into: %xd = ADDIS8 %xs, sym@dtprel@ha case PPC::ADDISdtprelHA32: { // Transform: %rd = ADDISdtprelHA32 %rs, @sym // Into: %rd = ADDIS %rs, sym@dtprel@ha const MachineOperand &MO = MI->getOperand(2); const GlobalValue *GValue = MO.getGlobal(); MCSymbol *MOSymbol = getSymbol(GValue); const MCExpr *SymDtprel = MCSymbolRefExpr::create(MOSymbol, MCSymbolRefExpr::VK_PPC_DTPREL_HA, OutContext); EmitToStreamer( *OutStreamer, MCInstBuilder(IsPPC64 ? PPC::ADDIS8 : PPC::ADDIS) .addReg(MI->getOperand(0).getReg()) .addReg(MI->getOperand(1).getReg()) .addExpr(SymDtprel)); return; } case PPC::ADDIdtprelL: // Transform: %xd = ADDIdtprelL %xs, @sym // Into: %xd = ADDI8 %xs, sym@dtprel@l case PPC::ADDIdtprelL32: { // Transform: %rd = ADDIdtprelL32 %rs, @sym // Into: %rd = ADDI %rs, sym@dtprel@l const MachineOperand &MO = MI->getOperand(2); const GlobalValue *GValue = MO.getGlobal(); MCSymbol *MOSymbol = getSymbol(GValue); const MCExpr *SymDtprel = MCSymbolRefExpr::create(MOSymbol, MCSymbolRefExpr::VK_PPC_DTPREL_LO, OutContext); EmitToStreamer(*OutStreamer, MCInstBuilder(IsPPC64 ? PPC::ADDI8 : PPC::ADDI) .addReg(MI->getOperand(0).getReg()) .addReg(MI->getOperand(1).getReg()) .addExpr(SymDtprel)); return; } case PPC::MFOCRF: case PPC::MFOCRF8: if (!Subtarget->hasMFOCRF()) { // Transform: %r3 = MFOCRF %cr7 // Into: %r3 = MFCR ;; cr7 unsigned NewOpcode = MI->getOpcode() == PPC::MFOCRF ? PPC::MFCR : PPC::MFCR8; OutStreamer->AddComment(PPCInstPrinter:: getRegisterName(MI->getOperand(1).getReg())); EmitToStreamer(*OutStreamer, MCInstBuilder(NewOpcode) .addReg(MI->getOperand(0).getReg())); return; } break; case PPC::MTOCRF: case PPC::MTOCRF8: if (!Subtarget->hasMFOCRF()) { // Transform: %cr7 = MTOCRF %r3 // Into: MTCRF mask, %r3 ;; cr7 unsigned NewOpcode = MI->getOpcode() == PPC::MTOCRF ? PPC::MTCRF : PPC::MTCRF8; unsigned Mask = 0x80 >> OutContext.getRegisterInfo() ->getEncodingValue(MI->getOperand(0).getReg()); OutStreamer->AddComment(PPCInstPrinter:: getRegisterName(MI->getOperand(0).getReg())); EmitToStreamer(*OutStreamer, MCInstBuilder(NewOpcode) .addImm(Mask) .addReg(MI->getOperand(1).getReg())); return; } break; case PPC::LD: case PPC::STD: case PPC::LWA_32: case PPC::LWA: { // Verify alignment is legal, so we don't create relocations // that can't be supported. // FIXME: This test is currently disabled for Darwin. The test // suite shows a handful of test cases that fail this check for // Darwin. Those need to be investigated before this sanity test // can be enabled for those subtargets. unsigned OpNum = (MI->getOpcode() == PPC::STD) ? 2 : 1; const MachineOperand &MO = MI->getOperand(OpNum); if (MO.isGlobal() && MO.getGlobal()->getAlignment() < 4) llvm_unreachable("Global must be word-aligned for LD, STD, LWA!"); // Now process the instruction normally. break; } } LowerPPCMachineInstrToMCInst(MI, TmpInst, *this); EmitToStreamer(*OutStreamer, TmpInst); } void PPCLinuxAsmPrinter::emitInstruction(const MachineInstr *MI) { if (!Subtarget->isPPC64()) return PPCAsmPrinter::emitInstruction(MI); switch (MI->getOpcode()) { default: return PPCAsmPrinter::emitInstruction(MI); case TargetOpcode::PATCHABLE_FUNCTION_ENTER: { // .begin: // b .end # lis 0, FuncId[16..32] // nop # li 0, FuncId[0..15] // std 0, -8(1) // mflr 0 // bl __xray_FunctionEntry // mtlr 0 // .end: // // Update compiler-rt/lib/xray/xray_powerpc64.cc accordingly when number // of instructions change. MCSymbol *BeginOfSled = OutContext.createTempSymbol(); MCSymbol *EndOfSled = OutContext.createTempSymbol(); OutStreamer->emitLabel(BeginOfSled); EmitToStreamer(*OutStreamer, MCInstBuilder(PPC::B).addExpr( MCSymbolRefExpr::create(EndOfSled, OutContext))); EmitToStreamer(*OutStreamer, MCInstBuilder(PPC::NOP)); EmitToStreamer( *OutStreamer, MCInstBuilder(PPC::STD).addReg(PPC::X0).addImm(-8).addReg(PPC::X1)); EmitToStreamer(*OutStreamer, MCInstBuilder(PPC::MFLR8).addReg(PPC::X0)); EmitToStreamer(*OutStreamer, MCInstBuilder(PPC::BL8_NOP) .addExpr(MCSymbolRefExpr::create( OutContext.getOrCreateSymbol("__xray_FunctionEntry"), OutContext))); EmitToStreamer(*OutStreamer, MCInstBuilder(PPC::MTLR8).addReg(PPC::X0)); OutStreamer->emitLabel(EndOfSled); recordSled(BeginOfSled, *MI, SledKind::FUNCTION_ENTER); break; } case TargetOpcode::PATCHABLE_RET: { unsigned RetOpcode = MI->getOperand(0).getImm(); MCInst RetInst; RetInst.setOpcode(RetOpcode); for (const auto &MO : make_range(std::next(MI->operands_begin()), MI->operands_end())) { MCOperand MCOp; if (LowerPPCMachineOperandToMCOperand(MO, MCOp, *this)) RetInst.addOperand(MCOp); } bool IsConditional; if (RetOpcode == PPC::BCCLR) { IsConditional = true; } else if (RetOpcode == PPC::TCRETURNdi8 || RetOpcode == PPC::TCRETURNri8 || RetOpcode == PPC::TCRETURNai8) { break; } else if (RetOpcode == PPC::BLR8 || RetOpcode == PPC::TAILB8) { IsConditional = false; } else { EmitToStreamer(*OutStreamer, RetInst); break; } MCSymbol *FallthroughLabel; if (IsConditional) { // Before: // bgtlr cr0 // // After: // ble cr0, .end // .p2align 3 // .begin: // blr # lis 0, FuncId[16..32] // nop # li 0, FuncId[0..15] // std 0, -8(1) // mflr 0 // bl __xray_FunctionExit // mtlr 0 // blr // .end: // // Update compiler-rt/lib/xray/xray_powerpc64.cc accordingly when number // of instructions change. FallthroughLabel = OutContext.createTempSymbol(); EmitToStreamer( *OutStreamer, MCInstBuilder(PPC::BCC) .addImm(PPC::InvertPredicate( static_cast(MI->getOperand(1).getImm()))) .addReg(MI->getOperand(2).getReg()) .addExpr(MCSymbolRefExpr::create(FallthroughLabel, OutContext))); RetInst = MCInst(); RetInst.setOpcode(PPC::BLR8); } // .p2align 3 // .begin: // b(lr)? # lis 0, FuncId[16..32] // nop # li 0, FuncId[0..15] // std 0, -8(1) // mflr 0 // bl __xray_FunctionExit // mtlr 0 // b(lr)? // // Update compiler-rt/lib/xray/xray_powerpc64.cc accordingly when number // of instructions change. OutStreamer->emitCodeAlignment(8); MCSymbol *BeginOfSled = OutContext.createTempSymbol(); OutStreamer->emitLabel(BeginOfSled); EmitToStreamer(*OutStreamer, RetInst); EmitToStreamer(*OutStreamer, MCInstBuilder(PPC::NOP)); EmitToStreamer( *OutStreamer, MCInstBuilder(PPC::STD).addReg(PPC::X0).addImm(-8).addReg(PPC::X1)); EmitToStreamer(*OutStreamer, MCInstBuilder(PPC::MFLR8).addReg(PPC::X0)); EmitToStreamer(*OutStreamer, MCInstBuilder(PPC::BL8_NOP) .addExpr(MCSymbolRefExpr::create( OutContext.getOrCreateSymbol("__xray_FunctionExit"), OutContext))); EmitToStreamer(*OutStreamer, MCInstBuilder(PPC::MTLR8).addReg(PPC::X0)); EmitToStreamer(*OutStreamer, RetInst); if (IsConditional) OutStreamer->emitLabel(FallthroughLabel); recordSled(BeginOfSled, *MI, SledKind::FUNCTION_EXIT); break; } case TargetOpcode::PATCHABLE_FUNCTION_EXIT: llvm_unreachable("PATCHABLE_FUNCTION_EXIT should never be emitted"); case TargetOpcode::PATCHABLE_TAIL_CALL: // TODO: Define a trampoline `__xray_FunctionTailExit` and differentiate a // normal function exit from a tail exit. llvm_unreachable("Tail call is handled in the normal case. See comments " "around this assert."); } } void PPCLinuxAsmPrinter::emitStartOfAsmFile(Module &M) { if (static_cast(TM).isELFv2ABI()) { PPCTargetStreamer *TS = static_cast(OutStreamer->getTargetStreamer()); if (TS) TS->emitAbiVersion(2); } if (static_cast(TM).isPPC64() || !isPositionIndependent()) return AsmPrinter::emitStartOfAsmFile(M); if (M.getPICLevel() == PICLevel::SmallPIC) return AsmPrinter::emitStartOfAsmFile(M); OutStreamer->SwitchSection(OutContext.getELFSection( ".got2", ELF::SHT_PROGBITS, ELF::SHF_WRITE | ELF::SHF_ALLOC)); MCSymbol *TOCSym = OutContext.getOrCreateSymbol(Twine(".LTOC")); MCSymbol *CurrentPos = OutContext.createTempSymbol(); OutStreamer->emitLabel(CurrentPos); // The GOT pointer points to the middle of the GOT, in order to reference the // entire 64kB range. 0x8000 is the midpoint. const MCExpr *tocExpr = MCBinaryExpr::createAdd(MCSymbolRefExpr::create(CurrentPos, OutContext), MCConstantExpr::create(0x8000, OutContext), OutContext); OutStreamer->emitAssignment(TOCSym, tocExpr); OutStreamer->SwitchSection(getObjFileLowering().getTextSection()); } void PPCLinuxAsmPrinter::emitFunctionEntryLabel() { // linux/ppc32 - Normal entry label. if (!Subtarget->isPPC64() && (!isPositionIndependent() || MF->getFunction().getParent()->getPICLevel() == PICLevel::SmallPIC)) return AsmPrinter::emitFunctionEntryLabel(); if (!Subtarget->isPPC64()) { const PPCFunctionInfo *PPCFI = MF->getInfo(); if (PPCFI->usesPICBase() && !Subtarget->isSecurePlt()) { MCSymbol *RelocSymbol = PPCFI->getPICOffsetSymbol(); MCSymbol *PICBase = MF->getPICBaseSymbol(); OutStreamer->emitLabel(RelocSymbol); const MCExpr *OffsExpr = MCBinaryExpr::createSub( MCSymbolRefExpr::create(OutContext.getOrCreateSymbol(Twine(".LTOC")), OutContext), MCSymbolRefExpr::create(PICBase, OutContext), OutContext); OutStreamer->emitValue(OffsExpr, 4); OutStreamer->emitLabel(CurrentFnSym); return; } else return AsmPrinter::emitFunctionEntryLabel(); } // ELFv2 ABI - Normal entry label. if (Subtarget->isELFv2ABI()) { // In the Large code model, we allow arbitrary displacements between // the text section and its associated TOC section. We place the // full 8-byte offset to the TOC in memory immediately preceding // the function global entry point. if (TM.getCodeModel() == CodeModel::Large && !MF->getRegInfo().use_empty(PPC::X2)) { const PPCFunctionInfo *PPCFI = MF->getInfo(); MCSymbol *TOCSymbol = OutContext.getOrCreateSymbol(StringRef(".TOC.")); MCSymbol *GlobalEPSymbol = PPCFI->getGlobalEPSymbol(); const MCExpr *TOCDeltaExpr = MCBinaryExpr::createSub(MCSymbolRefExpr::create(TOCSymbol, OutContext), MCSymbolRefExpr::create(GlobalEPSymbol, OutContext), OutContext); OutStreamer->emitLabel(PPCFI->getTOCOffsetSymbol()); OutStreamer->emitValue(TOCDeltaExpr, 8); } return AsmPrinter::emitFunctionEntryLabel(); } // Emit an official procedure descriptor. MCSectionSubPair Current = OutStreamer->getCurrentSection(); MCSectionELF *Section = OutStreamer->getContext().getELFSection( ".opd", ELF::SHT_PROGBITS, ELF::SHF_WRITE | ELF::SHF_ALLOC); OutStreamer->SwitchSection(Section); OutStreamer->emitLabel(CurrentFnSym); OutStreamer->emitValueToAlignment(8); MCSymbol *Symbol1 = CurrentFnSymForSize; // Generates a R_PPC64_ADDR64 (from FK_DATA_8) relocation for the function // entry point. OutStreamer->emitValue(MCSymbolRefExpr::create(Symbol1, OutContext), 8 /*size*/); MCSymbol *Symbol2 = OutContext.getOrCreateSymbol(StringRef(".TOC.")); // Generates a R_PPC64_TOC relocation for TOC base insertion. OutStreamer->emitValue( MCSymbolRefExpr::create(Symbol2, MCSymbolRefExpr::VK_PPC_TOCBASE, OutContext), 8/*size*/); // Emit a null environment pointer. OutStreamer->emitIntValue(0, 8 /* size */); OutStreamer->SwitchSection(Current.first, Current.second); } void PPCLinuxAsmPrinter::emitEndOfAsmFile(Module &M) { const DataLayout &DL = getDataLayout(); bool isPPC64 = DL.getPointerSizeInBits() == 64; PPCTargetStreamer &TS = static_cast(*OutStreamer->getTargetStreamer()); if (!TOC.empty()) { const char *Name = isPPC64 ? ".toc" : ".got2"; MCSectionELF *Section = OutContext.getELFSection( Name, ELF::SHT_PROGBITS, ELF::SHF_WRITE | ELF::SHF_ALLOC); OutStreamer->SwitchSection(Section); if (!isPPC64) OutStreamer->emitValueToAlignment(4); for (const auto &TOCMapPair : TOC) { const MCSymbol *const TOCEntryTarget = TOCMapPair.first; MCSymbol *const TOCEntryLabel = TOCMapPair.second; OutStreamer->emitLabel(TOCEntryLabel); if (isPPC64) TS.emitTCEntry(*TOCEntryTarget); else OutStreamer->emitSymbolValue(TOCEntryTarget, 4); } } PPCAsmPrinter::emitEndOfAsmFile(M); } /// EmitFunctionBodyStart - Emit a global entry point prefix for ELFv2. void PPCLinuxAsmPrinter::emitFunctionBodyStart() { // In the ELFv2 ABI, in functions that use the TOC register, we need to // provide two entry points. The ABI guarantees that when calling the // local entry point, r2 is set up by the caller to contain the TOC base // for this function, and when calling the global entry point, r12 is set // up by the caller to hold the address of the global entry point. We // thus emit a prefix sequence along the following lines: // // func: // .Lfunc_gepNN: // # global entry point // addis r2,r12,(.TOC.-.Lfunc_gepNN)@ha // addi r2,r2,(.TOC.-.Lfunc_gepNN)@l // .Lfunc_lepNN: // .localentry func, .Lfunc_lepNN-.Lfunc_gepNN // # local entry point, followed by function body // // For the Large code model, we create // // .Lfunc_tocNN: // .quad .TOC.-.Lfunc_gepNN # done by EmitFunctionEntryLabel // func: // .Lfunc_gepNN: // # global entry point // ld r2,.Lfunc_tocNN-.Lfunc_gepNN(r12) // add r2,r2,r12 // .Lfunc_lepNN: // .localentry func, .Lfunc_lepNN-.Lfunc_gepNN // # local entry point, followed by function body // // This ensures we have r2 set up correctly while executing the function // body, no matter which entry point is called. - if (Subtarget->isELFv2ABI() - // Only do all that if the function uses r2 in the first place. - && !MF->getRegInfo().use_empty(PPC::X2)) { + const PPCFunctionInfo *PPCFI = MF->getInfo(); + const bool UsesX2OrR2 = !MF->getRegInfo().use_empty(PPC::X2) || + !MF->getRegInfo().use_empty(PPC::R2); + // Only do all that if the function uses R2 as the TOC pointer + // in the first place. We don't need the global entry point if the + // function uses R2 as an allocatable register. + if (Subtarget->isELFv2ABI() && UsesX2OrR2 && PPCFI->usesTOCBasePtr()) { // Note: The logic here must be synchronized with the code in the // branch-selection pass which sets the offset of the first block in the // function. This matters because it affects the alignment. - const PPCFunctionInfo *PPCFI = MF->getInfo(); - MCSymbol *GlobalEntryLabel = PPCFI->getGlobalEPSymbol(); OutStreamer->emitLabel(GlobalEntryLabel); const MCSymbolRefExpr *GlobalEntryLabelExp = MCSymbolRefExpr::create(GlobalEntryLabel, OutContext); if (TM.getCodeModel() != CodeModel::Large) { MCSymbol *TOCSymbol = OutContext.getOrCreateSymbol(StringRef(".TOC.")); const MCExpr *TOCDeltaExpr = MCBinaryExpr::createSub(MCSymbolRefExpr::create(TOCSymbol, OutContext), GlobalEntryLabelExp, OutContext); const MCExpr *TOCDeltaHi = PPCMCExpr::createHa(TOCDeltaExpr, OutContext); EmitToStreamer(*OutStreamer, MCInstBuilder(PPC::ADDIS) .addReg(PPC::X2) .addReg(PPC::X12) .addExpr(TOCDeltaHi)); const MCExpr *TOCDeltaLo = PPCMCExpr::createLo(TOCDeltaExpr, OutContext); EmitToStreamer(*OutStreamer, MCInstBuilder(PPC::ADDI) .addReg(PPC::X2) .addReg(PPC::X2) .addExpr(TOCDeltaLo)); } else { MCSymbol *TOCOffset = PPCFI->getTOCOffsetSymbol(); const MCExpr *TOCOffsetDeltaExpr = MCBinaryExpr::createSub(MCSymbolRefExpr::create(TOCOffset, OutContext), GlobalEntryLabelExp, OutContext); EmitToStreamer(*OutStreamer, MCInstBuilder(PPC::LD) .addReg(PPC::X2) .addExpr(TOCOffsetDeltaExpr) .addReg(PPC::X12)); EmitToStreamer(*OutStreamer, MCInstBuilder(PPC::ADD8) .addReg(PPC::X2) .addReg(PPC::X2) .addReg(PPC::X12)); } MCSymbol *LocalEntryLabel = PPCFI->getLocalEPSymbol(); OutStreamer->emitLabel(LocalEntryLabel); const MCSymbolRefExpr *LocalEntryLabelExp = MCSymbolRefExpr::create(LocalEntryLabel, OutContext); const MCExpr *LocalOffsetExp = MCBinaryExpr::createSub(LocalEntryLabelExp, GlobalEntryLabelExp, OutContext); PPCTargetStreamer *TS = static_cast(OutStreamer->getTargetStreamer()); if (TS) TS->emitLocalEntry(cast(CurrentFnSym), LocalOffsetExp); + } else if (Subtarget->isELFv2ABI()) { + // When generating the entry point for a function we have a few scenarios + // based on whether or not that function uses R2 and whether or not that + // function makes calls (or is a leaf function). + // 1) A leaf function that does not use R2 (or treats it as callee-saved + // and preserves it). In this case st_other=0 and both + // the local and global entry points for the function are the same. + // No special entry point code is required. + // 2) A function uses the TOC pointer R2. This function may or may not have + // calls. In this case st_other=[2,6] and the global and local entry + // points are different. Code to correctly setup the TOC pointer in R2 + // is put between the global and local entry points. This case is + // covered by the if statatement above. + // 3) A function does not use the TOC pointer R2 but does have calls. + // In this case st_other=1 since we do not know whether or not any + // of the callees clobber R2. This case is dealt with in this else if + // block. + // 4) The function does not use the TOC pointer but R2 is used inside + // the function. In this case st_other=1 once again. + // 5) This function uses inline asm. We mark R2 as reserved if the function + // has inline asm so we have to assume that it may be used. + if (MF->getFrameInfo().hasCalls() || MF->hasInlineAsm() || + (!PPCFI->usesTOCBasePtr() && UsesX2OrR2)) { + PPCTargetStreamer *TS = + static_cast(OutStreamer->getTargetStreamer()); + if (TS) + TS->emitLocalEntry(cast(CurrentFnSym), + MCConstantExpr::create(1, OutContext)); + } } } /// EmitFunctionBodyEnd - Print the traceback table before the .size /// directive. /// void PPCLinuxAsmPrinter::emitFunctionBodyEnd() { // Only the 64-bit target requires a traceback table. For now, // we only emit the word of zeroes that GDB requires to find // the end of the function, and zeroes for the eight-byte // mandatory fields. // FIXME: We should fill in the eight-byte mandatory fields as described in // the PPC64 ELF ABI (this is a low-priority item because GDB does not // currently make use of these fields). if (Subtarget->isPPC64()) { OutStreamer->emitIntValue(0, 4/*size*/); OutStreamer->emitIntValue(0, 8/*size*/); } } void PPCAIXAsmPrinter::SetupMachineFunction(MachineFunction &MF) { // Setup CurrentFnDescSym and its containing csect. MCSectionXCOFF *FnDescSec = cast(getObjFileLowering().getSectionForFunctionDescriptor( &MF.getFunction(), TM)); FnDescSec->setAlignment(Align(Subtarget->isPPC64() ? 8 : 4)); CurrentFnDescSym = FnDescSec->getQualNameSymbol(); return AsmPrinter::SetupMachineFunction(MF); } void PPCAIXAsmPrinter::ValidateGV(const GlobalVariable *GV) { // Early error checking limiting what is supported. if (GV->isThreadLocal()) report_fatal_error("Thread local not yet supported on AIX."); if (GV->hasSection()) report_fatal_error("Custom section for Data not yet supported."); if (GV->hasComdat()) report_fatal_error("COMDAT not yet supported by AIX."); } const MCExpr *PPCAIXAsmPrinter::lowerConstant(const Constant *CV) { if (const Function *F = dyn_cast(CV)) { MCSectionXCOFF *Csect = cast( F->isDeclaration() ? getObjFileLowering().getSectionForExternalReference(F, TM) : getObjFileLowering().getSectionForFunctionDescriptor(F, TM)); return MCSymbolRefExpr::create(Csect->getQualNameSymbol(), OutContext); } return PPCAsmPrinter::lowerConstant(CV); } static bool isSpecialLLVMGlobalArrayForStaticInit(const GlobalVariable *GV) { return StringSwitch(GV->getName()) .Cases("llvm.global_ctors", "llvm.global_dtors", true) .Default(false); } void PPCAIXAsmPrinter::emitGlobalVariable(const GlobalVariable *GV) { ValidateGV(GV); // TODO: Update the handling of global arrays for static init when we support // the ".ref" directive. // Otherwise, we can skip these arrays, because the AIX linker collects // static init functions simply based on their name. if (isSpecialLLVMGlobalArrayForStaticInit(GV)) return; // Create the symbol, set its storage class. MCSymbolXCOFF *GVSym = cast(getSymbol(GV)); GVSym->setStorageClass( TargetLoweringObjectFileXCOFF::getStorageClassForGlobal(GV)); SectionKind GVKind; // Create the containing csect and set it. We set it for externals as well, // since this may not have been set elsewhere depending on how they are used. MCSectionXCOFF *Csect = cast( GV->isDeclaration() ? getObjFileLowering().getSectionForExternalReference(GV, TM) : getObjFileLowering().SectionForGlobal( GV, GVKind = getObjFileLowering().getKindForGlobal(GV, TM), TM)); // External global variables are already handled. if (GV->isDeclaration()) return; if (!GVKind.isGlobalWriteableData() && !GVKind.isReadOnly()) report_fatal_error("Encountered a global variable kind that is " "not supported yet."); // Switch to the containing csect. OutStreamer->SwitchSection(Csect); const DataLayout &DL = GV->getParent()->getDataLayout(); // Handle common symbols. if (GVKind.isCommon() || GVKind.isBSSLocal()) { unsigned Align = GV->getAlignment() ? GV->getAlignment() : DL.getPreferredAlignment(GV); uint64_t Size = DL.getTypeAllocSize(GV->getType()->getElementType()); if (GVKind.isBSSLocal()) OutStreamer->emitXCOFFLocalCommonSymbol( GVSym, Size, Csect->getQualNameSymbol(), Align); else OutStreamer->emitCommonSymbol(Csect->getQualNameSymbol(), Size, Align); return; } MCSymbol *EmittedInitSym = GVSym; emitLinkage(GV, EmittedInitSym); emitAlignment(getGVAlignment(GV, DL), GV); OutStreamer->emitLabel(EmittedInitSym); emitGlobalConstant(GV->getParent()->getDataLayout(), GV->getInitializer()); } void PPCAIXAsmPrinter::emitFunctionDescriptor() { const DataLayout &DL = getDataLayout(); const unsigned PointerSize = DL.getPointerSizeInBits() == 64 ? 8 : 4; MCSectionSubPair Current = OutStreamer->getCurrentSection(); // Emit function descriptor. OutStreamer->SwitchSection( cast(CurrentFnDescSym)->getRepresentedCsect()); // Emit function entry point address. OutStreamer->emitValue(MCSymbolRefExpr::create(CurrentFnSym, OutContext), PointerSize); // Emit TOC base address. const MCSymbol *TOCBaseSym = cast(getObjFileLowering().getTOCBaseSection()) ->getQualNameSymbol(); OutStreamer->emitValue(MCSymbolRefExpr::create(TOCBaseSym, OutContext), PointerSize); // Emit a null environment pointer. OutStreamer->emitIntValue(0, PointerSize); OutStreamer->SwitchSection(Current.first, Current.second); } void PPCAIXAsmPrinter::emitEndOfAsmFile(Module &M) { // If there are no functions in this module, we will never need to reference // the TOC base. if (M.empty()) return; // Switch to section to emit TOC base. OutStreamer->SwitchSection(getObjFileLowering().getTOCBaseSection()); PPCTargetStreamer &TS = static_cast(*OutStreamer->getTargetStreamer()); const unsigned EntryByteSize = Subtarget->isPPC64() ? 8 : 4; const unsigned TOCEntriesByteSize = TOC.size() * EntryByteSize; // TODO: If TOC entries' size is larger than 32768, then we run out of // positive displacement to reach the TOC entry. We need to decide how to // handle entries' size larger than that later. if (TOCEntriesByteSize > 32767) { report_fatal_error("Handling of TOC entry displacement larger than 32767 " "is not yet implemented."); } for (auto &I : TOC) { // Setup the csect for the current TC entry. MCSectionXCOFF *TCEntry = cast( getObjFileLowering().getSectionForTOCEntry(I.first)); OutStreamer->SwitchSection(TCEntry); OutStreamer->emitLabel(I.second); TS.emitTCEntry(*I.first); } } MCSymbol * PPCAIXAsmPrinter::getMCSymbolForTOCPseudoMO(const MachineOperand &MO) { const GlobalObject *GO = nullptr; // If the MO is a function or certain kind of globals, we want to make sure to // refer to the csect symbol, otherwise we can just do the default handling. if (MO.getType() != MachineOperand::MO_GlobalAddress || !(GO = dyn_cast(MO.getGlobal()))) return PPCAsmPrinter::getMCSymbolForTOCPseudoMO(MO); // Do an early error check for globals we don't support. This will go away // eventually. const auto *GV = dyn_cast(GO); if (GV) { ValidateGV(GV); } // If the global object is a global variable without initializer or is a // declaration of a function, then XSym is an external referenced symbol. // Hence we may need to explictly create a MCSectionXCOFF for it so that we // can return its symbol later. if (GO->isDeclaration()) { return cast( getObjFileLowering().getSectionForExternalReference(GO, TM)) ->getQualNameSymbol(); } // Handle initialized global variables and defined functions. SectionKind GOKind = getObjFileLowering().getKindForGlobal(GO, TM); if (GOKind.isText()) { // If the MO is a function, we want to make sure to refer to the function // descriptor csect. return cast( getObjFileLowering().getSectionForFunctionDescriptor( cast(GO), TM)) ->getQualNameSymbol(); } else if (GOKind.isCommon() || GOKind.isBSSLocal()) { // If the operand is a common then we should refer to the csect symbol. return cast( getObjFileLowering().SectionForGlobal(GO, GOKind, TM)) ->getQualNameSymbol(); } // Other global variables are refered to by labels inside of a single csect, // so refer to the label directly. return getSymbol(GV); } /// createPPCAsmPrinterPass - Returns a pass that prints the PPC assembly code /// for a MachineFunction to the given output stream, in a format that the /// Darwin assembler can deal with. /// static AsmPrinter * createPPCAsmPrinterPass(TargetMachine &tm, std::unique_ptr &&Streamer) { if (tm.getTargetTriple().isOSAIX()) return new PPCAIXAsmPrinter(tm, std::move(Streamer)); return new PPCLinuxAsmPrinter(tm, std::move(Streamer)); } // Force static initialization. extern "C" LLVM_EXTERNAL_VISIBILITY void LLVMInitializePowerPCAsmPrinter() { TargetRegistry::RegisterAsmPrinter(getThePPC32Target(), createPPCAsmPrinterPass); TargetRegistry::RegisterAsmPrinter(getThePPC64Target(), createPPCAsmPrinterPass); TargetRegistry::RegisterAsmPrinter(getThePPC64LETarget(), createPPCAsmPrinterPass); } diff --git a/llvm/lib/Target/PowerPC/PPCISelLowering.cpp b/llvm/lib/Target/PowerPC/PPCISelLowering.cpp index 3d8441907c04..db81a6c2cb70 100644 --- a/llvm/lib/Target/PowerPC/PPCISelLowering.cpp +++ b/llvm/lib/Target/PowerPC/PPCISelLowering.cpp @@ -1,16111 +1,16135 @@ //===-- PPCISelLowering.cpp - PPC DAG Lowering Implementation -------------===// // // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. // See https://llvm.org/LICENSE.txt for license information. // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // // This file implements the PPCISelLowering class. // //===----------------------------------------------------------------------===// #include "PPCISelLowering.h" #include "MCTargetDesc/PPCPredicates.h" #include "PPC.h" #include "PPCCCState.h" #include "PPCCallingConv.h" #include "PPCFrameLowering.h" #include "PPCInstrInfo.h" #include "PPCMachineFunctionInfo.h" #include "PPCPerfectShuffle.h" #include "PPCRegisterInfo.h" #include "PPCSubtarget.h" #include "PPCTargetMachine.h" #include "llvm/ADT/APFloat.h" #include "llvm/ADT/APInt.h" #include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/DenseMap.h" #include "llvm/ADT/None.h" #include "llvm/ADT/STLExtras.h" #include "llvm/ADT/SmallPtrSet.h" #include "llvm/ADT/SmallSet.h" #include "llvm/ADT/SmallVector.h" #include "llvm/ADT/Statistic.h" #include "llvm/ADT/StringRef.h" #include "llvm/ADT/StringSwitch.h" #include "llvm/CodeGen/CallingConvLower.h" #include "llvm/CodeGen/ISDOpcodes.h" #include "llvm/CodeGen/MachineBasicBlock.h" #include "llvm/CodeGen/MachineFrameInfo.h" #include "llvm/CodeGen/MachineFunction.h" #include "llvm/CodeGen/MachineInstr.h" #include "llvm/CodeGen/MachineInstrBuilder.h" #include "llvm/CodeGen/MachineJumpTableInfo.h" #include "llvm/CodeGen/MachineLoopInfo.h" #include "llvm/CodeGen/MachineMemOperand.h" #include "llvm/CodeGen/MachineModuleInfo.h" #include "llvm/CodeGen/MachineOperand.h" #include "llvm/CodeGen/MachineRegisterInfo.h" #include "llvm/CodeGen/RuntimeLibcalls.h" #include "llvm/CodeGen/SelectionDAG.h" #include "llvm/CodeGen/SelectionDAGNodes.h" #include "llvm/CodeGen/TargetInstrInfo.h" #include "llvm/CodeGen/TargetLowering.h" #include "llvm/CodeGen/TargetLoweringObjectFileImpl.h" #include "llvm/CodeGen/TargetRegisterInfo.h" #include "llvm/CodeGen/ValueTypes.h" #include "llvm/IR/CallSite.h" #include "llvm/IR/CallingConv.h" #include "llvm/IR/Constant.h" #include "llvm/IR/Constants.h" #include "llvm/IR/DataLayout.h" #include "llvm/IR/DebugLoc.h" #include "llvm/IR/DerivedTypes.h" #include "llvm/IR/Function.h" #include "llvm/IR/GlobalValue.h" #include "llvm/IR/IRBuilder.h" #include "llvm/IR/Instructions.h" #include "llvm/IR/Intrinsics.h" #include "llvm/IR/IntrinsicsPowerPC.h" #include "llvm/IR/Module.h" #include "llvm/IR/Type.h" #include "llvm/IR/Use.h" #include "llvm/IR/Value.h" #include "llvm/MC/MCContext.h" #include "llvm/MC/MCExpr.h" #include "llvm/MC/MCRegisterInfo.h" #include "llvm/MC/MCSymbolXCOFF.h" #include "llvm/Support/AtomicOrdering.h" #include "llvm/Support/BranchProbability.h" #include "llvm/Support/Casting.h" #include "llvm/Support/CodeGen.h" #include "llvm/Support/CommandLine.h" #include "llvm/Support/Compiler.h" #include "llvm/Support/Debug.h" #include "llvm/Support/ErrorHandling.h" #include "llvm/Support/Format.h" #include "llvm/Support/KnownBits.h" #include "llvm/Support/MachineValueType.h" #include "llvm/Support/MathExtras.h" #include "llvm/Support/raw_ostream.h" #include "llvm/Target/TargetMachine.h" #include "llvm/Target/TargetOptions.h" #include #include #include #include #include #include #include using namespace llvm; #define DEBUG_TYPE "ppc-lowering" static cl::opt DisablePPCPreinc("disable-ppc-preinc", cl::desc("disable preincrement load/store generation on PPC"), cl::Hidden); static cl::opt DisableILPPref("disable-ppc-ilp-pref", cl::desc("disable setting the node scheduling preference to ILP on PPC"), cl::Hidden); static cl::opt DisablePPCUnaligned("disable-ppc-unaligned", cl::desc("disable unaligned load/store generation on PPC"), cl::Hidden); static cl::opt DisableSCO("disable-ppc-sco", cl::desc("disable sibling call optimization on ppc"), cl::Hidden); static cl::opt DisableInnermostLoopAlign32("disable-ppc-innermost-loop-align32", cl::desc("don't always align innermost loop to 32 bytes on ppc"), cl::Hidden); static cl::opt EnableQuadPrecision("enable-ppc-quad-precision", cl::desc("enable quad precision float support on ppc"), cl::Hidden); static cl::opt UseAbsoluteJumpTables("ppc-use-absolute-jumptables", cl::desc("use absolute jump tables on ppc"), cl::Hidden); STATISTIC(NumTailCalls, "Number of tail calls"); STATISTIC(NumSiblingCalls, "Number of sibling calls"); static bool isNByteElemShuffleMask(ShuffleVectorSDNode *, unsigned, int); static SDValue widenVec(SelectionDAG &DAG, SDValue Vec, const SDLoc &dl); // FIXME: Remove this once the bug has been fixed! extern cl::opt ANDIGlueBug; PPCTargetLowering::PPCTargetLowering(const PPCTargetMachine &TM, const PPCSubtarget &STI) : TargetLowering(TM), Subtarget(STI) { // On PPC32/64, arguments smaller than 4/8 bytes are extended, so all // arguments are at least 4/8 bytes aligned. bool isPPC64 = Subtarget.isPPC64(); setMinStackArgumentAlignment(isPPC64 ? Align(8) : Align(4)); // Set up the register classes. addRegisterClass(MVT::i32, &PPC::GPRCRegClass); if (!useSoftFloat()) { if (hasSPE()) { addRegisterClass(MVT::f32, &PPC::GPRCRegClass); addRegisterClass(MVT::f64, &PPC::SPERCRegClass); } else { addRegisterClass(MVT::f32, &PPC::F4RCRegClass); addRegisterClass(MVT::f64, &PPC::F8RCRegClass); } } // Match BITREVERSE to customized fast code sequence in the td file. setOperationAction(ISD::BITREVERSE, MVT::i32, Legal); setOperationAction(ISD::BITREVERSE, MVT::i64, Legal); // Sub-word ATOMIC_CMP_SWAP need to ensure that the input is zero-extended. setOperationAction(ISD::ATOMIC_CMP_SWAP, MVT::i32, Custom); // PowerPC has an i16 but no i8 (or i1) SEXTLOAD. for (MVT VT : MVT::integer_valuetypes()) { setLoadExtAction(ISD::SEXTLOAD, VT, MVT::i1, Promote); setLoadExtAction(ISD::SEXTLOAD, VT, MVT::i8, Expand); } setTruncStoreAction(MVT::f64, MVT::f32, Expand); // PowerPC has pre-inc load and store's. setIndexedLoadAction(ISD::PRE_INC, MVT::i1, Legal); setIndexedLoadAction(ISD::PRE_INC, MVT::i8, Legal); setIndexedLoadAction(ISD::PRE_INC, MVT::i16, Legal); setIndexedLoadAction(ISD::PRE_INC, MVT::i32, Legal); setIndexedLoadAction(ISD::PRE_INC, MVT::i64, Legal); setIndexedStoreAction(ISD::PRE_INC, MVT::i1, Legal); setIndexedStoreAction(ISD::PRE_INC, MVT::i8, Legal); setIndexedStoreAction(ISD::PRE_INC, MVT::i16, Legal); setIndexedStoreAction(ISD::PRE_INC, MVT::i32, Legal); setIndexedStoreAction(ISD::PRE_INC, MVT::i64, Legal); if (!Subtarget.hasSPE()) { setIndexedLoadAction(ISD::PRE_INC, MVT::f32, Legal); setIndexedLoadAction(ISD::PRE_INC, MVT::f64, Legal); setIndexedStoreAction(ISD::PRE_INC, MVT::f32, Legal); setIndexedStoreAction(ISD::PRE_INC, MVT::f64, Legal); } // PowerPC uses ADDC/ADDE/SUBC/SUBE to propagate carry. const MVT ScalarIntVTs[] = { MVT::i32, MVT::i64 }; for (MVT VT : ScalarIntVTs) { setOperationAction(ISD::ADDC, VT, Legal); setOperationAction(ISD::ADDE, VT, Legal); setOperationAction(ISD::SUBC, VT, Legal); setOperationAction(ISD::SUBE, VT, Legal); } if (Subtarget.useCRBits()) { setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i1, Expand); if (isPPC64 || Subtarget.hasFPCVT()) { setOperationAction(ISD::SINT_TO_FP, MVT::i1, Promote); AddPromotedToType (ISD::SINT_TO_FP, MVT::i1, isPPC64 ? MVT::i64 : MVT::i32); setOperationAction(ISD::UINT_TO_FP, MVT::i1, Promote); AddPromotedToType(ISD::UINT_TO_FP, MVT::i1, isPPC64 ? MVT::i64 : MVT::i32); } else { setOperationAction(ISD::SINT_TO_FP, MVT::i1, Custom); setOperationAction(ISD::UINT_TO_FP, MVT::i1, Custom); } // PowerPC does not support direct load/store of condition registers. setOperationAction(ISD::LOAD, MVT::i1, Custom); setOperationAction(ISD::STORE, MVT::i1, Custom); // FIXME: Remove this once the ANDI glue bug is fixed: if (ANDIGlueBug) setOperationAction(ISD::TRUNCATE, MVT::i1, Custom); for (MVT VT : MVT::integer_valuetypes()) { setLoadExtAction(ISD::SEXTLOAD, VT, MVT::i1, Promote); setLoadExtAction(ISD::ZEXTLOAD, VT, MVT::i1, Promote); setTruncStoreAction(VT, MVT::i1, Expand); } addRegisterClass(MVT::i1, &PPC::CRBITRCRegClass); } // Expand ppcf128 to i32 by hand for the benefit of llvm-gcc bootstrap on // PPC (the libcall is not available). setOperationAction(ISD::FP_TO_SINT, MVT::ppcf128, Custom); setOperationAction(ISD::FP_TO_UINT, MVT::ppcf128, Custom); // We do not currently implement these libm ops for PowerPC. setOperationAction(ISD::FFLOOR, MVT::ppcf128, Expand); setOperationAction(ISD::FCEIL, MVT::ppcf128, Expand); setOperationAction(ISD::FTRUNC, MVT::ppcf128, Expand); setOperationAction(ISD::FRINT, MVT::ppcf128, Expand); setOperationAction(ISD::FNEARBYINT, MVT::ppcf128, Expand); setOperationAction(ISD::FREM, MVT::ppcf128, Expand); // PowerPC has no SREM/UREM instructions unless we are on P9 // On P9 we may use a hardware instruction to compute the remainder. // The instructions are not legalized directly because in the cases where the // result of both the remainder and the division is required it is more // efficient to compute the remainder from the result of the division rather // than use the remainder instruction. if (Subtarget.isISA3_0()) { setOperationAction(ISD::SREM, MVT::i32, Custom); setOperationAction(ISD::UREM, MVT::i32, Custom); setOperationAction(ISD::SREM, MVT::i64, Custom); setOperationAction(ISD::UREM, MVT::i64, Custom); } else { setOperationAction(ISD::SREM, MVT::i32, Expand); setOperationAction(ISD::UREM, MVT::i32, Expand); setOperationAction(ISD::SREM, MVT::i64, Expand); setOperationAction(ISD::UREM, MVT::i64, Expand); } // Don't use SMUL_LOHI/UMUL_LOHI or SDIVREM/UDIVREM to lower SREM/UREM. setOperationAction(ISD::UMUL_LOHI, MVT::i32, Expand); setOperationAction(ISD::SMUL_LOHI, MVT::i32, Expand); setOperationAction(ISD::UMUL_LOHI, MVT::i64, Expand); setOperationAction(ISD::SMUL_LOHI, MVT::i64, Expand); setOperationAction(ISD::UDIVREM, MVT::i32, Expand); setOperationAction(ISD::SDIVREM, MVT::i32, Expand); setOperationAction(ISD::UDIVREM, MVT::i64, Expand); setOperationAction(ISD::SDIVREM, MVT::i64, Expand); // Handle constrained floating-point operations of scalar. // TODO: Handle SPE specific operation. setOperationAction(ISD::STRICT_FADD, MVT::f32, Legal); setOperationAction(ISD::STRICT_FSUB, MVT::f32, Legal); setOperationAction(ISD::STRICT_FMUL, MVT::f32, Legal); setOperationAction(ISD::STRICT_FDIV, MVT::f32, Legal); setOperationAction(ISD::STRICT_FADD, MVT::f64, Legal); setOperationAction(ISD::STRICT_FSUB, MVT::f64, Legal); setOperationAction(ISD::STRICT_FMUL, MVT::f64, Legal); setOperationAction(ISD::STRICT_FDIV, MVT::f64, Legal); // We don't support sin/cos/sqrt/fmod/pow setOperationAction(ISD::FSIN , MVT::f64, Expand); setOperationAction(ISD::FCOS , MVT::f64, Expand); setOperationAction(ISD::FSINCOS, MVT::f64, Expand); setOperationAction(ISD::FREM , MVT::f64, Expand); setOperationAction(ISD::FPOW , MVT::f64, Expand); setOperationAction(ISD::FSIN , MVT::f32, Expand); setOperationAction(ISD::FCOS , MVT::f32, Expand); setOperationAction(ISD::FSINCOS, MVT::f32, Expand); setOperationAction(ISD::FREM , MVT::f32, Expand); setOperationAction(ISD::FPOW , MVT::f32, Expand); if (Subtarget.hasSPE()) { setOperationAction(ISD::FMA , MVT::f64, Expand); setOperationAction(ISD::FMA , MVT::f32, Expand); } else { setOperationAction(ISD::FMA , MVT::f64, Legal); setOperationAction(ISD::FMA , MVT::f32, Legal); } setOperationAction(ISD::FLT_ROUNDS_, MVT::i32, Custom); // If we're enabling GP optimizations, use hardware square root if (!Subtarget.hasFSQRT() && !(TM.Options.UnsafeFPMath && Subtarget.hasFRSQRTE() && Subtarget.hasFRE())) setOperationAction(ISD::FSQRT, MVT::f64, Expand); if (!Subtarget.hasFSQRT() && !(TM.Options.UnsafeFPMath && Subtarget.hasFRSQRTES() && Subtarget.hasFRES())) setOperationAction(ISD::FSQRT, MVT::f32, Expand); if (Subtarget.hasFCPSGN()) { setOperationAction(ISD::FCOPYSIGN, MVT::f64, Legal); setOperationAction(ISD::FCOPYSIGN, MVT::f32, Legal); } else { setOperationAction(ISD::FCOPYSIGN, MVT::f64, Expand); setOperationAction(ISD::FCOPYSIGN, MVT::f32, Expand); } if (Subtarget.hasFPRND()) { setOperationAction(ISD::FFLOOR, MVT::f64, Legal); setOperationAction(ISD::FCEIL, MVT::f64, Legal); setOperationAction(ISD::FTRUNC, MVT::f64, Legal); setOperationAction(ISD::FROUND, MVT::f64, Legal); setOperationAction(ISD::FFLOOR, MVT::f32, Legal); setOperationAction(ISD::FCEIL, MVT::f32, Legal); setOperationAction(ISD::FTRUNC, MVT::f32, Legal); setOperationAction(ISD::FROUND, MVT::f32, Legal); } // PowerPC does not have BSWAP, but we can use vector BSWAP instruction xxbrd // to speed up scalar BSWAP64. // CTPOP or CTTZ were introduced in P8/P9 respectively setOperationAction(ISD::BSWAP, MVT::i32 , Expand); if (Subtarget.hasP9Vector()) setOperationAction(ISD::BSWAP, MVT::i64 , Custom); else setOperationAction(ISD::BSWAP, MVT::i64 , Expand); if (Subtarget.isISA3_0()) { setOperationAction(ISD::CTTZ , MVT::i32 , Legal); setOperationAction(ISD::CTTZ , MVT::i64 , Legal); } else { setOperationAction(ISD::CTTZ , MVT::i32 , Expand); setOperationAction(ISD::CTTZ , MVT::i64 , Expand); } if (Subtarget.hasPOPCNTD() == PPCSubtarget::POPCNTD_Fast) { setOperationAction(ISD::CTPOP, MVT::i32 , Legal); setOperationAction(ISD::CTPOP, MVT::i64 , Legal); } else { setOperationAction(ISD::CTPOP, MVT::i32 , Expand); setOperationAction(ISD::CTPOP, MVT::i64 , Expand); } // PowerPC does not have ROTR setOperationAction(ISD::ROTR, MVT::i32 , Expand); setOperationAction(ISD::ROTR, MVT::i64 , Expand); if (!Subtarget.useCRBits()) { // PowerPC does not have Select setOperationAction(ISD::SELECT, MVT::i32, Expand); setOperationAction(ISD::SELECT, MVT::i64, Expand); setOperationAction(ISD::SELECT, MVT::f32, Expand); setOperationAction(ISD::SELECT, MVT::f64, Expand); } // PowerPC wants to turn select_cc of FP into fsel when possible. setOperationAction(ISD::SELECT_CC, MVT::f32, Custom); setOperationAction(ISD::SELECT_CC, MVT::f64, Custom); // PowerPC wants to optimize integer setcc a bit if (!Subtarget.useCRBits()) setOperationAction(ISD::SETCC, MVT::i32, Custom); // PowerPC does not have BRCOND which requires SetCC if (!Subtarget.useCRBits()) setOperationAction(ISD::BRCOND, MVT::Other, Expand); setOperationAction(ISD::BR_JT, MVT::Other, Expand); if (Subtarget.hasSPE()) { // SPE has built-in conversions setOperationAction(ISD::FP_TO_SINT, MVT::i32, Legal); setOperationAction(ISD::SINT_TO_FP, MVT::i32, Legal); setOperationAction(ISD::UINT_TO_FP, MVT::i32, Legal); } else { // PowerPC turns FP_TO_SINT into FCTIWZ and some load/stores. setOperationAction(ISD::FP_TO_SINT, MVT::i32, Custom); // PowerPC does not have [U|S]INT_TO_FP setOperationAction(ISD::SINT_TO_FP, MVT::i32, Expand); setOperationAction(ISD::UINT_TO_FP, MVT::i32, Expand); } if (Subtarget.hasDirectMove() && isPPC64) { setOperationAction(ISD::BITCAST, MVT::f32, Legal); setOperationAction(ISD::BITCAST, MVT::i32, Legal); setOperationAction(ISD::BITCAST, MVT::i64, Legal); setOperationAction(ISD::BITCAST, MVT::f64, Legal); if (TM.Options.UnsafeFPMath) { setOperationAction(ISD::LRINT, MVT::f64, Legal); setOperationAction(ISD::LRINT, MVT::f32, Legal); setOperationAction(ISD::LLRINT, MVT::f64, Legal); setOperationAction(ISD::LLRINT, MVT::f32, Legal); setOperationAction(ISD::LROUND, MVT::f64, Legal); setOperationAction(ISD::LROUND, MVT::f32, Legal); setOperationAction(ISD::LLROUND, MVT::f64, Legal); setOperationAction(ISD::LLROUND, MVT::f32, Legal); } } else { setOperationAction(ISD::BITCAST, MVT::f32, Expand); setOperationAction(ISD::BITCAST, MVT::i32, Expand); setOperationAction(ISD::BITCAST, MVT::i64, Expand); setOperationAction(ISD::BITCAST, MVT::f64, Expand); } // We cannot sextinreg(i1). Expand to shifts. setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i1, Expand); // NOTE: EH_SJLJ_SETJMP/_LONGJMP supported here is NOT intended to support // SjLj exception handling but a light-weight setjmp/longjmp replacement to // support continuation, user-level threading, and etc.. As a result, no // other SjLj exception interfaces are implemented and please don't build // your own exception handling based on them. // LLVM/Clang supports zero-cost DWARF exception handling. setOperationAction(ISD::EH_SJLJ_SETJMP, MVT::i32, Custom); setOperationAction(ISD::EH_SJLJ_LONGJMP, MVT::Other, Custom); // We want to legalize GlobalAddress and ConstantPool nodes into the // appropriate instructions to materialize the address. setOperationAction(ISD::GlobalAddress, MVT::i32, Custom); setOperationAction(ISD::GlobalTLSAddress, MVT::i32, Custom); setOperationAction(ISD::BlockAddress, MVT::i32, Custom); setOperationAction(ISD::ConstantPool, MVT::i32, Custom); setOperationAction(ISD::JumpTable, MVT::i32, Custom); setOperationAction(ISD::GlobalAddress, MVT::i64, Custom); setOperationAction(ISD::GlobalTLSAddress, MVT::i64, Custom); setOperationAction(ISD::BlockAddress, MVT::i64, Custom); setOperationAction(ISD::ConstantPool, MVT::i64, Custom); setOperationAction(ISD::JumpTable, MVT::i64, Custom); // TRAP is legal. setOperationAction(ISD::TRAP, MVT::Other, Legal); // TRAMPOLINE is custom lowered. setOperationAction(ISD::INIT_TRAMPOLINE, MVT::Other, Custom); setOperationAction(ISD::ADJUST_TRAMPOLINE, MVT::Other, Custom); // VASTART needs to be custom lowered to use the VarArgsFrameIndex setOperationAction(ISD::VASTART , MVT::Other, Custom); if (Subtarget.is64BitELFABI()) { // VAARG always uses double-word chunks, so promote anything smaller. setOperationAction(ISD::VAARG, MVT::i1, Promote); AddPromotedToType(ISD::VAARG, MVT::i1, MVT::i64); setOperationAction(ISD::VAARG, MVT::i8, Promote); AddPromotedToType(ISD::VAARG, MVT::i8, MVT::i64); setOperationAction(ISD::VAARG, MVT::i16, Promote); AddPromotedToType(ISD::VAARG, MVT::i16, MVT::i64); setOperationAction(ISD::VAARG, MVT::i32, Promote); AddPromotedToType(ISD::VAARG, MVT::i32, MVT::i64); setOperationAction(ISD::VAARG, MVT::Other, Expand); } else if (Subtarget.is32BitELFABI()) { // VAARG is custom lowered with the 32-bit SVR4 ABI. setOperationAction(ISD::VAARG, MVT::Other, Custom); setOperationAction(ISD::VAARG, MVT::i64, Custom); } else setOperationAction(ISD::VAARG, MVT::Other, Expand); // VACOPY is custom lowered with the 32-bit SVR4 ABI. if (Subtarget.is32BitELFABI()) setOperationAction(ISD::VACOPY , MVT::Other, Custom); else setOperationAction(ISD::VACOPY , MVT::Other, Expand); // Use the default implementation. setOperationAction(ISD::VAEND , MVT::Other, Expand); setOperationAction(ISD::STACKSAVE , MVT::Other, Expand); setOperationAction(ISD::STACKRESTORE , MVT::Other, Custom); setOperationAction(ISD::DYNAMIC_STACKALLOC, MVT::i32 , Custom); setOperationAction(ISD::DYNAMIC_STACKALLOC, MVT::i64 , Custom); setOperationAction(ISD::GET_DYNAMIC_AREA_OFFSET, MVT::i32, Custom); setOperationAction(ISD::GET_DYNAMIC_AREA_OFFSET, MVT::i64, Custom); setOperationAction(ISD::EH_DWARF_CFA, MVT::i32, Custom); setOperationAction(ISD::EH_DWARF_CFA, MVT::i64, Custom); // We want to custom lower some of our intrinsics. setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::Other, Custom); // To handle counter-based loop conditions. setOperationAction(ISD::INTRINSIC_W_CHAIN, MVT::i1, Custom); setOperationAction(ISD::INTRINSIC_VOID, MVT::i8, Custom); setOperationAction(ISD::INTRINSIC_VOID, MVT::i16, Custom); setOperationAction(ISD::INTRINSIC_VOID, MVT::i32, Custom); setOperationAction(ISD::INTRINSIC_VOID, MVT::Other, Custom); // Comparisons that require checking two conditions. if (Subtarget.hasSPE()) { setCondCodeAction(ISD::SETO, MVT::f32, Expand); setCondCodeAction(ISD::SETO, MVT::f64, Expand); setCondCodeAction(ISD::SETUO, MVT::f32, Expand); setCondCodeAction(ISD::SETUO, MVT::f64, Expand); } setCondCodeAction(ISD::SETULT, MVT::f32, Expand); setCondCodeAction(ISD::SETULT, MVT::f64, Expand); setCondCodeAction(ISD::SETUGT, MVT::f32, Expand); setCondCodeAction(ISD::SETUGT, MVT::f64, Expand); setCondCodeAction(ISD::SETUEQ, MVT::f32, Expand); setCondCodeAction(ISD::SETUEQ, MVT::f64, Expand); setCondCodeAction(ISD::SETOGE, MVT::f32, Expand); setCondCodeAction(ISD::SETOGE, MVT::f64, Expand); setCondCodeAction(ISD::SETOLE, MVT::f32, Expand); setCondCodeAction(ISD::SETOLE, MVT::f64, Expand); setCondCodeAction(ISD::SETONE, MVT::f32, Expand); setCondCodeAction(ISD::SETONE, MVT::f64, Expand); if (Subtarget.has64BitSupport()) { // They also have instructions for converting between i64 and fp. setOperationAction(ISD::FP_TO_SINT, MVT::i64, Custom); setOperationAction(ISD::FP_TO_UINT, MVT::i64, Expand); setOperationAction(ISD::SINT_TO_FP, MVT::i64, Custom); setOperationAction(ISD::UINT_TO_FP, MVT::i64, Expand); // This is just the low 32 bits of a (signed) fp->i64 conversion. // We cannot do this with Promote because i64 is not a legal type. setOperationAction(ISD::FP_TO_UINT, MVT::i32, Custom); if (Subtarget.hasLFIWAX() || Subtarget.isPPC64()) setOperationAction(ISD::SINT_TO_FP, MVT::i32, Custom); } else { // PowerPC does not have FP_TO_UINT on 32-bit implementations. if (Subtarget.hasSPE()) setOperationAction(ISD::FP_TO_UINT, MVT::i32, Legal); else setOperationAction(ISD::FP_TO_UINT, MVT::i32, Expand); } // With the instructions enabled under FPCVT, we can do everything. if (Subtarget.hasFPCVT()) { if (Subtarget.has64BitSupport()) { setOperationAction(ISD::FP_TO_SINT, MVT::i64, Custom); setOperationAction(ISD::FP_TO_UINT, MVT::i64, Custom); setOperationAction(ISD::SINT_TO_FP, MVT::i64, Custom); setOperationAction(ISD::UINT_TO_FP, MVT::i64, Custom); } setOperationAction(ISD::FP_TO_SINT, MVT::i32, Custom); setOperationAction(ISD::FP_TO_UINT, MVT::i32, Custom); setOperationAction(ISD::SINT_TO_FP, MVT::i32, Custom); setOperationAction(ISD::UINT_TO_FP, MVT::i32, Custom); } if (Subtarget.use64BitRegs()) { // 64-bit PowerPC implementations can support i64 types directly addRegisterClass(MVT::i64, &PPC::G8RCRegClass); // BUILD_PAIR can't be handled natively, and should be expanded to shl/or setOperationAction(ISD::BUILD_PAIR, MVT::i64, Expand); // 64-bit PowerPC wants to expand i128 shifts itself. setOperationAction(ISD::SHL_PARTS, MVT::i64, Custom); setOperationAction(ISD::SRA_PARTS, MVT::i64, Custom); setOperationAction(ISD::SRL_PARTS, MVT::i64, Custom); } else { // 32-bit PowerPC wants to expand i64 shifts itself. setOperationAction(ISD::SHL_PARTS, MVT::i32, Custom); setOperationAction(ISD::SRA_PARTS, MVT::i32, Custom); setOperationAction(ISD::SRL_PARTS, MVT::i32, Custom); } if (Subtarget.hasVSX()) { setOperationAction(ISD::FMAXNUM_IEEE, MVT::f64, Legal); setOperationAction(ISD::FMAXNUM_IEEE, MVT::f32, Legal); setOperationAction(ISD::FMINNUM_IEEE, MVT::f64, Legal); setOperationAction(ISD::FMINNUM_IEEE, MVT::f32, Legal); } if (Subtarget.hasAltivec()) { for (MVT VT : { MVT::v16i8, MVT::v8i16, MVT::v4i32 }) { setOperationAction(ISD::SADDSAT, VT, Legal); setOperationAction(ISD::SSUBSAT, VT, Legal); setOperationAction(ISD::UADDSAT, VT, Legal); setOperationAction(ISD::USUBSAT, VT, Legal); } // First set operation action for all vector types to expand. Then we // will selectively turn on ones that can be effectively codegen'd. for (MVT VT : MVT::fixedlen_vector_valuetypes()) { // add/sub are legal for all supported vector VT's. setOperationAction(ISD::ADD, VT, Legal); setOperationAction(ISD::SUB, VT, Legal); // For v2i64, these are only valid with P8Vector. This is corrected after // the loop. if (VT.getSizeInBits() <= 128 && VT.getScalarSizeInBits() <= 64) { setOperationAction(ISD::SMAX, VT, Legal); setOperationAction(ISD::SMIN, VT, Legal); setOperationAction(ISD::UMAX, VT, Legal); setOperationAction(ISD::UMIN, VT, Legal); } else { setOperationAction(ISD::SMAX, VT, Expand); setOperationAction(ISD::SMIN, VT, Expand); setOperationAction(ISD::UMAX, VT, Expand); setOperationAction(ISD::UMIN, VT, Expand); } if (Subtarget.hasVSX()) { setOperationAction(ISD::FMAXNUM, VT, Legal); setOperationAction(ISD::FMINNUM, VT, Legal); } // Vector instructions introduced in P8 if (Subtarget.hasP8Altivec() && (VT.SimpleTy != MVT::v1i128)) { setOperationAction(ISD::CTPOP, VT, Legal); setOperationAction(ISD::CTLZ, VT, Legal); } else { setOperationAction(ISD::CTPOP, VT, Expand); setOperationAction(ISD::CTLZ, VT, Expand); } // Vector instructions introduced in P9 if (Subtarget.hasP9Altivec() && (VT.SimpleTy != MVT::v1i128)) setOperationAction(ISD::CTTZ, VT, Legal); else setOperationAction(ISD::CTTZ, VT, Expand); // We promote all shuffles to v16i8. setOperationAction(ISD::VECTOR_SHUFFLE, VT, Promote); AddPromotedToType (ISD::VECTOR_SHUFFLE, VT, MVT::v16i8); // We promote all non-typed operations to v4i32. setOperationAction(ISD::AND , VT, Promote); AddPromotedToType (ISD::AND , VT, MVT::v4i32); setOperationAction(ISD::OR , VT, Promote); AddPromotedToType (ISD::OR , VT, MVT::v4i32); setOperationAction(ISD::XOR , VT, Promote); AddPromotedToType (ISD::XOR , VT, MVT::v4i32); setOperationAction(ISD::LOAD , VT, Promote); AddPromotedToType (ISD::LOAD , VT, MVT::v4i32); setOperationAction(ISD::SELECT, VT, Promote); AddPromotedToType (ISD::SELECT, VT, MVT::v4i32); setOperationAction(ISD::VSELECT, VT, Legal); setOperationAction(ISD::SELECT_CC, VT, Promote); AddPromotedToType (ISD::SELECT_CC, VT, MVT::v4i32); setOperationAction(ISD::STORE, VT, Promote); AddPromotedToType (ISD::STORE, VT, MVT::v4i32); // No other operations are legal. setOperationAction(ISD::MUL , VT, Expand); setOperationAction(ISD::SDIV, VT, Expand); setOperationAction(ISD::SREM, VT, Expand); setOperationAction(ISD::UDIV, VT, Expand); setOperationAction(ISD::UREM, VT, Expand); setOperationAction(ISD::FDIV, VT, Expand); setOperationAction(ISD::FREM, VT, Expand); setOperationAction(ISD::FNEG, VT, Expand); setOperationAction(ISD::FSQRT, VT, Expand); setOperationAction(ISD::FLOG, VT, Expand); setOperationAction(ISD::FLOG10, VT, Expand); setOperationAction(ISD::FLOG2, VT, Expand); setOperationAction(ISD::FEXP, VT, Expand); setOperationAction(ISD::FEXP2, VT, Expand); setOperationAction(ISD::FSIN, VT, Expand); setOperationAction(ISD::FCOS, VT, Expand); setOperationAction(ISD::FABS, VT, Expand); setOperationAction(ISD::FFLOOR, VT, Expand); setOperationAction(ISD::FCEIL, VT, Expand); setOperationAction(ISD::FTRUNC, VT, Expand); setOperationAction(ISD::FRINT, VT, Expand); setOperationAction(ISD::FNEARBYINT, VT, Expand); setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT, Expand); setOperationAction(ISD::INSERT_VECTOR_ELT, VT, Expand); setOperationAction(ISD::BUILD_VECTOR, VT, Expand); setOperationAction(ISD::MULHU, VT, Expand); setOperationAction(ISD::MULHS, VT, Expand); setOperationAction(ISD::UMUL_LOHI, VT, Expand); setOperationAction(ISD::SMUL_LOHI, VT, Expand); setOperationAction(ISD::UDIVREM, VT, Expand); setOperationAction(ISD::SDIVREM, VT, Expand); setOperationAction(ISD::SCALAR_TO_VECTOR, VT, Expand); setOperationAction(ISD::FPOW, VT, Expand); setOperationAction(ISD::BSWAP, VT, Expand); setOperationAction(ISD::SIGN_EXTEND_INREG, VT, Expand); setOperationAction(ISD::ROTL, VT, Expand); setOperationAction(ISD::ROTR, VT, Expand); for (MVT InnerVT : MVT::fixedlen_vector_valuetypes()) { setTruncStoreAction(VT, InnerVT, Expand); setLoadExtAction(ISD::SEXTLOAD, VT, InnerVT, Expand); setLoadExtAction(ISD::ZEXTLOAD, VT, InnerVT, Expand); setLoadExtAction(ISD::EXTLOAD, VT, InnerVT, Expand); } } if (!Subtarget.hasP8Vector()) { setOperationAction(ISD::SMAX, MVT::v2i64, Expand); setOperationAction(ISD::SMIN, MVT::v2i64, Expand); setOperationAction(ISD::UMAX, MVT::v2i64, Expand); setOperationAction(ISD::UMIN, MVT::v2i64, Expand); } for (auto VT : {MVT::v2i64, MVT::v4i32, MVT::v8i16, MVT::v16i8}) setOperationAction(ISD::ABS, VT, Custom); // We can custom expand all VECTOR_SHUFFLEs to VPERM, others we can handle // with merges, splats, etc. setOperationAction(ISD::VECTOR_SHUFFLE, MVT::v16i8, Custom); // Vector truncates to sub-word integer that fit in an Altivec/VSX register // are cheap, so handle them before they get expanded to scalar. setOperationAction(ISD::TRUNCATE, MVT::v8i8, Custom); setOperationAction(ISD::TRUNCATE, MVT::v4i8, Custom); setOperationAction(ISD::TRUNCATE, MVT::v2i8, Custom); setOperationAction(ISD::TRUNCATE, MVT::v4i16, Custom); setOperationAction(ISD::TRUNCATE, MVT::v2i16, Custom); setOperationAction(ISD::AND , MVT::v4i32, Legal); setOperationAction(ISD::OR , MVT::v4i32, Legal); setOperationAction(ISD::XOR , MVT::v4i32, Legal); setOperationAction(ISD::LOAD , MVT::v4i32, Legal); setOperationAction(ISD::SELECT, MVT::v4i32, Subtarget.useCRBits() ? Legal : Expand); setOperationAction(ISD::STORE , MVT::v4i32, Legal); setOperationAction(ISD::FP_TO_SINT, MVT::v4i32, Legal); setOperationAction(ISD::FP_TO_UINT, MVT::v4i32, Legal); setOperationAction(ISD::SINT_TO_FP, MVT::v4i32, Legal); setOperationAction(ISD::UINT_TO_FP, MVT::v4i32, Legal); setOperationAction(ISD::FFLOOR, MVT::v4f32, Legal); setOperationAction(ISD::FCEIL, MVT::v4f32, Legal); setOperationAction(ISD::FTRUNC, MVT::v4f32, Legal); setOperationAction(ISD::FNEARBYINT, MVT::v4f32, Legal); // Without hasP8Altivec set, v2i64 SMAX isn't available. // But ABS custom lowering requires SMAX support. if (!Subtarget.hasP8Altivec()) setOperationAction(ISD::ABS, MVT::v2i64, Expand); // With hasAltivec set, we can lower ISD::ROTL to vrl(b|h|w). if (Subtarget.hasAltivec()) for (auto VT : {MVT::v4i32, MVT::v8i16, MVT::v16i8}) setOperationAction(ISD::ROTL, VT, Legal); // With hasP8Altivec set, we can lower ISD::ROTL to vrld. if (Subtarget.hasP8Altivec()) setOperationAction(ISD::ROTL, MVT::v2i64, Legal); addRegisterClass(MVT::v4f32, &PPC::VRRCRegClass); addRegisterClass(MVT::v4i32, &PPC::VRRCRegClass); addRegisterClass(MVT::v8i16, &PPC::VRRCRegClass); addRegisterClass(MVT::v16i8, &PPC::VRRCRegClass); setOperationAction(ISD::MUL, MVT::v4f32, Legal); setOperationAction(ISD::FMA, MVT::v4f32, Legal); if (TM.Options.UnsafeFPMath || Subtarget.hasVSX()) { setOperationAction(ISD::FDIV, MVT::v4f32, Legal); setOperationAction(ISD::FSQRT, MVT::v4f32, Legal); } if (Subtarget.hasP8Altivec()) setOperationAction(ISD::MUL, MVT::v4i32, Legal); else setOperationAction(ISD::MUL, MVT::v4i32, Custom); setOperationAction(ISD::MUL, MVT::v8i16, Legal); setOperationAction(ISD::MUL, MVT::v16i8, Custom); setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v4f32, Custom); setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v4i32, Custom); setOperationAction(ISD::BUILD_VECTOR, MVT::v16i8, Custom); setOperationAction(ISD::BUILD_VECTOR, MVT::v8i16, Custom); setOperationAction(ISD::BUILD_VECTOR, MVT::v4i32, Custom); setOperationAction(ISD::BUILD_VECTOR, MVT::v4f32, Custom); // Altivec does not contain unordered floating-point compare instructions setCondCodeAction(ISD::SETUO, MVT::v4f32, Expand); setCondCodeAction(ISD::SETUEQ, MVT::v4f32, Expand); setCondCodeAction(ISD::SETO, MVT::v4f32, Expand); setCondCodeAction(ISD::SETONE, MVT::v4f32, Expand); if (Subtarget.hasVSX()) { setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v2f64, Legal); setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v2f64, Legal); if (Subtarget.hasP8Vector()) { setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v4f32, Legal); setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v4f32, Legal); } if (Subtarget.hasDirectMove() && isPPC64) { setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v16i8, Legal); setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v8i16, Legal); setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v4i32, Legal); setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v2i64, Legal); setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v16i8, Legal); setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v8i16, Legal); setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v4i32, Legal); setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v2i64, Legal); } setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v2f64, Legal); // The nearbyint variants are not allowed to raise the inexact exception // so we can only code-gen them with unsafe math. if (TM.Options.UnsafeFPMath) { setOperationAction(ISD::FNEARBYINT, MVT::f64, Legal); setOperationAction(ISD::FNEARBYINT, MVT::f32, Legal); } setOperationAction(ISD::FFLOOR, MVT::v2f64, Legal); setOperationAction(ISD::FCEIL, MVT::v2f64, Legal); setOperationAction(ISD::FTRUNC, MVT::v2f64, Legal); setOperationAction(ISD::FNEARBYINT, MVT::v2f64, Legal); setOperationAction(ISD::FRINT, MVT::v2f64, Legal); setOperationAction(ISD::FROUND, MVT::v2f64, Legal); setOperationAction(ISD::FROUND, MVT::f64, Legal); setOperationAction(ISD::FRINT, MVT::f64, Legal); setOperationAction(ISD::FNEARBYINT, MVT::v4f32, Legal); setOperationAction(ISD::FRINT, MVT::v4f32, Legal); setOperationAction(ISD::FROUND, MVT::v4f32, Legal); setOperationAction(ISD::FROUND, MVT::f32, Legal); setOperationAction(ISD::FRINT, MVT::f32, Legal); setOperationAction(ISD::MUL, MVT::v2f64, Legal); setOperationAction(ISD::FMA, MVT::v2f64, Legal); setOperationAction(ISD::FDIV, MVT::v2f64, Legal); setOperationAction(ISD::FSQRT, MVT::v2f64, Legal); // Share the Altivec comparison restrictions. setCondCodeAction(ISD::SETUO, MVT::v2f64, Expand); setCondCodeAction(ISD::SETUEQ, MVT::v2f64, Expand); setCondCodeAction(ISD::SETO, MVT::v2f64, Expand); setCondCodeAction(ISD::SETONE, MVT::v2f64, Expand); setOperationAction(ISD::LOAD, MVT::v2f64, Legal); setOperationAction(ISD::STORE, MVT::v2f64, Legal); setOperationAction(ISD::VECTOR_SHUFFLE, MVT::v2f64, Legal); if (Subtarget.hasP8Vector()) addRegisterClass(MVT::f32, &PPC::VSSRCRegClass); addRegisterClass(MVT::f64, &PPC::VSFRCRegClass); addRegisterClass(MVT::v4i32, &PPC::VSRCRegClass); addRegisterClass(MVT::v4f32, &PPC::VSRCRegClass); addRegisterClass(MVT::v2f64, &PPC::VSRCRegClass); if (Subtarget.hasP8Altivec()) { setOperationAction(ISD::SHL, MVT::v2i64, Legal); setOperationAction(ISD::SRA, MVT::v2i64, Legal); setOperationAction(ISD::SRL, MVT::v2i64, Legal); // 128 bit shifts can be accomplished via 3 instructions for SHL and // SRL, but not for SRA because of the instructions available: // VS{RL} and VS{RL}O. However due to direct move costs, it's not worth // doing setOperationAction(ISD::SHL, MVT::v1i128, Expand); setOperationAction(ISD::SRL, MVT::v1i128, Expand); setOperationAction(ISD::SRA, MVT::v1i128, Expand); setOperationAction(ISD::SETCC, MVT::v2i64, Legal); } else { setOperationAction(ISD::SHL, MVT::v2i64, Expand); setOperationAction(ISD::SRA, MVT::v2i64, Expand); setOperationAction(ISD::SRL, MVT::v2i64, Expand); setOperationAction(ISD::SETCC, MVT::v2i64, Custom); // VSX v2i64 only supports non-arithmetic operations. setOperationAction(ISD::ADD, MVT::v2i64, Expand); setOperationAction(ISD::SUB, MVT::v2i64, Expand); } setOperationAction(ISD::LOAD, MVT::v2i64, Promote); AddPromotedToType (ISD::LOAD, MVT::v2i64, MVT::v2f64); setOperationAction(ISD::STORE, MVT::v2i64, Promote); AddPromotedToType (ISD::STORE, MVT::v2i64, MVT::v2f64); setOperationAction(ISD::VECTOR_SHUFFLE, MVT::v2i64, Legal); setOperationAction(ISD::SINT_TO_FP, MVT::v2i64, Legal); setOperationAction(ISD::UINT_TO_FP, MVT::v2i64, Legal); setOperationAction(ISD::FP_TO_SINT, MVT::v2i64, Legal); setOperationAction(ISD::FP_TO_UINT, MVT::v2i64, Legal); // Custom handling for partial vectors of integers converted to // floating point. We already have optimal handling for v2i32 through // the DAG combine, so those aren't necessary. setOperationAction(ISD::UINT_TO_FP, MVT::v2i8, Custom); setOperationAction(ISD::UINT_TO_FP, MVT::v4i8, Custom); setOperationAction(ISD::UINT_TO_FP, MVT::v2i16, Custom); setOperationAction(ISD::UINT_TO_FP, MVT::v4i16, Custom); setOperationAction(ISD::SINT_TO_FP, MVT::v2i8, Custom); setOperationAction(ISD::SINT_TO_FP, MVT::v4i8, Custom); setOperationAction(ISD::SINT_TO_FP, MVT::v2i16, Custom); setOperationAction(ISD::SINT_TO_FP, MVT::v4i16, Custom); setOperationAction(ISD::FNEG, MVT::v4f32, Legal); setOperationAction(ISD::FNEG, MVT::v2f64, Legal); setOperationAction(ISD::FABS, MVT::v4f32, Legal); setOperationAction(ISD::FABS, MVT::v2f64, Legal); setOperationAction(ISD::FCOPYSIGN, MVT::v4f32, Legal); setOperationAction(ISD::FCOPYSIGN, MVT::v2f64, Legal); if (Subtarget.hasDirectMove()) setOperationAction(ISD::BUILD_VECTOR, MVT::v2i64, Custom); setOperationAction(ISD::BUILD_VECTOR, MVT::v2f64, Custom); // Handle constrained floating-point operations of vector. // The predictor is `hasVSX` because altivec instruction has // no exception but VSX vector instruction has. setOperationAction(ISD::STRICT_FADD, MVT::v4f32, Legal); setOperationAction(ISD::STRICT_FSUB, MVT::v4f32, Legal); setOperationAction(ISD::STRICT_FMUL, MVT::v4f32, Legal); setOperationAction(ISD::STRICT_FDIV, MVT::v4f32, Legal); setOperationAction(ISD::STRICT_FADD, MVT::v2f64, Legal); setOperationAction(ISD::STRICT_FSUB, MVT::v2f64, Legal); setOperationAction(ISD::STRICT_FMUL, MVT::v2f64, Legal); setOperationAction(ISD::STRICT_FDIV, MVT::v2f64, Legal); addRegisterClass(MVT::v2i64, &PPC::VSRCRegClass); } if (Subtarget.hasP8Altivec()) { addRegisterClass(MVT::v2i64, &PPC::VRRCRegClass); addRegisterClass(MVT::v1i128, &PPC::VRRCRegClass); } if (Subtarget.hasP9Vector()) { setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v4i32, Custom); setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v4f32, Custom); // 128 bit shifts can be accomplished via 3 instructions for SHL and // SRL, but not for SRA because of the instructions available: // VS{RL} and VS{RL}O. setOperationAction(ISD::SHL, MVT::v1i128, Legal); setOperationAction(ISD::SRL, MVT::v1i128, Legal); setOperationAction(ISD::SRA, MVT::v1i128, Expand); if (EnableQuadPrecision) { addRegisterClass(MVT::f128, &PPC::VRRCRegClass); setOperationAction(ISD::FADD, MVT::f128, Legal); setOperationAction(ISD::FSUB, MVT::f128, Legal); setOperationAction(ISD::FDIV, MVT::f128, Legal); setOperationAction(ISD::FMUL, MVT::f128, Legal); setOperationAction(ISD::FP_EXTEND, MVT::f128, Legal); // No extending loads to f128 on PPC. for (MVT FPT : MVT::fp_valuetypes()) setLoadExtAction(ISD::EXTLOAD, MVT::f128, FPT, Expand); setOperationAction(ISD::FMA, MVT::f128, Legal); setCondCodeAction(ISD::SETULT, MVT::f128, Expand); setCondCodeAction(ISD::SETUGT, MVT::f128, Expand); setCondCodeAction(ISD::SETUEQ, MVT::f128, Expand); setCondCodeAction(ISD::SETOGE, MVT::f128, Expand); setCondCodeAction(ISD::SETOLE, MVT::f128, Expand); setCondCodeAction(ISD::SETONE, MVT::f128, Expand); setOperationAction(ISD::FTRUNC, MVT::f128, Legal); setOperationAction(ISD::FRINT, MVT::f128, Legal); setOperationAction(ISD::FFLOOR, MVT::f128, Legal); setOperationAction(ISD::FCEIL, MVT::f128, Legal); setOperationAction(ISD::FNEARBYINT, MVT::f128, Legal); setOperationAction(ISD::FROUND, MVT::f128, Legal); setOperationAction(ISD::SELECT, MVT::f128, Expand); setOperationAction(ISD::FP_ROUND, MVT::f64, Legal); setOperationAction(ISD::FP_ROUND, MVT::f32, Legal); setTruncStoreAction(MVT::f128, MVT::f64, Expand); setTruncStoreAction(MVT::f128, MVT::f32, Expand); setOperationAction(ISD::BITCAST, MVT::i128, Custom); // No implementation for these ops for PowerPC. setOperationAction(ISD::FSIN , MVT::f128, Expand); setOperationAction(ISD::FCOS , MVT::f128, Expand); setOperationAction(ISD::FPOW, MVT::f128, Expand); setOperationAction(ISD::FPOWI, MVT::f128, Expand); setOperationAction(ISD::FREM, MVT::f128, Expand); // Handle constrained floating-point operations of fp128 setOperationAction(ISD::STRICT_FADD, MVT::f128, Legal); setOperationAction(ISD::STRICT_FSUB, MVT::f128, Legal); setOperationAction(ISD::STRICT_FMUL, MVT::f128, Legal); setOperationAction(ISD::STRICT_FDIV, MVT::f128, Legal); } setOperationAction(ISD::FP_EXTEND, MVT::v2f32, Custom); setOperationAction(ISD::BSWAP, MVT::v8i16, Legal); setOperationAction(ISD::BSWAP, MVT::v4i32, Legal); setOperationAction(ISD::BSWAP, MVT::v2i64, Legal); setOperationAction(ISD::BSWAP, MVT::v1i128, Legal); } if (Subtarget.hasP9Altivec()) { setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v8i16, Custom); setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v16i8, Custom); setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::v4i8, Legal); setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::v4i16, Legal); setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::v4i32, Legal); setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::v2i8, Legal); setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::v2i16, Legal); setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::v2i32, Legal); setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::v2i64, Legal); } } if (Subtarget.hasQPX()) { setOperationAction(ISD::FADD, MVT::v4f64, Legal); setOperationAction(ISD::FSUB, MVT::v4f64, Legal); setOperationAction(ISD::FMUL, MVT::v4f64, Legal); setOperationAction(ISD::FREM, MVT::v4f64, Expand); setOperationAction(ISD::FCOPYSIGN, MVT::v4f64, Legal); setOperationAction(ISD::FGETSIGN, MVT::v4f64, Expand); setOperationAction(ISD::LOAD , MVT::v4f64, Custom); setOperationAction(ISD::STORE , MVT::v4f64, Custom); setTruncStoreAction(MVT::v4f64, MVT::v4f32, Custom); setLoadExtAction(ISD::EXTLOAD, MVT::v4f64, MVT::v4f32, Custom); if (!Subtarget.useCRBits()) setOperationAction(ISD::SELECT, MVT::v4f64, Expand); setOperationAction(ISD::VSELECT, MVT::v4f64, Legal); setOperationAction(ISD::EXTRACT_VECTOR_ELT , MVT::v4f64, Legal); setOperationAction(ISD::INSERT_VECTOR_ELT , MVT::v4f64, Expand); setOperationAction(ISD::CONCAT_VECTORS , MVT::v4f64, Expand); setOperationAction(ISD::EXTRACT_SUBVECTOR , MVT::v4f64, Expand); setOperationAction(ISD::VECTOR_SHUFFLE , MVT::v4f64, Custom); setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v4f64, Legal); setOperationAction(ISD::BUILD_VECTOR, MVT::v4f64, Custom); setOperationAction(ISD::FP_TO_SINT , MVT::v4f64, Legal); setOperationAction(ISD::FP_TO_UINT , MVT::v4f64, Expand); setOperationAction(ISD::FP_ROUND , MVT::v4f32, Legal); setOperationAction(ISD::FP_EXTEND, MVT::v4f64, Legal); setOperationAction(ISD::FNEG , MVT::v4f64, Legal); setOperationAction(ISD::FABS , MVT::v4f64, Legal); setOperationAction(ISD::FSIN , MVT::v4f64, Expand); setOperationAction(ISD::FCOS , MVT::v4f64, Expand); setOperationAction(ISD::FPOW , MVT::v4f64, Expand); setOperationAction(ISD::FLOG , MVT::v4f64, Expand); setOperationAction(ISD::FLOG2 , MVT::v4f64, Expand); setOperationAction(ISD::FLOG10 , MVT::v4f64, Expand); setOperationAction(ISD::FEXP , MVT::v4f64, Expand); setOperationAction(ISD::FEXP2 , MVT::v4f64, Expand); setOperationAction(ISD::FMINNUM, MVT::v4f64, Legal); setOperationAction(ISD::FMAXNUM, MVT::v4f64, Legal); setIndexedLoadAction(ISD::PRE_INC, MVT::v4f64, Legal); setIndexedStoreAction(ISD::PRE_INC, MVT::v4f64, Legal); addRegisterClass(MVT::v4f64, &PPC::QFRCRegClass); setOperationAction(ISD::FADD, MVT::v4f32, Legal); setOperationAction(ISD::FSUB, MVT::v4f32, Legal); setOperationAction(ISD::FMUL, MVT::v4f32, Legal); setOperationAction(ISD::FREM, MVT::v4f32, Expand); setOperationAction(ISD::FCOPYSIGN, MVT::v4f32, Legal); setOperationAction(ISD::FGETSIGN, MVT::v4f32, Expand); setOperationAction(ISD::LOAD , MVT::v4f32, Custom); setOperationAction(ISD::STORE , MVT::v4f32, Custom); if (!Subtarget.useCRBits()) setOperationAction(ISD::SELECT, MVT::v4f32, Expand); setOperationAction(ISD::VSELECT, MVT::v4f32, Legal); setOperationAction(ISD::EXTRACT_VECTOR_ELT , MVT::v4f32, Legal); setOperationAction(ISD::INSERT_VECTOR_ELT , MVT::v4f32, Expand); setOperationAction(ISD::CONCAT_VECTORS , MVT::v4f32, Expand); setOperationAction(ISD::EXTRACT_SUBVECTOR , MVT::v4f32, Expand); setOperationAction(ISD::VECTOR_SHUFFLE , MVT::v4f32, Custom); setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v4f32, Legal); setOperationAction(ISD::BUILD_VECTOR, MVT::v4f32, Custom); setOperationAction(ISD::FP_TO_SINT , MVT::v4f32, Legal); setOperationAction(ISD::FP_TO_UINT , MVT::v4f32, Expand); setOperationAction(ISD::FNEG , MVT::v4f32, Legal); setOperationAction(ISD::FABS , MVT::v4f32, Legal); setOperationAction(ISD::FSIN , MVT::v4f32, Expand); setOperationAction(ISD::FCOS , MVT::v4f32, Expand); setOperationAction(ISD::FPOW , MVT::v4f32, Expand); setOperationAction(ISD::FLOG , MVT::v4f32, Expand); setOperationAction(ISD::FLOG2 , MVT::v4f32, Expand); setOperationAction(ISD::FLOG10 , MVT::v4f32, Expand); setOperationAction(ISD::FEXP , MVT::v4f32, Expand); setOperationAction(ISD::FEXP2 , MVT::v4f32, Expand); setOperationAction(ISD::FMINNUM, MVT::v4f32, Legal); setOperationAction(ISD::FMAXNUM, MVT::v4f32, Legal); setIndexedLoadAction(ISD::PRE_INC, MVT::v4f32, Legal); setIndexedStoreAction(ISD::PRE_INC, MVT::v4f32, Legal); addRegisterClass(MVT::v4f32, &PPC::QSRCRegClass); setOperationAction(ISD::AND , MVT::v4i1, Legal); setOperationAction(ISD::OR , MVT::v4i1, Legal); setOperationAction(ISD::XOR , MVT::v4i1, Legal); if (!Subtarget.useCRBits()) setOperationAction(ISD::SELECT, MVT::v4i1, Expand); setOperationAction(ISD::VSELECT, MVT::v4i1, Legal); setOperationAction(ISD::LOAD , MVT::v4i1, Custom); setOperationAction(ISD::STORE , MVT::v4i1, Custom); setOperationAction(ISD::EXTRACT_VECTOR_ELT , MVT::v4i1, Custom); setOperationAction(ISD::INSERT_VECTOR_ELT , MVT::v4i1, Expand); setOperationAction(ISD::CONCAT_VECTORS , MVT::v4i1, Expand); setOperationAction(ISD::EXTRACT_SUBVECTOR , MVT::v4i1, Expand); setOperationAction(ISD::VECTOR_SHUFFLE , MVT::v4i1, Custom); setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v4i1, Expand); setOperationAction(ISD::BUILD_VECTOR, MVT::v4i1, Custom); setOperationAction(ISD::SINT_TO_FP, MVT::v4i1, Custom); setOperationAction(ISD::UINT_TO_FP, MVT::v4i1, Custom); addRegisterClass(MVT::v4i1, &PPC::QBRCRegClass); setOperationAction(ISD::FFLOOR, MVT::v4f64, Legal); setOperationAction(ISD::FCEIL, MVT::v4f64, Legal); setOperationAction(ISD::FTRUNC, MVT::v4f64, Legal); setOperationAction(ISD::FROUND, MVT::v4f64, Legal); setOperationAction(ISD::FFLOOR, MVT::v4f32, Legal); setOperationAction(ISD::FCEIL, MVT::v4f32, Legal); setOperationAction(ISD::FTRUNC, MVT::v4f32, Legal); setOperationAction(ISD::FROUND, MVT::v4f32, Legal); setOperationAction(ISD::FNEARBYINT, MVT::v4f64, Expand); setOperationAction(ISD::FNEARBYINT, MVT::v4f32, Expand); // These need to set FE_INEXACT, and so cannot be vectorized here. setOperationAction(ISD::FRINT, MVT::v4f64, Expand); setOperationAction(ISD::FRINT, MVT::v4f32, Expand); if (TM.Options.UnsafeFPMath) { setOperationAction(ISD::FDIV, MVT::v4f64, Legal); setOperationAction(ISD::FSQRT, MVT::v4f64, Legal); setOperationAction(ISD::FDIV, MVT::v4f32, Legal); setOperationAction(ISD::FSQRT, MVT::v4f32, Legal); } else { setOperationAction(ISD::FDIV, MVT::v4f64, Expand); setOperationAction(ISD::FSQRT, MVT::v4f64, Expand); setOperationAction(ISD::FDIV, MVT::v4f32, Expand); setOperationAction(ISD::FSQRT, MVT::v4f32, Expand); } // TODO: Handle constrained floating-point operations of v4f64 } if (Subtarget.has64BitSupport()) setOperationAction(ISD::PREFETCH, MVT::Other, Legal); setOperationAction(ISD::READCYCLECOUNTER, MVT::i64, isPPC64 ? Legal : Custom); if (!isPPC64) { setOperationAction(ISD::ATOMIC_LOAD, MVT::i64, Expand); setOperationAction(ISD::ATOMIC_STORE, MVT::i64, Expand); } setBooleanContents(ZeroOrOneBooleanContent); if (Subtarget.hasAltivec()) { // Altivec instructions set fields to all zeros or all ones. setBooleanVectorContents(ZeroOrNegativeOneBooleanContent); } if (!isPPC64) { // These libcalls are not available in 32-bit. setLibcallName(RTLIB::SHL_I128, nullptr); setLibcallName(RTLIB::SRL_I128, nullptr); setLibcallName(RTLIB::SRA_I128, nullptr); } setStackPointerRegisterToSaveRestore(isPPC64 ? PPC::X1 : PPC::R1); // We have target-specific dag combine patterns for the following nodes: setTargetDAGCombine(ISD::ADD); setTargetDAGCombine(ISD::SHL); setTargetDAGCombine(ISD::SRA); setTargetDAGCombine(ISD::SRL); setTargetDAGCombine(ISD::MUL); setTargetDAGCombine(ISD::SINT_TO_FP); setTargetDAGCombine(ISD::BUILD_VECTOR); if (Subtarget.hasFPCVT()) setTargetDAGCombine(ISD::UINT_TO_FP); setTargetDAGCombine(ISD::LOAD); setTargetDAGCombine(ISD::STORE); setTargetDAGCombine(ISD::BR_CC); if (Subtarget.useCRBits()) setTargetDAGCombine(ISD::BRCOND); setTargetDAGCombine(ISD::BSWAP); setTargetDAGCombine(ISD::INTRINSIC_WO_CHAIN); setTargetDAGCombine(ISD::INTRINSIC_W_CHAIN); setTargetDAGCombine(ISD::INTRINSIC_VOID); setTargetDAGCombine(ISD::SIGN_EXTEND); setTargetDAGCombine(ISD::ZERO_EXTEND); setTargetDAGCombine(ISD::ANY_EXTEND); setTargetDAGCombine(ISD::TRUNCATE); setTargetDAGCombine(ISD::VECTOR_SHUFFLE); if (Subtarget.useCRBits()) { setTargetDAGCombine(ISD::TRUNCATE); setTargetDAGCombine(ISD::SETCC); setTargetDAGCombine(ISD::SELECT_CC); } // Use reciprocal estimates. if (TM.Options.UnsafeFPMath) { setTargetDAGCombine(ISD::FDIV); setTargetDAGCombine(ISD::FSQRT); } if (Subtarget.hasP9Altivec()) { setTargetDAGCombine(ISD::ABS); setTargetDAGCombine(ISD::VSELECT); } if (EnableQuadPrecision) { setLibcallName(RTLIB::LOG_F128, "logf128"); setLibcallName(RTLIB::LOG2_F128, "log2f128"); setLibcallName(RTLIB::LOG10_F128, "log10f128"); setLibcallName(RTLIB::EXP_F128, "expf128"); setLibcallName(RTLIB::EXP2_F128, "exp2f128"); setLibcallName(RTLIB::SIN_F128, "sinf128"); setLibcallName(RTLIB::COS_F128, "cosf128"); setLibcallName(RTLIB::POW_F128, "powf128"); setLibcallName(RTLIB::FMIN_F128, "fminf128"); setLibcallName(RTLIB::FMAX_F128, "fmaxf128"); setLibcallName(RTLIB::POWI_F128, "__powikf2"); setLibcallName(RTLIB::REM_F128, "fmodf128"); } // With 32 condition bits, we don't need to sink (and duplicate) compares // aggressively in CodeGenPrep. if (Subtarget.useCRBits()) { setHasMultipleConditionRegisters(); setJumpIsExpensive(); } setMinFunctionAlignment(Align(4)); switch (Subtarget.getCPUDirective()) { default: break; case PPC::DIR_970: case PPC::DIR_A2: case PPC::DIR_E500: case PPC::DIR_E500mc: case PPC::DIR_E5500: case PPC::DIR_PWR4: case PPC::DIR_PWR5: case PPC::DIR_PWR5X: case PPC::DIR_PWR6: case PPC::DIR_PWR6X: case PPC::DIR_PWR7: case PPC::DIR_PWR8: case PPC::DIR_PWR9: case PPC::DIR_PWR_FUTURE: setPrefLoopAlignment(Align(16)); setPrefFunctionAlignment(Align(16)); break; } if (Subtarget.enableMachineScheduler()) setSchedulingPreference(Sched::Source); else setSchedulingPreference(Sched::Hybrid); computeRegisterProperties(STI.getRegisterInfo()); // The Freescale cores do better with aggressive inlining of memcpy and // friends. GCC uses same threshold of 128 bytes (= 32 word stores). if (Subtarget.getCPUDirective() == PPC::DIR_E500mc || Subtarget.getCPUDirective() == PPC::DIR_E5500) { MaxStoresPerMemset = 32; MaxStoresPerMemsetOptSize = 16; MaxStoresPerMemcpy = 32; MaxStoresPerMemcpyOptSize = 8; MaxStoresPerMemmove = 32; MaxStoresPerMemmoveOptSize = 8; } else if (Subtarget.getCPUDirective() == PPC::DIR_A2) { // The A2 also benefits from (very) aggressive inlining of memcpy and // friends. The overhead of a the function call, even when warm, can be // over one hundred cycles. MaxStoresPerMemset = 128; MaxStoresPerMemcpy = 128; MaxStoresPerMemmove = 128; MaxLoadsPerMemcmp = 128; } else { MaxLoadsPerMemcmp = 8; MaxLoadsPerMemcmpOptSize = 4; } } /// getMaxByValAlign - Helper for getByValTypeAlignment to determine /// the desired ByVal argument alignment. static void getMaxByValAlign(Type *Ty, unsigned &MaxAlign, unsigned MaxMaxAlign) { if (MaxAlign == MaxMaxAlign) return; if (VectorType *VTy = dyn_cast(Ty)) { if (MaxMaxAlign >= 32 && VTy->getBitWidth() >= 256) MaxAlign = 32; else if (VTy->getBitWidth() >= 128 && MaxAlign < 16) MaxAlign = 16; } else if (ArrayType *ATy = dyn_cast(Ty)) { unsigned EltAlign = 0; getMaxByValAlign(ATy->getElementType(), EltAlign, MaxMaxAlign); if (EltAlign > MaxAlign) MaxAlign = EltAlign; } else if (StructType *STy = dyn_cast(Ty)) { for (auto *EltTy : STy->elements()) { unsigned EltAlign = 0; getMaxByValAlign(EltTy, EltAlign, MaxMaxAlign); if (EltAlign > MaxAlign) MaxAlign = EltAlign; if (MaxAlign == MaxMaxAlign) break; } } } /// getByValTypeAlignment - Return the desired alignment for ByVal aggregate /// function arguments in the caller parameter area. unsigned PPCTargetLowering::getByValTypeAlignment(Type *Ty, const DataLayout &DL) const { // 16byte and wider vectors are passed on 16byte boundary. // The rest is 8 on PPC64 and 4 on PPC32 boundary. unsigned Align = Subtarget.isPPC64() ? 8 : 4; if (Subtarget.hasAltivec() || Subtarget.hasQPX()) getMaxByValAlign(Ty, Align, Subtarget.hasQPX() ? 32 : 16); return Align; } bool PPCTargetLowering::useSoftFloat() const { return Subtarget.useSoftFloat(); } bool PPCTargetLowering::hasSPE() const { return Subtarget.hasSPE(); } bool PPCTargetLowering::preferIncOfAddToSubOfNot(EVT VT) const { return VT.isScalarInteger(); } const char *PPCTargetLowering::getTargetNodeName(unsigned Opcode) const { switch ((PPCISD::NodeType)Opcode) { case PPCISD::FIRST_NUMBER: break; case PPCISD::FSEL: return "PPCISD::FSEL"; case PPCISD::XSMAXCDP: return "PPCISD::XSMAXCDP"; case PPCISD::XSMINCDP: return "PPCISD::XSMINCDP"; case PPCISD::FCFID: return "PPCISD::FCFID"; case PPCISD::FCFIDU: return "PPCISD::FCFIDU"; case PPCISD::FCFIDS: return "PPCISD::FCFIDS"; case PPCISD::FCFIDUS: return "PPCISD::FCFIDUS"; case PPCISD::FCTIDZ: return "PPCISD::FCTIDZ"; case PPCISD::FCTIWZ: return "PPCISD::FCTIWZ"; case PPCISD::FCTIDUZ: return "PPCISD::FCTIDUZ"; case PPCISD::FCTIWUZ: return "PPCISD::FCTIWUZ"; case PPCISD::FP_TO_UINT_IN_VSR: return "PPCISD::FP_TO_UINT_IN_VSR,"; case PPCISD::FP_TO_SINT_IN_VSR: return "PPCISD::FP_TO_SINT_IN_VSR"; case PPCISD::FRE: return "PPCISD::FRE"; case PPCISD::FRSQRTE: return "PPCISD::FRSQRTE"; case PPCISD::STFIWX: return "PPCISD::STFIWX"; case PPCISD::VMADDFP: return "PPCISD::VMADDFP"; case PPCISD::VNMSUBFP: return "PPCISD::VNMSUBFP"; case PPCISD::VPERM: return "PPCISD::VPERM"; case PPCISD::XXSPLT: return "PPCISD::XXSPLT"; case PPCISD::VECINSERT: return "PPCISD::VECINSERT"; case PPCISD::XXPERMDI: return "PPCISD::XXPERMDI"; case PPCISD::VECSHL: return "PPCISD::VECSHL"; case PPCISD::CMPB: return "PPCISD::CMPB"; case PPCISD::Hi: return "PPCISD::Hi"; case PPCISD::Lo: return "PPCISD::Lo"; case PPCISD::TOC_ENTRY: return "PPCISD::TOC_ENTRY"; case PPCISD::ATOMIC_CMP_SWAP_8: return "PPCISD::ATOMIC_CMP_SWAP_8"; case PPCISD::ATOMIC_CMP_SWAP_16: return "PPCISD::ATOMIC_CMP_SWAP_16"; case PPCISD::DYNALLOC: return "PPCISD::DYNALLOC"; case PPCISD::DYNAREAOFFSET: return "PPCISD::DYNAREAOFFSET"; case PPCISD::GlobalBaseReg: return "PPCISD::GlobalBaseReg"; case PPCISD::SRL: return "PPCISD::SRL"; case PPCISD::SRA: return "PPCISD::SRA"; case PPCISD::SHL: return "PPCISD::SHL"; case PPCISD::SRA_ADDZE: return "PPCISD::SRA_ADDZE"; case PPCISD::CALL: return "PPCISD::CALL"; case PPCISD::CALL_NOP: return "PPCISD::CALL_NOP"; + case PPCISD::CALL_NOTOC: return "PPCISD::CALL_NOTOC"; case PPCISD::MTCTR: return "PPCISD::MTCTR"; case PPCISD::BCTRL: return "PPCISD::BCTRL"; case PPCISD::BCTRL_LOAD_TOC: return "PPCISD::BCTRL_LOAD_TOC"; case PPCISD::RET_FLAG: return "PPCISD::RET_FLAG"; case PPCISD::READ_TIME_BASE: return "PPCISD::READ_TIME_BASE"; case PPCISD::EH_SJLJ_SETJMP: return "PPCISD::EH_SJLJ_SETJMP"; case PPCISD::EH_SJLJ_LONGJMP: return "PPCISD::EH_SJLJ_LONGJMP"; case PPCISD::MFOCRF: return "PPCISD::MFOCRF"; case PPCISD::MFVSR: return "PPCISD::MFVSR"; case PPCISD::MTVSRA: return "PPCISD::MTVSRA"; case PPCISD::MTVSRZ: return "PPCISD::MTVSRZ"; case PPCISD::SINT_VEC_TO_FP: return "PPCISD::SINT_VEC_TO_FP"; case PPCISD::UINT_VEC_TO_FP: return "PPCISD::UINT_VEC_TO_FP"; case PPCISD::ANDI_rec_1_EQ_BIT: return "PPCISD::ANDI_rec_1_EQ_BIT"; case PPCISD::ANDI_rec_1_GT_BIT: return "PPCISD::ANDI_rec_1_GT_BIT"; case PPCISD::VCMP: return "PPCISD::VCMP"; case PPCISD::VCMPo: return "PPCISD::VCMPo"; case PPCISD::LBRX: return "PPCISD::LBRX"; case PPCISD::STBRX: return "PPCISD::STBRX"; case PPCISD::LFIWAX: return "PPCISD::LFIWAX"; case PPCISD::LFIWZX: return "PPCISD::LFIWZX"; case PPCISD::LXSIZX: return "PPCISD::LXSIZX"; case PPCISD::STXSIX: return "PPCISD::STXSIX"; case PPCISD::VEXTS: return "PPCISD::VEXTS"; case PPCISD::LXVD2X: return "PPCISD::LXVD2X"; case PPCISD::STXVD2X: return "PPCISD::STXVD2X"; case PPCISD::LOAD_VEC_BE: return "PPCISD::LOAD_VEC_BE"; case PPCISD::STORE_VEC_BE: return "PPCISD::STORE_VEC_BE"; case PPCISD::ST_VSR_SCAL_INT: return "PPCISD::ST_VSR_SCAL_INT"; case PPCISD::COND_BRANCH: return "PPCISD::COND_BRANCH"; case PPCISD::BDNZ: return "PPCISD::BDNZ"; case PPCISD::BDZ: return "PPCISD::BDZ"; case PPCISD::MFFS: return "PPCISD::MFFS"; case PPCISD::FADDRTZ: return "PPCISD::FADDRTZ"; case PPCISD::TC_RETURN: return "PPCISD::TC_RETURN"; case PPCISD::CR6SET: return "PPCISD::CR6SET"; case PPCISD::CR6UNSET: return "PPCISD::CR6UNSET"; case PPCISD::PPC32_GOT: return "PPCISD::PPC32_GOT"; case PPCISD::PPC32_PICGOT: return "PPCISD::PPC32_PICGOT"; case PPCISD::ADDIS_GOT_TPREL_HA: return "PPCISD::ADDIS_GOT_TPREL_HA"; case PPCISD::LD_GOT_TPREL_L: return "PPCISD::LD_GOT_TPREL_L"; case PPCISD::ADD_TLS: return "PPCISD::ADD_TLS"; case PPCISD::ADDIS_TLSGD_HA: return "PPCISD::ADDIS_TLSGD_HA"; case PPCISD::ADDI_TLSGD_L: return "PPCISD::ADDI_TLSGD_L"; case PPCISD::GET_TLS_ADDR: return "PPCISD::GET_TLS_ADDR"; case PPCISD::ADDI_TLSGD_L_ADDR: return "PPCISD::ADDI_TLSGD_L_ADDR"; case PPCISD::ADDIS_TLSLD_HA: return "PPCISD::ADDIS_TLSLD_HA"; case PPCISD::ADDI_TLSLD_L: return "PPCISD::ADDI_TLSLD_L"; case PPCISD::GET_TLSLD_ADDR: return "PPCISD::GET_TLSLD_ADDR"; case PPCISD::ADDI_TLSLD_L_ADDR: return "PPCISD::ADDI_TLSLD_L_ADDR"; case PPCISD::ADDIS_DTPREL_HA: return "PPCISD::ADDIS_DTPREL_HA"; case PPCISD::ADDI_DTPREL_L: return "PPCISD::ADDI_DTPREL_L"; case PPCISD::VADD_SPLAT: return "PPCISD::VADD_SPLAT"; case PPCISD::SC: return "PPCISD::SC"; case PPCISD::CLRBHRB: return "PPCISD::CLRBHRB"; case PPCISD::MFBHRBE: return "PPCISD::MFBHRBE"; case PPCISD::RFEBB: return "PPCISD::RFEBB"; case PPCISD::XXSWAPD: return "PPCISD::XXSWAPD"; case PPCISD::SWAP_NO_CHAIN: return "PPCISD::SWAP_NO_CHAIN"; case PPCISD::VABSD: return "PPCISD::VABSD"; case PPCISD::QVFPERM: return "PPCISD::QVFPERM"; case PPCISD::QVGPCI: return "PPCISD::QVGPCI"; case PPCISD::QVALIGNI: return "PPCISD::QVALIGNI"; case PPCISD::QVESPLATI: return "PPCISD::QVESPLATI"; case PPCISD::QBFLT: return "PPCISD::QBFLT"; case PPCISD::QVLFSb: return "PPCISD::QVLFSb"; case PPCISD::BUILD_FP128: return "PPCISD::BUILD_FP128"; case PPCISD::BUILD_SPE64: return "PPCISD::BUILD_SPE64"; case PPCISD::EXTRACT_SPE: return "PPCISD::EXTRACT_SPE"; case PPCISD::EXTSWSLI: return "PPCISD::EXTSWSLI"; case PPCISD::LD_VSX_LH: return "PPCISD::LD_VSX_LH"; case PPCISD::FP_EXTEND_HALF: return "PPCISD::FP_EXTEND_HALF"; case PPCISD::LD_SPLAT: return "PPCISD::LD_SPLAT"; } return nullptr; } EVT PPCTargetLowering::getSetCCResultType(const DataLayout &DL, LLVMContext &C, EVT VT) const { if (!VT.isVector()) return Subtarget.useCRBits() ? MVT::i1 : MVT::i32; if (Subtarget.hasQPX()) return EVT::getVectorVT(C, MVT::i1, VT.getVectorNumElements()); return VT.changeVectorElementTypeToInteger(); } bool PPCTargetLowering::enableAggressiveFMAFusion(EVT VT) const { assert(VT.isFloatingPoint() && "Non-floating-point FMA?"); return true; } //===----------------------------------------------------------------------===// // Node matching predicates, for use by the tblgen matching code. //===----------------------------------------------------------------------===// /// isFloatingPointZero - Return true if this is 0.0 or -0.0. static bool isFloatingPointZero(SDValue Op) { if (ConstantFPSDNode *CFP = dyn_cast(Op)) return CFP->getValueAPF().isZero(); else if (ISD::isEXTLoad(Op.getNode()) || ISD::isNON_EXTLoad(Op.getNode())) { // Maybe this has already been legalized into the constant pool? if (ConstantPoolSDNode *CP = dyn_cast(Op.getOperand(1))) if (const ConstantFP *CFP = dyn_cast(CP->getConstVal())) return CFP->getValueAPF().isZero(); } return false; } /// isConstantOrUndef - Op is either an undef node or a ConstantSDNode. Return /// true if Op is undef or if it matches the specified value. static bool isConstantOrUndef(int Op, int Val) { return Op < 0 || Op == Val; } /// isVPKUHUMShuffleMask - Return true if this is the shuffle mask for a /// VPKUHUM instruction. /// The ShuffleKind distinguishes between big-endian operations with /// two different inputs (0), either-endian operations with two identical /// inputs (1), and little-endian operations with two different inputs (2). /// For the latter, the input operands are swapped (see PPCInstrAltivec.td). bool PPC::isVPKUHUMShuffleMask(ShuffleVectorSDNode *N, unsigned ShuffleKind, SelectionDAG &DAG) { bool IsLE = DAG.getDataLayout().isLittleEndian(); if (ShuffleKind == 0) { if (IsLE) return false; for (unsigned i = 0; i != 16; ++i) if (!isConstantOrUndef(N->getMaskElt(i), i*2+1)) return false; } else if (ShuffleKind == 2) { if (!IsLE) return false; for (unsigned i = 0; i != 16; ++i) if (!isConstantOrUndef(N->getMaskElt(i), i*2)) return false; } else if (ShuffleKind == 1) { unsigned j = IsLE ? 0 : 1; for (unsigned i = 0; i != 8; ++i) if (!isConstantOrUndef(N->getMaskElt(i), i*2+j) || !isConstantOrUndef(N->getMaskElt(i+8), i*2+j)) return false; } return true; } /// isVPKUWUMShuffleMask - Return true if this is the shuffle mask for a /// VPKUWUM instruction. /// The ShuffleKind distinguishes between big-endian operations with /// two different inputs (0), either-endian operations with two identical /// inputs (1), and little-endian operations with two different inputs (2). /// For the latter, the input operands are swapped (see PPCInstrAltivec.td). bool PPC::isVPKUWUMShuffleMask(ShuffleVectorSDNode *N, unsigned ShuffleKind, SelectionDAG &DAG) { bool IsLE = DAG.getDataLayout().isLittleEndian(); if (ShuffleKind == 0) { if (IsLE) return false; for (unsigned i = 0; i != 16; i += 2) if (!isConstantOrUndef(N->getMaskElt(i ), i*2+2) || !isConstantOrUndef(N->getMaskElt(i+1), i*2+3)) return false; } else if (ShuffleKind == 2) { if (!IsLE) return false; for (unsigned i = 0; i != 16; i += 2) if (!isConstantOrUndef(N->getMaskElt(i ), i*2) || !isConstantOrUndef(N->getMaskElt(i+1), i*2+1)) return false; } else if (ShuffleKind == 1) { unsigned j = IsLE ? 0 : 2; for (unsigned i = 0; i != 8; i += 2) if (!isConstantOrUndef(N->getMaskElt(i ), i*2+j) || !isConstantOrUndef(N->getMaskElt(i+1), i*2+j+1) || !isConstantOrUndef(N->getMaskElt(i+8), i*2+j) || !isConstantOrUndef(N->getMaskElt(i+9), i*2+j+1)) return false; } return true; } /// isVPKUDUMShuffleMask - Return true if this is the shuffle mask for a /// VPKUDUM instruction, AND the VPKUDUM instruction exists for the /// current subtarget. /// /// The ShuffleKind distinguishes between big-endian operations with /// two different inputs (0), either-endian operations with two identical /// inputs (1), and little-endian operations with two different inputs (2). /// For the latter, the input operands are swapped (see PPCInstrAltivec.td). bool PPC::isVPKUDUMShuffleMask(ShuffleVectorSDNode *N, unsigned ShuffleKind, SelectionDAG &DAG) { const PPCSubtarget& Subtarget = static_cast(DAG.getSubtarget()); if (!Subtarget.hasP8Vector()) return false; bool IsLE = DAG.getDataLayout().isLittleEndian(); if (ShuffleKind == 0) { if (IsLE) return false; for (unsigned i = 0; i != 16; i += 4) if (!isConstantOrUndef(N->getMaskElt(i ), i*2+4) || !isConstantOrUndef(N->getMaskElt(i+1), i*2+5) || !isConstantOrUndef(N->getMaskElt(i+2), i*2+6) || !isConstantOrUndef(N->getMaskElt(i+3), i*2+7)) return false; } else if (ShuffleKind == 2) { if (!IsLE) return false; for (unsigned i = 0; i != 16; i += 4) if (!isConstantOrUndef(N->getMaskElt(i ), i*2) || !isConstantOrUndef(N->getMaskElt(i+1), i*2+1) || !isConstantOrUndef(N->getMaskElt(i+2), i*2+2) || !isConstantOrUndef(N->getMaskElt(i+3), i*2+3)) return false; } else if (ShuffleKind == 1) { unsigned j = IsLE ? 0 : 4; for (unsigned i = 0; i != 8; i += 4) if (!isConstantOrUndef(N->getMaskElt(i ), i*2+j) || !isConstantOrUndef(N->getMaskElt(i+1), i*2+j+1) || !isConstantOrUndef(N->getMaskElt(i+2), i*2+j+2) || !isConstantOrUndef(N->getMaskElt(i+3), i*2+j+3) || !isConstantOrUndef(N->getMaskElt(i+8), i*2+j) || !isConstantOrUndef(N->getMaskElt(i+9), i*2+j+1) || !isConstantOrUndef(N->getMaskElt(i+10), i*2+j+2) || !isConstantOrUndef(N->getMaskElt(i+11), i*2+j+3)) return false; } return true; } /// isVMerge - Common function, used to match vmrg* shuffles. /// static bool isVMerge(ShuffleVectorSDNode *N, unsigned UnitSize, unsigned LHSStart, unsigned RHSStart) { if (N->getValueType(0) != MVT::v16i8) return false; assert((UnitSize == 1 || UnitSize == 2 || UnitSize == 4) && "Unsupported merge size!"); for (unsigned i = 0; i != 8/UnitSize; ++i) // Step over units for (unsigned j = 0; j != UnitSize; ++j) { // Step over bytes within unit if (!isConstantOrUndef(N->getMaskElt(i*UnitSize*2+j), LHSStart+j+i*UnitSize) || !isConstantOrUndef(N->getMaskElt(i*UnitSize*2+UnitSize+j), RHSStart+j+i*UnitSize)) return false; } return true; } /// isVMRGLShuffleMask - Return true if this is a shuffle mask suitable for /// a VMRGL* instruction with the specified unit size (1,2 or 4 bytes). /// The ShuffleKind distinguishes between big-endian merges with two /// different inputs (0), either-endian merges with two identical inputs (1), /// and little-endian merges with two different inputs (2). For the latter, /// the input operands are swapped (see PPCInstrAltivec.td). bool PPC::isVMRGLShuffleMask(ShuffleVectorSDNode *N, unsigned UnitSize, unsigned ShuffleKind, SelectionDAG &DAG) { if (DAG.getDataLayout().isLittleEndian()) { if (ShuffleKind == 1) // unary return isVMerge(N, UnitSize, 0, 0); else if (ShuffleKind == 2) // swapped return isVMerge(N, UnitSize, 0, 16); else return false; } else { if (ShuffleKind == 1) // unary return isVMerge(N, UnitSize, 8, 8); else if (ShuffleKind == 0) // normal return isVMerge(N, UnitSize, 8, 24); else return false; } } /// isVMRGHShuffleMask - Return true if this is a shuffle mask suitable for /// a VMRGH* instruction with the specified unit size (1,2 or 4 bytes). /// The ShuffleKind distinguishes between big-endian merges with two /// different inputs (0), either-endian merges with two identical inputs (1), /// and little-endian merges with two different inputs (2). For the latter, /// the input operands are swapped (see PPCInstrAltivec.td). bool PPC::isVMRGHShuffleMask(ShuffleVectorSDNode *N, unsigned UnitSize, unsigned ShuffleKind, SelectionDAG &DAG) { if (DAG.getDataLayout().isLittleEndian()) { if (ShuffleKind == 1) // unary return isVMerge(N, UnitSize, 8, 8); else if (ShuffleKind == 2) // swapped return isVMerge(N, UnitSize, 8, 24); else return false; } else { if (ShuffleKind == 1) // unary return isVMerge(N, UnitSize, 0, 0); else if (ShuffleKind == 0) // normal return isVMerge(N, UnitSize, 0, 16); else return false; } } /** * Common function used to match vmrgew and vmrgow shuffles * * The indexOffset determines whether to look for even or odd words in * the shuffle mask. This is based on the of the endianness of the target * machine. * - Little Endian: * - Use offset of 0 to check for odd elements * - Use offset of 4 to check for even elements * - Big Endian: * - Use offset of 0 to check for even elements * - Use offset of 4 to check for odd elements * A detailed description of the vector element ordering for little endian and * big endian can be found at * http://www.ibm.com/developerworks/library/l-ibm-xl-c-cpp-compiler/index.html * Targeting your applications - what little endian and big endian IBM XL C/C++ * compiler differences mean to you * * The mask to the shuffle vector instruction specifies the indices of the * elements from the two input vectors to place in the result. The elements are * numbered in array-access order, starting with the first vector. These vectors * are always of type v16i8, thus each vector will contain 16 elements of size * 8. More info on the shuffle vector can be found in the * http://llvm.org/docs/LangRef.html#shufflevector-instruction * Language Reference. * * The RHSStartValue indicates whether the same input vectors are used (unary) * or two different input vectors are used, based on the following: * - If the instruction uses the same vector for both inputs, the range of the * indices will be 0 to 15. In this case, the RHSStart value passed should * be 0. * - If the instruction has two different vectors then the range of the * indices will be 0 to 31. In this case, the RHSStart value passed should * be 16 (indices 0-15 specify elements in the first vector while indices 16 * to 31 specify elements in the second vector). * * \param[in] N The shuffle vector SD Node to analyze * \param[in] IndexOffset Specifies whether to look for even or odd elements * \param[in] RHSStartValue Specifies the starting index for the righthand input * vector to the shuffle_vector instruction * \return true iff this shuffle vector represents an even or odd word merge */ static bool isVMerge(ShuffleVectorSDNode *N, unsigned IndexOffset, unsigned RHSStartValue) { if (N->getValueType(0) != MVT::v16i8) return false; for (unsigned i = 0; i < 2; ++i) for (unsigned j = 0; j < 4; ++j) if (!isConstantOrUndef(N->getMaskElt(i*4+j), i*RHSStartValue+j+IndexOffset) || !isConstantOrUndef(N->getMaskElt(i*4+j+8), i*RHSStartValue+j+IndexOffset+8)) return false; return true; } /** * Determine if the specified shuffle mask is suitable for the vmrgew or * vmrgow instructions. * * \param[in] N The shuffle vector SD Node to analyze * \param[in] CheckEven Check for an even merge (true) or an odd merge (false) * \param[in] ShuffleKind Identify the type of merge: * - 0 = big-endian merge with two different inputs; * - 1 = either-endian merge with two identical inputs; * - 2 = little-endian merge with two different inputs (inputs are swapped for * little-endian merges). * \param[in] DAG The current SelectionDAG * \return true iff this shuffle mask */ bool PPC::isVMRGEOShuffleMask(ShuffleVectorSDNode *N, bool CheckEven, unsigned ShuffleKind, SelectionDAG &DAG) { if (DAG.getDataLayout().isLittleEndian()) { unsigned indexOffset = CheckEven ? 4 : 0; if (ShuffleKind == 1) // Unary return isVMerge(N, indexOffset, 0); else if (ShuffleKind == 2) // swapped return isVMerge(N, indexOffset, 16); else return false; } else { unsigned indexOffset = CheckEven ? 0 : 4; if (ShuffleKind == 1) // Unary return isVMerge(N, indexOffset, 0); else if (ShuffleKind == 0) // Normal return isVMerge(N, indexOffset, 16); else return false; } return false; } /// isVSLDOIShuffleMask - If this is a vsldoi shuffle mask, return the shift /// amount, otherwise return -1. /// The ShuffleKind distinguishes between big-endian operations with two /// different inputs (0), either-endian operations with two identical inputs /// (1), and little-endian operations with two different inputs (2). For the /// latter, the input operands are swapped (see PPCInstrAltivec.td). int PPC::isVSLDOIShuffleMask(SDNode *N, unsigned ShuffleKind, SelectionDAG &DAG) { if (N->getValueType(0) != MVT::v16i8) return -1; ShuffleVectorSDNode *SVOp = cast(N); // Find the first non-undef value in the shuffle mask. unsigned i; for (i = 0; i != 16 && SVOp->getMaskElt(i) < 0; ++i) /*search*/; if (i == 16) return -1; // all undef. // Otherwise, check to see if the rest of the elements are consecutively // numbered from this value. unsigned ShiftAmt = SVOp->getMaskElt(i); if (ShiftAmt < i) return -1; ShiftAmt -= i; bool isLE = DAG.getDataLayout().isLittleEndian(); if ((ShuffleKind == 0 && !isLE) || (ShuffleKind == 2 && isLE)) { // Check the rest of the elements to see if they are consecutive. for (++i; i != 16; ++i) if (!isConstantOrUndef(SVOp->getMaskElt(i), ShiftAmt+i)) return -1; } else if (ShuffleKind == 1) { // Check the rest of the elements to see if they are consecutive. for (++i; i != 16; ++i) if (!isConstantOrUndef(SVOp->getMaskElt(i), (ShiftAmt+i) & 15)) return -1; } else return -1; if (isLE) ShiftAmt = 16 - ShiftAmt; return ShiftAmt; } /// isSplatShuffleMask - Return true if the specified VECTOR_SHUFFLE operand /// specifies a splat of a single element that is suitable for input to /// one of the splat operations (VSPLTB/VSPLTH/VSPLTW/XXSPLTW/LXVDSX/etc.). bool PPC::isSplatShuffleMask(ShuffleVectorSDNode *N, unsigned EltSize) { assert(N->getValueType(0) == MVT::v16i8 && isPowerOf2_32(EltSize) && EltSize <= 8 && "Can only handle 1,2,4,8 byte element sizes"); // The consecutive indices need to specify an element, not part of two // different elements. So abandon ship early if this isn't the case. if (N->getMaskElt(0) % EltSize != 0) return false; // This is a splat operation if each element of the permute is the same, and // if the value doesn't reference the second vector. unsigned ElementBase = N->getMaskElt(0); // FIXME: Handle UNDEF elements too! if (ElementBase >= 16) return false; // Check that the indices are consecutive, in the case of a multi-byte element // splatted with a v16i8 mask. for (unsigned i = 1; i != EltSize; ++i) if (N->getMaskElt(i) < 0 || N->getMaskElt(i) != (int)(i+ElementBase)) return false; for (unsigned i = EltSize, e = 16; i != e; i += EltSize) { if (N->getMaskElt(i) < 0) continue; for (unsigned j = 0; j != EltSize; ++j) if (N->getMaskElt(i+j) != N->getMaskElt(j)) return false; } return true; } /// Check that the mask is shuffling N byte elements. Within each N byte /// element of the mask, the indices could be either in increasing or /// decreasing order as long as they are consecutive. /// \param[in] N the shuffle vector SD Node to analyze /// \param[in] Width the element width in bytes, could be 2/4/8/16 (HalfWord/ /// Word/DoubleWord/QuadWord). /// \param[in] StepLen the delta indices number among the N byte element, if /// the mask is in increasing/decreasing order then it is 1/-1. /// \return true iff the mask is shuffling N byte elements. static bool isNByteElemShuffleMask(ShuffleVectorSDNode *N, unsigned Width, int StepLen) { assert((Width == 2 || Width == 4 || Width == 8 || Width == 16) && "Unexpected element width."); assert((StepLen == 1 || StepLen == -1) && "Unexpected element width."); unsigned NumOfElem = 16 / Width; unsigned MaskVal[16]; // Width is never greater than 16 for (unsigned i = 0; i < NumOfElem; ++i) { MaskVal[0] = N->getMaskElt(i * Width); if ((StepLen == 1) && (MaskVal[0] % Width)) { return false; } else if ((StepLen == -1) && ((MaskVal[0] + 1) % Width)) { return false; } for (unsigned int j = 1; j < Width; ++j) { MaskVal[j] = N->getMaskElt(i * Width + j); if (MaskVal[j] != MaskVal[j-1] + StepLen) { return false; } } } return true; } bool PPC::isXXINSERTWMask(ShuffleVectorSDNode *N, unsigned &ShiftElts, unsigned &InsertAtByte, bool &Swap, bool IsLE) { if (!isNByteElemShuffleMask(N, 4, 1)) return false; // Now we look at mask elements 0,4,8,12 unsigned M0 = N->getMaskElt(0) / 4; unsigned M1 = N->getMaskElt(4) / 4; unsigned M2 = N->getMaskElt(8) / 4; unsigned M3 = N->getMaskElt(12) / 4; unsigned LittleEndianShifts[] = { 2, 1, 0, 3 }; unsigned BigEndianShifts[] = { 3, 0, 1, 2 }; // Below, let H and L be arbitrary elements of the shuffle mask // where H is in the range [4,7] and L is in the range [0,3]. // H, 1, 2, 3 or L, 5, 6, 7 if ((M0 > 3 && M1 == 1 && M2 == 2 && M3 == 3) || (M0 < 4 && M1 == 5 && M2 == 6 && M3 == 7)) { ShiftElts = IsLE ? LittleEndianShifts[M0 & 0x3] : BigEndianShifts[M0 & 0x3]; InsertAtByte = IsLE ? 12 : 0; Swap = M0 < 4; return true; } // 0, H, 2, 3 or 4, L, 6, 7 if ((M1 > 3 && M0 == 0 && M2 == 2 && M3 == 3) || (M1 < 4 && M0 == 4 && M2 == 6 && M3 == 7)) { ShiftElts = IsLE ? LittleEndianShifts[M1 & 0x3] : BigEndianShifts[M1 & 0x3]; InsertAtByte = IsLE ? 8 : 4; Swap = M1 < 4; return true; } // 0, 1, H, 3 or 4, 5, L, 7 if ((M2 > 3 && M0 == 0 && M1 == 1 && M3 == 3) || (M2 < 4 && M0 == 4 && M1 == 5 && M3 == 7)) { ShiftElts = IsLE ? LittleEndianShifts[M2 & 0x3] : BigEndianShifts[M2 & 0x3]; InsertAtByte = IsLE ? 4 : 8; Swap = M2 < 4; return true; } // 0, 1, 2, H or 4, 5, 6, L if ((M3 > 3 && M0 == 0 && M1 == 1 && M2 == 2) || (M3 < 4 && M0 == 4 && M1 == 5 && M2 == 6)) { ShiftElts = IsLE ? LittleEndianShifts[M3 & 0x3] : BigEndianShifts[M3 & 0x3]; InsertAtByte = IsLE ? 0 : 12; Swap = M3 < 4; return true; } // If both vector operands for the shuffle are the same vector, the mask will // contain only elements from the first one and the second one will be undef. if (N->getOperand(1).isUndef()) { ShiftElts = 0; Swap = true; unsigned XXINSERTWSrcElem = IsLE ? 2 : 1; if (M0 == XXINSERTWSrcElem && M1 == 1 && M2 == 2 && M3 == 3) { InsertAtByte = IsLE ? 12 : 0; return true; } if (M0 == 0 && M1 == XXINSERTWSrcElem && M2 == 2 && M3 == 3) { InsertAtByte = IsLE ? 8 : 4; return true; } if (M0 == 0 && M1 == 1 && M2 == XXINSERTWSrcElem && M3 == 3) { InsertAtByte = IsLE ? 4 : 8; return true; } if (M0 == 0 && M1 == 1 && M2 == 2 && M3 == XXINSERTWSrcElem) { InsertAtByte = IsLE ? 0 : 12; return true; } } return false; } bool PPC::isXXSLDWIShuffleMask(ShuffleVectorSDNode *N, unsigned &ShiftElts, bool &Swap, bool IsLE) { assert(N->getValueType(0) == MVT::v16i8 && "Shuffle vector expects v16i8"); // Ensure each byte index of the word is consecutive. if (!isNByteElemShuffleMask(N, 4, 1)) return false; // Now we look at mask elements 0,4,8,12, which are the beginning of words. unsigned M0 = N->getMaskElt(0) / 4; unsigned M1 = N->getMaskElt(4) / 4; unsigned M2 = N->getMaskElt(8) / 4; unsigned M3 = N->getMaskElt(12) / 4; // If both vector operands for the shuffle are the same vector, the mask will // contain only elements from the first one and the second one will be undef. if (N->getOperand(1).isUndef()) { assert(M0 < 4 && "Indexing into an undef vector?"); if (M1 != (M0 + 1) % 4 || M2 != (M1 + 1) % 4 || M3 != (M2 + 1) % 4) return false; ShiftElts = IsLE ? (4 - M0) % 4 : M0; Swap = false; return true; } // Ensure each word index of the ShuffleVector Mask is consecutive. if (M1 != (M0 + 1) % 8 || M2 != (M1 + 1) % 8 || M3 != (M2 + 1) % 8) return false; if (IsLE) { if (M0 == 0 || M0 == 7 || M0 == 6 || M0 == 5) { // Input vectors don't need to be swapped if the leading element // of the result is one of the 3 left elements of the second vector // (or if there is no shift to be done at all). Swap = false; ShiftElts = (8 - M0) % 8; } else if (M0 == 4 || M0 == 3 || M0 == 2 || M0 == 1) { // Input vectors need to be swapped if the leading element // of the result is one of the 3 left elements of the first vector // (or if we're shifting by 4 - thereby simply swapping the vectors). Swap = true; ShiftElts = (4 - M0) % 4; } return true; } else { // BE if (M0 == 0 || M0 == 1 || M0 == 2 || M0 == 3) { // Input vectors don't need to be swapped if the leading element // of the result is one of the 4 elements of the first vector. Swap = false; ShiftElts = M0; } else if (M0 == 4 || M0 == 5 || M0 == 6 || M0 == 7) { // Input vectors need to be swapped if the leading element // of the result is one of the 4 elements of the right vector. Swap = true; ShiftElts = M0 - 4; } return true; } } bool static isXXBRShuffleMaskHelper(ShuffleVectorSDNode *N, int Width) { assert(N->getValueType(0) == MVT::v16i8 && "Shuffle vector expects v16i8"); if (!isNByteElemShuffleMask(N, Width, -1)) return false; for (int i = 0; i < 16; i += Width) if (N->getMaskElt(i) != i + Width - 1) return false; return true; } bool PPC::isXXBRHShuffleMask(ShuffleVectorSDNode *N) { return isXXBRShuffleMaskHelper(N, 2); } bool PPC::isXXBRWShuffleMask(ShuffleVectorSDNode *N) { return isXXBRShuffleMaskHelper(N, 4); } bool PPC::isXXBRDShuffleMask(ShuffleVectorSDNode *N) { return isXXBRShuffleMaskHelper(N, 8); } bool PPC::isXXBRQShuffleMask(ShuffleVectorSDNode *N) { return isXXBRShuffleMaskHelper(N, 16); } /// Can node \p N be lowered to an XXPERMDI instruction? If so, set \p Swap /// if the inputs to the instruction should be swapped and set \p DM to the /// value for the immediate. /// Specifically, set \p Swap to true only if \p N can be lowered to XXPERMDI /// AND element 0 of the result comes from the first input (LE) or second input /// (BE). Set \p DM to the calculated result (0-3) only if \p N can be lowered. /// \return true iff the given mask of shuffle node \p N is a XXPERMDI shuffle /// mask. bool PPC::isXXPERMDIShuffleMask(ShuffleVectorSDNode *N, unsigned &DM, bool &Swap, bool IsLE) { assert(N->getValueType(0) == MVT::v16i8 && "Shuffle vector expects v16i8"); // Ensure each byte index of the double word is consecutive. if (!isNByteElemShuffleMask(N, 8, 1)) return false; unsigned M0 = N->getMaskElt(0) / 8; unsigned M1 = N->getMaskElt(8) / 8; assert(((M0 | M1) < 4) && "A mask element out of bounds?"); // If both vector operands for the shuffle are the same vector, the mask will // contain only elements from the first one and the second one will be undef. if (N->getOperand(1).isUndef()) { if ((M0 | M1) < 2) { DM = IsLE ? (((~M1) & 1) << 1) + ((~M0) & 1) : (M0 << 1) + (M1 & 1); Swap = false; return true; } else return false; } if (IsLE) { if (M0 > 1 && M1 < 2) { Swap = false; } else if (M0 < 2 && M1 > 1) { M0 = (M0 + 2) % 4; M1 = (M1 + 2) % 4; Swap = true; } else return false; // Note: if control flow comes here that means Swap is already set above DM = (((~M1) & 1) << 1) + ((~M0) & 1); return true; } else { // BE if (M0 < 2 && M1 > 1) { Swap = false; } else if (M0 > 1 && M1 < 2) { M0 = (M0 + 2) % 4; M1 = (M1 + 2) % 4; Swap = true; } else return false; // Note: if control flow comes here that means Swap is already set above DM = (M0 << 1) + (M1 & 1); return true; } } /// getSplatIdxForPPCMnemonics - Return the splat index as a value that is /// appropriate for PPC mnemonics (which have a big endian bias - namely /// elements are counted from the left of the vector register). unsigned PPC::getSplatIdxForPPCMnemonics(SDNode *N, unsigned EltSize, SelectionDAG &DAG) { ShuffleVectorSDNode *SVOp = cast(N); assert(isSplatShuffleMask(SVOp, EltSize)); if (DAG.getDataLayout().isLittleEndian()) return (16 / EltSize) - 1 - (SVOp->getMaskElt(0) / EltSize); else return SVOp->getMaskElt(0) / EltSize; } /// get_VSPLTI_elt - If this is a build_vector of constants which can be formed /// by using a vspltis[bhw] instruction of the specified element size, return /// the constant being splatted. The ByteSize field indicates the number of /// bytes of each element [124] -> [bhw]. SDValue PPC::get_VSPLTI_elt(SDNode *N, unsigned ByteSize, SelectionDAG &DAG) { SDValue OpVal(nullptr, 0); // If ByteSize of the splat is bigger than the element size of the // build_vector, then we have a case where we are checking for a splat where // multiple elements of the buildvector are folded together into a single // logical element of the splat (e.g. "vsplish 1" to splat {0,1}*8). unsigned EltSize = 16/N->getNumOperands(); if (EltSize < ByteSize) { unsigned Multiple = ByteSize/EltSize; // Number of BV entries per spltval. SDValue UniquedVals[4]; assert(Multiple > 1 && Multiple <= 4 && "How can this happen?"); // See if all of the elements in the buildvector agree across. for (unsigned i = 0, e = N->getNumOperands(); i != e; ++i) { if (N->getOperand(i).isUndef()) continue; // If the element isn't a constant, bail fully out. if (!isa(N->getOperand(i))) return SDValue(); if (!UniquedVals[i&(Multiple-1)].getNode()) UniquedVals[i&(Multiple-1)] = N->getOperand(i); else if (UniquedVals[i&(Multiple-1)] != N->getOperand(i)) return SDValue(); // no match. } // Okay, if we reached this point, UniquedVals[0..Multiple-1] contains // either constant or undef values that are identical for each chunk. See // if these chunks can form into a larger vspltis*. // Check to see if all of the leading entries are either 0 or -1. If // neither, then this won't fit into the immediate field. bool LeadingZero = true; bool LeadingOnes = true; for (unsigned i = 0; i != Multiple-1; ++i) { if (!UniquedVals[i].getNode()) continue; // Must have been undefs. LeadingZero &= isNullConstant(UniquedVals[i]); LeadingOnes &= isAllOnesConstant(UniquedVals[i]); } // Finally, check the least significant entry. if (LeadingZero) { if (!UniquedVals[Multiple-1].getNode()) return DAG.getTargetConstant(0, SDLoc(N), MVT::i32); // 0,0,0,undef int Val = cast(UniquedVals[Multiple-1])->getZExtValue(); if (Val < 16) // 0,0,0,4 -> vspltisw(4) return DAG.getTargetConstant(Val, SDLoc(N), MVT::i32); } if (LeadingOnes) { if (!UniquedVals[Multiple-1].getNode()) return DAG.getTargetConstant(~0U, SDLoc(N), MVT::i32); // -1,-1,-1,undef int Val =cast(UniquedVals[Multiple-1])->getSExtValue(); if (Val >= -16) // -1,-1,-1,-2 -> vspltisw(-2) return DAG.getTargetConstant(Val, SDLoc(N), MVT::i32); } return SDValue(); } // Check to see if this buildvec has a single non-undef value in its elements. for (unsigned i = 0, e = N->getNumOperands(); i != e; ++i) { if (N->getOperand(i).isUndef()) continue; if (!OpVal.getNode()) OpVal = N->getOperand(i); else if (OpVal != N->getOperand(i)) return SDValue(); } if (!OpVal.getNode()) return SDValue(); // All UNDEF: use implicit def. unsigned ValSizeInBytes = EltSize; uint64_t Value = 0; if (ConstantSDNode *CN = dyn_cast(OpVal)) { Value = CN->getZExtValue(); } else if (ConstantFPSDNode *CN = dyn_cast(OpVal)) { assert(CN->getValueType(0) == MVT::f32 && "Only one legal FP vector type!"); Value = FloatToBits(CN->getValueAPF().convertToFloat()); } // If the splat value is larger than the element value, then we can never do // this splat. The only case that we could fit the replicated bits into our // immediate field for would be zero, and we prefer to use vxor for it. if (ValSizeInBytes < ByteSize) return SDValue(); // If the element value is larger than the splat value, check if it consists // of a repeated bit pattern of size ByteSize. if (!APInt(ValSizeInBytes * 8, Value).isSplat(ByteSize * 8)) return SDValue(); // Properly sign extend the value. int MaskVal = SignExtend32(Value, ByteSize * 8); // If this is zero, don't match, zero matches ISD::isBuildVectorAllZeros. if (MaskVal == 0) return SDValue(); // Finally, if this value fits in a 5 bit sext field, return it if (SignExtend32<5>(MaskVal) == MaskVal) return DAG.getTargetConstant(MaskVal, SDLoc(N), MVT::i32); return SDValue(); } /// isQVALIGNIShuffleMask - If this is a qvaligni shuffle mask, return the shift /// amount, otherwise return -1. int PPC::isQVALIGNIShuffleMask(SDNode *N) { EVT VT = N->getValueType(0); if (VT != MVT::v4f64 && VT != MVT::v4f32 && VT != MVT::v4i1) return -1; ShuffleVectorSDNode *SVOp = cast(N); // Find the first non-undef value in the shuffle mask. unsigned i; for (i = 0; i != 4 && SVOp->getMaskElt(i) < 0; ++i) /*search*/; if (i == 4) return -1; // all undef. // Otherwise, check to see if the rest of the elements are consecutively // numbered from this value. unsigned ShiftAmt = SVOp->getMaskElt(i); if (ShiftAmt < i) return -1; ShiftAmt -= i; // Check the rest of the elements to see if they are consecutive. for (++i; i != 4; ++i) if (!isConstantOrUndef(SVOp->getMaskElt(i), ShiftAmt+i)) return -1; return ShiftAmt; } //===----------------------------------------------------------------------===// // Addressing Mode Selection //===----------------------------------------------------------------------===// /// isIntS16Immediate - This method tests to see if the node is either a 32-bit /// or 64-bit immediate, and if the value can be accurately represented as a /// sign extension from a 16-bit value. If so, this returns true and the /// immediate. bool llvm::isIntS16Immediate(SDNode *N, int16_t &Imm) { if (!isa(N)) return false; Imm = (int16_t)cast(N)->getZExtValue(); if (N->getValueType(0) == MVT::i32) return Imm == (int32_t)cast(N)->getZExtValue(); else return Imm == (int64_t)cast(N)->getZExtValue(); } bool llvm::isIntS16Immediate(SDValue Op, int16_t &Imm) { return isIntS16Immediate(Op.getNode(), Imm); } /// SelectAddressEVXRegReg - Given the specified address, check to see if it can /// be represented as an indexed [r+r] operation. bool PPCTargetLowering::SelectAddressEVXRegReg(SDValue N, SDValue &Base, SDValue &Index, SelectionDAG &DAG) const { for (SDNode::use_iterator UI = N->use_begin(), E = N->use_end(); UI != E; ++UI) { if (MemSDNode *Memop = dyn_cast(*UI)) { if (Memop->getMemoryVT() == MVT::f64) { Base = N.getOperand(0); Index = N.getOperand(1); return true; } } } return false; } /// SelectAddressRegReg - Given the specified addressed, check to see if it /// can be represented as an indexed [r+r] operation. Returns false if it /// can be more efficiently represented as [r+imm]. If \p EncodingAlignment is /// non-zero and N can be represented by a base register plus a signed 16-bit /// displacement, make a more precise judgement by checking (displacement % \p /// EncodingAlignment). bool PPCTargetLowering::SelectAddressRegReg(SDValue N, SDValue &Base, SDValue &Index, SelectionDAG &DAG, unsigned EncodingAlignment) const { int16_t imm = 0; if (N.getOpcode() == ISD::ADD) { // Is there any SPE load/store (f64), which can't handle 16bit offset? // SPE load/store can only handle 8-bit offsets. if (hasSPE() && SelectAddressEVXRegReg(N, Base, Index, DAG)) return true; if (isIntS16Immediate(N.getOperand(1), imm) && (!EncodingAlignment || !(imm % EncodingAlignment))) return false; // r+i if (N.getOperand(1).getOpcode() == PPCISD::Lo) return false; // r+i Base = N.getOperand(0); Index = N.getOperand(1); return true; } else if (N.getOpcode() == ISD::OR) { if (isIntS16Immediate(N.getOperand(1), imm) && (!EncodingAlignment || !(imm % EncodingAlignment))) return false; // r+i can fold it if we can. // If this is an or of disjoint bitfields, we can codegen this as an add // (for better address arithmetic) if the LHS and RHS of the OR are provably // disjoint. KnownBits LHSKnown = DAG.computeKnownBits(N.getOperand(0)); if (LHSKnown.Zero.getBoolValue()) { KnownBits RHSKnown = DAG.computeKnownBits(N.getOperand(1)); // If all of the bits are known zero on the LHS or RHS, the add won't // carry. if (~(LHSKnown.Zero | RHSKnown.Zero) == 0) { Base = N.getOperand(0); Index = N.getOperand(1); return true; } } } return false; } // If we happen to be doing an i64 load or store into a stack slot that has // less than a 4-byte alignment, then the frame-index elimination may need to // use an indexed load or store instruction (because the offset may not be a // multiple of 4). The extra register needed to hold the offset comes from the // register scavenger, and it is possible that the scavenger will need to use // an emergency spill slot. As a result, we need to make sure that a spill slot // is allocated when doing an i64 load/store into a less-than-4-byte-aligned // stack slot. static void fixupFuncForFI(SelectionDAG &DAG, int FrameIdx, EVT VT) { // FIXME: This does not handle the LWA case. if (VT != MVT::i64) return; // NOTE: We'll exclude negative FIs here, which come from argument // lowering, because there are no known test cases triggering this problem // using packed structures (or similar). We can remove this exclusion if // we find such a test case. The reason why this is so test-case driven is // because this entire 'fixup' is only to prevent crashes (from the // register scavenger) on not-really-valid inputs. For example, if we have: // %a = alloca i1 // %b = bitcast i1* %a to i64* // store i64* a, i64 b // then the store should really be marked as 'align 1', but is not. If it // were marked as 'align 1' then the indexed form would have been // instruction-selected initially, and the problem this 'fixup' is preventing // won't happen regardless. if (FrameIdx < 0) return; MachineFunction &MF = DAG.getMachineFunction(); MachineFrameInfo &MFI = MF.getFrameInfo(); if (MFI.getObjectAlign(FrameIdx) >= Align(4)) return; PPCFunctionInfo *FuncInfo = MF.getInfo(); FuncInfo->setHasNonRISpills(); } /// Returns true if the address N can be represented by a base register plus /// a signed 16-bit displacement [r+imm], and if it is not better /// represented as reg+reg. If \p EncodingAlignment is non-zero, only accept /// displacements that are multiples of that value. bool PPCTargetLowering::SelectAddressRegImm(SDValue N, SDValue &Disp, SDValue &Base, SelectionDAG &DAG, unsigned EncodingAlignment) const { // FIXME dl should come from parent load or store, not from address SDLoc dl(N); // If this can be more profitably realized as r+r, fail. if (SelectAddressRegReg(N, Disp, Base, DAG, EncodingAlignment)) return false; if (N.getOpcode() == ISD::ADD) { int16_t imm = 0; if (isIntS16Immediate(N.getOperand(1), imm) && (!EncodingAlignment || (imm % EncodingAlignment) == 0)) { Disp = DAG.getTargetConstant(imm, dl, N.getValueType()); if (FrameIndexSDNode *FI = dyn_cast(N.getOperand(0))) { Base = DAG.getTargetFrameIndex(FI->getIndex(), N.getValueType()); fixupFuncForFI(DAG, FI->getIndex(), N.getValueType()); } else { Base = N.getOperand(0); } return true; // [r+i] } else if (N.getOperand(1).getOpcode() == PPCISD::Lo) { // Match LOAD (ADD (X, Lo(G))). assert(!cast(N.getOperand(1).getOperand(1))->getZExtValue() && "Cannot handle constant offsets yet!"); Disp = N.getOperand(1).getOperand(0); // The global address. assert(Disp.getOpcode() == ISD::TargetGlobalAddress || Disp.getOpcode() == ISD::TargetGlobalTLSAddress || Disp.getOpcode() == ISD::TargetConstantPool || Disp.getOpcode() == ISD::TargetJumpTable); Base = N.getOperand(0); return true; // [&g+r] } } else if (N.getOpcode() == ISD::OR) { int16_t imm = 0; if (isIntS16Immediate(N.getOperand(1), imm) && (!EncodingAlignment || (imm % EncodingAlignment) == 0)) { // If this is an or of disjoint bitfields, we can codegen this as an add // (for better address arithmetic) if the LHS and RHS of the OR are // provably disjoint. KnownBits LHSKnown = DAG.computeKnownBits(N.getOperand(0)); if ((LHSKnown.Zero.getZExtValue()|~(uint64_t)imm) == ~0ULL) { // If all of the bits are known zero on the LHS or RHS, the add won't // carry. if (FrameIndexSDNode *FI = dyn_cast(N.getOperand(0))) { Base = DAG.getTargetFrameIndex(FI->getIndex(), N.getValueType()); fixupFuncForFI(DAG, FI->getIndex(), N.getValueType()); } else { Base = N.getOperand(0); } Disp = DAG.getTargetConstant(imm, dl, N.getValueType()); return true; } } } else if (ConstantSDNode *CN = dyn_cast(N)) { // Loading from a constant address. // If this address fits entirely in a 16-bit sext immediate field, codegen // this as "d, 0" int16_t Imm; if (isIntS16Immediate(CN, Imm) && (!EncodingAlignment || (Imm % EncodingAlignment) == 0)) { Disp = DAG.getTargetConstant(Imm, dl, CN->getValueType(0)); Base = DAG.getRegister(Subtarget.isPPC64() ? PPC::ZERO8 : PPC::ZERO, CN->getValueType(0)); return true; } // Handle 32-bit sext immediates with LIS + addr mode. if ((CN->getValueType(0) == MVT::i32 || (int64_t)CN->getZExtValue() == (int)CN->getZExtValue()) && (!EncodingAlignment || (CN->getZExtValue() % EncodingAlignment) == 0)) { int Addr = (int)CN->getZExtValue(); // Otherwise, break this down into an LIS + disp. Disp = DAG.getTargetConstant((short)Addr, dl, MVT::i32); Base = DAG.getTargetConstant((Addr - (signed short)Addr) >> 16, dl, MVT::i32); unsigned Opc = CN->getValueType(0) == MVT::i32 ? PPC::LIS : PPC::LIS8; Base = SDValue(DAG.getMachineNode(Opc, dl, CN->getValueType(0), Base), 0); return true; } } Disp = DAG.getTargetConstant(0, dl, getPointerTy(DAG.getDataLayout())); if (FrameIndexSDNode *FI = dyn_cast(N)) { Base = DAG.getTargetFrameIndex(FI->getIndex(), N.getValueType()); fixupFuncForFI(DAG, FI->getIndex(), N.getValueType()); } else Base = N; return true; // [r+0] } /// SelectAddressRegRegOnly - Given the specified addressed, force it to be /// represented as an indexed [r+r] operation. bool PPCTargetLowering::SelectAddressRegRegOnly(SDValue N, SDValue &Base, SDValue &Index, SelectionDAG &DAG) const { // Check to see if we can easily represent this as an [r+r] address. This // will fail if it thinks that the address is more profitably represented as // reg+imm, e.g. where imm = 0. if (SelectAddressRegReg(N, Base, Index, DAG)) return true; // If the address is the result of an add, we will utilize the fact that the // address calculation includes an implicit add. However, we can reduce // register pressure if we do not materialize a constant just for use as the // index register. We only get rid of the add if it is not an add of a // value and a 16-bit signed constant and both have a single use. int16_t imm = 0; if (N.getOpcode() == ISD::ADD && (!isIntS16Immediate(N.getOperand(1), imm) || !N.getOperand(1).hasOneUse() || !N.getOperand(0).hasOneUse())) { Base = N.getOperand(0); Index = N.getOperand(1); return true; } // Otherwise, do it the hard way, using R0 as the base register. Base = DAG.getRegister(Subtarget.isPPC64() ? PPC::ZERO8 : PPC::ZERO, N.getValueType()); Index = N; return true; } /// Returns true if we should use a direct load into vector instruction /// (such as lxsd or lfd), instead of a load into gpr + direct move sequence. static bool usePartialVectorLoads(SDNode *N, const PPCSubtarget& ST) { // If there are any other uses other than scalar to vector, then we should // keep it as a scalar load -> direct move pattern to prevent multiple // loads. LoadSDNode *LD = dyn_cast(N); if (!LD) return false; EVT MemVT = LD->getMemoryVT(); if (!MemVT.isSimple()) return false; switch(MemVT.getSimpleVT().SimpleTy) { case MVT::i64: break; case MVT::i32: if (!ST.hasP8Vector()) return false; break; case MVT::i16: case MVT::i8: if (!ST.hasP9Vector()) return false; break; default: return false; } SDValue LoadedVal(N, 0); if (!LoadedVal.hasOneUse()) return false; for (SDNode::use_iterator UI = LD->use_begin(), UE = LD->use_end(); UI != UE; ++UI) if (UI.getUse().get().getResNo() == 0 && UI->getOpcode() != ISD::SCALAR_TO_VECTOR) return false; return true; } /// getPreIndexedAddressParts - returns true by value, base pointer and /// offset pointer and addressing mode by reference if the node's address /// can be legally represented as pre-indexed load / store address. bool PPCTargetLowering::getPreIndexedAddressParts(SDNode *N, SDValue &Base, SDValue &Offset, ISD::MemIndexedMode &AM, SelectionDAG &DAG) const { if (DisablePPCPreinc) return false; bool isLoad = true; SDValue Ptr; EVT VT; unsigned Alignment; if (LoadSDNode *LD = dyn_cast(N)) { Ptr = LD->getBasePtr(); VT = LD->getMemoryVT(); Alignment = LD->getAlignment(); } else if (StoreSDNode *ST = dyn_cast(N)) { Ptr = ST->getBasePtr(); VT = ST->getMemoryVT(); Alignment = ST->getAlignment(); isLoad = false; } else return false; // Do not generate pre-inc forms for specific loads that feed scalar_to_vector // instructions because we can fold these into a more efficient instruction // instead, (such as LXSD). if (isLoad && usePartialVectorLoads(N, Subtarget)) { return false; } // PowerPC doesn't have preinc load/store instructions for vectors (except // for QPX, which does have preinc r+r forms). if (VT.isVector()) { if (!Subtarget.hasQPX() || (VT != MVT::v4f64 && VT != MVT::v4f32)) { return false; } else if (SelectAddressRegRegOnly(Ptr, Offset, Base, DAG)) { AM = ISD::PRE_INC; return true; } } if (SelectAddressRegReg(Ptr, Base, Offset, DAG)) { // Common code will reject creating a pre-inc form if the base pointer // is a frame index, or if N is a store and the base pointer is either // the same as or a predecessor of the value being stored. Check for // those situations here, and try with swapped Base/Offset instead. bool Swap = false; if (isa(Base) || isa(Base)) Swap = true; else if (!isLoad) { SDValue Val = cast(N)->getValue(); if (Val == Base || Base.getNode()->isPredecessorOf(Val.getNode())) Swap = true; } if (Swap) std::swap(Base, Offset); AM = ISD::PRE_INC; return true; } // LDU/STU can only handle immediates that are a multiple of 4. if (VT != MVT::i64) { if (!SelectAddressRegImm(Ptr, Offset, Base, DAG, 0)) return false; } else { // LDU/STU need an address with at least 4-byte alignment. if (Alignment < 4) return false; if (!SelectAddressRegImm(Ptr, Offset, Base, DAG, 4)) return false; } if (LoadSDNode *LD = dyn_cast(N)) { // PPC64 doesn't have lwau, but it does have lwaux. Reject preinc load of // sext i32 to i64 when addr mode is r+i. if (LD->getValueType(0) == MVT::i64 && LD->getMemoryVT() == MVT::i32 && LD->getExtensionType() == ISD::SEXTLOAD && isa(Offset)) return false; } AM = ISD::PRE_INC; return true; } //===----------------------------------------------------------------------===// // LowerOperation implementation //===----------------------------------------------------------------------===// /// Return true if we should reference labels using a PICBase, set the HiOpFlags /// and LoOpFlags to the target MO flags. static void getLabelAccessInfo(bool IsPIC, const PPCSubtarget &Subtarget, unsigned &HiOpFlags, unsigned &LoOpFlags, const GlobalValue *GV = nullptr) { HiOpFlags = PPCII::MO_HA; LoOpFlags = PPCII::MO_LO; // Don't use the pic base if not in PIC relocation model. if (IsPIC) { HiOpFlags |= PPCII::MO_PIC_FLAG; LoOpFlags |= PPCII::MO_PIC_FLAG; } } static SDValue LowerLabelRef(SDValue HiPart, SDValue LoPart, bool isPIC, SelectionDAG &DAG) { SDLoc DL(HiPart); EVT PtrVT = HiPart.getValueType(); SDValue Zero = DAG.getConstant(0, DL, PtrVT); SDValue Hi = DAG.getNode(PPCISD::Hi, DL, PtrVT, HiPart, Zero); SDValue Lo = DAG.getNode(PPCISD::Lo, DL, PtrVT, LoPart, Zero); // With PIC, the first instruction is actually "GR+hi(&G)". if (isPIC) Hi = DAG.getNode(ISD::ADD, DL, PtrVT, DAG.getNode(PPCISD::GlobalBaseReg, DL, PtrVT), Hi); // Generate non-pic code that has direct accesses to the constant pool. // The address of the global is just (hi(&g)+lo(&g)). return DAG.getNode(ISD::ADD, DL, PtrVT, Hi, Lo); } static void setUsesTOCBasePtr(MachineFunction &MF) { PPCFunctionInfo *FuncInfo = MF.getInfo(); FuncInfo->setUsesTOCBasePtr(); } static void setUsesTOCBasePtr(SelectionDAG &DAG) { setUsesTOCBasePtr(DAG.getMachineFunction()); } SDValue PPCTargetLowering::getTOCEntry(SelectionDAG &DAG, const SDLoc &dl, SDValue GA) const { const bool Is64Bit = Subtarget.isPPC64(); EVT VT = Is64Bit ? MVT::i64 : MVT::i32; SDValue Reg = Is64Bit ? DAG.getRegister(PPC::X2, VT) : Subtarget.isAIXABI() ? DAG.getRegister(PPC::R2, VT) : DAG.getNode(PPCISD::GlobalBaseReg, dl, VT); SDValue Ops[] = { GA, Reg }; return DAG.getMemIntrinsicNode( PPCISD::TOC_ENTRY, dl, DAG.getVTList(VT, MVT::Other), Ops, VT, MachinePointerInfo::getGOT(DAG.getMachineFunction()), None, MachineMemOperand::MOLoad); } SDValue PPCTargetLowering::LowerConstantPool(SDValue Op, SelectionDAG &DAG) const { EVT PtrVT = Op.getValueType(); ConstantPoolSDNode *CP = cast(Op); const Constant *C = CP->getConstVal(); // 64-bit SVR4 ABI and AIX ABI code are always position-independent. // The actual address of the GlobalValue is stored in the TOC. if (Subtarget.is64BitELFABI() || Subtarget.isAIXABI()) { setUsesTOCBasePtr(DAG); SDValue GA = DAG.getTargetConstantPool(C, PtrVT, CP->getAlignment(), 0); return getTOCEntry(DAG, SDLoc(CP), GA); } unsigned MOHiFlag, MOLoFlag; bool IsPIC = isPositionIndependent(); getLabelAccessInfo(IsPIC, Subtarget, MOHiFlag, MOLoFlag); if (IsPIC && Subtarget.isSVR4ABI()) { SDValue GA = DAG.getTargetConstantPool(C, PtrVT, CP->getAlignment(), PPCII::MO_PIC_FLAG); return getTOCEntry(DAG, SDLoc(CP), GA); } SDValue CPIHi = DAG.getTargetConstantPool(C, PtrVT, CP->getAlignment(), 0, MOHiFlag); SDValue CPILo = DAG.getTargetConstantPool(C, PtrVT, CP->getAlignment(), 0, MOLoFlag); return LowerLabelRef(CPIHi, CPILo, IsPIC, DAG); } // For 64-bit PowerPC, prefer the more compact relative encodings. // This trades 32 bits per jump table entry for one or two instructions // on the jump site. unsigned PPCTargetLowering::getJumpTableEncoding() const { if (isJumpTableRelative()) return MachineJumpTableInfo::EK_LabelDifference32; return TargetLowering::getJumpTableEncoding(); } bool PPCTargetLowering::isJumpTableRelative() const { if (UseAbsoluteJumpTables) return false; if (Subtarget.isPPC64() || Subtarget.isAIXABI()) return true; return TargetLowering::isJumpTableRelative(); } SDValue PPCTargetLowering::getPICJumpTableRelocBase(SDValue Table, SelectionDAG &DAG) const { if (!Subtarget.isPPC64() || Subtarget.isAIXABI()) return TargetLowering::getPICJumpTableRelocBase(Table, DAG); switch (getTargetMachine().getCodeModel()) { case CodeModel::Small: case CodeModel::Medium: return TargetLowering::getPICJumpTableRelocBase(Table, DAG); default: return DAG.getNode(PPCISD::GlobalBaseReg, SDLoc(), getPointerTy(DAG.getDataLayout())); } } const MCExpr * PPCTargetLowering::getPICJumpTableRelocBaseExpr(const MachineFunction *MF, unsigned JTI, MCContext &Ctx) const { if (!Subtarget.isPPC64() || Subtarget.isAIXABI()) return TargetLowering::getPICJumpTableRelocBaseExpr(MF, JTI, Ctx); switch (getTargetMachine().getCodeModel()) { case CodeModel::Small: case CodeModel::Medium: return TargetLowering::getPICJumpTableRelocBaseExpr(MF, JTI, Ctx); default: return MCSymbolRefExpr::create(MF->getPICBaseSymbol(), Ctx); } } SDValue PPCTargetLowering::LowerJumpTable(SDValue Op, SelectionDAG &DAG) const { EVT PtrVT = Op.getValueType(); JumpTableSDNode *JT = cast(Op); // 64-bit SVR4 ABI and AIX ABI code are always position-independent. // The actual address of the GlobalValue is stored in the TOC. if (Subtarget.is64BitELFABI() || Subtarget.isAIXABI()) { setUsesTOCBasePtr(DAG); SDValue GA = DAG.getTargetJumpTable(JT->getIndex(), PtrVT); return getTOCEntry(DAG, SDLoc(JT), GA); } unsigned MOHiFlag, MOLoFlag; bool IsPIC = isPositionIndependent(); getLabelAccessInfo(IsPIC, Subtarget, MOHiFlag, MOLoFlag); if (IsPIC && Subtarget.isSVR4ABI()) { SDValue GA = DAG.getTargetJumpTable(JT->getIndex(), PtrVT, PPCII::MO_PIC_FLAG); return getTOCEntry(DAG, SDLoc(GA), GA); } SDValue JTIHi = DAG.getTargetJumpTable(JT->getIndex(), PtrVT, MOHiFlag); SDValue JTILo = DAG.getTargetJumpTable(JT->getIndex(), PtrVT, MOLoFlag); return LowerLabelRef(JTIHi, JTILo, IsPIC, DAG); } SDValue PPCTargetLowering::LowerBlockAddress(SDValue Op, SelectionDAG &DAG) const { EVT PtrVT = Op.getValueType(); BlockAddressSDNode *BASDN = cast(Op); const BlockAddress *BA = BASDN->getBlockAddress(); // 64-bit SVR4 ABI and AIX ABI code are always position-independent. // The actual BlockAddress is stored in the TOC. if (Subtarget.is64BitELFABI() || Subtarget.isAIXABI()) { setUsesTOCBasePtr(DAG); SDValue GA = DAG.getTargetBlockAddress(BA, PtrVT, BASDN->getOffset()); return getTOCEntry(DAG, SDLoc(BASDN), GA); } // 32-bit position-independent ELF stores the BlockAddress in the .got. if (Subtarget.is32BitELFABI() && isPositionIndependent()) return getTOCEntry( DAG, SDLoc(BASDN), DAG.getTargetBlockAddress(BA, PtrVT, BASDN->getOffset())); unsigned MOHiFlag, MOLoFlag; bool IsPIC = isPositionIndependent(); getLabelAccessInfo(IsPIC, Subtarget, MOHiFlag, MOLoFlag); SDValue TgtBAHi = DAG.getTargetBlockAddress(BA, PtrVT, 0, MOHiFlag); SDValue TgtBALo = DAG.getTargetBlockAddress(BA, PtrVT, 0, MOLoFlag); return LowerLabelRef(TgtBAHi, TgtBALo, IsPIC, DAG); } SDValue PPCTargetLowering::LowerGlobalTLSAddress(SDValue Op, SelectionDAG &DAG) const { // FIXME: TLS addresses currently use medium model code sequences, // which is the most useful form. Eventually support for small and // large models could be added if users need it, at the cost of // additional complexity. GlobalAddressSDNode *GA = cast(Op); if (DAG.getTarget().useEmulatedTLS()) return LowerToTLSEmulatedModel(GA, DAG); SDLoc dl(GA); const GlobalValue *GV = GA->getGlobal(); EVT PtrVT = getPointerTy(DAG.getDataLayout()); bool is64bit = Subtarget.isPPC64(); const Module *M = DAG.getMachineFunction().getFunction().getParent(); PICLevel::Level picLevel = M->getPICLevel(); const TargetMachine &TM = getTargetMachine(); TLSModel::Model Model = TM.getTLSModel(GV); if (Model == TLSModel::LocalExec) { SDValue TGAHi = DAG.getTargetGlobalAddress(GV, dl, PtrVT, 0, PPCII::MO_TPREL_HA); SDValue TGALo = DAG.getTargetGlobalAddress(GV, dl, PtrVT, 0, PPCII::MO_TPREL_LO); SDValue TLSReg = is64bit ? DAG.getRegister(PPC::X13, MVT::i64) : DAG.getRegister(PPC::R2, MVT::i32); SDValue Hi = DAG.getNode(PPCISD::Hi, dl, PtrVT, TGAHi, TLSReg); return DAG.getNode(PPCISD::Lo, dl, PtrVT, TGALo, Hi); } if (Model == TLSModel::InitialExec) { SDValue TGA = DAG.getTargetGlobalAddress(GV, dl, PtrVT, 0, 0); SDValue TGATLS = DAG.getTargetGlobalAddress(GV, dl, PtrVT, 0, PPCII::MO_TLS); SDValue GOTPtr; if (is64bit) { setUsesTOCBasePtr(DAG); SDValue GOTReg = DAG.getRegister(PPC::X2, MVT::i64); GOTPtr = DAG.getNode(PPCISD::ADDIS_GOT_TPREL_HA, dl, PtrVT, GOTReg, TGA); } else { if (!TM.isPositionIndependent()) GOTPtr = DAG.getNode(PPCISD::PPC32_GOT, dl, PtrVT); else if (picLevel == PICLevel::SmallPIC) GOTPtr = DAG.getNode(PPCISD::GlobalBaseReg, dl, PtrVT); else GOTPtr = DAG.getNode(PPCISD::PPC32_PICGOT, dl, PtrVT); } SDValue TPOffset = DAG.getNode(PPCISD::LD_GOT_TPREL_L, dl, PtrVT, TGA, GOTPtr); return DAG.getNode(PPCISD::ADD_TLS, dl, PtrVT, TPOffset, TGATLS); } if (Model == TLSModel::GeneralDynamic) { SDValue TGA = DAG.getTargetGlobalAddress(GV, dl, PtrVT, 0, 0); SDValue GOTPtr; if (is64bit) { setUsesTOCBasePtr(DAG); SDValue GOTReg = DAG.getRegister(PPC::X2, MVT::i64); GOTPtr = DAG.getNode(PPCISD::ADDIS_TLSGD_HA, dl, PtrVT, GOTReg, TGA); } else { if (picLevel == PICLevel::SmallPIC) GOTPtr = DAG.getNode(PPCISD::GlobalBaseReg, dl, PtrVT); else GOTPtr = DAG.getNode(PPCISD::PPC32_PICGOT, dl, PtrVT); } return DAG.getNode(PPCISD::ADDI_TLSGD_L_ADDR, dl, PtrVT, GOTPtr, TGA, TGA); } if (Model == TLSModel::LocalDynamic) { SDValue TGA = DAG.getTargetGlobalAddress(GV, dl, PtrVT, 0, 0); SDValue GOTPtr; if (is64bit) { setUsesTOCBasePtr(DAG); SDValue GOTReg = DAG.getRegister(PPC::X2, MVT::i64); GOTPtr = DAG.getNode(PPCISD::ADDIS_TLSLD_HA, dl, PtrVT, GOTReg, TGA); } else { if (picLevel == PICLevel::SmallPIC) GOTPtr = DAG.getNode(PPCISD::GlobalBaseReg, dl, PtrVT); else GOTPtr = DAG.getNode(PPCISD::PPC32_PICGOT, dl, PtrVT); } SDValue TLSAddr = DAG.getNode(PPCISD::ADDI_TLSLD_L_ADDR, dl, PtrVT, GOTPtr, TGA, TGA); SDValue DtvOffsetHi = DAG.getNode(PPCISD::ADDIS_DTPREL_HA, dl, PtrVT, TLSAddr, TGA); return DAG.getNode(PPCISD::ADDI_DTPREL_L, dl, PtrVT, DtvOffsetHi, TGA); } llvm_unreachable("Unknown TLS model!"); } SDValue PPCTargetLowering::LowerGlobalAddress(SDValue Op, SelectionDAG &DAG) const { EVT PtrVT = Op.getValueType(); GlobalAddressSDNode *GSDN = cast(Op); SDLoc DL(GSDN); const GlobalValue *GV = GSDN->getGlobal(); // 64-bit SVR4 ABI & AIX ABI code is always position-independent. // The actual address of the GlobalValue is stored in the TOC. if (Subtarget.is64BitELFABI() || Subtarget.isAIXABI()) { setUsesTOCBasePtr(DAG); SDValue GA = DAG.getTargetGlobalAddress(GV, DL, PtrVT, GSDN->getOffset()); return getTOCEntry(DAG, DL, GA); } unsigned MOHiFlag, MOLoFlag; bool IsPIC = isPositionIndependent(); getLabelAccessInfo(IsPIC, Subtarget, MOHiFlag, MOLoFlag, GV); if (IsPIC && Subtarget.isSVR4ABI()) { SDValue GA = DAG.getTargetGlobalAddress(GV, DL, PtrVT, GSDN->getOffset(), PPCII::MO_PIC_FLAG); return getTOCEntry(DAG, DL, GA); } SDValue GAHi = DAG.getTargetGlobalAddress(GV, DL, PtrVT, GSDN->getOffset(), MOHiFlag); SDValue GALo = DAG.getTargetGlobalAddress(GV, DL, PtrVT, GSDN->getOffset(), MOLoFlag); return LowerLabelRef(GAHi, GALo, IsPIC, DAG); } SDValue PPCTargetLowering::LowerSETCC(SDValue Op, SelectionDAG &DAG) const { ISD::CondCode CC = cast(Op.getOperand(2))->get(); SDLoc dl(Op); if (Op.getValueType() == MVT::v2i64) { // When the operands themselves are v2i64 values, we need to do something // special because VSX has no underlying comparison operations for these. if (Op.getOperand(0).getValueType() == MVT::v2i64) { // Equality can be handled by casting to the legal type for Altivec // comparisons, everything else needs to be expanded. if (CC == ISD::SETEQ || CC == ISD::SETNE) { return DAG.getNode(ISD::BITCAST, dl, MVT::v2i64, DAG.getSetCC(dl, MVT::v4i32, DAG.getNode(ISD::BITCAST, dl, MVT::v4i32, Op.getOperand(0)), DAG.getNode(ISD::BITCAST, dl, MVT::v4i32, Op.getOperand(1)), CC)); } return SDValue(); } // We handle most of these in the usual way. return Op; } // If we're comparing for equality to zero, expose the fact that this is // implemented as a ctlz/srl pair on ppc, so that the dag combiner can // fold the new nodes. if (SDValue V = lowerCmpEqZeroToCtlzSrl(Op, DAG)) return V; if (ConstantSDNode *C = dyn_cast(Op.getOperand(1))) { // Leave comparisons against 0 and -1 alone for now, since they're usually // optimized. FIXME: revisit this when we can custom lower all setcc // optimizations. if (C->isAllOnesValue() || C->isNullValue()) return SDValue(); } // If we have an integer seteq/setne, turn it into a compare against zero // by xor'ing the rhs with the lhs, which is faster than setting a // condition register, reading it back out, and masking the correct bit. The // normal approach here uses sub to do this instead of xor. Using xor exposes // the result to other bit-twiddling opportunities. EVT LHSVT = Op.getOperand(0).getValueType(); if (LHSVT.isInteger() && (CC == ISD::SETEQ || CC == ISD::SETNE)) { EVT VT = Op.getValueType(); SDValue Sub = DAG.getNode(ISD::XOR, dl, LHSVT, Op.getOperand(0), Op.getOperand(1)); return DAG.getSetCC(dl, VT, Sub, DAG.getConstant(0, dl, LHSVT), CC); } return SDValue(); } SDValue PPCTargetLowering::LowerVAARG(SDValue Op, SelectionDAG &DAG) const { SDNode *Node = Op.getNode(); EVT VT = Node->getValueType(0); EVT PtrVT = getPointerTy(DAG.getDataLayout()); SDValue InChain = Node->getOperand(0); SDValue VAListPtr = Node->getOperand(1); const Value *SV = cast(Node->getOperand(2))->getValue(); SDLoc dl(Node); assert(!Subtarget.isPPC64() && "LowerVAARG is PPC32 only"); // gpr_index SDValue GprIndex = DAG.getExtLoad(ISD::ZEXTLOAD, dl, MVT::i32, InChain, VAListPtr, MachinePointerInfo(SV), MVT::i8); InChain = GprIndex.getValue(1); if (VT == MVT::i64) { // Check if GprIndex is even SDValue GprAnd = DAG.getNode(ISD::AND, dl, MVT::i32, GprIndex, DAG.getConstant(1, dl, MVT::i32)); SDValue CC64 = DAG.getSetCC(dl, MVT::i32, GprAnd, DAG.getConstant(0, dl, MVT::i32), ISD::SETNE); SDValue GprIndexPlusOne = DAG.getNode(ISD::ADD, dl, MVT::i32, GprIndex, DAG.getConstant(1, dl, MVT::i32)); // Align GprIndex to be even if it isn't GprIndex = DAG.getNode(ISD::SELECT, dl, MVT::i32, CC64, GprIndexPlusOne, GprIndex); } // fpr index is 1 byte after gpr SDValue FprPtr = DAG.getNode(ISD::ADD, dl, PtrVT, VAListPtr, DAG.getConstant(1, dl, MVT::i32)); // fpr SDValue FprIndex = DAG.getExtLoad(ISD::ZEXTLOAD, dl, MVT::i32, InChain, FprPtr, MachinePointerInfo(SV), MVT::i8); InChain = FprIndex.getValue(1); SDValue RegSaveAreaPtr = DAG.getNode(ISD::ADD, dl, PtrVT, VAListPtr, DAG.getConstant(8, dl, MVT::i32)); SDValue OverflowAreaPtr = DAG.getNode(ISD::ADD, dl, PtrVT, VAListPtr, DAG.getConstant(4, dl, MVT::i32)); // areas SDValue OverflowArea = DAG.getLoad(MVT::i32, dl, InChain, OverflowAreaPtr, MachinePointerInfo()); InChain = OverflowArea.getValue(1); SDValue RegSaveArea = DAG.getLoad(MVT::i32, dl, InChain, RegSaveAreaPtr, MachinePointerInfo()); InChain = RegSaveArea.getValue(1); // select overflow_area if index > 8 SDValue CC = DAG.getSetCC(dl, MVT::i32, VT.isInteger() ? GprIndex : FprIndex, DAG.getConstant(8, dl, MVT::i32), ISD::SETLT); // adjustment constant gpr_index * 4/8 SDValue RegConstant = DAG.getNode(ISD::MUL, dl, MVT::i32, VT.isInteger() ? GprIndex : FprIndex, DAG.getConstant(VT.isInteger() ? 4 : 8, dl, MVT::i32)); // OurReg = RegSaveArea + RegConstant SDValue OurReg = DAG.getNode(ISD::ADD, dl, PtrVT, RegSaveArea, RegConstant); // Floating types are 32 bytes into RegSaveArea if (VT.isFloatingPoint()) OurReg = DAG.getNode(ISD::ADD, dl, PtrVT, OurReg, DAG.getConstant(32, dl, MVT::i32)); // increase {f,g}pr_index by 1 (or 2 if VT is i64) SDValue IndexPlus1 = DAG.getNode(ISD::ADD, dl, MVT::i32, VT.isInteger() ? GprIndex : FprIndex, DAG.getConstant(VT == MVT::i64 ? 2 : 1, dl, MVT::i32)); InChain = DAG.getTruncStore(InChain, dl, IndexPlus1, VT.isInteger() ? VAListPtr : FprPtr, MachinePointerInfo(SV), MVT::i8); // determine if we should load from reg_save_area or overflow_area SDValue Result = DAG.getNode(ISD::SELECT, dl, PtrVT, CC, OurReg, OverflowArea); // increase overflow_area by 4/8 if gpr/fpr > 8 SDValue OverflowAreaPlusN = DAG.getNode(ISD::ADD, dl, PtrVT, OverflowArea, DAG.getConstant(VT.isInteger() ? 4 : 8, dl, MVT::i32)); OverflowArea = DAG.getNode(ISD::SELECT, dl, MVT::i32, CC, OverflowArea, OverflowAreaPlusN); InChain = DAG.getTruncStore(InChain, dl, OverflowArea, OverflowAreaPtr, MachinePointerInfo(), MVT::i32); return DAG.getLoad(VT, dl, InChain, Result, MachinePointerInfo()); } SDValue PPCTargetLowering::LowerVACOPY(SDValue Op, SelectionDAG &DAG) const { assert(!Subtarget.isPPC64() && "LowerVACOPY is PPC32 only"); // We have to copy the entire va_list struct: // 2*sizeof(char) + 2 Byte alignment + 2*sizeof(char*) = 12 Byte return DAG.getMemcpy(Op.getOperand(0), Op, Op.getOperand(1), Op.getOperand(2), DAG.getConstant(12, SDLoc(Op), MVT::i32), Align(8), false, true, false, MachinePointerInfo(), MachinePointerInfo()); } SDValue PPCTargetLowering::LowerADJUST_TRAMPOLINE(SDValue Op, SelectionDAG &DAG) const { if (Subtarget.isAIXABI()) report_fatal_error("ADJUST_TRAMPOLINE operation is not supported on AIX."); return Op.getOperand(0); } SDValue PPCTargetLowering::LowerINIT_TRAMPOLINE(SDValue Op, SelectionDAG &DAG) const { if (Subtarget.isAIXABI()) report_fatal_error("INIT_TRAMPOLINE operation is not supported on AIX."); SDValue Chain = Op.getOperand(0); SDValue Trmp = Op.getOperand(1); // trampoline SDValue FPtr = Op.getOperand(2); // nested function SDValue Nest = Op.getOperand(3); // 'nest' parameter value SDLoc dl(Op); EVT PtrVT = getPointerTy(DAG.getDataLayout()); bool isPPC64 = (PtrVT == MVT::i64); Type *IntPtrTy = DAG.getDataLayout().getIntPtrType(*DAG.getContext()); TargetLowering::ArgListTy Args; TargetLowering::ArgListEntry Entry; Entry.Ty = IntPtrTy; Entry.Node = Trmp; Args.push_back(Entry); // TrampSize == (isPPC64 ? 48 : 40); Entry.Node = DAG.getConstant(isPPC64 ? 48 : 40, dl, isPPC64 ? MVT::i64 : MVT::i32); Args.push_back(Entry); Entry.Node = FPtr; Args.push_back(Entry); Entry.Node = Nest; Args.push_back(Entry); // Lower to a call to __trampoline_setup(Trmp, TrampSize, FPtr, ctx_reg) TargetLowering::CallLoweringInfo CLI(DAG); CLI.setDebugLoc(dl).setChain(Chain).setLibCallee( CallingConv::C, Type::getVoidTy(*DAG.getContext()), DAG.getExternalSymbol("__trampoline_setup", PtrVT), std::move(Args)); std::pair CallResult = LowerCallTo(CLI); return CallResult.second; } SDValue PPCTargetLowering::LowerVASTART(SDValue Op, SelectionDAG &DAG) const { MachineFunction &MF = DAG.getMachineFunction(); PPCFunctionInfo *FuncInfo = MF.getInfo(); EVT PtrVT = getPointerTy(MF.getDataLayout()); SDLoc dl(Op); if (Subtarget.isPPC64()) { // vastart just stores the address of the VarArgsFrameIndex slot into the // memory location argument. SDValue FR = DAG.getFrameIndex(FuncInfo->getVarArgsFrameIndex(), PtrVT); const Value *SV = cast(Op.getOperand(2))->getValue(); return DAG.getStore(Op.getOperand(0), dl, FR, Op.getOperand(1), MachinePointerInfo(SV)); } // For the 32-bit SVR4 ABI we follow the layout of the va_list struct. // We suppose the given va_list is already allocated. // // typedef struct { // char gpr; /* index into the array of 8 GPRs // * stored in the register save area // * gpr=0 corresponds to r3, // * gpr=1 to r4, etc. // */ // char fpr; /* index into the array of 8 FPRs // * stored in the register save area // * fpr=0 corresponds to f1, // * fpr=1 to f2, etc. // */ // char *overflow_arg_area; // /* location on stack that holds // * the next overflow argument // */ // char *reg_save_area; // /* where r3:r10 and f1:f8 (if saved) // * are stored // */ // } va_list[1]; SDValue ArgGPR = DAG.getConstant(FuncInfo->getVarArgsNumGPR(), dl, MVT::i32); SDValue ArgFPR = DAG.getConstant(FuncInfo->getVarArgsNumFPR(), dl, MVT::i32); SDValue StackOffsetFI = DAG.getFrameIndex(FuncInfo->getVarArgsStackOffset(), PtrVT); SDValue FR = DAG.getFrameIndex(FuncInfo->getVarArgsFrameIndex(), PtrVT); uint64_t FrameOffset = PtrVT.getSizeInBits()/8; SDValue ConstFrameOffset = DAG.getConstant(FrameOffset, dl, PtrVT); uint64_t StackOffset = PtrVT.getSizeInBits()/8 - 1; SDValue ConstStackOffset = DAG.getConstant(StackOffset, dl, PtrVT); uint64_t FPROffset = 1; SDValue ConstFPROffset = DAG.getConstant(FPROffset, dl, PtrVT); const Value *SV = cast(Op.getOperand(2))->getValue(); // Store first byte : number of int regs SDValue firstStore = DAG.getTruncStore(Op.getOperand(0), dl, ArgGPR, Op.getOperand(1), MachinePointerInfo(SV), MVT::i8); uint64_t nextOffset = FPROffset; SDValue nextPtr = DAG.getNode(ISD::ADD, dl, PtrVT, Op.getOperand(1), ConstFPROffset); // Store second byte : number of float regs SDValue secondStore = DAG.getTruncStore(firstStore, dl, ArgFPR, nextPtr, MachinePointerInfo(SV, nextOffset), MVT::i8); nextOffset += StackOffset; nextPtr = DAG.getNode(ISD::ADD, dl, PtrVT, nextPtr, ConstStackOffset); // Store second word : arguments given on stack SDValue thirdStore = DAG.getStore(secondStore, dl, StackOffsetFI, nextPtr, MachinePointerInfo(SV, nextOffset)); nextOffset += FrameOffset; nextPtr = DAG.getNode(ISD::ADD, dl, PtrVT, nextPtr, ConstFrameOffset); // Store third word : arguments given in registers return DAG.getStore(thirdStore, dl, FR, nextPtr, MachinePointerInfo(SV, nextOffset)); } /// FPR - The set of FP registers that should be allocated for arguments /// on Darwin and AIX. static const MCPhysReg FPR[] = {PPC::F1, PPC::F2, PPC::F3, PPC::F4, PPC::F5, PPC::F6, PPC::F7, PPC::F8, PPC::F9, PPC::F10, PPC::F11, PPC::F12, PPC::F13}; /// QFPR - The set of QPX registers that should be allocated for arguments. static const MCPhysReg QFPR[] = { PPC::QF1, PPC::QF2, PPC::QF3, PPC::QF4, PPC::QF5, PPC::QF6, PPC::QF7, PPC::QF8, PPC::QF9, PPC::QF10, PPC::QF11, PPC::QF12, PPC::QF13}; /// CalculateStackSlotSize - Calculates the size reserved for this argument on /// the stack. static unsigned CalculateStackSlotSize(EVT ArgVT, ISD::ArgFlagsTy Flags, unsigned PtrByteSize) { unsigned ArgSize = ArgVT.getStoreSize(); if (Flags.isByVal()) ArgSize = Flags.getByValSize(); // Round up to multiples of the pointer size, except for array members, // which are always packed. if (!Flags.isInConsecutiveRegs()) ArgSize = ((ArgSize + PtrByteSize - 1)/PtrByteSize) * PtrByteSize; return ArgSize; } /// CalculateStackSlotAlignment - Calculates the alignment of this argument /// on the stack. static Align CalculateStackSlotAlignment(EVT ArgVT, EVT OrigVT, ISD::ArgFlagsTy Flags, unsigned PtrByteSize) { Align Alignment(PtrByteSize); // Altivec parameters are padded to a 16 byte boundary. if (ArgVT == MVT::v4f32 || ArgVT == MVT::v4i32 || ArgVT == MVT::v8i16 || ArgVT == MVT::v16i8 || ArgVT == MVT::v2f64 || ArgVT == MVT::v2i64 || ArgVT == MVT::v1i128 || ArgVT == MVT::f128) Alignment = Align(16); // QPX vector types stored in double-precision are padded to a 32 byte // boundary. else if (ArgVT == MVT::v4f64 || ArgVT == MVT::v4i1) Alignment = Align(32); // ByVal parameters are aligned as requested. if (Flags.isByVal()) { auto BVAlign = Flags.getNonZeroByValAlign(); if (BVAlign > PtrByteSize) { if (BVAlign.value() % PtrByteSize != 0) llvm_unreachable( "ByVal alignment is not a multiple of the pointer size"); Alignment = BVAlign; } } // Array members are always packed to their original alignment. if (Flags.isInConsecutiveRegs()) { // If the array member was split into multiple registers, the first // needs to be aligned to the size of the full type. (Except for // ppcf128, which is only aligned as its f64 components.) if (Flags.isSplit() && OrigVT != MVT::ppcf128) Alignment = Align(OrigVT.getStoreSize()); else Alignment = Align(ArgVT.getStoreSize()); } return Alignment; } /// CalculateStackSlotUsed - Return whether this argument will use its /// stack slot (instead of being passed in registers). ArgOffset, /// AvailableFPRs, and AvailableVRs must hold the current argument /// position, and will be updated to account for this argument. static bool CalculateStackSlotUsed(EVT ArgVT, EVT OrigVT, ISD::ArgFlagsTy Flags, unsigned PtrByteSize, unsigned LinkageSize, unsigned ParamAreaSize, unsigned &ArgOffset, unsigned &AvailableFPRs, unsigned &AvailableVRs, bool HasQPX) { bool UseMemory = false; // Respect alignment of argument on the stack. Align Alignment = CalculateStackSlotAlignment(ArgVT, OrigVT, Flags, PtrByteSize); ArgOffset = alignTo(ArgOffset, Alignment); // If there's no space left in the argument save area, we must // use memory (this check also catches zero-sized arguments). if (ArgOffset >= LinkageSize + ParamAreaSize) UseMemory = true; // Allocate argument on the stack. ArgOffset += CalculateStackSlotSize(ArgVT, Flags, PtrByteSize); if (Flags.isInConsecutiveRegsLast()) ArgOffset = ((ArgOffset + PtrByteSize - 1)/PtrByteSize) * PtrByteSize; // If we overran the argument save area, we must use memory // (this check catches arguments passed partially in memory) if (ArgOffset > LinkageSize + ParamAreaSize) UseMemory = true; // However, if the argument is actually passed in an FPR or a VR, // we don't use memory after all. if (!Flags.isByVal()) { if (ArgVT == MVT::f32 || ArgVT == MVT::f64 || // QPX registers overlap with the scalar FP registers. (HasQPX && (ArgVT == MVT::v4f32 || ArgVT == MVT::v4f64 || ArgVT == MVT::v4i1))) if (AvailableFPRs > 0) { --AvailableFPRs; return false; } if (ArgVT == MVT::v4f32 || ArgVT == MVT::v4i32 || ArgVT == MVT::v8i16 || ArgVT == MVT::v16i8 || ArgVT == MVT::v2f64 || ArgVT == MVT::v2i64 || ArgVT == MVT::v1i128 || ArgVT == MVT::f128) if (AvailableVRs > 0) { --AvailableVRs; return false; } } return UseMemory; } /// EnsureStackAlignment - Round stack frame size up from NumBytes to /// ensure minimum alignment required for target. static unsigned EnsureStackAlignment(const PPCFrameLowering *Lowering, unsigned NumBytes) { return alignTo(NumBytes, Lowering->getStackAlign()); } SDValue PPCTargetLowering::LowerFormalArguments( SDValue Chain, CallingConv::ID CallConv, bool isVarArg, const SmallVectorImpl &Ins, const SDLoc &dl, SelectionDAG &DAG, SmallVectorImpl &InVals) const { if (Subtarget.isAIXABI()) return LowerFormalArguments_AIX(Chain, CallConv, isVarArg, Ins, dl, DAG, InVals); if (Subtarget.is64BitELFABI()) return LowerFormalArguments_64SVR4(Chain, CallConv, isVarArg, Ins, dl, DAG, InVals); if (Subtarget.is32BitELFABI()) return LowerFormalArguments_32SVR4(Chain, CallConv, isVarArg, Ins, dl, DAG, InVals); return LowerFormalArguments_Darwin(Chain, CallConv, isVarArg, Ins, dl, DAG, InVals); } SDValue PPCTargetLowering::LowerFormalArguments_32SVR4( SDValue Chain, CallingConv::ID CallConv, bool isVarArg, const SmallVectorImpl &Ins, const SDLoc &dl, SelectionDAG &DAG, SmallVectorImpl &InVals) const { // 32-bit SVR4 ABI Stack Frame Layout: // +-----------------------------------+ // +--> | Back chain | // | +-----------------------------------+ // | | Floating-point register save area | // | +-----------------------------------+ // | | General register save area | // | +-----------------------------------+ // | | CR save word | // | +-----------------------------------+ // | | VRSAVE save word | // | +-----------------------------------+ // | | Alignment padding | // | +-----------------------------------+ // | | Vector register save area | // | +-----------------------------------+ // | | Local variable space | // | +-----------------------------------+ // | | Parameter list area | // | +-----------------------------------+ // | | LR save word | // | +-----------------------------------+ // SP--> +--- | Back chain | // +-----------------------------------+ // // Specifications: // System V Application Binary Interface PowerPC Processor Supplement // AltiVec Technology Programming Interface Manual MachineFunction &MF = DAG.getMachineFunction(); MachineFrameInfo &MFI = MF.getFrameInfo(); PPCFunctionInfo *FuncInfo = MF.getInfo(); EVT PtrVT = getPointerTy(MF.getDataLayout()); // Potential tail calls could cause overwriting of argument stack slots. bool isImmutable = !(getTargetMachine().Options.GuaranteedTailCallOpt && (CallConv == CallingConv::Fast)); unsigned PtrByteSize = 4; // Assign locations to all of the incoming arguments. SmallVector ArgLocs; PPCCCState CCInfo(CallConv, isVarArg, DAG.getMachineFunction(), ArgLocs, *DAG.getContext()); // Reserve space for the linkage area on the stack. unsigned LinkageSize = Subtarget.getFrameLowering()->getLinkageSize(); CCInfo.AllocateStack(LinkageSize, PtrByteSize); if (useSoftFloat()) CCInfo.PreAnalyzeFormalArguments(Ins); CCInfo.AnalyzeFormalArguments(Ins, CC_PPC32_SVR4); CCInfo.clearWasPPCF128(); for (unsigned i = 0, e = ArgLocs.size(); i != e; ++i) { CCValAssign &VA = ArgLocs[i]; // Arguments stored in registers. if (VA.isRegLoc()) { const TargetRegisterClass *RC; EVT ValVT = VA.getValVT(); switch (ValVT.getSimpleVT().SimpleTy) { default: llvm_unreachable("ValVT not supported by formal arguments Lowering"); case MVT::i1: case MVT::i32: RC = &PPC::GPRCRegClass; break; case MVT::f32: if (Subtarget.hasP8Vector()) RC = &PPC::VSSRCRegClass; else if (Subtarget.hasSPE()) RC = &PPC::GPRCRegClass; else RC = &PPC::F4RCRegClass; break; case MVT::f64: if (Subtarget.hasVSX()) RC = &PPC::VSFRCRegClass; else if (Subtarget.hasSPE()) // SPE passes doubles in GPR pairs. RC = &PPC::GPRCRegClass; else RC = &PPC::F8RCRegClass; break; case MVT::v16i8: case MVT::v8i16: case MVT::v4i32: RC = &PPC::VRRCRegClass; break; case MVT::v4f32: RC = Subtarget.hasQPX() ? &PPC::QSRCRegClass : &PPC::VRRCRegClass; break; case MVT::v2f64: case MVT::v2i64: RC = &PPC::VRRCRegClass; break; case MVT::v4f64: RC = &PPC::QFRCRegClass; break; case MVT::v4i1: RC = &PPC::QBRCRegClass; break; } SDValue ArgValue; // Transform the arguments stored in physical registers into // virtual ones. if (VA.getLocVT() == MVT::f64 && Subtarget.hasSPE()) { assert(i + 1 < e && "No second half of double precision argument"); unsigned RegLo = MF.addLiveIn(VA.getLocReg(), RC); unsigned RegHi = MF.addLiveIn(ArgLocs[++i].getLocReg(), RC); SDValue ArgValueLo = DAG.getCopyFromReg(Chain, dl, RegLo, MVT::i32); SDValue ArgValueHi = DAG.getCopyFromReg(Chain, dl, RegHi, MVT::i32); if (!Subtarget.isLittleEndian()) std::swap (ArgValueLo, ArgValueHi); ArgValue = DAG.getNode(PPCISD::BUILD_SPE64, dl, MVT::f64, ArgValueLo, ArgValueHi); } else { unsigned Reg = MF.addLiveIn(VA.getLocReg(), RC); ArgValue = DAG.getCopyFromReg(Chain, dl, Reg, ValVT == MVT::i1 ? MVT::i32 : ValVT); if (ValVT == MVT::i1) ArgValue = DAG.getNode(ISD::TRUNCATE, dl, MVT::i1, ArgValue); } InVals.push_back(ArgValue); } else { // Argument stored in memory. assert(VA.isMemLoc()); // Get the extended size of the argument type in stack unsigned ArgSize = VA.getLocVT().getStoreSize(); // Get the actual size of the argument type unsigned ObjSize = VA.getValVT().getStoreSize(); unsigned ArgOffset = VA.getLocMemOffset(); // Stack objects in PPC32 are right justified. ArgOffset += ArgSize - ObjSize; int FI = MFI.CreateFixedObject(ArgSize, ArgOffset, isImmutable); // Create load nodes to retrieve arguments from the stack. SDValue FIN = DAG.getFrameIndex(FI, PtrVT); InVals.push_back( DAG.getLoad(VA.getValVT(), dl, Chain, FIN, MachinePointerInfo())); } } // Assign locations to all of the incoming aggregate by value arguments. // Aggregates passed by value are stored in the local variable space of the // caller's stack frame, right above the parameter list area. SmallVector ByValArgLocs; CCState CCByValInfo(CallConv, isVarArg, DAG.getMachineFunction(), ByValArgLocs, *DAG.getContext()); // Reserve stack space for the allocations in CCInfo. CCByValInfo.AllocateStack(CCInfo.getNextStackOffset(), PtrByteSize); CCByValInfo.AnalyzeFormalArguments(Ins, CC_PPC32_SVR4_ByVal); // Area that is at least reserved in the caller of this function. unsigned MinReservedArea = CCByValInfo.getNextStackOffset(); MinReservedArea = std::max(MinReservedArea, LinkageSize); // Set the size that is at least reserved in caller of this function. Tail // call optimized function's reserved stack space needs to be aligned so that // taking the difference between two stack areas will result in an aligned // stack. MinReservedArea = EnsureStackAlignment(Subtarget.getFrameLowering(), MinReservedArea); FuncInfo->setMinReservedArea(MinReservedArea); SmallVector MemOps; // If the function takes variable number of arguments, make a frame index for // the start of the first vararg value... for expansion of llvm.va_start. if (isVarArg) { static const MCPhysReg GPArgRegs[] = { PPC::R3, PPC::R4, PPC::R5, PPC::R6, PPC::R7, PPC::R8, PPC::R9, PPC::R10, }; const unsigned NumGPArgRegs = array_lengthof(GPArgRegs); static const MCPhysReg FPArgRegs[] = { PPC::F1, PPC::F2, PPC::F3, PPC::F4, PPC::F5, PPC::F6, PPC::F7, PPC::F8 }; unsigned NumFPArgRegs = array_lengthof(FPArgRegs); if (useSoftFloat() || hasSPE()) NumFPArgRegs = 0; FuncInfo->setVarArgsNumGPR(CCInfo.getFirstUnallocated(GPArgRegs)); FuncInfo->setVarArgsNumFPR(CCInfo.getFirstUnallocated(FPArgRegs)); // Make room for NumGPArgRegs and NumFPArgRegs. int Depth = NumGPArgRegs * PtrVT.getSizeInBits()/8 + NumFPArgRegs * MVT(MVT::f64).getSizeInBits()/8; FuncInfo->setVarArgsStackOffset( MFI.CreateFixedObject(PtrVT.getSizeInBits()/8, CCInfo.getNextStackOffset(), true)); FuncInfo->setVarArgsFrameIndex(MFI.CreateStackObject(Depth, 8, false)); SDValue FIN = DAG.getFrameIndex(FuncInfo->getVarArgsFrameIndex(), PtrVT); // The fixed integer arguments of a variadic function are stored to the // VarArgsFrameIndex on the stack so that they may be loaded by // dereferencing the result of va_next. for (unsigned GPRIndex = 0; GPRIndex != NumGPArgRegs; ++GPRIndex) { // Get an existing live-in vreg, or add a new one. unsigned VReg = MF.getRegInfo().getLiveInVirtReg(GPArgRegs[GPRIndex]); if (!VReg) VReg = MF.addLiveIn(GPArgRegs[GPRIndex], &PPC::GPRCRegClass); SDValue Val = DAG.getCopyFromReg(Chain, dl, VReg, PtrVT); SDValue Store = DAG.getStore(Val.getValue(1), dl, Val, FIN, MachinePointerInfo()); MemOps.push_back(Store); // Increment the address by four for the next argument to store SDValue PtrOff = DAG.getConstant(PtrVT.getSizeInBits()/8, dl, PtrVT); FIN = DAG.getNode(ISD::ADD, dl, PtrOff.getValueType(), FIN, PtrOff); } // FIXME 32-bit SVR4: We only need to save FP argument registers if CR bit 6 // is set. // The double arguments are stored to the VarArgsFrameIndex // on the stack. for (unsigned FPRIndex = 0; FPRIndex != NumFPArgRegs; ++FPRIndex) { // Get an existing live-in vreg, or add a new one. unsigned VReg = MF.getRegInfo().getLiveInVirtReg(FPArgRegs[FPRIndex]); if (!VReg) VReg = MF.addLiveIn(FPArgRegs[FPRIndex], &PPC::F8RCRegClass); SDValue Val = DAG.getCopyFromReg(Chain, dl, VReg, MVT::f64); SDValue Store = DAG.getStore(Val.getValue(1), dl, Val, FIN, MachinePointerInfo()); MemOps.push_back(Store); // Increment the address by eight for the next argument to store SDValue PtrOff = DAG.getConstant(MVT(MVT::f64).getSizeInBits()/8, dl, PtrVT); FIN = DAG.getNode(ISD::ADD, dl, PtrOff.getValueType(), FIN, PtrOff); } } if (!MemOps.empty()) Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, MemOps); return Chain; } // PPC64 passes i8, i16, and i32 values in i64 registers. Promote // value to MVT::i64 and then truncate to the correct register size. SDValue PPCTargetLowering::extendArgForPPC64(ISD::ArgFlagsTy Flags, EVT ObjectVT, SelectionDAG &DAG, SDValue ArgVal, const SDLoc &dl) const { if (Flags.isSExt()) ArgVal = DAG.getNode(ISD::AssertSext, dl, MVT::i64, ArgVal, DAG.getValueType(ObjectVT)); else if (Flags.isZExt()) ArgVal = DAG.getNode(ISD::AssertZext, dl, MVT::i64, ArgVal, DAG.getValueType(ObjectVT)); return DAG.getNode(ISD::TRUNCATE, dl, ObjectVT, ArgVal); } SDValue PPCTargetLowering::LowerFormalArguments_64SVR4( SDValue Chain, CallingConv::ID CallConv, bool isVarArg, const SmallVectorImpl &Ins, const SDLoc &dl, SelectionDAG &DAG, SmallVectorImpl &InVals) const { // TODO: add description of PPC stack frame format, or at least some docs. // bool isELFv2ABI = Subtarget.isELFv2ABI(); bool isLittleEndian = Subtarget.isLittleEndian(); MachineFunction &MF = DAG.getMachineFunction(); MachineFrameInfo &MFI = MF.getFrameInfo(); PPCFunctionInfo *FuncInfo = MF.getInfo(); assert(!(CallConv == CallingConv::Fast && isVarArg) && "fastcc not supported on varargs functions"); EVT PtrVT = getPointerTy(MF.getDataLayout()); // Potential tail calls could cause overwriting of argument stack slots. bool isImmutable = !(getTargetMachine().Options.GuaranteedTailCallOpt && (CallConv == CallingConv::Fast)); unsigned PtrByteSize = 8; unsigned LinkageSize = Subtarget.getFrameLowering()->getLinkageSize(); static const MCPhysReg GPR[] = { PPC::X3, PPC::X4, PPC::X5, PPC::X6, PPC::X7, PPC::X8, PPC::X9, PPC::X10, }; static const MCPhysReg VR[] = { PPC::V2, PPC::V3, PPC::V4, PPC::V5, PPC::V6, PPC::V7, PPC::V8, PPC::V9, PPC::V10, PPC::V11, PPC::V12, PPC::V13 }; const unsigned Num_GPR_Regs = array_lengthof(GPR); const unsigned Num_FPR_Regs = useSoftFloat() ? 0 : 13; const unsigned Num_VR_Regs = array_lengthof(VR); const unsigned Num_QFPR_Regs = Num_FPR_Regs; // Do a first pass over the arguments to determine whether the ABI // guarantees that our caller has allocated the parameter save area // on its stack frame. In the ELFv1 ABI, this is always the case; // in the ELFv2 ABI, it is true if this is a vararg function or if // any parameter is located in a stack slot. bool HasParameterArea = !isELFv2ABI || isVarArg; unsigned ParamAreaSize = Num_GPR_Regs * PtrByteSize; unsigned NumBytes = LinkageSize; unsigned AvailableFPRs = Num_FPR_Regs; unsigned AvailableVRs = Num_VR_Regs; for (unsigned i = 0, e = Ins.size(); i != e; ++i) { if (Ins[i].Flags.isNest()) continue; if (CalculateStackSlotUsed(Ins[i].VT, Ins[i].ArgVT, Ins[i].Flags, PtrByteSize, LinkageSize, ParamAreaSize, NumBytes, AvailableFPRs, AvailableVRs, Subtarget.hasQPX())) HasParameterArea = true; } // Add DAG nodes to load the arguments or copy them out of registers. On // entry to a function on PPC, the arguments start after the linkage area, // although the first ones are often in registers. unsigned ArgOffset = LinkageSize; unsigned GPR_idx = 0, FPR_idx = 0, VR_idx = 0; unsigned &QFPR_idx = FPR_idx; SmallVector MemOps; Function::const_arg_iterator FuncArg = MF.getFunction().arg_begin(); unsigned CurArgIdx = 0; for (unsigned ArgNo = 0, e = Ins.size(); ArgNo != e; ++ArgNo) { SDValue ArgVal; bool needsLoad = false; EVT ObjectVT = Ins[ArgNo].VT; EVT OrigVT = Ins[ArgNo].ArgVT; unsigned ObjSize = ObjectVT.getStoreSize(); unsigned ArgSize = ObjSize; ISD::ArgFlagsTy Flags = Ins[ArgNo].Flags; if (Ins[ArgNo].isOrigArg()) { std::advance(FuncArg, Ins[ArgNo].getOrigArgIndex() - CurArgIdx); CurArgIdx = Ins[ArgNo].getOrigArgIndex(); } // We re-align the argument offset for each argument, except when using the // fast calling convention, when we need to make sure we do that only when // we'll actually use a stack slot. unsigned CurArgOffset; Align Alignment; auto ComputeArgOffset = [&]() { /* Respect alignment of argument on the stack. */ Alignment = CalculateStackSlotAlignment(ObjectVT, OrigVT, Flags, PtrByteSize); ArgOffset = alignTo(ArgOffset, Alignment); CurArgOffset = ArgOffset; }; if (CallConv != CallingConv::Fast) { ComputeArgOffset(); /* Compute GPR index associated with argument offset. */ GPR_idx = (ArgOffset - LinkageSize) / PtrByteSize; GPR_idx = std::min(GPR_idx, Num_GPR_Regs); } // FIXME the codegen can be much improved in some cases. // We do not have to keep everything in memory. if (Flags.isByVal()) { assert(Ins[ArgNo].isOrigArg() && "Byval arguments cannot be implicit"); if (CallConv == CallingConv::Fast) ComputeArgOffset(); // ObjSize is the true size, ArgSize rounded up to multiple of registers. ObjSize = Flags.getByValSize(); ArgSize = ((ObjSize + PtrByteSize - 1)/PtrByteSize) * PtrByteSize; // Empty aggregate parameters do not take up registers. Examples: // struct { } a; // union { } b; // int c[0]; // etc. However, we have to provide a place-holder in InVals, so // pretend we have an 8-byte item at the current address for that // purpose. if (!ObjSize) { int FI = MFI.CreateFixedObject(PtrByteSize, ArgOffset, true); SDValue FIN = DAG.getFrameIndex(FI, PtrVT); InVals.push_back(FIN); continue; } // Create a stack object covering all stack doublewords occupied // by the argument. If the argument is (fully or partially) on // the stack, or if the argument is fully in registers but the // caller has allocated the parameter save anyway, we can refer // directly to the caller's stack frame. Otherwise, create a // local copy in our own frame. int FI; if (HasParameterArea || ArgSize + ArgOffset > LinkageSize + Num_GPR_Regs * PtrByteSize) FI = MFI.CreateFixedObject(ArgSize, ArgOffset, false, true); else FI = MFI.CreateStackObject(ArgSize, Alignment, false); SDValue FIN = DAG.getFrameIndex(FI, PtrVT); // Handle aggregates smaller than 8 bytes. if (ObjSize < PtrByteSize) { // The value of the object is its address, which differs from the // address of the enclosing doubleword on big-endian systems. SDValue Arg = FIN; if (!isLittleEndian) { SDValue ArgOff = DAG.getConstant(PtrByteSize - ObjSize, dl, PtrVT); Arg = DAG.getNode(ISD::ADD, dl, ArgOff.getValueType(), Arg, ArgOff); } InVals.push_back(Arg); if (GPR_idx != Num_GPR_Regs) { unsigned VReg = MF.addLiveIn(GPR[GPR_idx++], &PPC::G8RCRegClass); FuncInfo->addLiveInAttr(VReg, Flags); SDValue Val = DAG.getCopyFromReg(Chain, dl, VReg, PtrVT); SDValue Store; if (ObjSize==1 || ObjSize==2 || ObjSize==4) { EVT ObjType = (ObjSize == 1 ? MVT::i8 : (ObjSize == 2 ? MVT::i16 : MVT::i32)); Store = DAG.getTruncStore(Val.getValue(1), dl, Val, Arg, MachinePointerInfo(&*FuncArg), ObjType); } else { // For sizes that don't fit a truncating store (3, 5, 6, 7), // store the whole register as-is to the parameter save area // slot. Store = DAG.getStore(Val.getValue(1), dl, Val, FIN, MachinePointerInfo(&*FuncArg)); } MemOps.push_back(Store); } // Whether we copied from a register or not, advance the offset // into the parameter save area by a full doubleword. ArgOffset += PtrByteSize; continue; } // The value of the object is its address, which is the address of // its first stack doubleword. InVals.push_back(FIN); // Store whatever pieces of the object are in registers to memory. for (unsigned j = 0; j < ArgSize; j += PtrByteSize) { if (GPR_idx == Num_GPR_Regs) break; unsigned VReg = MF.addLiveIn(GPR[GPR_idx], &PPC::G8RCRegClass); FuncInfo->addLiveInAttr(VReg, Flags); SDValue Val = DAG.getCopyFromReg(Chain, dl, VReg, PtrVT); SDValue Addr = FIN; if (j) { SDValue Off = DAG.getConstant(j, dl, PtrVT); Addr = DAG.getNode(ISD::ADD, dl, Off.getValueType(), Addr, Off); } SDValue Store = DAG.getStore(Val.getValue(1), dl, Val, Addr, MachinePointerInfo(&*FuncArg, j)); MemOps.push_back(Store); ++GPR_idx; } ArgOffset += ArgSize; continue; } switch (ObjectVT.getSimpleVT().SimpleTy) { default: llvm_unreachable("Unhandled argument type!"); case MVT::i1: case MVT::i32: case MVT::i64: if (Flags.isNest()) { // The 'nest' parameter, if any, is passed in R11. unsigned VReg = MF.addLiveIn(PPC::X11, &PPC::G8RCRegClass); ArgVal = DAG.getCopyFromReg(Chain, dl, VReg, MVT::i64); if (ObjectVT == MVT::i32 || ObjectVT == MVT::i1) ArgVal = extendArgForPPC64(Flags, ObjectVT, DAG, ArgVal, dl); break; } // These can be scalar arguments or elements of an integer array type // passed directly. Clang may use those instead of "byval" aggregate // types to avoid forcing arguments to memory unnecessarily. if (GPR_idx != Num_GPR_Regs) { unsigned VReg = MF.addLiveIn(GPR[GPR_idx++], &PPC::G8RCRegClass); FuncInfo->addLiveInAttr(VReg, Flags); ArgVal = DAG.getCopyFromReg(Chain, dl, VReg, MVT::i64); if (ObjectVT == MVT::i32 || ObjectVT == MVT::i1) // PPC64 passes i8, i16, and i32 values in i64 registers. Promote // value to MVT::i64 and then truncate to the correct register size. ArgVal = extendArgForPPC64(Flags, ObjectVT, DAG, ArgVal, dl); } else { if (CallConv == CallingConv::Fast) ComputeArgOffset(); needsLoad = true; ArgSize = PtrByteSize; } if (CallConv != CallingConv::Fast || needsLoad) ArgOffset += 8; break; case MVT::f32: case MVT::f64: // These can be scalar arguments or elements of a float array type // passed directly. The latter are used to implement ELFv2 homogenous // float aggregates. if (FPR_idx != Num_FPR_Regs) { unsigned VReg; if (ObjectVT == MVT::f32) VReg = MF.addLiveIn(FPR[FPR_idx], Subtarget.hasP8Vector() ? &PPC::VSSRCRegClass : &PPC::F4RCRegClass); else VReg = MF.addLiveIn(FPR[FPR_idx], Subtarget.hasVSX() ? &PPC::VSFRCRegClass : &PPC::F8RCRegClass); ArgVal = DAG.getCopyFromReg(Chain, dl, VReg, ObjectVT); ++FPR_idx; } else if (GPR_idx != Num_GPR_Regs && CallConv != CallingConv::Fast) { // FIXME: We may want to re-enable this for CallingConv::Fast on the P8 // once we support fp <-> gpr moves. // This can only ever happen in the presence of f32 array types, // since otherwise we never run out of FPRs before running out // of GPRs. unsigned VReg = MF.addLiveIn(GPR[GPR_idx++], &PPC::G8RCRegClass); FuncInfo->addLiveInAttr(VReg, Flags); ArgVal = DAG.getCopyFromReg(Chain, dl, VReg, MVT::i64); if (ObjectVT == MVT::f32) { if ((ArgOffset % PtrByteSize) == (isLittleEndian ? 4 : 0)) ArgVal = DAG.getNode(ISD::SRL, dl, MVT::i64, ArgVal, DAG.getConstant(32, dl, MVT::i32)); ArgVal = DAG.getNode(ISD::TRUNCATE, dl, MVT::i32, ArgVal); } ArgVal = DAG.getNode(ISD::BITCAST, dl, ObjectVT, ArgVal); } else { if (CallConv == CallingConv::Fast) ComputeArgOffset(); needsLoad = true; } // When passing an array of floats, the array occupies consecutive // space in the argument area; only round up to the next doubleword // at the end of the array. Otherwise, each float takes 8 bytes. if (CallConv != CallingConv::Fast || needsLoad) { ArgSize = Flags.isInConsecutiveRegs() ? ObjSize : PtrByteSize; ArgOffset += ArgSize; if (Flags.isInConsecutiveRegsLast()) ArgOffset = ((ArgOffset + PtrByteSize - 1)/PtrByteSize) * PtrByteSize; } break; case MVT::v4f32: case MVT::v4i32: case MVT::v8i16: case MVT::v16i8: case MVT::v2f64: case MVT::v2i64: case MVT::v1i128: case MVT::f128: if (!Subtarget.hasQPX()) { // These can be scalar arguments or elements of a vector array type // passed directly. The latter are used to implement ELFv2 homogenous // vector aggregates. if (VR_idx != Num_VR_Regs) { unsigned VReg = MF.addLiveIn(VR[VR_idx], &PPC::VRRCRegClass); ArgVal = DAG.getCopyFromReg(Chain, dl, VReg, ObjectVT); ++VR_idx; } else { if (CallConv == CallingConv::Fast) ComputeArgOffset(); needsLoad = true; } if (CallConv != CallingConv::Fast || needsLoad) ArgOffset += 16; break; } // not QPX assert(ObjectVT.getSimpleVT().SimpleTy == MVT::v4f32 && "Invalid QPX parameter type"); LLVM_FALLTHROUGH; case MVT::v4f64: case MVT::v4i1: // QPX vectors are treated like their scalar floating-point subregisters // (except that they're larger). unsigned Sz = ObjectVT.getSimpleVT().SimpleTy == MVT::v4f32 ? 16 : 32; if (QFPR_idx != Num_QFPR_Regs) { const TargetRegisterClass *RC; switch (ObjectVT.getSimpleVT().SimpleTy) { case MVT::v4f64: RC = &PPC::QFRCRegClass; break; case MVT::v4f32: RC = &PPC::QSRCRegClass; break; default: RC = &PPC::QBRCRegClass; break; } unsigned VReg = MF.addLiveIn(QFPR[QFPR_idx], RC); ArgVal = DAG.getCopyFromReg(Chain, dl, VReg, ObjectVT); ++QFPR_idx; } else { if (CallConv == CallingConv::Fast) ComputeArgOffset(); needsLoad = true; } if (CallConv != CallingConv::Fast || needsLoad) ArgOffset += Sz; break; } // We need to load the argument to a virtual register if we determined // above that we ran out of physical registers of the appropriate type. if (needsLoad) { if (ObjSize < ArgSize && !isLittleEndian) CurArgOffset += ArgSize - ObjSize; int FI = MFI.CreateFixedObject(ObjSize, CurArgOffset, isImmutable); SDValue FIN = DAG.getFrameIndex(FI, PtrVT); ArgVal = DAG.getLoad(ObjectVT, dl, Chain, FIN, MachinePointerInfo()); } InVals.push_back(ArgVal); } // Area that is at least reserved in the caller of this function. unsigned MinReservedArea; if (HasParameterArea) MinReservedArea = std::max(ArgOffset, LinkageSize + 8 * PtrByteSize); else MinReservedArea = LinkageSize; // Set the size that is at least reserved in caller of this function. Tail // call optimized functions' reserved stack space needs to be aligned so that // taking the difference between two stack areas will result in an aligned // stack. MinReservedArea = EnsureStackAlignment(Subtarget.getFrameLowering(), MinReservedArea); FuncInfo->setMinReservedArea(MinReservedArea); // If the function takes variable number of arguments, make a frame index for // the start of the first vararg value... for expansion of llvm.va_start. if (isVarArg) { int Depth = ArgOffset; FuncInfo->setVarArgsFrameIndex( MFI.CreateFixedObject(PtrByteSize, Depth, true)); SDValue FIN = DAG.getFrameIndex(FuncInfo->getVarArgsFrameIndex(), PtrVT); // If this function is vararg, store any remaining integer argument regs // to their spots on the stack so that they may be loaded by dereferencing // the result of va_next. for (GPR_idx = (ArgOffset - LinkageSize) / PtrByteSize; GPR_idx < Num_GPR_Regs; ++GPR_idx) { unsigned VReg = MF.addLiveIn(GPR[GPR_idx], &PPC::G8RCRegClass); SDValue Val = DAG.getCopyFromReg(Chain, dl, VReg, PtrVT); SDValue Store = DAG.getStore(Val.getValue(1), dl, Val, FIN, MachinePointerInfo()); MemOps.push_back(Store); // Increment the address by four for the next argument to store SDValue PtrOff = DAG.getConstant(PtrByteSize, dl, PtrVT); FIN = DAG.getNode(ISD::ADD, dl, PtrOff.getValueType(), FIN, PtrOff); } } if (!MemOps.empty()) Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, MemOps); return Chain; } SDValue PPCTargetLowering::LowerFormalArguments_Darwin( SDValue Chain, CallingConv::ID CallConv, bool isVarArg, const SmallVectorImpl &Ins, const SDLoc &dl, SelectionDAG &DAG, SmallVectorImpl &InVals) const { // TODO: add description of PPC stack frame format, or at least some docs. // MachineFunction &MF = DAG.getMachineFunction(); MachineFrameInfo &MFI = MF.getFrameInfo(); PPCFunctionInfo *FuncInfo = MF.getInfo(); EVT PtrVT = getPointerTy(MF.getDataLayout()); bool isPPC64 = PtrVT == MVT::i64; // Potential tail calls could cause overwriting of argument stack slots. bool isImmutable = !(getTargetMachine().Options.GuaranteedTailCallOpt && (CallConv == CallingConv::Fast)); unsigned PtrByteSize = isPPC64 ? 8 : 4; unsigned LinkageSize = Subtarget.getFrameLowering()->getLinkageSize(); unsigned ArgOffset = LinkageSize; // Area that is at least reserved in caller of this function. unsigned MinReservedArea = ArgOffset; static const MCPhysReg GPR_32[] = { // 32-bit registers. PPC::R3, PPC::R4, PPC::R5, PPC::R6, PPC::R7, PPC::R8, PPC::R9, PPC::R10, }; static const MCPhysReg GPR_64[] = { // 64-bit registers. PPC::X3, PPC::X4, PPC::X5, PPC::X6, PPC::X7, PPC::X8, PPC::X9, PPC::X10, }; static const MCPhysReg VR[] = { PPC::V2, PPC::V3, PPC::V4, PPC::V5, PPC::V6, PPC::V7, PPC::V8, PPC::V9, PPC::V10, PPC::V11, PPC::V12, PPC::V13 }; const unsigned Num_GPR_Regs = array_lengthof(GPR_32); const unsigned Num_FPR_Regs = useSoftFloat() ? 0 : 13; const unsigned Num_VR_Regs = array_lengthof( VR); unsigned GPR_idx = 0, FPR_idx = 0, VR_idx = 0; const MCPhysReg *GPR = isPPC64 ? GPR_64 : GPR_32; // In 32-bit non-varargs functions, the stack space for vectors is after the // stack space for non-vectors. We do not use this space unless we have // too many vectors to fit in registers, something that only occurs in // constructed examples:), but we have to walk the arglist to figure // that out...for the pathological case, compute VecArgOffset as the // start of the vector parameter area. Computing VecArgOffset is the // entire point of the following loop. unsigned VecArgOffset = ArgOffset; if (!isVarArg && !isPPC64) { for (unsigned ArgNo = 0, e = Ins.size(); ArgNo != e; ++ArgNo) { EVT ObjectVT = Ins[ArgNo].VT; ISD::ArgFlagsTy Flags = Ins[ArgNo].Flags; if (Flags.isByVal()) { // ObjSize is the true size, ArgSize rounded up to multiple of regs. unsigned ObjSize = Flags.getByValSize(); unsigned ArgSize = ((ObjSize + PtrByteSize - 1)/PtrByteSize) * PtrByteSize; VecArgOffset += ArgSize; continue; } switch(ObjectVT.getSimpleVT().SimpleTy) { default: llvm_unreachable("Unhandled argument type!"); case MVT::i1: case MVT::i32: case MVT::f32: VecArgOffset += 4; break; case MVT::i64: // PPC64 case MVT::f64: // FIXME: We are guaranteed to be !isPPC64 at this point. // Does MVT::i64 apply? VecArgOffset += 8; break; case MVT::v4f32: case MVT::v4i32: case MVT::v8i16: case MVT::v16i8: // Nothing to do, we're only looking at Nonvector args here. break; } } } // We've found where the vector parameter area in memory is. Skip the // first 12 parameters; these don't use that memory. VecArgOffset = ((VecArgOffset+15)/16)*16; VecArgOffset += 12*16; // Add DAG nodes to load the arguments or copy them out of registers. On // entry to a function on PPC, the arguments start after the linkage area, // although the first ones are often in registers. SmallVector MemOps; unsigned nAltivecParamsAtEnd = 0; Function::const_arg_iterator FuncArg = MF.getFunction().arg_begin(); unsigned CurArgIdx = 0; for (unsigned ArgNo = 0, e = Ins.size(); ArgNo != e; ++ArgNo) { SDValue ArgVal; bool needsLoad = false; EVT ObjectVT = Ins[ArgNo].VT; unsigned ObjSize = ObjectVT.getSizeInBits()/8; unsigned ArgSize = ObjSize; ISD::ArgFlagsTy Flags = Ins[ArgNo].Flags; if (Ins[ArgNo].isOrigArg()) { std::advance(FuncArg, Ins[ArgNo].getOrigArgIndex() - CurArgIdx); CurArgIdx = Ins[ArgNo].getOrigArgIndex(); } unsigned CurArgOffset = ArgOffset; // Varargs or 64 bit Altivec parameters are padded to a 16 byte boundary. if (ObjectVT==MVT::v4f32 || ObjectVT==MVT::v4i32 || ObjectVT==MVT::v8i16 || ObjectVT==MVT::v16i8) { if (isVarArg || isPPC64) { MinReservedArea = ((MinReservedArea+15)/16)*16; MinReservedArea += CalculateStackSlotSize(ObjectVT, Flags, PtrByteSize); } else nAltivecParamsAtEnd++; } else // Calculate min reserved area. MinReservedArea += CalculateStackSlotSize(Ins[ArgNo].VT, Flags, PtrByteSize); // FIXME the codegen can be much improved in some cases. // We do not have to keep everything in memory. if (Flags.isByVal()) { assert(Ins[ArgNo].isOrigArg() && "Byval arguments cannot be implicit"); // ObjSize is the true size, ArgSize rounded up to multiple of registers. ObjSize = Flags.getByValSize(); ArgSize = ((ObjSize + PtrByteSize - 1)/PtrByteSize) * PtrByteSize; // Objects of size 1 and 2 are right justified, everything else is // left justified. This means the memory address is adjusted forwards. if (ObjSize==1 || ObjSize==2) { CurArgOffset = CurArgOffset + (4 - ObjSize); } // The value of the object is its address. int FI = MFI.CreateFixedObject(ObjSize, CurArgOffset, false, true); SDValue FIN = DAG.getFrameIndex(FI, PtrVT); InVals.push_back(FIN); if (ObjSize==1 || ObjSize==2) { if (GPR_idx != Num_GPR_Regs) { unsigned VReg; if (isPPC64) VReg = MF.addLiveIn(GPR[GPR_idx], &PPC::G8RCRegClass); else VReg = MF.addLiveIn(GPR[GPR_idx], &PPC::GPRCRegClass); SDValue Val = DAG.getCopyFromReg(Chain, dl, VReg, PtrVT); EVT ObjType = ObjSize == 1 ? MVT::i8 : MVT::i16; SDValue Store = DAG.getTruncStore(Val.getValue(1), dl, Val, FIN, MachinePointerInfo(&*FuncArg), ObjType); MemOps.push_back(Store); ++GPR_idx; } ArgOffset += PtrByteSize; continue; } for (unsigned j = 0; j < ArgSize; j += PtrByteSize) { // Store whatever pieces of the object are in registers // to memory. ArgOffset will be the address of the beginning // of the object. if (GPR_idx != Num_GPR_Regs) { unsigned VReg; if (isPPC64) VReg = MF.addLiveIn(GPR[GPR_idx], &PPC::G8RCRegClass); else VReg = MF.addLiveIn(GPR[GPR_idx], &PPC::GPRCRegClass); int FI = MFI.CreateFixedObject(PtrByteSize, ArgOffset, true); SDValue FIN = DAG.getFrameIndex(FI, PtrVT); SDValue Val = DAG.getCopyFromReg(Chain, dl, VReg, PtrVT); SDValue Store = DAG.getStore(Val.getValue(1), dl, Val, FIN, MachinePointerInfo(&*FuncArg, j)); MemOps.push_back(Store); ++GPR_idx; ArgOffset += PtrByteSize; } else { ArgOffset += ArgSize - (ArgOffset-CurArgOffset); break; } } continue; } switch (ObjectVT.getSimpleVT().SimpleTy) { default: llvm_unreachable("Unhandled argument type!"); case MVT::i1: case MVT::i32: if (!isPPC64) { if (GPR_idx != Num_GPR_Regs) { unsigned VReg = MF.addLiveIn(GPR[GPR_idx], &PPC::GPRCRegClass); ArgVal = DAG.getCopyFromReg(Chain, dl, VReg, MVT::i32); if (ObjectVT == MVT::i1) ArgVal = DAG.getNode(ISD::TRUNCATE, dl, MVT::i1, ArgVal); ++GPR_idx; } else { needsLoad = true; ArgSize = PtrByteSize; } // All int arguments reserve stack space in the Darwin ABI. ArgOffset += PtrByteSize; break; } LLVM_FALLTHROUGH; case MVT::i64: // PPC64 if (GPR_idx != Num_GPR_Regs) { unsigned VReg = MF.addLiveIn(GPR[GPR_idx], &PPC::G8RCRegClass); ArgVal = DAG.getCopyFromReg(Chain, dl, VReg, MVT::i64); if (ObjectVT == MVT::i32 || ObjectVT == MVT::i1) // PPC64 passes i8, i16, and i32 values in i64 registers. Promote // value to MVT::i64 and then truncate to the correct register size. ArgVal = extendArgForPPC64(Flags, ObjectVT, DAG, ArgVal, dl); ++GPR_idx; } else { needsLoad = true; ArgSize = PtrByteSize; } // All int arguments reserve stack space in the Darwin ABI. ArgOffset += 8; break; case MVT::f32: case MVT::f64: // Every 4 bytes of argument space consumes one of the GPRs available for // argument passing. if (GPR_idx != Num_GPR_Regs) { ++GPR_idx; if (ObjSize == 8 && GPR_idx != Num_GPR_Regs && !isPPC64) ++GPR_idx; } if (FPR_idx != Num_FPR_Regs) { unsigned VReg; if (ObjectVT == MVT::f32) VReg = MF.addLiveIn(FPR[FPR_idx], &PPC::F4RCRegClass); else VReg = MF.addLiveIn(FPR[FPR_idx], &PPC::F8RCRegClass); ArgVal = DAG.getCopyFromReg(Chain, dl, VReg, ObjectVT); ++FPR_idx; } else { needsLoad = true; } // All FP arguments reserve stack space in the Darwin ABI. ArgOffset += isPPC64 ? 8 : ObjSize; break; case MVT::v4f32: case MVT::v4i32: case MVT::v8i16: case MVT::v16i8: // Note that vector arguments in registers don't reserve stack space, // except in varargs functions. if (VR_idx != Num_VR_Regs) { unsigned VReg = MF.addLiveIn(VR[VR_idx], &PPC::VRRCRegClass); ArgVal = DAG.getCopyFromReg(Chain, dl, VReg, ObjectVT); if (isVarArg) { while ((ArgOffset % 16) != 0) { ArgOffset += PtrByteSize; if (GPR_idx != Num_GPR_Regs) GPR_idx++; } ArgOffset += 16; GPR_idx = std::min(GPR_idx+4, Num_GPR_Regs); // FIXME correct for ppc64? } ++VR_idx; } else { if (!isVarArg && !isPPC64) { // Vectors go after all the nonvectors. CurArgOffset = VecArgOffset; VecArgOffset += 16; } else { // Vectors are aligned. ArgOffset = ((ArgOffset+15)/16)*16; CurArgOffset = ArgOffset; ArgOffset += 16; } needsLoad = true; } break; } // We need to load the argument to a virtual register if we determined above // that we ran out of physical registers of the appropriate type. if (needsLoad) { int FI = MFI.CreateFixedObject(ObjSize, CurArgOffset + (ArgSize - ObjSize), isImmutable); SDValue FIN = DAG.getFrameIndex(FI, PtrVT); ArgVal = DAG.getLoad(ObjectVT, dl, Chain, FIN, MachinePointerInfo()); } InVals.push_back(ArgVal); } // Allow for Altivec parameters at the end, if needed. if (nAltivecParamsAtEnd) { MinReservedArea = ((MinReservedArea+15)/16)*16; MinReservedArea += 16*nAltivecParamsAtEnd; } // Area that is at least reserved in the caller of this function. MinReservedArea = std::max(MinReservedArea, LinkageSize + 8 * PtrByteSize); // Set the size that is at least reserved in caller of this function. Tail // call optimized functions' reserved stack space needs to be aligned so that // taking the difference between two stack areas will result in an aligned // stack. MinReservedArea = EnsureStackAlignment(Subtarget.getFrameLowering(), MinReservedArea); FuncInfo->setMinReservedArea(MinReservedArea); // If the function takes variable number of arguments, make a frame index for // the start of the first vararg value... for expansion of llvm.va_start. if (isVarArg) { int Depth = ArgOffset; FuncInfo->setVarArgsFrameIndex( MFI.CreateFixedObject(PtrVT.getSizeInBits()/8, Depth, true)); SDValue FIN = DAG.getFrameIndex(FuncInfo->getVarArgsFrameIndex(), PtrVT); // If this function is vararg, store any remaining integer argument regs // to their spots on the stack so that they may be loaded by dereferencing // the result of va_next. for (; GPR_idx != Num_GPR_Regs; ++GPR_idx) { unsigned VReg; if (isPPC64) VReg = MF.addLiveIn(GPR[GPR_idx], &PPC::G8RCRegClass); else VReg = MF.addLiveIn(GPR[GPR_idx], &PPC::GPRCRegClass); SDValue Val = DAG.getCopyFromReg(Chain, dl, VReg, PtrVT); SDValue Store = DAG.getStore(Val.getValue(1), dl, Val, FIN, MachinePointerInfo()); MemOps.push_back(Store); // Increment the address by four for the next argument to store SDValue PtrOff = DAG.getConstant(PtrVT.getSizeInBits()/8, dl, PtrVT); FIN = DAG.getNode(ISD::ADD, dl, PtrOff.getValueType(), FIN, PtrOff); } } if (!MemOps.empty()) Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, MemOps); return Chain; } /// CalculateTailCallSPDiff - Get the amount the stack pointer has to be /// adjusted to accommodate the arguments for the tailcall. static int CalculateTailCallSPDiff(SelectionDAG& DAG, bool isTailCall, unsigned ParamSize) { if (!isTailCall) return 0; PPCFunctionInfo *FI = DAG.getMachineFunction().getInfo(); unsigned CallerMinReservedArea = FI->getMinReservedArea(); int SPDiff = (int)CallerMinReservedArea - (int)ParamSize; // Remember only if the new adjustment is bigger. if (SPDiff < FI->getTailCallSPDelta()) FI->setTailCallSPDelta(SPDiff); return SPDiff; } static bool isFunctionGlobalAddress(SDValue Callee); static bool callsShareTOCBase(const Function *Caller, SDValue Callee, const TargetMachine &TM) { // Callee is either a GlobalAddress or an ExternalSymbol. ExternalSymbols // don't have enough information to determine if the caller and calle share // the same TOC base, so we have to pessimistically assume they don't for // correctness. GlobalAddressSDNode *G = dyn_cast(Callee); if (!G) return false; const GlobalValue *GV = G->getGlobal(); // The medium and large code models are expected to provide a sufficiently // large TOC to provide all data addressing needs of a module with a // single TOC. Since each module will be addressed with a single TOC then we // only need to check that caller and callee don't cross dso boundaries. if (CodeModel::Medium == TM.getCodeModel() || CodeModel::Large == TM.getCodeModel()) return TM.shouldAssumeDSOLocal(*Caller->getParent(), GV); // Otherwise we need to ensure callee and caller are in the same section, // since the linker may allocate multiple TOCs, and we don't know which // sections will belong to the same TOC base. if (!GV->isStrongDefinitionForLinker()) return false; // Any explicitly-specified sections and section prefixes must also match. // Also, if we're using -ffunction-sections, then each function is always in // a different section (the same is true for COMDAT functions). if (TM.getFunctionSections() || GV->hasComdat() || Caller->hasComdat() || GV->getSection() != Caller->getSection()) return false; if (const auto *F = dyn_cast(GV)) { if (F->getSectionPrefix() != Caller->getSectionPrefix()) return false; } // If the callee might be interposed, then we can't assume the ultimate call // target will be in the same section. Even in cases where we can assume that // interposition won't happen, in any case where the linker might insert a // stub to allow for interposition, we must generate code as though // interposition might occur. To understand why this matters, consider a // situation where: a -> b -> c where the arrows indicate calls. b and c are // in the same section, but a is in a different module (i.e. has a different // TOC base pointer). If the linker allows for interposition between b and c, // then it will generate a stub for the call edge between b and c which will // save the TOC pointer into the designated stack slot allocated by b. If we // return true here, and therefore allow a tail call between b and c, that // stack slot won't exist and the b -> c stub will end up saving b'c TOC base // pointer into the stack slot allocated by a (where the a -> b stub saved // a's TOC base pointer). If we're not considering a tail call, but rather, // whether a nop is needed after the call instruction in b, because the linker // will insert a stub, it might complain about a missing nop if we omit it // (although many don't complain in this case). if (!TM.shouldAssumeDSOLocal(*Caller->getParent(), GV)) return false; return true; } static bool needStackSlotPassParameters(const PPCSubtarget &Subtarget, const SmallVectorImpl &Outs) { assert(Subtarget.is64BitELFABI()); const unsigned PtrByteSize = 8; const unsigned LinkageSize = Subtarget.getFrameLowering()->getLinkageSize(); static const MCPhysReg GPR[] = { PPC::X3, PPC::X4, PPC::X5, PPC::X6, PPC::X7, PPC::X8, PPC::X9, PPC::X10, }; static const MCPhysReg VR[] = { PPC::V2, PPC::V3, PPC::V4, PPC::V5, PPC::V6, PPC::V7, PPC::V8, PPC::V9, PPC::V10, PPC::V11, PPC::V12, PPC::V13 }; const unsigned NumGPRs = array_lengthof(GPR); const unsigned NumFPRs = 13; const unsigned NumVRs = array_lengthof(VR); const unsigned ParamAreaSize = NumGPRs * PtrByteSize; unsigned NumBytes = LinkageSize; unsigned AvailableFPRs = NumFPRs; unsigned AvailableVRs = NumVRs; for (const ISD::OutputArg& Param : Outs) { if (Param.Flags.isNest()) continue; if (CalculateStackSlotUsed(Param.VT, Param.ArgVT, Param.Flags, PtrByteSize, LinkageSize, ParamAreaSize, NumBytes, AvailableFPRs, AvailableVRs, Subtarget.hasQPX())) return true; } return false; } static bool hasSameArgumentList(const Function *CallerFn, ImmutableCallSite CS) { if (CS.arg_size() != CallerFn->arg_size()) return false; ImmutableCallSite::arg_iterator CalleeArgIter = CS.arg_begin(); ImmutableCallSite::arg_iterator CalleeArgEnd = CS.arg_end(); Function::const_arg_iterator CallerArgIter = CallerFn->arg_begin(); for (; CalleeArgIter != CalleeArgEnd; ++CalleeArgIter, ++CallerArgIter) { const Value* CalleeArg = *CalleeArgIter; const Value* CallerArg = &(*CallerArgIter); if (CalleeArg == CallerArg) continue; // e.g. @caller([4 x i64] %a, [4 x i64] %b) { // tail call @callee([4 x i64] undef, [4 x i64] %b) // } // 1st argument of callee is undef and has the same type as caller. if (CalleeArg->getType() == CallerArg->getType() && isa(CalleeArg)) continue; return false; } return true; } // Returns true if TCO is possible between the callers and callees // calling conventions. static bool areCallingConvEligibleForTCO_64SVR4(CallingConv::ID CallerCC, CallingConv::ID CalleeCC) { // Tail calls are possible with fastcc and ccc. auto isTailCallableCC = [] (CallingConv::ID CC){ return CC == CallingConv::C || CC == CallingConv::Fast; }; if (!isTailCallableCC(CallerCC) || !isTailCallableCC(CalleeCC)) return false; // We can safely tail call both fastcc and ccc callees from a c calling // convention caller. If the caller is fastcc, we may have less stack space // than a non-fastcc caller with the same signature so disable tail-calls in // that case. return CallerCC == CallingConv::C || CallerCC == CalleeCC; } bool PPCTargetLowering::IsEligibleForTailCallOptimization_64SVR4( SDValue Callee, CallingConv::ID CalleeCC, ImmutableCallSite CS, bool isVarArg, const SmallVectorImpl &Outs, const SmallVectorImpl &Ins, SelectionDAG& DAG) const { bool TailCallOpt = getTargetMachine().Options.GuaranteedTailCallOpt; + // FIXME: Tail calls are currently disabled when using PC Relative addressing. + // The issue is that PC Relative is only partially implemented and so there + // is currently a mix of functions that require the TOC and functions that do + // not require it. If we have A calls B calls C and both A and B require the + // TOC and C does not and is marked as clobbering R2 then it is not safe for + // B to tail call C. Since we do not have the information of whether or not + // a funciton needs to use the TOC here in this function we need to be + // conservatively safe and disable all tail calls for now. + if (Subtarget.isUsingPCRelativeCalls()) return false; + if (DisableSCO && !TailCallOpt) return false; // Variadic argument functions are not supported. if (isVarArg) return false; auto &Caller = DAG.getMachineFunction().getFunction(); // Check that the calling conventions are compatible for tco. if (!areCallingConvEligibleForTCO_64SVR4(Caller.getCallingConv(), CalleeCC)) return false; // Caller contains any byval parameter is not supported. if (any_of(Ins, [](const ISD::InputArg &IA) { return IA.Flags.isByVal(); })) return false; // Callee contains any byval parameter is not supported, too. // Note: This is a quick work around, because in some cases, e.g. // caller's stack size > callee's stack size, we are still able to apply // sibling call optimization. For example, gcc is able to do SCO for caller1 // in the following example, but not for caller2. // struct test { // long int a; // char ary[56]; // } gTest; // __attribute__((noinline)) int callee(struct test v, struct test *b) { // b->a = v.a; // return 0; // } // void caller1(struct test a, struct test c, struct test *b) { // callee(gTest, b); } // void caller2(struct test *b) { callee(gTest, b); } if (any_of(Outs, [](const ISD::OutputArg& OA) { return OA.Flags.isByVal(); })) return false; // If callee and caller use different calling conventions, we cannot pass // parameters on stack since offsets for the parameter area may be different. if (Caller.getCallingConv() != CalleeCC && needStackSlotPassParameters(Subtarget, Outs)) return false; // No TCO/SCO on indirect call because Caller have to restore its TOC if (!isFunctionGlobalAddress(Callee) && !isa(Callee)) return false; // If the caller and callee potentially have different TOC bases then we // cannot tail call since we need to restore the TOC pointer after the call. // ref: https://bugzilla.mozilla.org/show_bug.cgi?id=973977 if (!callsShareTOCBase(&Caller, Callee, getTargetMachine())) return false; // TCO allows altering callee ABI, so we don't have to check further. if (CalleeCC == CallingConv::Fast && TailCallOpt) return true; if (DisableSCO) return false; // If callee use the same argument list that caller is using, then we can // apply SCO on this case. If it is not, then we need to check if callee needs // stack for passing arguments. if (!hasSameArgumentList(&Caller, CS) && needStackSlotPassParameters(Subtarget, Outs)) { return false; } return true; } /// IsEligibleForTailCallOptimization - Check whether the call is eligible /// for tail call optimization. Targets which want to do tail call /// optimization should implement this function. bool PPCTargetLowering::IsEligibleForTailCallOptimization(SDValue Callee, CallingConv::ID CalleeCC, bool isVarArg, const SmallVectorImpl &Ins, SelectionDAG& DAG) const { if (!getTargetMachine().Options.GuaranteedTailCallOpt) return false; // Variable argument functions are not supported. if (isVarArg) return false; MachineFunction &MF = DAG.getMachineFunction(); CallingConv::ID CallerCC = MF.getFunction().getCallingConv(); if (CalleeCC == CallingConv::Fast && CallerCC == CalleeCC) { // Functions containing by val parameters are not supported. for (unsigned i = 0; i != Ins.size(); i++) { ISD::ArgFlagsTy Flags = Ins[i].Flags; if (Flags.isByVal()) return false; } // Non-PIC/GOT tail calls are supported. if (getTargetMachine().getRelocationModel() != Reloc::PIC_) return true; // At the moment we can only do local tail calls (in same module, hidden // or protected) if we are generating PIC. if (GlobalAddressSDNode *G = dyn_cast(Callee)) return G->getGlobal()->hasHiddenVisibility() || G->getGlobal()->hasProtectedVisibility(); } return false; } /// isCallCompatibleAddress - Return the immediate to use if the specified /// 32-bit value is representable in the immediate field of a BxA instruction. static SDNode *isBLACompatibleAddress(SDValue Op, SelectionDAG &DAG) { ConstantSDNode *C = dyn_cast(Op); if (!C) return nullptr; int Addr = C->getZExtValue(); if ((Addr & 3) != 0 || // Low 2 bits are implicitly zero. SignExtend32<26>(Addr) != Addr) return nullptr; // Top 6 bits have to be sext of immediate. return DAG .getConstant( (int)C->getZExtValue() >> 2, SDLoc(Op), DAG.getTargetLoweringInfo().getPointerTy(DAG.getDataLayout())) .getNode(); } namespace { struct TailCallArgumentInfo { SDValue Arg; SDValue FrameIdxOp; int FrameIdx = 0; TailCallArgumentInfo() = default; }; } // end anonymous namespace /// StoreTailCallArgumentsToStackSlot - Stores arguments to their stack slot. static void StoreTailCallArgumentsToStackSlot( SelectionDAG &DAG, SDValue Chain, const SmallVectorImpl &TailCallArgs, SmallVectorImpl &MemOpChains, const SDLoc &dl) { for (unsigned i = 0, e = TailCallArgs.size(); i != e; ++i) { SDValue Arg = TailCallArgs[i].Arg; SDValue FIN = TailCallArgs[i].FrameIdxOp; int FI = TailCallArgs[i].FrameIdx; // Store relative to framepointer. MemOpChains.push_back(DAG.getStore( Chain, dl, Arg, FIN, MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FI))); } } /// EmitTailCallStoreFPAndRetAddr - Move the frame pointer and return address to /// the appropriate stack slot for the tail call optimized function call. static SDValue EmitTailCallStoreFPAndRetAddr(SelectionDAG &DAG, SDValue Chain, SDValue OldRetAddr, SDValue OldFP, int SPDiff, const SDLoc &dl) { if (SPDiff) { // Calculate the new stack slot for the return address. MachineFunction &MF = DAG.getMachineFunction(); const PPCSubtarget &Subtarget = MF.getSubtarget(); const PPCFrameLowering *FL = Subtarget.getFrameLowering(); bool isPPC64 = Subtarget.isPPC64(); int SlotSize = isPPC64 ? 8 : 4; int NewRetAddrLoc = SPDiff + FL->getReturnSaveOffset(); int NewRetAddr = MF.getFrameInfo().CreateFixedObject(SlotSize, NewRetAddrLoc, true); EVT VT = isPPC64 ? MVT::i64 : MVT::i32; SDValue NewRetAddrFrIdx = DAG.getFrameIndex(NewRetAddr, VT); Chain = DAG.getStore(Chain, dl, OldRetAddr, NewRetAddrFrIdx, MachinePointerInfo::getFixedStack(MF, NewRetAddr)); } return Chain; } /// CalculateTailCallArgDest - Remember Argument for later processing. Calculate /// the position of the argument. static void CalculateTailCallArgDest(SelectionDAG &DAG, MachineFunction &MF, bool isPPC64, SDValue Arg, int SPDiff, unsigned ArgOffset, SmallVectorImpl& TailCallArguments) { int Offset = ArgOffset + SPDiff; uint32_t OpSize = (Arg.getValueSizeInBits() + 7) / 8; int FI = MF.getFrameInfo().CreateFixedObject(OpSize, Offset, true); EVT VT = isPPC64 ? MVT::i64 : MVT::i32; SDValue FIN = DAG.getFrameIndex(FI, VT); TailCallArgumentInfo Info; Info.Arg = Arg; Info.FrameIdxOp = FIN; Info.FrameIdx = FI; TailCallArguments.push_back(Info); } /// EmitTCFPAndRetAddrLoad - Emit load from frame pointer and return address /// stack slot. Returns the chain as result and the loaded frame pointers in /// LROpOut/FPOpout. Used when tail calling. SDValue PPCTargetLowering::EmitTailCallLoadFPAndRetAddr( SelectionDAG &DAG, int SPDiff, SDValue Chain, SDValue &LROpOut, SDValue &FPOpOut, const SDLoc &dl) const { if (SPDiff) { // Load the LR and FP stack slot for later adjusting. EVT VT = Subtarget.isPPC64() ? MVT::i64 : MVT::i32; LROpOut = getReturnAddrFrameIndex(DAG); LROpOut = DAG.getLoad(VT, dl, Chain, LROpOut, MachinePointerInfo()); Chain = SDValue(LROpOut.getNode(), 1); } return Chain; } /// CreateCopyOfByValArgument - Make a copy of an aggregate at address specified /// by "Src" to address "Dst" of size "Size". Alignment information is /// specified by the specific parameter attribute. The copy will be passed as /// a byval function parameter. /// Sometimes what we are copying is the end of a larger object, the part that /// does not fit in registers. static SDValue CreateCopyOfByValArgument(SDValue Src, SDValue Dst, SDValue Chain, ISD::ArgFlagsTy Flags, SelectionDAG &DAG, const SDLoc &dl) { SDValue SizeNode = DAG.getConstant(Flags.getByValSize(), dl, MVT::i32); return DAG.getMemcpy(Chain, dl, Dst, Src, SizeNode, Flags.getNonZeroByValAlign(), false, false, false, MachinePointerInfo(), MachinePointerInfo()); } /// LowerMemOpCallTo - Store the argument to the stack or remember it in case of /// tail calls. static void LowerMemOpCallTo( SelectionDAG &DAG, MachineFunction &MF, SDValue Chain, SDValue Arg, SDValue PtrOff, int SPDiff, unsigned ArgOffset, bool isPPC64, bool isTailCall, bool isVector, SmallVectorImpl &MemOpChains, SmallVectorImpl &TailCallArguments, const SDLoc &dl) { EVT PtrVT = DAG.getTargetLoweringInfo().getPointerTy(DAG.getDataLayout()); if (!isTailCall) { if (isVector) { SDValue StackPtr; if (isPPC64) StackPtr = DAG.getRegister(PPC::X1, MVT::i64); else StackPtr = DAG.getRegister(PPC::R1, MVT::i32); PtrOff = DAG.getNode(ISD::ADD, dl, PtrVT, StackPtr, DAG.getConstant(ArgOffset, dl, PtrVT)); } MemOpChains.push_back( DAG.getStore(Chain, dl, Arg, PtrOff, MachinePointerInfo())); // Calculate and remember argument location. } else CalculateTailCallArgDest(DAG, MF, isPPC64, Arg, SPDiff, ArgOffset, TailCallArguments); } static void PrepareTailCall(SelectionDAG &DAG, SDValue &InFlag, SDValue &Chain, const SDLoc &dl, int SPDiff, unsigned NumBytes, SDValue LROp, SDValue FPOp, SmallVectorImpl &TailCallArguments) { // Emit a sequence of copyto/copyfrom virtual registers for arguments that // might overwrite each other in case of tail call optimization. SmallVector MemOpChains2; // Do not flag preceding copytoreg stuff together with the following stuff. InFlag = SDValue(); StoreTailCallArgumentsToStackSlot(DAG, Chain, TailCallArguments, MemOpChains2, dl); if (!MemOpChains2.empty()) Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, MemOpChains2); // Store the return address to the appropriate stack slot. Chain = EmitTailCallStoreFPAndRetAddr(DAG, Chain, LROp, FPOp, SPDiff, dl); // Emit callseq_end just before tailcall node. Chain = DAG.getCALLSEQ_END(Chain, DAG.getIntPtrConstant(NumBytes, dl, true), DAG.getIntPtrConstant(0, dl, true), InFlag, dl); InFlag = Chain.getValue(1); } // Is this global address that of a function that can be called by name? (as // opposed to something that must hold a descriptor for an indirect call). static bool isFunctionGlobalAddress(SDValue Callee) { if (GlobalAddressSDNode *G = dyn_cast(Callee)) { if (Callee.getOpcode() == ISD::GlobalTLSAddress || Callee.getOpcode() == ISD::TargetGlobalTLSAddress) return false; return G->getGlobal()->getValueType()->isFunctionTy(); } return false; } SDValue PPCTargetLowering::LowerCallResult( SDValue Chain, SDValue InFlag, CallingConv::ID CallConv, bool isVarArg, const SmallVectorImpl &Ins, const SDLoc &dl, SelectionDAG &DAG, SmallVectorImpl &InVals) const { SmallVector RVLocs; CCState CCRetInfo(CallConv, isVarArg, DAG.getMachineFunction(), RVLocs, *DAG.getContext()); CCRetInfo.AnalyzeCallResult( Ins, (Subtarget.isSVR4ABI() && CallConv == CallingConv::Cold) ? RetCC_PPC_Cold : RetCC_PPC); // Copy all of the result registers out of their specified physreg. for (unsigned i = 0, e = RVLocs.size(); i != e; ++i) { CCValAssign &VA = RVLocs[i]; assert(VA.isRegLoc() && "Can only return in registers!"); SDValue Val; if (Subtarget.hasSPE() && VA.getLocVT() == MVT::f64) { SDValue Lo = DAG.getCopyFromReg(Chain, dl, VA.getLocReg(), MVT::i32, InFlag); Chain = Lo.getValue(1); InFlag = Lo.getValue(2); VA = RVLocs[++i]; // skip ahead to next loc SDValue Hi = DAG.getCopyFromReg(Chain, dl, VA.getLocReg(), MVT::i32, InFlag); Chain = Hi.getValue(1); InFlag = Hi.getValue(2); if (!Subtarget.isLittleEndian()) std::swap (Lo, Hi); Val = DAG.getNode(PPCISD::BUILD_SPE64, dl, MVT::f64, Lo, Hi); } else { Val = DAG.getCopyFromReg(Chain, dl, VA.getLocReg(), VA.getLocVT(), InFlag); Chain = Val.getValue(1); InFlag = Val.getValue(2); } switch (VA.getLocInfo()) { default: llvm_unreachable("Unknown loc info!"); case CCValAssign::Full: break; case CCValAssign::AExt: Val = DAG.getNode(ISD::TRUNCATE, dl, VA.getValVT(), Val); break; case CCValAssign::ZExt: Val = DAG.getNode(ISD::AssertZext, dl, VA.getLocVT(), Val, DAG.getValueType(VA.getValVT())); Val = DAG.getNode(ISD::TRUNCATE, dl, VA.getValVT(), Val); break; case CCValAssign::SExt: Val = DAG.getNode(ISD::AssertSext, dl, VA.getLocVT(), Val, DAG.getValueType(VA.getValVT())); Val = DAG.getNode(ISD::TRUNCATE, dl, VA.getValVT(), Val); break; } InVals.push_back(Val); } return Chain; } static bool isIndirectCall(const SDValue &Callee, SelectionDAG &DAG, const PPCSubtarget &Subtarget, bool isPatchPoint) { // PatchPoint calls are not indirect. if (isPatchPoint) return false; if (isFunctionGlobalAddress(Callee) || dyn_cast(Callee)) return false; // Darwin, and 32-bit ELF can use a BLA. The descriptor based ABIs can not // becuase the immediate function pointer points to a descriptor instead of // a function entry point. The ELFv2 ABI cannot use a BLA because the function // pointer immediate points to the global entry point, while the BLA would // need to jump to the local entry point (see rL211174). if (!Subtarget.usesFunctionDescriptors() && !Subtarget.isELFv2ABI() && isBLACompatibleAddress(Callee, DAG)) return false; return true; } static unsigned getCallOpcode(PPCTargetLowering::CallFlags CFlags, const Function &Caller, const SDValue &Callee, const PPCSubtarget &Subtarget, const TargetMachine &TM) { if (CFlags.IsTailCall) return PPCISD::TC_RETURN; // This is a call through a function pointer. if (CFlags.IsIndirect) { // AIX and the 64-bit ELF ABIs need to maintain the TOC pointer accross // indirect calls. The save of the caller's TOC pointer to the stack will be // inserted into the DAG as part of call lowering. The restore of the TOC // pointer is modeled by using a pseudo instruction for the call opcode that // represents the 2 instruction sequence of an indirect branch and link, // immediately followed by a load of the TOC pointer from the the stack save // slot into gpr2. if (Subtarget.isAIXABI() || Subtarget.is64BitELFABI()) return PPCISD::BCTRL_LOAD_TOC; // An indirect call that does not need a TOC restore. return PPCISD::BCTRL; } + // FIXME: At this moment indirect calls are treated ahead of the + // PC Relative condition because binaries can still contain a possible + // mix of functions that use a TOC and functions that do not use a TOC. + // Once the PC Relative feature is complete this condition should be moved + // up ahead of the indirect calls and should return a PPCISD::BCTRL for + // that case. + if (Subtarget.isUsingPCRelativeCalls()) { + assert(Subtarget.is64BitELFABI() && "PC Relative is only on ELF ABI."); + return PPCISD::CALL_NOTOC; + } + // The ABIs that maintain a TOC pointer accross calls need to have a nop // immediately following the call instruction if the caller and callee may // have different TOC bases. At link time if the linker determines the calls // may not share a TOC base, the call is redirected to a trampoline inserted // by the linker. The trampoline will (among other things) save the callers // TOC pointer at an ABI designated offset in the linkage area and the linker // will rewrite the nop to be a load of the TOC pointer from the linkage area // into gpr2. if (Subtarget.isAIXABI() || Subtarget.is64BitELFABI()) - return callsShareTOCBase(&Caller, Callee, TM) ? PPCISD::CALL - : PPCISD::CALL_NOP; + return callsShareTOCBase(&Caller, Callee, TM) ? PPCISD::CALL + : PPCISD::CALL_NOP; return PPCISD::CALL; } static SDValue transformCallee(const SDValue &Callee, SelectionDAG &DAG, const SDLoc &dl, const PPCSubtarget &Subtarget) { if (!Subtarget.usesFunctionDescriptors() && !Subtarget.isELFv2ABI()) if (SDNode *Dest = isBLACompatibleAddress(Callee, DAG)) return SDValue(Dest, 0); // Returns true if the callee is local, and false otherwise. auto isLocalCallee = [&]() { const GlobalAddressSDNode *G = dyn_cast(Callee); const Module *Mod = DAG.getMachineFunction().getFunction().getParent(); const GlobalValue *GV = G ? G->getGlobal() : nullptr; return DAG.getTarget().shouldAssumeDSOLocal(*Mod, GV) && !dyn_cast_or_null(GV); }; // The PLT is only used in 32-bit ELF PIC mode. Attempting to use the PLT in // a static relocation model causes some versions of GNU LD (2.17.50, at // least) to force BSS-PLT, instead of secure-PLT, even if all objects are // built with secure-PLT. bool UsePlt = Subtarget.is32BitELFABI() && !isLocalCallee() && Subtarget.getTargetMachine().getRelocationModel() == Reloc::PIC_; // On AIX, direct function calls reference the symbol for the function's // entry point, which is named by prepending a "." before the function's // C-linkage name. const auto getAIXFuncEntryPointSymbolSDNode = [&](StringRef FuncName, bool IsDeclaration, const XCOFF::StorageClass &SC) { auto &Context = DAG.getMachineFunction().getMMI().getContext(); MCSymbolXCOFF *S = cast( Context.getOrCreateSymbol(Twine(".") + Twine(FuncName))); if (IsDeclaration && !S->hasRepresentedCsectSet()) { // On AIX, an undefined symbol needs to be associated with a // MCSectionXCOFF to get the correct storage mapping class. // In this case, XCOFF::XMC_PR. MCSectionXCOFF *Sec = Context.getXCOFFSection( S->getName(), XCOFF::XMC_PR, XCOFF::XTY_ER, SC, SectionKind::getMetadata()); S->setRepresentedCsect(Sec); } MVT PtrVT = DAG.getTargetLoweringInfo().getPointerTy(DAG.getDataLayout()); return DAG.getMCSymbol(S, PtrVT); }; if (isFunctionGlobalAddress(Callee)) { const GlobalAddressSDNode *G = cast(Callee); const GlobalValue *GV = G->getGlobal(); if (!Subtarget.isAIXABI()) return DAG.getTargetGlobalAddress(GV, dl, Callee.getValueType(), 0, UsePlt ? PPCII::MO_PLT : 0); assert(!isa(GV) && "IFunc is not supported on AIX."); const GlobalObject *GO = cast(GV); const XCOFF::StorageClass SC = TargetLoweringObjectFileXCOFF::getStorageClassForGlobal(GO); return getAIXFuncEntryPointSymbolSDNode(GO->getName(), GO->isDeclaration(), SC); } if (ExternalSymbolSDNode *S = dyn_cast(Callee)) { const char *SymName = S->getSymbol(); if (!Subtarget.isAIXABI()) return DAG.getTargetExternalSymbol(SymName, Callee.getValueType(), UsePlt ? PPCII::MO_PLT : 0); // If there exists a user-declared function whose name is the same as the // ExternalSymbol's, then we pick up the user-declared version. const Module *Mod = DAG.getMachineFunction().getFunction().getParent(); if (const Function *F = dyn_cast_or_null(Mod->getNamedValue(SymName))) { const XCOFF::StorageClass SC = TargetLoweringObjectFileXCOFF::getStorageClassForGlobal(F); return getAIXFuncEntryPointSymbolSDNode(F->getName(), F->isDeclaration(), SC); } return getAIXFuncEntryPointSymbolSDNode(SymName, true, XCOFF::C_EXT); } // No transformation needed. assert(Callee.getNode() && "What no callee?"); return Callee; } static SDValue getOutputChainFromCallSeq(SDValue CallSeqStart) { assert(CallSeqStart.getOpcode() == ISD::CALLSEQ_START && "Expected a CALLSEQ_STARTSDNode."); // The last operand is the chain, except when the node has glue. If the node // has glue, then the last operand is the glue, and the chain is the second // last operand. SDValue LastValue = CallSeqStart.getValue(CallSeqStart->getNumValues() - 1); if (LastValue.getValueType() != MVT::Glue) return LastValue; return CallSeqStart.getValue(CallSeqStart->getNumValues() - 2); } // Creates the node that moves a functions address into the count register // to prepare for an indirect call instruction. static void prepareIndirectCall(SelectionDAG &DAG, SDValue &Callee, SDValue &Glue, SDValue &Chain, const SDLoc &dl) { SDValue MTCTROps[] = {Chain, Callee, Glue}; EVT ReturnTypes[] = {MVT::Other, MVT::Glue}; Chain = DAG.getNode(PPCISD::MTCTR, dl, makeArrayRef(ReturnTypes, 2), makeArrayRef(MTCTROps, Glue.getNode() ? 3 : 2)); // The glue is the second value produced. Glue = Chain.getValue(1); } static void prepareDescriptorIndirectCall(SelectionDAG &DAG, SDValue &Callee, SDValue &Glue, SDValue &Chain, SDValue CallSeqStart, ImmutableCallSite CS, const SDLoc &dl, bool hasNest, const PPCSubtarget &Subtarget) { // Function pointers in the 64-bit SVR4 ABI do not point to the function // entry point, but to the function descriptor (the function entry point // address is part of the function descriptor though). // The function descriptor is a three doubleword structure with the // following fields: function entry point, TOC base address and // environment pointer. // Thus for a call through a function pointer, the following actions need // to be performed: // 1. Save the TOC of the caller in the TOC save area of its stack // frame (this is done in LowerCall_Darwin() or LowerCall_64SVR4()). // 2. Load the address of the function entry point from the function // descriptor. // 3. Load the TOC of the callee from the function descriptor into r2. // 4. Load the environment pointer from the function descriptor into // r11. // 5. Branch to the function entry point address. // 6. On return of the callee, the TOC of the caller needs to be // restored (this is done in FinishCall()). // // The loads are scheduled at the beginning of the call sequence, and the // register copies are flagged together to ensure that no other // operations can be scheduled in between. E.g. without flagging the // copies together, a TOC access in the caller could be scheduled between // the assignment of the callee TOC and the branch to the callee, which leads // to incorrect code. // Start by loading the function address from the descriptor. SDValue LDChain = getOutputChainFromCallSeq(CallSeqStart); auto MMOFlags = Subtarget.hasInvariantFunctionDescriptors() ? (MachineMemOperand::MODereferenceable | MachineMemOperand::MOInvariant) : MachineMemOperand::MONone; MachinePointerInfo MPI(CS ? CS.getCalledValue() : nullptr); // Registers used in building the DAG. const MCRegister EnvPtrReg = Subtarget.getEnvironmentPointerRegister(); const MCRegister TOCReg = Subtarget.getTOCPointerRegister(); // Offsets of descriptor members. const unsigned TOCAnchorOffset = Subtarget.descriptorTOCAnchorOffset(); const unsigned EnvPtrOffset = Subtarget.descriptorEnvironmentPointerOffset(); const MVT RegVT = Subtarget.isPPC64() ? MVT::i64 : MVT::i32; const unsigned Alignment = Subtarget.isPPC64() ? 8 : 4; // One load for the functions entry point address. SDValue LoadFuncPtr = DAG.getLoad(RegVT, dl, LDChain, Callee, MPI, Alignment, MMOFlags); // One for loading the TOC anchor for the module that contains the called // function. SDValue TOCOff = DAG.getIntPtrConstant(TOCAnchorOffset, dl); SDValue AddTOC = DAG.getNode(ISD::ADD, dl, RegVT, Callee, TOCOff); SDValue TOCPtr = DAG.getLoad(RegVT, dl, LDChain, AddTOC, MPI.getWithOffset(TOCAnchorOffset), Alignment, MMOFlags); // One for loading the environment pointer. SDValue PtrOff = DAG.getIntPtrConstant(EnvPtrOffset, dl); SDValue AddPtr = DAG.getNode(ISD::ADD, dl, RegVT, Callee, PtrOff); SDValue LoadEnvPtr = DAG.getLoad(RegVT, dl, LDChain, AddPtr, MPI.getWithOffset(EnvPtrOffset), Alignment, MMOFlags); // Then copy the newly loaded TOC anchor to the TOC pointer. SDValue TOCVal = DAG.getCopyToReg(Chain, dl, TOCReg, TOCPtr, Glue); Chain = TOCVal.getValue(0); Glue = TOCVal.getValue(1); // If the function call has an explicit 'nest' parameter, it takes the // place of the environment pointer. assert((!hasNest || !Subtarget.isAIXABI()) && "Nest parameter is not supported on AIX."); if (!hasNest) { SDValue EnvVal = DAG.getCopyToReg(Chain, dl, EnvPtrReg, LoadEnvPtr, Glue); Chain = EnvVal.getValue(0); Glue = EnvVal.getValue(1); } // The rest of the indirect call sequence is the same as the non-descriptor // DAG. prepareIndirectCall(DAG, LoadFuncPtr, Glue, Chain, dl); } static void buildCallOperands(SmallVectorImpl &Ops, PPCTargetLowering::CallFlags CFlags, const SDLoc &dl, SelectionDAG &DAG, SmallVector, 8> &RegsToPass, SDValue Glue, SDValue Chain, SDValue &Callee, int SPDiff, const PPCSubtarget &Subtarget) { const bool IsPPC64 = Subtarget.isPPC64(); // MVT for a general purpose register. const MVT RegVT = IsPPC64 ? MVT::i64 : MVT::i32; // First operand is always the chain. Ops.push_back(Chain); // If it's a direct call pass the callee as the second operand. if (!CFlags.IsIndirect) Ops.push_back(Callee); else { assert(!CFlags.IsPatchPoint && "Patch point calls are not indirect."); // For the TOC based ABIs, we have saved the TOC pointer to the linkage area // on the stack (this would have been done in `LowerCall_64SVR4` or // `LowerCall_AIX`). The call instruction is a pseudo instruction that // represents both the indirect branch and a load that restores the TOC // pointer from the linkage area. The operand for the TOC restore is an add // of the TOC save offset to the stack pointer. This must be the second // operand: after the chain input but before any other variadic arguments. if (Subtarget.is64BitELFABI() || Subtarget.isAIXABI()) { const MCRegister StackPtrReg = Subtarget.getStackPointerRegister(); SDValue StackPtr = DAG.getRegister(StackPtrReg, RegVT); unsigned TOCSaveOffset = Subtarget.getFrameLowering()->getTOCSaveOffset(); SDValue TOCOff = DAG.getIntPtrConstant(TOCSaveOffset, dl); SDValue AddTOC = DAG.getNode(ISD::ADD, dl, RegVT, StackPtr, TOCOff); Ops.push_back(AddTOC); } // Add the register used for the environment pointer. if (Subtarget.usesFunctionDescriptors() && !CFlags.HasNest) Ops.push_back(DAG.getRegister(Subtarget.getEnvironmentPointerRegister(), RegVT)); // Add CTR register as callee so a bctr can be emitted later. if (CFlags.IsTailCall) Ops.push_back(DAG.getRegister(IsPPC64 ? PPC::CTR8 : PPC::CTR, RegVT)); } // If this is a tail call add stack pointer delta. if (CFlags.IsTailCall) Ops.push_back(DAG.getConstant(SPDiff, dl, MVT::i32)); // Add argument registers to the end of the list so that they are known live // into the call. for (unsigned i = 0, e = RegsToPass.size(); i != e; ++i) Ops.push_back(DAG.getRegister(RegsToPass[i].first, RegsToPass[i].second.getValueType())); // We cannot add R2/X2 as an operand here for PATCHPOINT, because there is // no way to mark dependencies as implicit here. // We will add the R2/X2 dependency in EmitInstrWithCustomInserter. if ((Subtarget.is64BitELFABI() || Subtarget.isAIXABI()) && - !CFlags.IsPatchPoint) + !CFlags.IsPatchPoint && !Subtarget.isUsingPCRelativeCalls()) Ops.push_back(DAG.getRegister(Subtarget.getTOCPointerRegister(), RegVT)); // Add implicit use of CR bit 6 for 32-bit SVR4 vararg calls if (CFlags.IsVarArg && Subtarget.is32BitELFABI()) Ops.push_back(DAG.getRegister(PPC::CR1EQ, MVT::i32)); // Add a register mask operand representing the call-preserved registers. const TargetRegisterInfo *TRI = Subtarget.getRegisterInfo(); const uint32_t *Mask = TRI->getCallPreservedMask(DAG.getMachineFunction(), CFlags.CallConv); assert(Mask && "Missing call preserved mask for calling convention"); Ops.push_back(DAG.getRegisterMask(Mask)); // If the glue is valid, it is the last operand. if (Glue.getNode()) Ops.push_back(Glue); } SDValue PPCTargetLowering::FinishCall( CallFlags CFlags, const SDLoc &dl, SelectionDAG &DAG, SmallVector, 8> &RegsToPass, SDValue Glue, SDValue Chain, SDValue CallSeqStart, SDValue &Callee, int SPDiff, unsigned NumBytes, const SmallVectorImpl &Ins, SmallVectorImpl &InVals, ImmutableCallSite CS) const { - if (Subtarget.is64BitELFABI() || Subtarget.isAIXABI()) + if ((Subtarget.is64BitELFABI() && !Subtarget.isUsingPCRelativeCalls()) || + Subtarget.isAIXABI()) setUsesTOCBasePtr(DAG); unsigned CallOpc = getCallOpcode(CFlags, DAG.getMachineFunction().getFunction(), Callee, Subtarget, DAG.getTarget()); if (!CFlags.IsIndirect) Callee = transformCallee(Callee, DAG, dl, Subtarget); else if (Subtarget.usesFunctionDescriptors()) prepareDescriptorIndirectCall(DAG, Callee, Glue, Chain, CallSeqStart, CS, dl, CFlags.HasNest, Subtarget); else prepareIndirectCall(DAG, Callee, Glue, Chain, dl); // Build the operand list for the call instruction. SmallVector Ops; buildCallOperands(Ops, CFlags, dl, DAG, RegsToPass, Glue, Chain, Callee, SPDiff, Subtarget); // Emit tail call. if (CFlags.IsTailCall) { assert(((Callee.getOpcode() == ISD::Register && cast(Callee)->getReg() == PPC::CTR) || Callee.getOpcode() == ISD::TargetExternalSymbol || Callee.getOpcode() == ISD::TargetGlobalAddress || isa(Callee)) && "Expecting a global address, external symbol, absolute value or " "register"); assert(CallOpc == PPCISD::TC_RETURN && "Unexpected call opcode for a tail call."); DAG.getMachineFunction().getFrameInfo().setHasTailCall(); return DAG.getNode(CallOpc, dl, MVT::Other, Ops); } std::array ReturnTypes = {{MVT::Other, MVT::Glue}}; Chain = DAG.getNode(CallOpc, dl, ReturnTypes, Ops); Glue = Chain.getValue(1); // When performing tail call optimization the callee pops its arguments off // the stack. Account for this here so these bytes can be pushed back on in // PPCFrameLowering::eliminateCallFramePseudoInstr. int BytesCalleePops = (CFlags.CallConv == CallingConv::Fast && getTargetMachine().Options.GuaranteedTailCallOpt) ? NumBytes : 0; Chain = DAG.getCALLSEQ_END(Chain, DAG.getIntPtrConstant(NumBytes, dl, true), DAG.getIntPtrConstant(BytesCalleePops, dl, true), Glue, dl); Glue = Chain.getValue(1); return LowerCallResult(Chain, Glue, CFlags.CallConv, CFlags.IsVarArg, Ins, dl, DAG, InVals); } SDValue PPCTargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI, SmallVectorImpl &InVals) const { SelectionDAG &DAG = CLI.DAG; SDLoc &dl = CLI.DL; SmallVectorImpl &Outs = CLI.Outs; SmallVectorImpl &OutVals = CLI.OutVals; SmallVectorImpl &Ins = CLI.Ins; SDValue Chain = CLI.Chain; SDValue Callee = CLI.Callee; bool &isTailCall = CLI.IsTailCall; CallingConv::ID CallConv = CLI.CallConv; bool isVarArg = CLI.IsVarArg; bool isPatchPoint = CLI.IsPatchPoint; ImmutableCallSite CS = CLI.CS; if (isTailCall) { if (Subtarget.useLongCalls() && !(CS && CS.isMustTailCall())) isTailCall = false; else if (Subtarget.isSVR4ABI() && Subtarget.isPPC64()) isTailCall = IsEligibleForTailCallOptimization_64SVR4(Callee, CallConv, CS, isVarArg, Outs, Ins, DAG); else isTailCall = IsEligibleForTailCallOptimization(Callee, CallConv, isVarArg, Ins, DAG); if (isTailCall) { ++NumTailCalls; if (!getTargetMachine().Options.GuaranteedTailCallOpt) ++NumSiblingCalls; assert(isa(Callee) && "Callee should be an llvm::Function object."); LLVM_DEBUG( const GlobalValue *GV = cast(Callee)->getGlobal(); const unsigned Width = 80 - strlen("TCO caller: ") - strlen(", callee linkage: 0, 0"); dbgs() << "TCO caller: " << left_justify(DAG.getMachineFunction().getName(), Width) << ", callee linkage: " << GV->getVisibility() << ", " << GV->getLinkage() << "\n"); } } if (!isTailCall && CS && CS.isMustTailCall()) report_fatal_error("failed to perform tail call elimination on a call " "site marked musttail"); // When long calls (i.e. indirect calls) are always used, calls are always // made via function pointer. If we have a function name, first translate it // into a pointer. if (Subtarget.useLongCalls() && isa(Callee) && !isTailCall) Callee = LowerGlobalAddress(Callee, DAG); CallFlags CFlags( CallConv, isTailCall, isVarArg, isPatchPoint, isIndirectCall(Callee, DAG, Subtarget, isPatchPoint), // hasNest Subtarget.is64BitELFABI() && any_of(Outs, [](ISD::OutputArg Arg) { return Arg.Flags.isNest(); })); if (Subtarget.isSVR4ABI() && Subtarget.isPPC64()) return LowerCall_64SVR4(Chain, Callee, CFlags, Outs, OutVals, Ins, dl, DAG, InVals, CS); if (Subtarget.isSVR4ABI()) return LowerCall_32SVR4(Chain, Callee, CFlags, Outs, OutVals, Ins, dl, DAG, InVals, CS); if (Subtarget.isAIXABI()) return LowerCall_AIX(Chain, Callee, CFlags, Outs, OutVals, Ins, dl, DAG, InVals, CS); return LowerCall_Darwin(Chain, Callee, CFlags, Outs, OutVals, Ins, dl, DAG, InVals, CS); } SDValue PPCTargetLowering::LowerCall_32SVR4( SDValue Chain, SDValue Callee, CallFlags CFlags, const SmallVectorImpl &Outs, const SmallVectorImpl &OutVals, const SmallVectorImpl &Ins, const SDLoc &dl, SelectionDAG &DAG, SmallVectorImpl &InVals, ImmutableCallSite CS) const { // See PPCTargetLowering::LowerFormalArguments_32SVR4() for a description // of the 32-bit SVR4 ABI stack frame layout. const CallingConv::ID CallConv = CFlags.CallConv; const bool IsVarArg = CFlags.IsVarArg; const bool IsTailCall = CFlags.IsTailCall; assert((CallConv == CallingConv::C || CallConv == CallingConv::Cold || CallConv == CallingConv::Fast) && "Unknown calling convention!"); unsigned PtrByteSize = 4; MachineFunction &MF = DAG.getMachineFunction(); // Mark this function as potentially containing a function that contains a // tail call. As a consequence the frame pointer will be used for dynamicalloc // and restoring the callers stack pointer in this functions epilog. This is // done because by tail calling the called function might overwrite the value // in this function's (MF) stack pointer stack slot 0(SP). if (getTargetMachine().Options.GuaranteedTailCallOpt && CallConv == CallingConv::Fast) MF.getInfo()->setHasFastCall(); // Count how many bytes are to be pushed on the stack, including the linkage // area, parameter list area and the part of the local variable space which // contains copies of aggregates which are passed by value. // Assign locations to all of the outgoing arguments. SmallVector ArgLocs; PPCCCState CCInfo(CallConv, IsVarArg, MF, ArgLocs, *DAG.getContext()); // Reserve space for the linkage area on the stack. CCInfo.AllocateStack(Subtarget.getFrameLowering()->getLinkageSize(), PtrByteSize); if (useSoftFloat()) CCInfo.PreAnalyzeCallOperands(Outs); if (IsVarArg) { // Handle fixed and variable vector arguments differently. // Fixed vector arguments go into registers as long as registers are // available. Variable vector arguments always go into memory. unsigned NumArgs = Outs.size(); for (unsigned i = 0; i != NumArgs; ++i) { MVT ArgVT = Outs[i].VT; ISD::ArgFlagsTy ArgFlags = Outs[i].Flags; bool Result; if (Outs[i].IsFixed) { Result = CC_PPC32_SVR4(i, ArgVT, ArgVT, CCValAssign::Full, ArgFlags, CCInfo); } else { Result = CC_PPC32_SVR4_VarArg(i, ArgVT, ArgVT, CCValAssign::Full, ArgFlags, CCInfo); } if (Result) { #ifndef NDEBUG errs() << "Call operand #" << i << " has unhandled type " << EVT(ArgVT).getEVTString() << "\n"; #endif llvm_unreachable(nullptr); } } } else { // All arguments are treated the same. CCInfo.AnalyzeCallOperands(Outs, CC_PPC32_SVR4); } CCInfo.clearWasPPCF128(); // Assign locations to all of the outgoing aggregate by value arguments. SmallVector ByValArgLocs; CCState CCByValInfo(CallConv, IsVarArg, MF, ByValArgLocs, *DAG.getContext()); // Reserve stack space for the allocations in CCInfo. CCByValInfo.AllocateStack(CCInfo.getNextStackOffset(), PtrByteSize); CCByValInfo.AnalyzeCallOperands(Outs, CC_PPC32_SVR4_ByVal); // Size of the linkage area, parameter list area and the part of the local // space variable where copies of aggregates which are passed by value are // stored. unsigned NumBytes = CCByValInfo.getNextStackOffset(); // Calculate by how many bytes the stack has to be adjusted in case of tail // call optimization. int SPDiff = CalculateTailCallSPDiff(DAG, IsTailCall, NumBytes); // Adjust the stack pointer for the new arguments... // These operations are automatically eliminated by the prolog/epilog pass Chain = DAG.getCALLSEQ_START(Chain, NumBytes, 0, dl); SDValue CallSeqStart = Chain; // Load the return address and frame pointer so it can be moved somewhere else // later. SDValue LROp, FPOp; Chain = EmitTailCallLoadFPAndRetAddr(DAG, SPDiff, Chain, LROp, FPOp, dl); // Set up a copy of the stack pointer for use loading and storing any // arguments that may not fit in the registers available for argument // passing. SDValue StackPtr = DAG.getRegister(PPC::R1, MVT::i32); SmallVector, 8> RegsToPass; SmallVector TailCallArguments; SmallVector MemOpChains; bool seenFloatArg = false; // Walk the register/memloc assignments, inserting copies/loads. // i - Tracks the index into the list of registers allocated for the call // RealArgIdx - Tracks the index into the list of actual function arguments // j - Tracks the index into the list of byval arguments for (unsigned i = 0, RealArgIdx = 0, j = 0, e = ArgLocs.size(); i != e; ++i, ++RealArgIdx) { CCValAssign &VA = ArgLocs[i]; SDValue Arg = OutVals[RealArgIdx]; ISD::ArgFlagsTy Flags = Outs[RealArgIdx].Flags; if (Flags.isByVal()) { // Argument is an aggregate which is passed by value, thus we need to // create a copy of it in the local variable space of the current stack // frame (which is the stack frame of the caller) and pass the address of // this copy to the callee. assert((j < ByValArgLocs.size()) && "Index out of bounds!"); CCValAssign &ByValVA = ByValArgLocs[j++]; assert((VA.getValNo() == ByValVA.getValNo()) && "ValNo mismatch!"); // Memory reserved in the local variable space of the callers stack frame. unsigned LocMemOffset = ByValVA.getLocMemOffset(); SDValue PtrOff = DAG.getIntPtrConstant(LocMemOffset, dl); PtrOff = DAG.getNode(ISD::ADD, dl, getPointerTy(MF.getDataLayout()), StackPtr, PtrOff); // Create a copy of the argument in the local area of the current // stack frame. SDValue MemcpyCall = CreateCopyOfByValArgument(Arg, PtrOff, CallSeqStart.getNode()->getOperand(0), Flags, DAG, dl); // This must go outside the CALLSEQ_START..END. SDValue NewCallSeqStart = DAG.getCALLSEQ_START(MemcpyCall, NumBytes, 0, SDLoc(MemcpyCall)); DAG.ReplaceAllUsesWith(CallSeqStart.getNode(), NewCallSeqStart.getNode()); Chain = CallSeqStart = NewCallSeqStart; // Pass the address of the aggregate copy on the stack either in a // physical register or in the parameter list area of the current stack // frame to the callee. Arg = PtrOff; } // When useCRBits() is true, there can be i1 arguments. // It is because getRegisterType(MVT::i1) => MVT::i1, // and for other integer types getRegisterType() => MVT::i32. // Extend i1 and ensure callee will get i32. if (Arg.getValueType() == MVT::i1) Arg = DAG.getNode(Flags.isSExt() ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND, dl, MVT::i32, Arg); if (VA.isRegLoc()) { seenFloatArg |= VA.getLocVT().isFloatingPoint(); // Put argument in a physical register. if (Subtarget.hasSPE() && Arg.getValueType() == MVT::f64) { bool IsLE = Subtarget.isLittleEndian(); SDValue SVal = DAG.getNode(PPCISD::EXTRACT_SPE, dl, MVT::i32, Arg, DAG.getIntPtrConstant(IsLE ? 0 : 1, dl)); RegsToPass.push_back(std::make_pair(VA.getLocReg(), SVal.getValue(0))); SVal = DAG.getNode(PPCISD::EXTRACT_SPE, dl, MVT::i32, Arg, DAG.getIntPtrConstant(IsLE ? 1 : 0, dl)); RegsToPass.push_back(std::make_pair(ArgLocs[++i].getLocReg(), SVal.getValue(0))); } else RegsToPass.push_back(std::make_pair(VA.getLocReg(), Arg)); } else { // Put argument in the parameter list area of the current stack frame. assert(VA.isMemLoc()); unsigned LocMemOffset = VA.getLocMemOffset(); if (!IsTailCall) { SDValue PtrOff = DAG.getIntPtrConstant(LocMemOffset, dl); PtrOff = DAG.getNode(ISD::ADD, dl, getPointerTy(MF.getDataLayout()), StackPtr, PtrOff); MemOpChains.push_back( DAG.getStore(Chain, dl, Arg, PtrOff, MachinePointerInfo())); } else { // Calculate and remember argument location. CalculateTailCallArgDest(DAG, MF, false, Arg, SPDiff, LocMemOffset, TailCallArguments); } } } if (!MemOpChains.empty()) Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, MemOpChains); // Build a sequence of copy-to-reg nodes chained together with token chain // and flag operands which copy the outgoing args into the appropriate regs. SDValue InFlag; for (unsigned i = 0, e = RegsToPass.size(); i != e; ++i) { Chain = DAG.getCopyToReg(Chain, dl, RegsToPass[i].first, RegsToPass[i].second, InFlag); InFlag = Chain.getValue(1); } // Set CR bit 6 to true if this is a vararg call with floating args passed in // registers. if (IsVarArg) { SDVTList VTs = DAG.getVTList(MVT::Other, MVT::Glue); SDValue Ops[] = { Chain, InFlag }; Chain = DAG.getNode(seenFloatArg ? PPCISD::CR6SET : PPCISD::CR6UNSET, dl, VTs, makeArrayRef(Ops, InFlag.getNode() ? 2 : 1)); InFlag = Chain.getValue(1); } if (IsTailCall) PrepareTailCall(DAG, InFlag, Chain, dl, SPDiff, NumBytes, LROp, FPOp, TailCallArguments); return FinishCall(CFlags, dl, DAG, RegsToPass, InFlag, Chain, CallSeqStart, Callee, SPDiff, NumBytes, Ins, InVals, CS); } // Copy an argument into memory, being careful to do this outside the // call sequence for the call to which the argument belongs. SDValue PPCTargetLowering::createMemcpyOutsideCallSeq( SDValue Arg, SDValue PtrOff, SDValue CallSeqStart, ISD::ArgFlagsTy Flags, SelectionDAG &DAG, const SDLoc &dl) const { SDValue MemcpyCall = CreateCopyOfByValArgument(Arg, PtrOff, CallSeqStart.getNode()->getOperand(0), Flags, DAG, dl); // The MEMCPY must go outside the CALLSEQ_START..END. int64_t FrameSize = CallSeqStart.getConstantOperandVal(1); SDValue NewCallSeqStart = DAG.getCALLSEQ_START(MemcpyCall, FrameSize, 0, SDLoc(MemcpyCall)); DAG.ReplaceAllUsesWith(CallSeqStart.getNode(), NewCallSeqStart.getNode()); return NewCallSeqStart; } SDValue PPCTargetLowering::LowerCall_64SVR4( SDValue Chain, SDValue Callee, CallFlags CFlags, const SmallVectorImpl &Outs, const SmallVectorImpl &OutVals, const SmallVectorImpl &Ins, const SDLoc &dl, SelectionDAG &DAG, SmallVectorImpl &InVals, ImmutableCallSite CS) const { bool isELFv2ABI = Subtarget.isELFv2ABI(); bool isLittleEndian = Subtarget.isLittleEndian(); unsigned NumOps = Outs.size(); bool IsSibCall = false; bool IsFastCall = CFlags.CallConv == CallingConv::Fast; EVT PtrVT = getPointerTy(DAG.getDataLayout()); unsigned PtrByteSize = 8; MachineFunction &MF = DAG.getMachineFunction(); if (CFlags.IsTailCall && !getTargetMachine().Options.GuaranteedTailCallOpt) IsSibCall = true; // Mark this function as potentially containing a function that contains a // tail call. As a consequence the frame pointer will be used for dynamicalloc // and restoring the callers stack pointer in this functions epilog. This is // done because by tail calling the called function might overwrite the value // in this function's (MF) stack pointer stack slot 0(SP). if (getTargetMachine().Options.GuaranteedTailCallOpt && IsFastCall) MF.getInfo()->setHasFastCall(); assert(!(IsFastCall && CFlags.IsVarArg) && "fastcc not supported on varargs functions"); // Count how many bytes are to be pushed on the stack, including the linkage // area, and parameter passing area. On ELFv1, the linkage area is 48 bytes // reserved space for [SP][CR][LR][2 x unused][TOC]; on ELFv2, the linkage // area is 32 bytes reserved space for [SP][CR][LR][TOC]. unsigned LinkageSize = Subtarget.getFrameLowering()->getLinkageSize(); unsigned NumBytes = LinkageSize; unsigned GPR_idx = 0, FPR_idx = 0, VR_idx = 0; unsigned &QFPR_idx = FPR_idx; static const MCPhysReg GPR[] = { PPC::X3, PPC::X4, PPC::X5, PPC::X6, PPC::X7, PPC::X8, PPC::X9, PPC::X10, }; static const MCPhysReg VR[] = { PPC::V2, PPC::V3, PPC::V4, PPC::V5, PPC::V6, PPC::V7, PPC::V8, PPC::V9, PPC::V10, PPC::V11, PPC::V12, PPC::V13 }; const unsigned NumGPRs = array_lengthof(GPR); const unsigned NumFPRs = useSoftFloat() ? 0 : 13; const unsigned NumVRs = array_lengthof(VR); const unsigned NumQFPRs = NumFPRs; // On ELFv2, we can avoid allocating the parameter area if all the arguments // can be passed to the callee in registers. // For the fast calling convention, there is another check below. // Note: We should keep consistent with LowerFormalArguments_64SVR4() bool HasParameterArea = !isELFv2ABI || CFlags.IsVarArg || IsFastCall; if (!HasParameterArea) { unsigned ParamAreaSize = NumGPRs * PtrByteSize; unsigned AvailableFPRs = NumFPRs; unsigned AvailableVRs = NumVRs; unsigned NumBytesTmp = NumBytes; for (unsigned i = 0; i != NumOps; ++i) { if (Outs[i].Flags.isNest()) continue; if (CalculateStackSlotUsed(Outs[i].VT, Outs[i].ArgVT, Outs[i].Flags, PtrByteSize, LinkageSize, ParamAreaSize, NumBytesTmp, AvailableFPRs, AvailableVRs, Subtarget.hasQPX())) HasParameterArea = true; } } // When using the fast calling convention, we don't provide backing for // arguments that will be in registers. unsigned NumGPRsUsed = 0, NumFPRsUsed = 0, NumVRsUsed = 0; // Avoid allocating parameter area for fastcc functions if all the arguments // can be passed in the registers. if (IsFastCall) HasParameterArea = false; // Add up all the space actually used. for (unsigned i = 0; i != NumOps; ++i) { ISD::ArgFlagsTy Flags = Outs[i].Flags; EVT ArgVT = Outs[i].VT; EVT OrigVT = Outs[i].ArgVT; if (Flags.isNest()) continue; if (IsFastCall) { if (Flags.isByVal()) { NumGPRsUsed += (Flags.getByValSize()+7)/8; if (NumGPRsUsed > NumGPRs) HasParameterArea = true; } else { switch (ArgVT.getSimpleVT().SimpleTy) { default: llvm_unreachable("Unexpected ValueType for argument!"); case MVT::i1: case MVT::i32: case MVT::i64: if (++NumGPRsUsed <= NumGPRs) continue; break; case MVT::v4i32: case MVT::v8i16: case MVT::v16i8: case MVT::v2f64: case MVT::v2i64: case MVT::v1i128: case MVT::f128: if (++NumVRsUsed <= NumVRs) continue; break; case MVT::v4f32: // When using QPX, this is handled like a FP register, otherwise, it // is an Altivec register. if (Subtarget.hasQPX()) { if (++NumFPRsUsed <= NumFPRs) continue; } else { if (++NumVRsUsed <= NumVRs) continue; } break; case MVT::f32: case MVT::f64: case MVT::v4f64: // QPX case MVT::v4i1: // QPX if (++NumFPRsUsed <= NumFPRs) continue; break; } HasParameterArea = true; } } /* Respect alignment of argument on the stack. */ auto Alignement = CalculateStackSlotAlignment(ArgVT, OrigVT, Flags, PtrByteSize); NumBytes = alignTo(NumBytes, Alignement); NumBytes += CalculateStackSlotSize(ArgVT, Flags, PtrByteSize); if (Flags.isInConsecutiveRegsLast()) NumBytes = ((NumBytes + PtrByteSize - 1)/PtrByteSize) * PtrByteSize; } unsigned NumBytesActuallyUsed = NumBytes; // In the old ELFv1 ABI, // the prolog code of the callee may store up to 8 GPR argument registers to // the stack, allowing va_start to index over them in memory if its varargs. // Because we cannot tell if this is needed on the caller side, we have to // conservatively assume that it is needed. As such, make sure we have at // least enough stack space for the caller to store the 8 GPRs. // In the ELFv2 ABI, we allocate the parameter area iff a callee // really requires memory operands, e.g. a vararg function. if (HasParameterArea) NumBytes = std::max(NumBytes, LinkageSize + 8 * PtrByteSize); else NumBytes = LinkageSize; // Tail call needs the stack to be aligned. if (getTargetMachine().Options.GuaranteedTailCallOpt && IsFastCall) NumBytes = EnsureStackAlignment(Subtarget.getFrameLowering(), NumBytes); int SPDiff = 0; // Calculate by how many bytes the stack has to be adjusted in case of tail // call optimization. if (!IsSibCall) SPDiff = CalculateTailCallSPDiff(DAG, CFlags.IsTailCall, NumBytes); // To protect arguments on the stack from being clobbered in a tail call, // force all the loads to happen before doing any other lowering. if (CFlags.IsTailCall) Chain = DAG.getStackArgumentTokenFactor(Chain); // Adjust the stack pointer for the new arguments... // These operations are automatically eliminated by the prolog/epilog pass if (!IsSibCall) Chain = DAG.getCALLSEQ_START(Chain, NumBytes, 0, dl); SDValue CallSeqStart = Chain; // Load the return address and frame pointer so it can be move somewhere else // later. SDValue LROp, FPOp; Chain = EmitTailCallLoadFPAndRetAddr(DAG, SPDiff, Chain, LROp, FPOp, dl); // Set up a copy of the stack pointer for use loading and storing any // arguments that may not fit in the registers available for argument // passing. SDValue StackPtr = DAG.getRegister(PPC::X1, MVT::i64); // Figure out which arguments are going to go in registers, and which in // memory. Also, if this is a vararg function, floating point operations // must be stored to our stack, and loaded into integer regs as well, if // any integer regs are available for argument passing. unsigned ArgOffset = LinkageSize; SmallVector, 8> RegsToPass; SmallVector TailCallArguments; SmallVector MemOpChains; for (unsigned i = 0; i != NumOps; ++i) { SDValue Arg = OutVals[i]; ISD::ArgFlagsTy Flags = Outs[i].Flags; EVT ArgVT = Outs[i].VT; EVT OrigVT = Outs[i].ArgVT; // PtrOff will be used to store the current argument to the stack if a // register cannot be found for it. SDValue PtrOff; // We re-align the argument offset for each argument, except when using the // fast calling convention, when we need to make sure we do that only when // we'll actually use a stack slot. auto ComputePtrOff = [&]() { /* Respect alignment of argument on the stack. */ auto Alignment = CalculateStackSlotAlignment(ArgVT, OrigVT, Flags, PtrByteSize); ArgOffset = alignTo(ArgOffset, Alignment); PtrOff = DAG.getConstant(ArgOffset, dl, StackPtr.getValueType()); PtrOff = DAG.getNode(ISD::ADD, dl, PtrVT, StackPtr, PtrOff); }; if (!IsFastCall) { ComputePtrOff(); /* Compute GPR index associated with argument offset. */ GPR_idx = (ArgOffset - LinkageSize) / PtrByteSize; GPR_idx = std::min(GPR_idx, NumGPRs); } // Promote integers to 64-bit values. if (Arg.getValueType() == MVT::i32 || Arg.getValueType() == MVT::i1) { // FIXME: Should this use ANY_EXTEND if neither sext nor zext? unsigned ExtOp = Flags.isSExt() ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND; Arg = DAG.getNode(ExtOp, dl, MVT::i64, Arg); } // FIXME memcpy is used way more than necessary. Correctness first. // Note: "by value" is code for passing a structure by value, not // basic types. if (Flags.isByVal()) { // Note: Size includes alignment padding, so // struct x { short a; char b; } // will have Size = 4. With #pragma pack(1), it will have Size = 3. // These are the proper values we need for right-justifying the // aggregate in a parameter register. unsigned Size = Flags.getByValSize(); // An empty aggregate parameter takes up no storage and no // registers. if (Size == 0) continue; if (IsFastCall) ComputePtrOff(); // All aggregates smaller than 8 bytes must be passed right-justified. if (Size==1 || Size==2 || Size==4) { EVT VT = (Size==1) ? MVT::i8 : ((Size==2) ? MVT::i16 : MVT::i32); if (GPR_idx != NumGPRs) { SDValue Load = DAG.getExtLoad(ISD::EXTLOAD, dl, PtrVT, Chain, Arg, MachinePointerInfo(), VT); MemOpChains.push_back(Load.getValue(1)); RegsToPass.push_back(std::make_pair(GPR[GPR_idx++], Load)); ArgOffset += PtrByteSize; continue; } } if (GPR_idx == NumGPRs && Size < 8) { SDValue AddPtr = PtrOff; if (!isLittleEndian) { SDValue Const = DAG.getConstant(PtrByteSize - Size, dl, PtrOff.getValueType()); AddPtr = DAG.getNode(ISD::ADD, dl, PtrVT, PtrOff, Const); } Chain = CallSeqStart = createMemcpyOutsideCallSeq(Arg, AddPtr, CallSeqStart, Flags, DAG, dl); ArgOffset += PtrByteSize; continue; } // Copy entire object into memory. There are cases where gcc-generated // code assumes it is there, even if it could be put entirely into // registers. (This is not what the doc says.) // FIXME: The above statement is likely due to a misunderstanding of the // documents. All arguments must be copied into the parameter area BY // THE CALLEE in the event that the callee takes the address of any // formal argument. That has not yet been implemented. However, it is // reasonable to use the stack area as a staging area for the register // load. // Skip this for small aggregates, as we will use the same slot for a // right-justified copy, below. if (Size >= 8) Chain = CallSeqStart = createMemcpyOutsideCallSeq(Arg, PtrOff, CallSeqStart, Flags, DAG, dl); // When a register is available, pass a small aggregate right-justified. if (Size < 8 && GPR_idx != NumGPRs) { // The easiest way to get this right-justified in a register // is to copy the structure into the rightmost portion of a // local variable slot, then load the whole slot into the // register. // FIXME: The memcpy seems to produce pretty awful code for // small aggregates, particularly for packed ones. // FIXME: It would be preferable to use the slot in the // parameter save area instead of a new local variable. SDValue AddPtr = PtrOff; if (!isLittleEndian) { SDValue Const = DAG.getConstant(8 - Size, dl, PtrOff.getValueType()); AddPtr = DAG.getNode(ISD::ADD, dl, PtrVT, PtrOff, Const); } Chain = CallSeqStart = createMemcpyOutsideCallSeq(Arg, AddPtr, CallSeqStart, Flags, DAG, dl); // Load the slot into the register. SDValue Load = DAG.getLoad(PtrVT, dl, Chain, PtrOff, MachinePointerInfo()); MemOpChains.push_back(Load.getValue(1)); RegsToPass.push_back(std::make_pair(GPR[GPR_idx++], Load)); // Done with this argument. ArgOffset += PtrByteSize; continue; } // For aggregates larger than PtrByteSize, copy the pieces of the // object that fit into registers from the parameter save area. for (unsigned j=0; j gpr moves. // In the non-vararg case, this can only ever happen in the // presence of f32 array types, since otherwise we never run // out of FPRs before running out of GPRs. SDValue ArgVal; // Double values are always passed in a single GPR. if (Arg.getValueType() != MVT::f32) { ArgVal = DAG.getNode(ISD::BITCAST, dl, MVT::i64, Arg); // Non-array float values are extended and passed in a GPR. } else if (!Flags.isInConsecutiveRegs()) { ArgVal = DAG.getNode(ISD::BITCAST, dl, MVT::i32, Arg); ArgVal = DAG.getNode(ISD::ANY_EXTEND, dl, MVT::i64, ArgVal); // If we have an array of floats, we collect every odd element // together with its predecessor into one GPR. } else if (ArgOffset % PtrByteSize != 0) { SDValue Lo, Hi; Lo = DAG.getNode(ISD::BITCAST, dl, MVT::i32, OutVals[i - 1]); Hi = DAG.getNode(ISD::BITCAST, dl, MVT::i32, Arg); if (!isLittleEndian) std::swap(Lo, Hi); ArgVal = DAG.getNode(ISD::BUILD_PAIR, dl, MVT::i64, Lo, Hi); // The final element, if even, goes into the first half of a GPR. } else if (Flags.isInConsecutiveRegsLast()) { ArgVal = DAG.getNode(ISD::BITCAST, dl, MVT::i32, Arg); ArgVal = DAG.getNode(ISD::ANY_EXTEND, dl, MVT::i64, ArgVal); if (!isLittleEndian) ArgVal = DAG.getNode(ISD::SHL, dl, MVT::i64, ArgVal, DAG.getConstant(32, dl, MVT::i32)); // Non-final even elements are skipped; they will be handled // together the with subsequent argument on the next go-around. } else ArgVal = SDValue(); if (ArgVal.getNode()) RegsToPass.push_back(std::make_pair(GPR[GPR_idx++], ArgVal)); } else { if (IsFastCall) ComputePtrOff(); // Single-precision floating-point values are mapped to the // second (rightmost) word of the stack doubleword. if (Arg.getValueType() == MVT::f32 && !isLittleEndian && !Flags.isInConsecutiveRegs()) { SDValue ConstFour = DAG.getConstant(4, dl, PtrOff.getValueType()); PtrOff = DAG.getNode(ISD::ADD, dl, PtrVT, PtrOff, ConstFour); } assert(HasParameterArea && "Parameter area must exist to pass an argument in memory."); LowerMemOpCallTo(DAG, MF, Chain, Arg, PtrOff, SPDiff, ArgOffset, true, CFlags.IsTailCall, false, MemOpChains, TailCallArguments, dl); NeededLoad = true; } // When passing an array of floats, the array occupies consecutive // space in the argument area; only round up to the next doubleword // at the end of the array. Otherwise, each float takes 8 bytes. if (!IsFastCall || NeededLoad) { ArgOffset += (Arg.getValueType() == MVT::f32 && Flags.isInConsecutiveRegs()) ? 4 : 8; if (Flags.isInConsecutiveRegsLast()) ArgOffset = ((ArgOffset + PtrByteSize - 1)/PtrByteSize) * PtrByteSize; } break; } case MVT::v4f32: case MVT::v4i32: case MVT::v8i16: case MVT::v16i8: case MVT::v2f64: case MVT::v2i64: case MVT::v1i128: case MVT::f128: if (!Subtarget.hasQPX()) { // These can be scalar arguments or elements of a vector array type // passed directly. The latter are used to implement ELFv2 homogenous // vector aggregates. // For a varargs call, named arguments go into VRs or on the stack as // usual; unnamed arguments always go to the stack or the corresponding // GPRs when within range. For now, we always put the value in both // locations (or even all three). if (CFlags.IsVarArg) { assert(HasParameterArea && "Parameter area must exist if we have a varargs call."); // We could elide this store in the case where the object fits // entirely in R registers. Maybe later. SDValue Store = DAG.getStore(Chain, dl, Arg, PtrOff, MachinePointerInfo()); MemOpChains.push_back(Store); if (VR_idx != NumVRs) { SDValue Load = DAG.getLoad(MVT::v4f32, dl, Store, PtrOff, MachinePointerInfo()); MemOpChains.push_back(Load.getValue(1)); RegsToPass.push_back(std::make_pair(VR[VR_idx++], Load)); } ArgOffset += 16; for (unsigned i=0; i<16; i+=PtrByteSize) { if (GPR_idx == NumGPRs) break; SDValue Ix = DAG.getNode(ISD::ADD, dl, PtrVT, PtrOff, DAG.getConstant(i, dl, PtrVT)); SDValue Load = DAG.getLoad(PtrVT, dl, Store, Ix, MachinePointerInfo()); MemOpChains.push_back(Load.getValue(1)); RegsToPass.push_back(std::make_pair(GPR[GPR_idx++], Load)); } break; } // Non-varargs Altivec params go into VRs or on the stack. if (VR_idx != NumVRs) { RegsToPass.push_back(std::make_pair(VR[VR_idx++], Arg)); } else { if (IsFastCall) ComputePtrOff(); assert(HasParameterArea && "Parameter area must exist to pass an argument in memory."); LowerMemOpCallTo(DAG, MF, Chain, Arg, PtrOff, SPDiff, ArgOffset, true, CFlags.IsTailCall, true, MemOpChains, TailCallArguments, dl); if (IsFastCall) ArgOffset += 16; } if (!IsFastCall) ArgOffset += 16; break; } // not QPX assert(Arg.getValueType().getSimpleVT().SimpleTy == MVT::v4f32 && "Invalid QPX parameter type"); LLVM_FALLTHROUGH; case MVT::v4f64: case MVT::v4i1: { bool IsF32 = Arg.getValueType().getSimpleVT().SimpleTy == MVT::v4f32; if (CFlags.IsVarArg) { assert(HasParameterArea && "Parameter area must exist if we have a varargs call."); // We could elide this store in the case where the object fits // entirely in R registers. Maybe later. SDValue Store = DAG.getStore(Chain, dl, Arg, PtrOff, MachinePointerInfo()); MemOpChains.push_back(Store); if (QFPR_idx != NumQFPRs) { SDValue Load = DAG.getLoad(IsF32 ? MVT::v4f32 : MVT::v4f64, dl, Store, PtrOff, MachinePointerInfo()); MemOpChains.push_back(Load.getValue(1)); RegsToPass.push_back(std::make_pair(QFPR[QFPR_idx++], Load)); } ArgOffset += (IsF32 ? 16 : 32); for (unsigned i = 0; i < (IsF32 ? 16U : 32U); i += PtrByteSize) { if (GPR_idx == NumGPRs) break; SDValue Ix = DAG.getNode(ISD::ADD, dl, PtrVT, PtrOff, DAG.getConstant(i, dl, PtrVT)); SDValue Load = DAG.getLoad(PtrVT, dl, Store, Ix, MachinePointerInfo()); MemOpChains.push_back(Load.getValue(1)); RegsToPass.push_back(std::make_pair(GPR[GPR_idx++], Load)); } break; } // Non-varargs QPX params go into registers or on the stack. if (QFPR_idx != NumQFPRs) { RegsToPass.push_back(std::make_pair(QFPR[QFPR_idx++], Arg)); } else { if (IsFastCall) ComputePtrOff(); assert(HasParameterArea && "Parameter area must exist to pass an argument in memory."); LowerMemOpCallTo(DAG, MF, Chain, Arg, PtrOff, SPDiff, ArgOffset, true, CFlags.IsTailCall, true, MemOpChains, TailCallArguments, dl); if (IsFastCall) ArgOffset += (IsF32 ? 16 : 32); } if (!IsFastCall) ArgOffset += (IsF32 ? 16 : 32); break; } } } assert((!HasParameterArea || NumBytesActuallyUsed == ArgOffset) && "mismatch in size of parameter area"); (void)NumBytesActuallyUsed; if (!MemOpChains.empty()) Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, MemOpChains); // Check if this is an indirect call (MTCTR/BCTRL). // See prepareDescriptorIndirectCall and buildCallOperands for more // information about calls through function pointers in the 64-bit SVR4 ABI. if (CFlags.IsIndirect) { assert(!CFlags.IsTailCall && "Indirect tails calls not supported"); // Load r2 into a virtual register and store it to the TOC save area. setUsesTOCBasePtr(DAG); SDValue Val = DAG.getCopyFromReg(Chain, dl, PPC::X2, MVT::i64); // TOC save area offset. unsigned TOCSaveOffset = Subtarget.getFrameLowering()->getTOCSaveOffset(); SDValue PtrOff = DAG.getIntPtrConstant(TOCSaveOffset, dl); SDValue AddPtr = DAG.getNode(ISD::ADD, dl, PtrVT, StackPtr, PtrOff); Chain = DAG.getStore( Val.getValue(1), dl, Val, AddPtr, MachinePointerInfo::getStack(DAG.getMachineFunction(), TOCSaveOffset)); // In the ELFv2 ABI, R12 must contain the address of an indirect callee. // This does not mean the MTCTR instruction must use R12; it's easier // to model this as an extra parameter, so do that. if (isELFv2ABI && !CFlags.IsPatchPoint) RegsToPass.push_back(std::make_pair((unsigned)PPC::X12, Callee)); } // Build a sequence of copy-to-reg nodes chained together with token chain // and flag operands which copy the outgoing args into the appropriate regs. SDValue InFlag; for (unsigned i = 0, e = RegsToPass.size(); i != e; ++i) { Chain = DAG.getCopyToReg(Chain, dl, RegsToPass[i].first, RegsToPass[i].second, InFlag); InFlag = Chain.getValue(1); } if (CFlags.IsTailCall && !IsSibCall) PrepareTailCall(DAG, InFlag, Chain, dl, SPDiff, NumBytes, LROp, FPOp, TailCallArguments); return FinishCall(CFlags, dl, DAG, RegsToPass, InFlag, Chain, CallSeqStart, Callee, SPDiff, NumBytes, Ins, InVals, CS); } SDValue PPCTargetLowering::LowerCall_Darwin( SDValue Chain, SDValue Callee, CallFlags CFlags, const SmallVectorImpl &Outs, const SmallVectorImpl &OutVals, const SmallVectorImpl &Ins, const SDLoc &dl, SelectionDAG &DAG, SmallVectorImpl &InVals, ImmutableCallSite CS) const { unsigned NumOps = Outs.size(); EVT PtrVT = getPointerTy(DAG.getDataLayout()); bool isPPC64 = PtrVT == MVT::i64; unsigned PtrByteSize = isPPC64 ? 8 : 4; MachineFunction &MF = DAG.getMachineFunction(); // Mark this function as potentially containing a function that contains a // tail call. As a consequence the frame pointer will be used for dynamicalloc // and restoring the callers stack pointer in this functions epilog. This is // done because by tail calling the called function might overwrite the value // in this function's (MF) stack pointer stack slot 0(SP). if (getTargetMachine().Options.GuaranteedTailCallOpt && CFlags.CallConv == CallingConv::Fast) MF.getInfo()->setHasFastCall(); // Count how many bytes are to be pushed on the stack, including the linkage // area, and parameter passing area. We start with 24/48 bytes, which is // prereserved space for [SP][CR][LR][3 x unused]. unsigned LinkageSize = Subtarget.getFrameLowering()->getLinkageSize(); unsigned NumBytes = LinkageSize; // Add up all the space actually used. // In 32-bit non-varargs calls, Altivec parameters all go at the end; usually // they all go in registers, but we must reserve stack space for them for // possible use by the caller. In varargs or 64-bit calls, parameters are // assigned stack space in order, with padding so Altivec parameters are // 16-byte aligned. unsigned nAltivecParamsAtEnd = 0; for (unsigned i = 0; i != NumOps; ++i) { ISD::ArgFlagsTy Flags = Outs[i].Flags; EVT ArgVT = Outs[i].VT; // Varargs Altivec parameters are padded to a 16 byte boundary. if (ArgVT == MVT::v4f32 || ArgVT == MVT::v4i32 || ArgVT == MVT::v8i16 || ArgVT == MVT::v16i8 || ArgVT == MVT::v2f64 || ArgVT == MVT::v2i64) { if (!CFlags.IsVarArg && !isPPC64) { // Non-varargs Altivec parameters go after all the non-Altivec // parameters; handle those later so we know how much padding we need. nAltivecParamsAtEnd++; continue; } // Varargs and 64-bit Altivec parameters are padded to 16 byte boundary. NumBytes = ((NumBytes+15)/16)*16; } NumBytes += CalculateStackSlotSize(ArgVT, Flags, PtrByteSize); } // Allow for Altivec parameters at the end, if needed. if (nAltivecParamsAtEnd) { NumBytes = ((NumBytes+15)/16)*16; NumBytes += 16*nAltivecParamsAtEnd; } // The prolog code of the callee may store up to 8 GPR argument registers to // the stack, allowing va_start to index over them in memory if its varargs. // Because we cannot tell if this is needed on the caller side, we have to // conservatively assume that it is needed. As such, make sure we have at // least enough stack space for the caller to store the 8 GPRs. NumBytes = std::max(NumBytes, LinkageSize + 8 * PtrByteSize); // Tail call needs the stack to be aligned. if (getTargetMachine().Options.GuaranteedTailCallOpt && CFlags.CallConv == CallingConv::Fast) NumBytes = EnsureStackAlignment(Subtarget.getFrameLowering(), NumBytes); // Calculate by how many bytes the stack has to be adjusted in case of tail // call optimization. int SPDiff = CalculateTailCallSPDiff(DAG, CFlags.IsTailCall, NumBytes); // To protect arguments on the stack from being clobbered in a tail call, // force all the loads to happen before doing any other lowering. if (CFlags.IsTailCall) Chain = DAG.getStackArgumentTokenFactor(Chain); // Adjust the stack pointer for the new arguments... // These operations are automatically eliminated by the prolog/epilog pass Chain = DAG.getCALLSEQ_START(Chain, NumBytes, 0, dl); SDValue CallSeqStart = Chain; // Load the return address and frame pointer so it can be move somewhere else // later. SDValue LROp, FPOp; Chain = EmitTailCallLoadFPAndRetAddr(DAG, SPDiff, Chain, LROp, FPOp, dl); // Set up a copy of the stack pointer for use loading and storing any // arguments that may not fit in the registers available for argument // passing. SDValue StackPtr; if (isPPC64) StackPtr = DAG.getRegister(PPC::X1, MVT::i64); else StackPtr = DAG.getRegister(PPC::R1, MVT::i32); // Figure out which arguments are going to go in registers, and which in // memory. Also, if this is a vararg function, floating point operations // must be stored to our stack, and loaded into integer regs as well, if // any integer regs are available for argument passing. unsigned ArgOffset = LinkageSize; unsigned GPR_idx = 0, FPR_idx = 0, VR_idx = 0; static const MCPhysReg GPR_32[] = { // 32-bit registers. PPC::R3, PPC::R4, PPC::R5, PPC::R6, PPC::R7, PPC::R8, PPC::R9, PPC::R10, }; static const MCPhysReg GPR_64[] = { // 64-bit registers. PPC::X3, PPC::X4, PPC::X5, PPC::X6, PPC::X7, PPC::X8, PPC::X9, PPC::X10, }; static const MCPhysReg VR[] = { PPC::V2, PPC::V3, PPC::V4, PPC::V5, PPC::V6, PPC::V7, PPC::V8, PPC::V9, PPC::V10, PPC::V11, PPC::V12, PPC::V13 }; const unsigned NumGPRs = array_lengthof(GPR_32); const unsigned NumFPRs = 13; const unsigned NumVRs = array_lengthof(VR); const MCPhysReg *GPR = isPPC64 ? GPR_64 : GPR_32; SmallVector, 8> RegsToPass; SmallVector TailCallArguments; SmallVector MemOpChains; for (unsigned i = 0; i != NumOps; ++i) { SDValue Arg = OutVals[i]; ISD::ArgFlagsTy Flags = Outs[i].Flags; // PtrOff will be used to store the current argument to the stack if a // register cannot be found for it. SDValue PtrOff; PtrOff = DAG.getConstant(ArgOffset, dl, StackPtr.getValueType()); PtrOff = DAG.getNode(ISD::ADD, dl, PtrVT, StackPtr, PtrOff); // On PPC64, promote integers to 64-bit values. if (isPPC64 && Arg.getValueType() == MVT::i32) { // FIXME: Should this use ANY_EXTEND if neither sext nor zext? unsigned ExtOp = Flags.isSExt() ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND; Arg = DAG.getNode(ExtOp, dl, MVT::i64, Arg); } // FIXME memcpy is used way more than necessary. Correctness first. // Note: "by value" is code for passing a structure by value, not // basic types. if (Flags.isByVal()) { unsigned Size = Flags.getByValSize(); // Very small objects are passed right-justified. Everything else is // passed left-justified. if (Size==1 || Size==2) { EVT VT = (Size==1) ? MVT::i8 : MVT::i16; if (GPR_idx != NumGPRs) { SDValue Load = DAG.getExtLoad(ISD::EXTLOAD, dl, PtrVT, Chain, Arg, MachinePointerInfo(), VT); MemOpChains.push_back(Load.getValue(1)); RegsToPass.push_back(std::make_pair(GPR[GPR_idx++], Load)); ArgOffset += PtrByteSize; } else { SDValue Const = DAG.getConstant(PtrByteSize - Size, dl, PtrOff.getValueType()); SDValue AddPtr = DAG.getNode(ISD::ADD, dl, PtrVT, PtrOff, Const); Chain = CallSeqStart = createMemcpyOutsideCallSeq(Arg, AddPtr, CallSeqStart, Flags, DAG, dl); ArgOffset += PtrByteSize; } continue; } // Copy entire object into memory. There are cases where gcc-generated // code assumes it is there, even if it could be put entirely into // registers. (This is not what the doc says.) Chain = CallSeqStart = createMemcpyOutsideCallSeq(Arg, PtrOff, CallSeqStart, Flags, DAG, dl); // For small aggregates (Darwin only) and aggregates >= PtrByteSize, // copy the pieces of the object that fit into registers from the // parameter save area. for (unsigned j=0; j NumVRs) { unsigned j = 0; // Offset is aligned; skip 1st 12 params which go in V registers. ArgOffset = ((ArgOffset+15)/16)*16; ArgOffset += 12*16; for (unsigned i = 0; i != NumOps; ++i) { SDValue Arg = OutVals[i]; EVT ArgType = Outs[i].VT; if (ArgType==MVT::v4f32 || ArgType==MVT::v4i32 || ArgType==MVT::v8i16 || ArgType==MVT::v16i8) { if (++j > NumVRs) { SDValue PtrOff; // We are emitting Altivec params in order. LowerMemOpCallTo(DAG, MF, Chain, Arg, PtrOff, SPDiff, ArgOffset, isPPC64, CFlags.IsTailCall, true, MemOpChains, TailCallArguments, dl); ArgOffset += 16; } } } } if (!MemOpChains.empty()) Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, MemOpChains); // On Darwin, R12 must contain the address of an indirect callee. This does // not mean the MTCTR instruction must use R12; it's easier to model this as // an extra parameter, so do that. if (CFlags.IsIndirect) { assert(!CFlags.IsTailCall && "Indirect tail-calls not supported."); RegsToPass.push_back(std::make_pair((unsigned)(isPPC64 ? PPC::X12 : PPC::R12), Callee)); } // Build a sequence of copy-to-reg nodes chained together with token chain // and flag operands which copy the outgoing args into the appropriate regs. SDValue InFlag; for (unsigned i = 0, e = RegsToPass.size(); i != e; ++i) { Chain = DAG.getCopyToReg(Chain, dl, RegsToPass[i].first, RegsToPass[i].second, InFlag); InFlag = Chain.getValue(1); } if (CFlags.IsTailCall) PrepareTailCall(DAG, InFlag, Chain, dl, SPDiff, NumBytes, LROp, FPOp, TailCallArguments); return FinishCall(CFlags, dl, DAG, RegsToPass, InFlag, Chain, CallSeqStart, Callee, SPDiff, NumBytes, Ins, InVals, CS); } static bool CC_AIX(unsigned ValNo, MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags, CCState &State) { const PPCSubtarget &Subtarget = static_cast( State.getMachineFunction().getSubtarget()); const bool IsPPC64 = Subtarget.isPPC64(); const unsigned PtrByteSize = IsPPC64 ? 8 : 4; const MVT RegVT = IsPPC64 ? MVT::i64 : MVT::i32; assert((!ValVT.isInteger() || (ValVT.getSizeInBits() <= RegVT.getSizeInBits())) && "Integer argument exceeds register size: should have been legalized"); if (ValVT == MVT::f128) report_fatal_error("f128 is unimplemented on AIX."); if (ArgFlags.isNest()) report_fatal_error("Nest arguments are unimplemented."); if (ValVT.isVector() || LocVT.isVector()) report_fatal_error("Vector arguments are unimplemented on AIX."); static const MCPhysReg GPR_32[] = {// 32-bit registers. PPC::R3, PPC::R4, PPC::R5, PPC::R6, PPC::R7, PPC::R8, PPC::R9, PPC::R10}; static const MCPhysReg GPR_64[] = {// 64-bit registers. PPC::X3, PPC::X4, PPC::X5, PPC::X6, PPC::X7, PPC::X8, PPC::X9, PPC::X10}; if (ArgFlags.isByVal()) { if (ArgFlags.getNonZeroByValAlign() > PtrByteSize) report_fatal_error("Pass-by-value arguments with alignment greater than " "register width are not supported."); const unsigned ByValSize = ArgFlags.getByValSize(); // An empty aggregate parameter takes up no storage and no registers, // but needs a MemLoc for a stack slot for the formal arguments side. if (ByValSize == 0) { State.addLoc(CCValAssign::getMem(ValNo, MVT::INVALID_SIMPLE_VALUE_TYPE, State.getNextStackOffset(), RegVT, LocInfo)); return false; } State.AllocateStack(alignTo(ByValSize, PtrByteSize), PtrByteSize); for (unsigned I = 0, E = ByValSize; I < E; I += PtrByteSize) { if (unsigned Reg = State.AllocateReg(IsPPC64 ? GPR_64 : GPR_32)) State.addLoc(CCValAssign::getReg(ValNo, ValVT, Reg, RegVT, LocInfo)); else report_fatal_error( "Pass-by-value arguments are only supported in registers."); } return false; } // Arguments always reserve parameter save area. switch (ValVT.SimpleTy) { default: report_fatal_error("Unhandled value type for argument."); case MVT::i64: // i64 arguments should have been split to i32 for PPC32. assert(IsPPC64 && "PPC32 should have split i64 values."); LLVM_FALLTHROUGH; case MVT::i1: case MVT::i32: { const unsigned Offset = State.AllocateStack(PtrByteSize, PtrByteSize); // AIX integer arguments are always passed in register width. if (ValVT.getSizeInBits() < RegVT.getSizeInBits()) LocInfo = ArgFlags.isSExt() ? CCValAssign::LocInfo::SExt : CCValAssign::LocInfo::ZExt; if (unsigned Reg = State.AllocateReg(IsPPC64 ? GPR_64 : GPR_32)) State.addLoc(CCValAssign::getReg(ValNo, ValVT, Reg, RegVT, LocInfo)); else State.addLoc(CCValAssign::getMem(ValNo, ValVT, Offset, RegVT, LocInfo)); return false; } case MVT::f32: case MVT::f64: { // Parameter save area (PSA) is reserved even if the float passes in fpr. const unsigned StoreSize = LocVT.getStoreSize(); // Floats are always 4-byte aligned in the PSA on AIX. // This includes f64 in 64-bit mode for ABI compatibility. const unsigned Offset = State.AllocateStack(IsPPC64 ? 8 : StoreSize, 4); unsigned FReg = State.AllocateReg(FPR); if (FReg) State.addLoc(CCValAssign::getReg(ValNo, ValVT, FReg, LocVT, LocInfo)); // Reserve and initialize GPRs or initialize the PSA as required. for (unsigned I = 0; I < StoreSize; I += PtrByteSize) { if (unsigned Reg = State.AllocateReg(IsPPC64 ? GPR_64 : GPR_32)) { assert(FReg && "An FPR should be available when a GPR is reserved."); if (State.isVarArg()) { // Successfully reserved GPRs are only initialized for vararg calls. // Custom handling is required for: // f64 in PPC32 needs to be split into 2 GPRs. // f32 in PPC64 needs to occupy only lower 32 bits of 64-bit GPR. State.addLoc( CCValAssign::getCustomReg(ValNo, ValVT, Reg, RegVT, LocInfo)); } } else { // If there are insufficient GPRs, the PSA needs to be initialized. // Initialization occurs even if an FPR was initialized for // compatibility with the AIX XL compiler. The full memory for the // argument will be initialized even if a prior word is saved in GPR. // A custom memLoc is used when the argument also passes in FPR so // that the callee handling can skip over it easily. State.addLoc( FReg ? CCValAssign::getCustomMem(ValNo, ValVT, Offset, LocVT, LocInfo) : CCValAssign::getMem(ValNo, ValVT, Offset, LocVT, LocInfo)); break; } } return false; } } return true; } static const TargetRegisterClass *getRegClassForSVT(MVT::SimpleValueType SVT, bool IsPPC64) { assert((IsPPC64 || SVT != MVT::i64) && "i64 should have been split for 32-bit codegen."); switch (SVT) { default: report_fatal_error("Unexpected value type for formal argument"); case MVT::i1: case MVT::i32: case MVT::i64: return IsPPC64 ? &PPC::G8RCRegClass : &PPC::GPRCRegClass; case MVT::f32: return &PPC::F4RCRegClass; case MVT::f64: return &PPC::F8RCRegClass; } } static SDValue truncateScalarIntegerArg(ISD::ArgFlagsTy Flags, EVT ValVT, SelectionDAG &DAG, SDValue ArgValue, MVT LocVT, const SDLoc &dl) { assert(ValVT.isScalarInteger() && LocVT.isScalarInteger()); assert(ValVT.getSizeInBits() < LocVT.getSizeInBits()); if (Flags.isSExt()) ArgValue = DAG.getNode(ISD::AssertSext, dl, LocVT, ArgValue, DAG.getValueType(ValVT)); else if (Flags.isZExt()) ArgValue = DAG.getNode(ISD::AssertZext, dl, LocVT, ArgValue, DAG.getValueType(ValVT)); return DAG.getNode(ISD::TRUNCATE, dl, ValVT, ArgValue); } static unsigned mapArgRegToOffsetAIX(unsigned Reg, const PPCFrameLowering *FL) { const unsigned LASize = FL->getLinkageSize(); if (PPC::GPRCRegClass.contains(Reg)) { assert(Reg >= PPC::R3 && Reg <= PPC::R10 && "Reg must be a valid argument register!"); return LASize + 4 * (Reg - PPC::R3); } if (PPC::G8RCRegClass.contains(Reg)) { assert(Reg >= PPC::X3 && Reg <= PPC::X10 && "Reg must be a valid argument register!"); return LASize + 8 * (Reg - PPC::X3); } llvm_unreachable("Only general purpose registers expected."); } SDValue PPCTargetLowering::LowerFormalArguments_AIX( SDValue Chain, CallingConv::ID CallConv, bool isVarArg, const SmallVectorImpl &Ins, const SDLoc &dl, SelectionDAG &DAG, SmallVectorImpl &InVals) const { assert((CallConv == CallingConv::C || CallConv == CallingConv::Cold || CallConv == CallingConv::Fast) && "Unexpected calling convention!"); if (isVarArg) report_fatal_error("This call type is unimplemented on AIX."); if (getTargetMachine().Options.GuaranteedTailCallOpt) report_fatal_error("Tail call support is unimplemented on AIX."); if (useSoftFloat()) report_fatal_error("Soft float support is unimplemented on AIX."); const PPCSubtarget &Subtarget = static_cast(DAG.getSubtarget()); if (Subtarget.hasQPX()) report_fatal_error("QPX support is not supported on AIX."); const bool IsPPC64 = Subtarget.isPPC64(); const unsigned PtrByteSize = IsPPC64 ? 8 : 4; // Assign locations to all of the incoming arguments. SmallVector ArgLocs; MachineFunction &MF = DAG.getMachineFunction(); CCState CCInfo(CallConv, isVarArg, MF, ArgLocs, *DAG.getContext()); const EVT PtrVT = getPointerTy(MF.getDataLayout()); // Reserve space for the linkage area on the stack. const unsigned LinkageSize = Subtarget.getFrameLowering()->getLinkageSize(); CCInfo.AllocateStack(LinkageSize, PtrByteSize); CCInfo.AnalyzeFormalArguments(Ins, CC_AIX); SmallVector MemOps; for (CCValAssign &VA : ArgLocs) { EVT ValVT = VA.getValVT(); MVT LocVT = VA.getLocVT(); ISD::ArgFlagsTy Flags = Ins[VA.getValNo()].Flags; assert((VA.isRegLoc() || VA.isMemLoc()) && "Unexpected location for function call argument."); // For compatibility with the AIX XL compiler, the float args in the // parameter save area are initialized even if the argument is available // in register. The caller is required to initialize both the register // and memory, however, the callee can choose to expect it in either. // The memloc is dismissed here because the argument is retrieved from // the register. if (VA.isMemLoc() && VA.needsCustom()) continue; if (Flags.isByVal() && VA.isMemLoc()) { if (Flags.getByValSize() != 0) report_fatal_error( "ByVal arguments passed on stack not implemented yet"); const int FI = MF.getFrameInfo().CreateFixedObject( PtrByteSize, VA.getLocMemOffset(), /* IsImmutable */ false, /* IsAliased */ true); SDValue FIN = DAG.getFrameIndex(FI, PtrVT); InVals.push_back(FIN); continue; } if (Flags.isByVal()) { assert(VA.isRegLoc() && "MemLocs should already be handled."); const unsigned ByValSize = Flags.getByValSize(); if (ByValSize > PtrByteSize) report_fatal_error("Formal arguments greater then register size not " "implemented yet."); const MCPhysReg ArgReg = VA.getLocReg(); const PPCFrameLowering *FL = Subtarget.getFrameLowering(); const unsigned Offset = mapArgRegToOffsetAIX(ArgReg, FL); const unsigned StackSize = alignTo(ByValSize, PtrByteSize); const int FI = MF.getFrameInfo().CreateFixedObject( StackSize, Offset, /* IsImmutable */ false, /* IsAliased */ true); SDValue FIN = DAG.getFrameIndex(FI, PtrVT); InVals.push_back(FIN); const unsigned VReg = MF.addLiveIn(ArgReg, IsPPC64 ? &PPC::G8RCRegClass : &PPC::GPRCRegClass); // Since the callers side has left justified the aggregate in the // register, we can simply store the entire register into the stack // slot. // The store to the fixedstack object is needed becuase accessing a // field of the ByVal will use a gep and load. Ideally we will optimize // to extracting the value from the register directly, and elide the // stores when the arguments address is not taken, but that will need to // be future work. SDValue CopyFrom = DAG.getCopyFromReg(Chain, dl, VReg, LocVT); SDValue Store = DAG.getStore(CopyFrom.getValue(1), dl, CopyFrom, FIN, MachinePointerInfo::getFixedStack(MF, FI, 0)); MemOps.push_back(Store); continue; } if (VA.isRegLoc()) { MVT::SimpleValueType SVT = ValVT.getSimpleVT().SimpleTy; unsigned VReg = MF.addLiveIn(VA.getLocReg(), getRegClassForSVT(SVT, IsPPC64)); SDValue ArgValue = DAG.getCopyFromReg(Chain, dl, VReg, LocVT); if (ValVT.isScalarInteger() && (ValVT.getSizeInBits() < LocVT.getSizeInBits())) { ArgValue = truncateScalarIntegerArg(Flags, ValVT, DAG, ArgValue, LocVT, dl); } InVals.push_back(ArgValue); continue; } const unsigned LocSize = LocVT.getStoreSize(); const unsigned ValSize = ValVT.getStoreSize(); assert((ValSize <= LocSize) && "Object size is larger than size of MemLoc"); int CurArgOffset = VA.getLocMemOffset(); // Objects are right-justified because AIX is big-endian. if (LocSize > ValSize) CurArgOffset += LocSize - ValSize; MachineFrameInfo &MFI = MF.getFrameInfo(); // Potential tail calls could cause overwriting of argument stack slots. const bool IsImmutable = !(getTargetMachine().Options.GuaranteedTailCallOpt && (CallConv == CallingConv::Fast)); int FI = MFI.CreateFixedObject(ValSize, CurArgOffset, IsImmutable); SDValue FIN = DAG.getFrameIndex(FI, PtrVT); SDValue ArgValue = DAG.getLoad(ValVT, dl, Chain, FIN, MachinePointerInfo()); InVals.push_back(ArgValue); } // On AIX a minimum of 8 words is saved to the parameter save area. const unsigned MinParameterSaveArea = 8 * PtrByteSize; // Area that is at least reserved in the caller of this function. unsigned CallerReservedArea = std::max(CCInfo.getNextStackOffset(), LinkageSize + MinParameterSaveArea); // Set the size that is at least reserved in caller of this function. Tail // call optimized function's reserved stack space needs to be aligned so // that taking the difference between two stack areas will result in an // aligned stack. CallerReservedArea = EnsureStackAlignment(Subtarget.getFrameLowering(), CallerReservedArea); PPCFunctionInfo *FuncInfo = MF.getInfo(); FuncInfo->setMinReservedArea(CallerReservedArea); if (!MemOps.empty()) Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, MemOps); return Chain; } SDValue PPCTargetLowering::LowerCall_AIX( SDValue Chain, SDValue Callee, CallFlags CFlags, const SmallVectorImpl &Outs, const SmallVectorImpl &OutVals, const SmallVectorImpl &Ins, const SDLoc &dl, SelectionDAG &DAG, SmallVectorImpl &InVals, ImmutableCallSite CS) const { assert((CFlags.CallConv == CallingConv::C || CFlags.CallConv == CallingConv::Cold || CFlags.CallConv == CallingConv::Fast) && "Unexpected calling convention!"); if (CFlags.IsPatchPoint) report_fatal_error("This call type is unimplemented on AIX."); const PPCSubtarget& Subtarget = static_cast(DAG.getSubtarget()); if (Subtarget.hasQPX()) report_fatal_error("QPX is not supported on AIX."); if (Subtarget.hasAltivec()) report_fatal_error("Altivec support is unimplemented on AIX."); MachineFunction &MF = DAG.getMachineFunction(); SmallVector ArgLocs; CCState CCInfo(CFlags.CallConv, CFlags.IsVarArg, MF, ArgLocs, *DAG.getContext()); // Reserve space for the linkage save area (LSA) on the stack. // In both PPC32 and PPC64 there are 6 reserved slots in the LSA: // [SP][CR][LR][2 x reserved][TOC]. // The LSA is 24 bytes (6x4) in PPC32 and 48 bytes (6x8) in PPC64. const unsigned LinkageSize = Subtarget.getFrameLowering()->getLinkageSize(); const bool IsPPC64 = Subtarget.isPPC64(); const EVT PtrVT = getPointerTy(DAG.getDataLayout()); const unsigned PtrByteSize = IsPPC64 ? 8 : 4; CCInfo.AllocateStack(LinkageSize, PtrByteSize); CCInfo.AnalyzeCallOperands(Outs, CC_AIX); // The prolog code of the callee may store up to 8 GPR argument registers to // the stack, allowing va_start to index over them in memory if the callee // is variadic. // Because we cannot tell if this is needed on the caller side, we have to // conservatively assume that it is needed. As such, make sure we have at // least enough stack space for the caller to store the 8 GPRs. const unsigned MinParameterSaveAreaSize = 8 * PtrByteSize; const unsigned NumBytes = std::max(LinkageSize + MinParameterSaveAreaSize, CCInfo.getNextStackOffset()); // Adjust the stack pointer for the new arguments... // These operations are automatically eliminated by the prolog/epilog pass. Chain = DAG.getCALLSEQ_START(Chain, NumBytes, 0, dl); SDValue CallSeqStart = Chain; SmallVector, 8> RegsToPass; SmallVector MemOpChains; // Set up a copy of the stack pointer for loading and storing any // arguments that may not fit in the registers available for argument // passing. const SDValue StackPtr = IsPPC64 ? DAG.getRegister(PPC::X1, MVT::i64) : DAG.getRegister(PPC::R1, MVT::i32); for (unsigned I = 0, E = ArgLocs.size(); I != E;) { const unsigned ValNo = ArgLocs[I].getValNo(); SDValue Arg = OutVals[ValNo]; ISD::ArgFlagsTy Flags = Outs[ValNo].Flags; if (Flags.isByVal()) { const unsigned ByValSize = Flags.getByValSize(); // Nothing to do for zero-sized ByVals on the caller side. if (!ByValSize) { ++I; continue; } auto GetLoad = [&](EVT VT, unsigned LoadOffset) { return DAG.getExtLoad(ISD::ZEXTLOAD, dl, PtrVT, Chain, (LoadOffset != 0) ? DAG.getObjectPtrOffset(dl, Arg, LoadOffset) : Arg, MachinePointerInfo(), VT); }; unsigned LoadOffset = 0; // Initialize registers, which are fully occupied by the by-val argument. while (I != E && LoadOffset + PtrByteSize <= ByValSize) { SDValue Load = GetLoad(PtrVT, LoadOffset); MemOpChains.push_back(Load.getValue(1)); LoadOffset += PtrByteSize; const CCValAssign &ByValVA = ArgLocs[I++]; assert(ByValVA.isRegLoc() && ByValVA.getValNo() == ValNo && "Unexpected location for pass-by-value argument."); RegsToPass.push_back(std::make_pair(ByValVA.getLocReg(), Load)); } if (LoadOffset == ByValSize) continue; const unsigned ResidueBytes = ByValSize % PtrByteSize; assert(ResidueBytes != 0 && LoadOffset + PtrByteSize > ByValSize && "Unexpected register residue for by-value argument."); // Initialize the final register residue. // Any residue that occupies the final by-val arg register must be // left-justified on AIX. Loads must be a power-of-2 size and cannot be // larger than the ByValSize. For example: a 7 byte by-val arg requires 4, // 2 and 1 byte loads. SDValue ResidueVal; for (unsigned Bytes = 0; Bytes != ResidueBytes;) { const unsigned N = PowerOf2Floor(ResidueBytes - Bytes); const MVT VT = N == 1 ? MVT::i8 : ((N == 2) ? MVT::i16 : (N == 4 ? MVT::i32 : MVT::i64)); SDValue Load = GetLoad(VT, LoadOffset); MemOpChains.push_back(Load.getValue(1)); LoadOffset += N; Bytes += N; // By-val arguments are passed left-justfied in register. // Every load here needs to be shifted, otherwise a full register load // should have been used. assert(PtrVT.getSimpleVT().getSizeInBits() > (Bytes * 8) && "Unexpected load emitted during handling of pass-by-value " "argument."); unsigned NumSHLBits = PtrVT.getSimpleVT().getSizeInBits() - (Bytes * 8); EVT ShiftAmountTy = getShiftAmountTy(Load->getValueType(0), DAG.getDataLayout()); SDValue SHLAmt = DAG.getConstant(NumSHLBits, dl, ShiftAmountTy); SDValue ShiftedLoad = DAG.getNode(ISD::SHL, dl, Load.getValueType(), Load, SHLAmt); ResidueVal = ResidueVal ? DAG.getNode(ISD::OR, dl, PtrVT, ResidueVal, ShiftedLoad) : ShiftedLoad; } const CCValAssign &ByValVA = ArgLocs[I++]; assert(ByValVA.isRegLoc() && ByValVA.getValNo() == ValNo && "Additional register location expected for by-value argument."); RegsToPass.push_back(std::make_pair(ByValVA.getLocReg(), ResidueVal)); continue; } CCValAssign &VA = ArgLocs[I++]; const MVT LocVT = VA.getLocVT(); const MVT ValVT = VA.getValVT(); switch (VA.getLocInfo()) { default: report_fatal_error("Unexpected argument extension type."); case CCValAssign::Full: break; case CCValAssign::ZExt: Arg = DAG.getNode(ISD::ZERO_EXTEND, dl, VA.getLocVT(), Arg); break; case CCValAssign::SExt: Arg = DAG.getNode(ISD::SIGN_EXTEND, dl, VA.getLocVT(), Arg); break; } if (VA.isRegLoc() && !VA.needsCustom()) { RegsToPass.push_back(std::make_pair(VA.getLocReg(), Arg)); continue; } if (VA.isMemLoc()) { SDValue PtrOff = DAG.getConstant(VA.getLocMemOffset(), dl, StackPtr.getValueType()); PtrOff = DAG.getNode(ISD::ADD, dl, PtrVT, StackPtr, PtrOff); MemOpChains.push_back( DAG.getStore(Chain, dl, Arg, PtrOff, MachinePointerInfo())); continue; } // Custom handling is used for GPR initializations for vararg float // arguments. assert(VA.isRegLoc() && VA.needsCustom() && CFlags.IsVarArg && ValVT.isFloatingPoint() && LocVT.isInteger() && "Unexpected register handling for calling convention."); SDValue ArgAsInt = DAG.getBitcast(MVT::getIntegerVT(ValVT.getSizeInBits()), Arg); if (Arg.getValueType().getStoreSize() == LocVT.getStoreSize()) // f32 in 32-bit GPR // f64 in 64-bit GPR RegsToPass.push_back(std::make_pair(VA.getLocReg(), ArgAsInt)); else if (Arg.getValueType().getSizeInBits() < LocVT.getSizeInBits()) // f32 in 64-bit GPR. RegsToPass.push_back(std::make_pair( VA.getLocReg(), DAG.getZExtOrTrunc(ArgAsInt, dl, LocVT))); else { // f64 in two 32-bit GPRs // The 2 GPRs are marked custom and expected to be adjacent in ArgLocs. assert(Arg.getValueType() == MVT::f64 && CFlags.IsVarArg && !IsPPC64 && "Unexpected custom register for argument!"); CCValAssign &GPR1 = VA; SDValue MSWAsI64 = DAG.getNode(ISD::SRL, dl, MVT::i64, ArgAsInt, DAG.getConstant(32, dl, MVT::i8)); RegsToPass.push_back(std::make_pair( GPR1.getLocReg(), DAG.getZExtOrTrunc(MSWAsI64, dl, MVT::i32))); if (I != E) { // If only 1 GPR was available, there will only be one custom GPR and // the argument will also pass in memory. CCValAssign &PeekArg = ArgLocs[I]; if (PeekArg.isRegLoc() && PeekArg.getValNo() == PeekArg.getValNo()) { assert(PeekArg.needsCustom() && "A second custom GPR is expected."); CCValAssign &GPR2 = ArgLocs[I++]; RegsToPass.push_back(std::make_pair( GPR2.getLocReg(), DAG.getZExtOrTrunc(ArgAsInt, dl, MVT::i32))); } } } } if (!MemOpChains.empty()) Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, MemOpChains); // For indirect calls, we need to save the TOC base to the stack for // restoration after the call. if (CFlags.IsIndirect) { assert(!CFlags.IsTailCall && "Indirect tail-calls not supported."); const MCRegister TOCBaseReg = Subtarget.getTOCPointerRegister(); const MCRegister StackPtrReg = Subtarget.getStackPointerRegister(); const MVT PtrVT = Subtarget.isPPC64() ? MVT::i64 : MVT::i32; const unsigned TOCSaveOffset = Subtarget.getFrameLowering()->getTOCSaveOffset(); setUsesTOCBasePtr(DAG); SDValue Val = DAG.getCopyFromReg(Chain, dl, TOCBaseReg, PtrVT); SDValue PtrOff = DAG.getIntPtrConstant(TOCSaveOffset, dl); SDValue StackPtr = DAG.getRegister(StackPtrReg, PtrVT); SDValue AddPtr = DAG.getNode(ISD::ADD, dl, PtrVT, StackPtr, PtrOff); Chain = DAG.getStore( Val.getValue(1), dl, Val, AddPtr, MachinePointerInfo::getStack(DAG.getMachineFunction(), TOCSaveOffset)); } // Build a sequence of copy-to-reg nodes chained together with token chain // and flag operands which copy the outgoing args into the appropriate regs. SDValue InFlag; for (auto Reg : RegsToPass) { Chain = DAG.getCopyToReg(Chain, dl, Reg.first, Reg.second, InFlag); InFlag = Chain.getValue(1); } const int SPDiff = 0; return FinishCall(CFlags, dl, DAG, RegsToPass, InFlag, Chain, CallSeqStart, Callee, SPDiff, NumBytes, Ins, InVals, CS); } bool PPCTargetLowering::CanLowerReturn(CallingConv::ID CallConv, MachineFunction &MF, bool isVarArg, const SmallVectorImpl &Outs, LLVMContext &Context) const { SmallVector RVLocs; CCState CCInfo(CallConv, isVarArg, MF, RVLocs, Context); return CCInfo.CheckReturn( Outs, (Subtarget.isSVR4ABI() && CallConv == CallingConv::Cold) ? RetCC_PPC_Cold : RetCC_PPC); } SDValue PPCTargetLowering::LowerReturn(SDValue Chain, CallingConv::ID CallConv, bool isVarArg, const SmallVectorImpl &Outs, const SmallVectorImpl &OutVals, const SDLoc &dl, SelectionDAG &DAG) const { SmallVector RVLocs; CCState CCInfo(CallConv, isVarArg, DAG.getMachineFunction(), RVLocs, *DAG.getContext()); CCInfo.AnalyzeReturn(Outs, (Subtarget.isSVR4ABI() && CallConv == CallingConv::Cold) ? RetCC_PPC_Cold : RetCC_PPC); SDValue Flag; SmallVector RetOps(1, Chain); // Copy the result values into the output registers. for (unsigned i = 0, RealResIdx = 0; i != RVLocs.size(); ++i, ++RealResIdx) { CCValAssign &VA = RVLocs[i]; assert(VA.isRegLoc() && "Can only return in registers!"); SDValue Arg = OutVals[RealResIdx]; switch (VA.getLocInfo()) { default: llvm_unreachable("Unknown loc info!"); case CCValAssign::Full: break; case CCValAssign::AExt: Arg = DAG.getNode(ISD::ANY_EXTEND, dl, VA.getLocVT(), Arg); break; case CCValAssign::ZExt: Arg = DAG.getNode(ISD::ZERO_EXTEND, dl, VA.getLocVT(), Arg); break; case CCValAssign::SExt: Arg = DAG.getNode(ISD::SIGN_EXTEND, dl, VA.getLocVT(), Arg); break; } if (Subtarget.hasSPE() && VA.getLocVT() == MVT::f64) { bool isLittleEndian = Subtarget.isLittleEndian(); // Legalize ret f64 -> ret 2 x i32. SDValue SVal = DAG.getNode(PPCISD::EXTRACT_SPE, dl, MVT::i32, Arg, DAG.getIntPtrConstant(isLittleEndian ? 0 : 1, dl)); Chain = DAG.getCopyToReg(Chain, dl, VA.getLocReg(), SVal, Flag); RetOps.push_back(DAG.getRegister(VA.getLocReg(), VA.getLocVT())); SVal = DAG.getNode(PPCISD::EXTRACT_SPE, dl, MVT::i32, Arg, DAG.getIntPtrConstant(isLittleEndian ? 1 : 0, dl)); Flag = Chain.getValue(1); VA = RVLocs[++i]; // skip ahead to next loc Chain = DAG.getCopyToReg(Chain, dl, VA.getLocReg(), SVal, Flag); } else Chain = DAG.getCopyToReg(Chain, dl, VA.getLocReg(), Arg, Flag); Flag = Chain.getValue(1); RetOps.push_back(DAG.getRegister(VA.getLocReg(), VA.getLocVT())); } const PPCRegisterInfo *TRI = Subtarget.getRegisterInfo(); const MCPhysReg *I = TRI->getCalleeSavedRegsViaCopy(&DAG.getMachineFunction()); if (I) { for (; *I; ++I) { if (PPC::G8RCRegClass.contains(*I)) RetOps.push_back(DAG.getRegister(*I, MVT::i64)); else if (PPC::F8RCRegClass.contains(*I)) RetOps.push_back(DAG.getRegister(*I, MVT::getFloatingPointVT(64))); else if (PPC::CRRCRegClass.contains(*I)) RetOps.push_back(DAG.getRegister(*I, MVT::i1)); else if (PPC::VRRCRegClass.contains(*I)) RetOps.push_back(DAG.getRegister(*I, MVT::Other)); else llvm_unreachable("Unexpected register class in CSRsViaCopy!"); } } RetOps[0] = Chain; // Update chain. // Add the flag if we have it. if (Flag.getNode()) RetOps.push_back(Flag); return DAG.getNode(PPCISD::RET_FLAG, dl, MVT::Other, RetOps); } SDValue PPCTargetLowering::LowerGET_DYNAMIC_AREA_OFFSET(SDValue Op, SelectionDAG &DAG) const { SDLoc dl(Op); // Get the correct type for integers. EVT IntVT = Op.getValueType(); // Get the inputs. SDValue Chain = Op.getOperand(0); SDValue FPSIdx = getFramePointerFrameIndex(DAG); // Build a DYNAREAOFFSET node. SDValue Ops[2] = {Chain, FPSIdx}; SDVTList VTs = DAG.getVTList(IntVT); return DAG.getNode(PPCISD::DYNAREAOFFSET, dl, VTs, Ops); } SDValue PPCTargetLowering::LowerSTACKRESTORE(SDValue Op, SelectionDAG &DAG) const { // When we pop the dynamic allocation we need to restore the SP link. SDLoc dl(Op); // Get the correct type for pointers. EVT PtrVT = getPointerTy(DAG.getDataLayout()); // Construct the stack pointer operand. bool isPPC64 = Subtarget.isPPC64(); unsigned SP = isPPC64 ? PPC::X1 : PPC::R1; SDValue StackPtr = DAG.getRegister(SP, PtrVT); // Get the operands for the STACKRESTORE. SDValue Chain = Op.getOperand(0); SDValue SaveSP = Op.getOperand(1); // Load the old link SP. SDValue LoadLinkSP = DAG.getLoad(PtrVT, dl, Chain, StackPtr, MachinePointerInfo()); // Restore the stack pointer. Chain = DAG.getCopyToReg(LoadLinkSP.getValue(1), dl, SP, SaveSP); // Store the old link SP. return DAG.getStore(Chain, dl, LoadLinkSP, StackPtr, MachinePointerInfo()); } SDValue PPCTargetLowering::getReturnAddrFrameIndex(SelectionDAG &DAG) const { MachineFunction &MF = DAG.getMachineFunction(); bool isPPC64 = Subtarget.isPPC64(); EVT PtrVT = getPointerTy(MF.getDataLayout()); // Get current frame pointer save index. The users of this index will be // primarily DYNALLOC instructions. PPCFunctionInfo *FI = MF.getInfo(); int RASI = FI->getReturnAddrSaveIndex(); // If the frame pointer save index hasn't been defined yet. if (!RASI) { // Find out what the fix offset of the frame pointer save area. int LROffset = Subtarget.getFrameLowering()->getReturnSaveOffset(); // Allocate the frame index for frame pointer save area. RASI = MF.getFrameInfo().CreateFixedObject(isPPC64? 8 : 4, LROffset, false); // Save the result. FI->setReturnAddrSaveIndex(RASI); } return DAG.getFrameIndex(RASI, PtrVT); } SDValue PPCTargetLowering::getFramePointerFrameIndex(SelectionDAG & DAG) const { MachineFunction &MF = DAG.getMachineFunction(); bool isPPC64 = Subtarget.isPPC64(); EVT PtrVT = getPointerTy(MF.getDataLayout()); // Get current frame pointer save index. The users of this index will be // primarily DYNALLOC instructions. PPCFunctionInfo *FI = MF.getInfo(); int FPSI = FI->getFramePointerSaveIndex(); // If the frame pointer save index hasn't been defined yet. if (!FPSI) { // Find out what the fix offset of the frame pointer save area. int FPOffset = Subtarget.getFrameLowering()->getFramePointerSaveOffset(); // Allocate the frame index for frame pointer save area. FPSI = MF.getFrameInfo().CreateFixedObject(isPPC64? 8 : 4, FPOffset, true); // Save the result. FI->setFramePointerSaveIndex(FPSI); } return DAG.getFrameIndex(FPSI, PtrVT); } SDValue PPCTargetLowering::LowerDYNAMIC_STACKALLOC(SDValue Op, SelectionDAG &DAG) const { // Get the inputs. SDValue Chain = Op.getOperand(0); SDValue Size = Op.getOperand(1); SDLoc dl(Op); // Get the correct type for pointers. EVT PtrVT = getPointerTy(DAG.getDataLayout()); // Negate the size. SDValue NegSize = DAG.getNode(ISD::SUB, dl, PtrVT, DAG.getConstant(0, dl, PtrVT), Size); // Construct a node for the frame pointer save index. SDValue FPSIdx = getFramePointerFrameIndex(DAG); // Build a DYNALLOC node. SDValue Ops[3] = { Chain, NegSize, FPSIdx }; SDVTList VTs = DAG.getVTList(PtrVT, MVT::Other); return DAG.getNode(PPCISD::DYNALLOC, dl, VTs, Ops); } SDValue PPCTargetLowering::LowerEH_DWARF_CFA(SDValue Op, SelectionDAG &DAG) const { MachineFunction &MF = DAG.getMachineFunction(); bool isPPC64 = Subtarget.isPPC64(); EVT PtrVT = getPointerTy(DAG.getDataLayout()); int FI = MF.getFrameInfo().CreateFixedObject(isPPC64 ? 8 : 4, 0, false); return DAG.getFrameIndex(FI, PtrVT); } SDValue PPCTargetLowering::lowerEH_SJLJ_SETJMP(SDValue Op, SelectionDAG &DAG) const { SDLoc DL(Op); return DAG.getNode(PPCISD::EH_SJLJ_SETJMP, DL, DAG.getVTList(MVT::i32, MVT::Other), Op.getOperand(0), Op.getOperand(1)); } SDValue PPCTargetLowering::lowerEH_SJLJ_LONGJMP(SDValue Op, SelectionDAG &DAG) const { SDLoc DL(Op); return DAG.getNode(PPCISD::EH_SJLJ_LONGJMP, DL, MVT::Other, Op.getOperand(0), Op.getOperand(1)); } SDValue PPCTargetLowering::LowerLOAD(SDValue Op, SelectionDAG &DAG) const { if (Op.getValueType().isVector()) return LowerVectorLoad(Op, DAG); assert(Op.getValueType() == MVT::i1 && "Custom lowering only for i1 loads"); // First, load 8 bits into 32 bits, then truncate to 1 bit. SDLoc dl(Op); LoadSDNode *LD = cast(Op); SDValue Chain = LD->getChain(); SDValue BasePtr = LD->getBasePtr(); MachineMemOperand *MMO = LD->getMemOperand(); SDValue NewLD = DAG.getExtLoad(ISD::EXTLOAD, dl, getPointerTy(DAG.getDataLayout()), Chain, BasePtr, MVT::i8, MMO); SDValue Result = DAG.getNode(ISD::TRUNCATE, dl, MVT::i1, NewLD); SDValue Ops[] = { Result, SDValue(NewLD.getNode(), 1) }; return DAG.getMergeValues(Ops, dl); } SDValue PPCTargetLowering::LowerSTORE(SDValue Op, SelectionDAG &DAG) const { if (Op.getOperand(1).getValueType().isVector()) return LowerVectorStore(Op, DAG); assert(Op.getOperand(1).getValueType() == MVT::i1 && "Custom lowering only for i1 stores"); // First, zero extend to 32 bits, then use a truncating store to 8 bits. SDLoc dl(Op); StoreSDNode *ST = cast(Op); SDValue Chain = ST->getChain(); SDValue BasePtr = ST->getBasePtr(); SDValue Value = ST->getValue(); MachineMemOperand *MMO = ST->getMemOperand(); Value = DAG.getNode(ISD::ZERO_EXTEND, dl, getPointerTy(DAG.getDataLayout()), Value); return DAG.getTruncStore(Chain, dl, Value, BasePtr, MVT::i8, MMO); } // FIXME: Remove this once the ANDI glue bug is fixed: SDValue PPCTargetLowering::LowerTRUNCATE(SDValue Op, SelectionDAG &DAG) const { assert(Op.getValueType() == MVT::i1 && "Custom lowering only for i1 results"); SDLoc DL(Op); return DAG.getNode(PPCISD::ANDI_rec_1_GT_BIT, DL, MVT::i1, Op.getOperand(0)); } SDValue PPCTargetLowering::LowerTRUNCATEVector(SDValue Op, SelectionDAG &DAG) const { // Implements a vector truncate that fits in a vector register as a shuffle. // We want to legalize vector truncates down to where the source fits in // a vector register (and target is therefore smaller than vector register // size). At that point legalization will try to custom lower the sub-legal // result and get here - where we can contain the truncate as a single target // operation. // For example a trunc <2 x i16> to <2 x i8> could be visualized as follows: // to // // We will implement it for big-endian ordering as this (where x denotes // undefined): // < MSB1|LSB1, MSB2|LSB2, uu, uu, uu, uu, uu, uu> to // < LSB1, LSB2, u, u, u, u, u, u, u, u, u, u, u, u, u, u> // // The same operation in little-endian ordering will be: // to // assert(Op.getValueType().isVector() && "Vector type expected."); SDLoc DL(Op); SDValue N1 = Op.getOperand(0); unsigned SrcSize = N1.getValueType().getSizeInBits(); assert(SrcSize <= 128 && "Source must fit in an Altivec/VSX vector"); SDValue WideSrc = SrcSize == 128 ? N1 : widenVec(DAG, N1, DL); EVT TrgVT = Op.getValueType(); unsigned TrgNumElts = TrgVT.getVectorNumElements(); EVT EltVT = TrgVT.getVectorElementType(); unsigned WideNumElts = 128 / EltVT.getSizeInBits(); EVT WideVT = EVT::getVectorVT(*DAG.getContext(), EltVT, WideNumElts); // First list the elements we want to keep. unsigned SizeMult = SrcSize / TrgVT.getSizeInBits(); SmallVector ShuffV; if (Subtarget.isLittleEndian()) for (unsigned i = 0; i < TrgNumElts; ++i) ShuffV.push_back(i * SizeMult); else for (unsigned i = 1; i <= TrgNumElts; ++i) ShuffV.push_back(i * SizeMult - 1); // Populate the remaining elements with undefs. for (unsigned i = TrgNumElts; i < WideNumElts; ++i) // ShuffV.push_back(i + WideNumElts); ShuffV.push_back(WideNumElts + 1); SDValue Conv = DAG.getNode(ISD::BITCAST, DL, WideVT, WideSrc); return DAG.getVectorShuffle(WideVT, DL, Conv, DAG.getUNDEF(WideVT), ShuffV); } /// LowerSELECT_CC - Lower floating point select_cc's into fsel instruction when /// possible. SDValue PPCTargetLowering::LowerSELECT_CC(SDValue Op, SelectionDAG &DAG) const { // Not FP? Not a fsel. if (!Op.getOperand(0).getValueType().isFloatingPoint() || !Op.getOperand(2).getValueType().isFloatingPoint()) return Op; ISD::CondCode CC = cast(Op.getOperand(4))->get(); EVT ResVT = Op.getValueType(); EVT CmpVT = Op.getOperand(0).getValueType(); SDValue LHS = Op.getOperand(0), RHS = Op.getOperand(1); SDValue TV = Op.getOperand(2), FV = Op.getOperand(3); SDLoc dl(Op); // We have xsmaxcdp/xsmincdp which are OK to emit even in the // presence of infinities. if (Subtarget.hasP9Vector() && LHS == TV && RHS == FV) { switch (CC) { default: break; case ISD::SETOGT: case ISD::SETGT: return DAG.getNode(PPCISD::XSMAXCDP, dl, Op.getValueType(), LHS, RHS); case ISD::SETOLT: case ISD::SETLT: return DAG.getNode(PPCISD::XSMINCDP, dl, Op.getValueType(), LHS, RHS); } } // We might be able to do better than this under some circumstances, but in // general, fsel-based lowering of select is a finite-math-only optimization. // For more information, see section F.3 of the 2.06 ISA specification. // With ISA 3.0 if (!DAG.getTarget().Options.NoInfsFPMath || !DAG.getTarget().Options.NoNaNsFPMath) return Op; // TODO: Propagate flags from the select rather than global settings. SDNodeFlags Flags; Flags.setNoInfs(true); Flags.setNoNaNs(true); // If the RHS of the comparison is a 0.0, we don't need to do the // subtraction at all. SDValue Sel1; if (isFloatingPointZero(RHS)) switch (CC) { default: break; // SETUO etc aren't handled by fsel. case ISD::SETNE: std::swap(TV, FV); LLVM_FALLTHROUGH; case ISD::SETEQ: if (LHS.getValueType() == MVT::f32) // Comparison is always 64-bits LHS = DAG.getNode(ISD::FP_EXTEND, dl, MVT::f64, LHS); Sel1 = DAG.getNode(PPCISD::FSEL, dl, ResVT, LHS, TV, FV); if (Sel1.getValueType() == MVT::f32) // Comparison is always 64-bits Sel1 = DAG.getNode(ISD::FP_EXTEND, dl, MVT::f64, Sel1); return DAG.getNode(PPCISD::FSEL, dl, ResVT, DAG.getNode(ISD::FNEG, dl, MVT::f64, LHS), Sel1, FV); case ISD::SETULT: case ISD::SETLT: std::swap(TV, FV); // fsel is natively setge, swap operands for setlt LLVM_FALLTHROUGH; case ISD::SETOGE: case ISD::SETGE: if (LHS.getValueType() == MVT::f32) // Comparison is always 64-bits LHS = DAG.getNode(ISD::FP_EXTEND, dl, MVT::f64, LHS); return DAG.getNode(PPCISD::FSEL, dl, ResVT, LHS, TV, FV); case ISD::SETUGT: case ISD::SETGT: std::swap(TV, FV); // fsel is natively setge, swap operands for setlt LLVM_FALLTHROUGH; case ISD::SETOLE: case ISD::SETLE: if (LHS.getValueType() == MVT::f32) // Comparison is always 64-bits LHS = DAG.getNode(ISD::FP_EXTEND, dl, MVT::f64, LHS); return DAG.getNode(PPCISD::FSEL, dl, ResVT, DAG.getNode(ISD::FNEG, dl, MVT::f64, LHS), TV, FV); } SDValue Cmp; switch (CC) { default: break; // SETUO etc aren't handled by fsel. case ISD::SETNE: std::swap(TV, FV); LLVM_FALLTHROUGH; case ISD::SETEQ: Cmp = DAG.getNode(ISD::FSUB, dl, CmpVT, LHS, RHS, Flags); if (Cmp.getValueType() == MVT::f32) // Comparison is always 64-bits Cmp = DAG.getNode(ISD::FP_EXTEND, dl, MVT::f64, Cmp); Sel1 = DAG.getNode(PPCISD::FSEL, dl, ResVT, Cmp, TV, FV); if (Sel1.getValueType() == MVT::f32) // Comparison is always 64-bits Sel1 = DAG.getNode(ISD::FP_EXTEND, dl, MVT::f64, Sel1); return DAG.getNode(PPCISD::FSEL, dl, ResVT, DAG.getNode(ISD::FNEG, dl, MVT::f64, Cmp), Sel1, FV); case ISD::SETULT: case ISD::SETLT: Cmp = DAG.getNode(ISD::FSUB, dl, CmpVT, LHS, RHS, Flags); if (Cmp.getValueType() == MVT::f32) // Comparison is always 64-bits Cmp = DAG.getNode(ISD::FP_EXTEND, dl, MVT::f64, Cmp); return DAG.getNode(PPCISD::FSEL, dl, ResVT, Cmp, FV, TV); case ISD::SETOGE: case ISD::SETGE: Cmp = DAG.getNode(ISD::FSUB, dl, CmpVT, LHS, RHS, Flags); if (Cmp.getValueType() == MVT::f32) // Comparison is always 64-bits Cmp = DAG.getNode(ISD::FP_EXTEND, dl, MVT::f64, Cmp); return DAG.getNode(PPCISD::FSEL, dl, ResVT, Cmp, TV, FV); case ISD::SETUGT: case ISD::SETGT: Cmp = DAG.getNode(ISD::FSUB, dl, CmpVT, RHS, LHS, Flags); if (Cmp.getValueType() == MVT::f32) // Comparison is always 64-bits Cmp = DAG.getNode(ISD::FP_EXTEND, dl, MVT::f64, Cmp); return DAG.getNode(PPCISD::FSEL, dl, ResVT, Cmp, FV, TV); case ISD::SETOLE: case ISD::SETLE: Cmp = DAG.getNode(ISD::FSUB, dl, CmpVT, RHS, LHS, Flags); if (Cmp.getValueType() == MVT::f32) // Comparison is always 64-bits Cmp = DAG.getNode(ISD::FP_EXTEND, dl, MVT::f64, Cmp); return DAG.getNode(PPCISD::FSEL, dl, ResVT, Cmp, TV, FV); } return Op; } void PPCTargetLowering::LowerFP_TO_INTForReuse(SDValue Op, ReuseLoadInfo &RLI, SelectionDAG &DAG, const SDLoc &dl) const { assert(Op.getOperand(0).getValueType().isFloatingPoint()); SDValue Src = Op.getOperand(0); if (Src.getValueType() == MVT::f32) Src = DAG.getNode(ISD::FP_EXTEND, dl, MVT::f64, Src); SDValue Tmp; switch (Op.getSimpleValueType().SimpleTy) { default: llvm_unreachable("Unhandled FP_TO_INT type in custom expander!"); case MVT::i32: Tmp = DAG.getNode( Op.getOpcode() == ISD::FP_TO_SINT ? PPCISD::FCTIWZ : (Subtarget.hasFPCVT() ? PPCISD::FCTIWUZ : PPCISD::FCTIDZ), dl, MVT::f64, Src); break; case MVT::i64: assert((Op.getOpcode() == ISD::FP_TO_SINT || Subtarget.hasFPCVT()) && "i64 FP_TO_UINT is supported only with FPCVT"); Tmp = DAG.getNode(Op.getOpcode()==ISD::FP_TO_SINT ? PPCISD::FCTIDZ : PPCISD::FCTIDUZ, dl, MVT::f64, Src); break; } // Convert the FP value to an int value through memory. bool i32Stack = Op.getValueType() == MVT::i32 && Subtarget.hasSTFIWX() && (Op.getOpcode() == ISD::FP_TO_SINT || Subtarget.hasFPCVT()); SDValue FIPtr = DAG.CreateStackTemporary(i32Stack ? MVT::i32 : MVT::f64); int FI = cast(FIPtr)->getIndex(); MachinePointerInfo MPI = MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FI); // Emit a store to the stack slot. SDValue Chain; if (i32Stack) { MachineFunction &MF = DAG.getMachineFunction(); MachineMemOperand *MMO = MF.getMachineMemOperand(MPI, MachineMemOperand::MOStore, 4, Align(4)); SDValue Ops[] = { DAG.getEntryNode(), Tmp, FIPtr }; Chain = DAG.getMemIntrinsicNode(PPCISD::STFIWX, dl, DAG.getVTList(MVT::Other), Ops, MVT::i32, MMO); } else Chain = DAG.getStore(DAG.getEntryNode(), dl, Tmp, FIPtr, MPI); // Result is a load from the stack slot. If loading 4 bytes, make sure to // add in a bias on big endian. if (Op.getValueType() == MVT::i32 && !i32Stack) { FIPtr = DAG.getNode(ISD::ADD, dl, FIPtr.getValueType(), FIPtr, DAG.getConstant(4, dl, FIPtr.getValueType())); MPI = MPI.getWithOffset(Subtarget.isLittleEndian() ? 0 : 4); } RLI.Chain = Chain; RLI.Ptr = FIPtr; RLI.MPI = MPI; } /// Custom lowers floating point to integer conversions to use /// the direct move instructions available in ISA 2.07 to avoid the /// need for load/store combinations. SDValue PPCTargetLowering::LowerFP_TO_INTDirectMove(SDValue Op, SelectionDAG &DAG, const SDLoc &dl) const { assert(Op.getOperand(0).getValueType().isFloatingPoint()); SDValue Src = Op.getOperand(0); if (Src.getValueType() == MVT::f32) Src = DAG.getNode(ISD::FP_EXTEND, dl, MVT::f64, Src); SDValue Tmp; switch (Op.getSimpleValueType().SimpleTy) { default: llvm_unreachable("Unhandled FP_TO_INT type in custom expander!"); case MVT::i32: Tmp = DAG.getNode( Op.getOpcode() == ISD::FP_TO_SINT ? PPCISD::FCTIWZ : (Subtarget.hasFPCVT() ? PPCISD::FCTIWUZ : PPCISD::FCTIDZ), dl, MVT::f64, Src); Tmp = DAG.getNode(PPCISD::MFVSR, dl, MVT::i32, Tmp); break; case MVT::i64: assert((Op.getOpcode() == ISD::FP_TO_SINT || Subtarget.hasFPCVT()) && "i64 FP_TO_UINT is supported only with FPCVT"); Tmp = DAG.getNode(Op.getOpcode()==ISD::FP_TO_SINT ? PPCISD::FCTIDZ : PPCISD::FCTIDUZ, dl, MVT::f64, Src); Tmp = DAG.getNode(PPCISD::MFVSR, dl, MVT::i64, Tmp); break; } return Tmp; } SDValue PPCTargetLowering::LowerFP_TO_INT(SDValue Op, SelectionDAG &DAG, const SDLoc &dl) const { // FP to INT conversions are legal for f128. if (EnableQuadPrecision && (Op->getOperand(0).getValueType() == MVT::f128)) return Op; // Expand ppcf128 to i32 by hand for the benefit of llvm-gcc bootstrap on // PPC (the libcall is not available). if (Op.getOperand(0).getValueType() == MVT::ppcf128) { if (Op.getValueType() == MVT::i32) { if (Op.getOpcode() == ISD::FP_TO_SINT) { SDValue Lo = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, MVT::f64, Op.getOperand(0), DAG.getIntPtrConstant(0, dl)); SDValue Hi = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, MVT::f64, Op.getOperand(0), DAG.getIntPtrConstant(1, dl)); // Add the two halves of the long double in round-to-zero mode. SDValue Res = DAG.getNode(PPCISD::FADDRTZ, dl, MVT::f64, Lo, Hi); // Now use a smaller FP_TO_SINT. return DAG.getNode(ISD::FP_TO_SINT, dl, MVT::i32, Res); } if (Op.getOpcode() == ISD::FP_TO_UINT) { const uint64_t TwoE31[] = {0x41e0000000000000LL, 0}; APFloat APF = APFloat(APFloat::PPCDoubleDouble(), APInt(128, TwoE31)); SDValue Tmp = DAG.getConstantFP(APF, dl, MVT::ppcf128); // X>=2^31 ? (int)(X-2^31)+0x80000000 : (int)X // FIXME: generated code sucks. // TODO: Are there fast-math-flags to propagate to this FSUB? SDValue True = DAG.getNode(ISD::FSUB, dl, MVT::ppcf128, Op.getOperand(0), Tmp); True = DAG.getNode(ISD::FP_TO_SINT, dl, MVT::i32, True); True = DAG.getNode(ISD::ADD, dl, MVT::i32, True, DAG.getConstant(0x80000000, dl, MVT::i32)); SDValue False = DAG.getNode(ISD::FP_TO_SINT, dl, MVT::i32, Op.getOperand(0)); return DAG.getSelectCC(dl, Op.getOperand(0), Tmp, True, False, ISD::SETGE); } } return SDValue(); } if (Subtarget.hasDirectMove() && Subtarget.isPPC64()) return LowerFP_TO_INTDirectMove(Op, DAG, dl); ReuseLoadInfo RLI; LowerFP_TO_INTForReuse(Op, RLI, DAG, dl); return DAG.getLoad(Op.getValueType(), dl, RLI.Chain, RLI.Ptr, RLI.MPI, RLI.Alignment, RLI.MMOFlags(), RLI.AAInfo, RLI.Ranges); } // We're trying to insert a regular store, S, and then a load, L. If the // incoming value, O, is a load, we might just be able to have our load use the // address used by O. However, we don't know if anything else will store to // that address before we can load from it. To prevent this situation, we need // to insert our load, L, into the chain as a peer of O. To do this, we give L // the same chain operand as O, we create a token factor from the chain results // of O and L, and we replace all uses of O's chain result with that token // factor (see spliceIntoChain below for this last part). bool PPCTargetLowering::canReuseLoadAddress(SDValue Op, EVT MemVT, ReuseLoadInfo &RLI, SelectionDAG &DAG, ISD::LoadExtType ET) const { SDLoc dl(Op); if (ET == ISD::NON_EXTLOAD && (Op.getOpcode() == ISD::FP_TO_UINT || Op.getOpcode() == ISD::FP_TO_SINT) && isOperationLegalOrCustom(Op.getOpcode(), Op.getOperand(0).getValueType())) { LowerFP_TO_INTForReuse(Op, RLI, DAG, dl); return true; } LoadSDNode *LD = dyn_cast(Op); if (!LD || LD->getExtensionType() != ET || LD->isVolatile() || LD->isNonTemporal()) return false; if (LD->getMemoryVT() != MemVT) return false; RLI.Ptr = LD->getBasePtr(); if (LD->isIndexed() && !LD->getOffset().isUndef()) { assert(LD->getAddressingMode() == ISD::PRE_INC && "Non-pre-inc AM on PPC?"); RLI.Ptr = DAG.getNode(ISD::ADD, dl, RLI.Ptr.getValueType(), RLI.Ptr, LD->getOffset()); } RLI.Chain = LD->getChain(); RLI.MPI = LD->getPointerInfo(); RLI.IsDereferenceable = LD->isDereferenceable(); RLI.IsInvariant = LD->isInvariant(); RLI.Alignment = LD->getAlign(); RLI.AAInfo = LD->getAAInfo(); RLI.Ranges = LD->getRanges(); RLI.ResChain = SDValue(LD, LD->isIndexed() ? 2 : 1); return true; } // Given the head of the old chain, ResChain, insert a token factor containing // it and NewResChain, and make users of ResChain now be users of that token // factor. // TODO: Remove and use DAG::makeEquivalentMemoryOrdering() instead. void PPCTargetLowering::spliceIntoChain(SDValue ResChain, SDValue NewResChain, SelectionDAG &DAG) const { if (!ResChain) return; SDLoc dl(NewResChain); SDValue TF = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, NewResChain, DAG.getUNDEF(MVT::Other)); assert(TF.getNode() != NewResChain.getNode() && "A new TF really is required here"); DAG.ReplaceAllUsesOfValueWith(ResChain, TF); DAG.UpdateNodeOperands(TF.getNode(), ResChain, NewResChain); } /// Analyze profitability of direct move /// prefer float load to int load plus direct move /// when there is no integer use of int load bool PPCTargetLowering::directMoveIsProfitable(const SDValue &Op) const { SDNode *Origin = Op.getOperand(0).getNode(); if (Origin->getOpcode() != ISD::LOAD) return true; // If there is no LXSIBZX/LXSIHZX, like Power8, // prefer direct move if the memory size is 1 or 2 bytes. MachineMemOperand *MMO = cast(Origin)->getMemOperand(); if (!Subtarget.hasP9Vector() && MMO->getSize() <= 2) return true; for (SDNode::use_iterator UI = Origin->use_begin(), UE = Origin->use_end(); UI != UE; ++UI) { // Only look at the users of the loaded value. if (UI.getUse().get().getResNo() != 0) continue; if (UI->getOpcode() != ISD::SINT_TO_FP && UI->getOpcode() != ISD::UINT_TO_FP) return true; } return false; } /// Custom lowers integer to floating point conversions to use /// the direct move instructions available in ISA 2.07 to avoid the /// need for load/store combinations. SDValue PPCTargetLowering::LowerINT_TO_FPDirectMove(SDValue Op, SelectionDAG &DAG, const SDLoc &dl) const { assert((Op.getValueType() == MVT::f32 || Op.getValueType() == MVT::f64) && "Invalid floating point type as target of conversion"); assert(Subtarget.hasFPCVT() && "Int to FP conversions with direct moves require FPCVT"); SDValue FP; SDValue Src = Op.getOperand(0); bool SinglePrec = Op.getValueType() == MVT::f32; bool WordInt = Src.getSimpleValueType().SimpleTy == MVT::i32; bool Signed = Op.getOpcode() == ISD::SINT_TO_FP; unsigned ConvOp = Signed ? (SinglePrec ? PPCISD::FCFIDS : PPCISD::FCFID) : (SinglePrec ? PPCISD::FCFIDUS : PPCISD::FCFIDU); if (WordInt) { FP = DAG.getNode(Signed ? PPCISD::MTVSRA : PPCISD::MTVSRZ, dl, MVT::f64, Src); FP = DAG.getNode(ConvOp, dl, SinglePrec ? MVT::f32 : MVT::f64, FP); } else { FP = DAG.getNode(PPCISD::MTVSRA, dl, MVT::f64, Src); FP = DAG.getNode(ConvOp, dl, SinglePrec ? MVT::f32 : MVT::f64, FP); } return FP; } static SDValue widenVec(SelectionDAG &DAG, SDValue Vec, const SDLoc &dl) { EVT VecVT = Vec.getValueType(); assert(VecVT.isVector() && "Expected a vector type."); assert(VecVT.getSizeInBits() < 128 && "Vector is already full width."); EVT EltVT = VecVT.getVectorElementType(); unsigned WideNumElts = 128 / EltVT.getSizeInBits(); EVT WideVT = EVT::getVectorVT(*DAG.getContext(), EltVT, WideNumElts); unsigned NumConcat = WideNumElts / VecVT.getVectorNumElements(); SmallVector Ops(NumConcat); Ops[0] = Vec; SDValue UndefVec = DAG.getUNDEF(VecVT); for (unsigned i = 1; i < NumConcat; ++i) Ops[i] = UndefVec; return DAG.getNode(ISD::CONCAT_VECTORS, dl, WideVT, Ops); } SDValue PPCTargetLowering::LowerINT_TO_FPVector(SDValue Op, SelectionDAG &DAG, const SDLoc &dl) const { unsigned Opc = Op.getOpcode(); assert((Opc == ISD::UINT_TO_FP || Opc == ISD::SINT_TO_FP) && "Unexpected conversion type"); assert((Op.getValueType() == MVT::v2f64 || Op.getValueType() == MVT::v4f32) && "Supports conversions to v2f64/v4f32 only."); bool SignedConv = Opc == ISD::SINT_TO_FP; bool FourEltRes = Op.getValueType() == MVT::v4f32; SDValue Wide = widenVec(DAG, Op.getOperand(0), dl); EVT WideVT = Wide.getValueType(); unsigned WideNumElts = WideVT.getVectorNumElements(); MVT IntermediateVT = FourEltRes ? MVT::v4i32 : MVT::v2i64; SmallVector ShuffV; for (unsigned i = 0; i < WideNumElts; ++i) ShuffV.push_back(i + WideNumElts); int Stride = FourEltRes ? WideNumElts / 4 : WideNumElts / 2; int SaveElts = FourEltRes ? 4 : 2; if (Subtarget.isLittleEndian()) for (int i = 0; i < SaveElts; i++) ShuffV[i * Stride] = i; else for (int i = 1; i <= SaveElts; i++) ShuffV[i * Stride - 1] = i - 1; SDValue ShuffleSrc2 = SignedConv ? DAG.getUNDEF(WideVT) : DAG.getConstant(0, dl, WideVT); SDValue Arrange = DAG.getVectorShuffle(WideVT, dl, Wide, ShuffleSrc2, ShuffV); SDValue Extend; if (SignedConv) { Arrange = DAG.getBitcast(IntermediateVT, Arrange); EVT ExtVT = Op.getOperand(0).getValueType(); if (Subtarget.hasP9Altivec()) ExtVT = EVT::getVectorVT(*DAG.getContext(), WideVT.getVectorElementType(), IntermediateVT.getVectorNumElements()); Extend = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, IntermediateVT, Arrange, DAG.getValueType(ExtVT)); } else Extend = DAG.getNode(ISD::BITCAST, dl, IntermediateVT, Arrange); return DAG.getNode(Opc, dl, Op.getValueType(), Extend); } SDValue PPCTargetLowering::LowerINT_TO_FP(SDValue Op, SelectionDAG &DAG) const { SDLoc dl(Op); EVT InVT = Op.getOperand(0).getValueType(); EVT OutVT = Op.getValueType(); if (OutVT.isVector() && OutVT.isFloatingPoint() && isOperationCustom(Op.getOpcode(), InVT)) return LowerINT_TO_FPVector(Op, DAG, dl); // Conversions to f128 are legal. if (EnableQuadPrecision && (Op.getValueType() == MVT::f128)) return Op; if (Subtarget.hasQPX() && Op.getOperand(0).getValueType() == MVT::v4i1) { if (Op.getValueType() != MVT::v4f32 && Op.getValueType() != MVT::v4f64) return SDValue(); SDValue Value = Op.getOperand(0); // The values are now known to be -1 (false) or 1 (true). To convert this // into 0 (false) and 1 (true), add 1 and then divide by 2 (multiply by 0.5). // This can be done with an fma and the 0.5 constant: (V+1.0)*0.5 = 0.5*V+0.5 Value = DAG.getNode(PPCISD::QBFLT, dl, MVT::v4f64, Value); SDValue FPHalfs = DAG.getConstantFP(0.5, dl, MVT::v4f64); Value = DAG.getNode(ISD::FMA, dl, MVT::v4f64, Value, FPHalfs, FPHalfs); if (Op.getValueType() != MVT::v4f64) Value = DAG.getNode(ISD::FP_ROUND, dl, Op.getValueType(), Value, DAG.getIntPtrConstant(1, dl)); return Value; } // Don't handle ppc_fp128 here; let it be lowered to a libcall. if (Op.getValueType() != MVT::f32 && Op.getValueType() != MVT::f64) return SDValue(); if (Op.getOperand(0).getValueType() == MVT::i1) return DAG.getNode(ISD::SELECT, dl, Op.getValueType(), Op.getOperand(0), DAG.getConstantFP(1.0, dl, Op.getValueType()), DAG.getConstantFP(0.0, dl, Op.getValueType())); // If we have direct moves, we can do all the conversion, skip the store/load // however, without FPCVT we can't do most conversions. if (Subtarget.hasDirectMove() && directMoveIsProfitable(Op) && Subtarget.isPPC64() && Subtarget.hasFPCVT()) return LowerINT_TO_FPDirectMove(Op, DAG, dl); assert((Op.getOpcode() == ISD::SINT_TO_FP || Subtarget.hasFPCVT()) && "UINT_TO_FP is supported only with FPCVT"); // If we have FCFIDS, then use it when converting to single-precision. // Otherwise, convert to double-precision and then round. unsigned FCFOp = (Subtarget.hasFPCVT() && Op.getValueType() == MVT::f32) ? (Op.getOpcode() == ISD::UINT_TO_FP ? PPCISD::FCFIDUS : PPCISD::FCFIDS) : (Op.getOpcode() == ISD::UINT_TO_FP ? PPCISD::FCFIDU : PPCISD::FCFID); MVT FCFTy = (Subtarget.hasFPCVT() && Op.getValueType() == MVT::f32) ? MVT::f32 : MVT::f64; if (Op.getOperand(0).getValueType() == MVT::i64) { SDValue SINT = Op.getOperand(0); // When converting to single-precision, we actually need to convert // to double-precision first and then round to single-precision. // To avoid double-rounding effects during that operation, we have // to prepare the input operand. Bits that might be truncated when // converting to double-precision are replaced by a bit that won't // be lost at this stage, but is below the single-precision rounding // position. // // However, if -enable-unsafe-fp-math is in effect, accept double // rounding to avoid the extra overhead. if (Op.getValueType() == MVT::f32 && !Subtarget.hasFPCVT() && !DAG.getTarget().Options.UnsafeFPMath) { // Twiddle input to make sure the low 11 bits are zero. (If this // is the case, we are guaranteed the value will fit into the 53 bit // mantissa of an IEEE double-precision value without rounding.) // If any of those low 11 bits were not zero originally, make sure // bit 12 (value 2048) is set instead, so that the final rounding // to single-precision gets the correct result. SDValue Round = DAG.getNode(ISD::AND, dl, MVT::i64, SINT, DAG.getConstant(2047, dl, MVT::i64)); Round = DAG.getNode(ISD::ADD, dl, MVT::i64, Round, DAG.getConstant(2047, dl, MVT::i64)); Round = DAG.getNode(ISD::OR, dl, MVT::i64, Round, SINT); Round = DAG.getNode(ISD::AND, dl, MVT::i64, Round, DAG.getConstant(-2048, dl, MVT::i64)); // However, we cannot use that value unconditionally: if the magnitude // of the input value is small, the bit-twiddling we did above might // end up visibly changing the output. Fortunately, in that case, we // don't need to twiddle bits since the original input will convert // exactly to double-precision floating-point already. Therefore, // construct a conditional to use the original value if the top 11 // bits are all sign-bit copies, and use the rounded value computed // above otherwise. SDValue Cond = DAG.getNode(ISD::SRA, dl, MVT::i64, SINT, DAG.getConstant(53, dl, MVT::i32)); Cond = DAG.getNode(ISD::ADD, dl, MVT::i64, Cond, DAG.getConstant(1, dl, MVT::i64)); Cond = DAG.getSetCC( dl, getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), MVT::i64), Cond, DAG.getConstant(1, dl, MVT::i64), ISD::SETUGT); SINT = DAG.getNode(ISD::SELECT, dl, MVT::i64, Cond, Round, SINT); } ReuseLoadInfo RLI; SDValue Bits; MachineFunction &MF = DAG.getMachineFunction(); if (canReuseLoadAddress(SINT, MVT::i64, RLI, DAG)) { Bits = DAG.getLoad(MVT::f64, dl, RLI.Chain, RLI.Ptr, RLI.MPI, RLI.Alignment, RLI.MMOFlags(), RLI.AAInfo, RLI.Ranges); spliceIntoChain(RLI.ResChain, Bits.getValue(1), DAG); } else if (Subtarget.hasLFIWAX() && canReuseLoadAddress(SINT, MVT::i32, RLI, DAG, ISD::SEXTLOAD)) { MachineMemOperand *MMO = MF.getMachineMemOperand(RLI.MPI, MachineMemOperand::MOLoad, 4, RLI.Alignment, RLI.AAInfo, RLI.Ranges); SDValue Ops[] = { RLI.Chain, RLI.Ptr }; Bits = DAG.getMemIntrinsicNode(PPCISD::LFIWAX, dl, DAG.getVTList(MVT::f64, MVT::Other), Ops, MVT::i32, MMO); spliceIntoChain(RLI.ResChain, Bits.getValue(1), DAG); } else if (Subtarget.hasFPCVT() && canReuseLoadAddress(SINT, MVT::i32, RLI, DAG, ISD::ZEXTLOAD)) { MachineMemOperand *MMO = MF.getMachineMemOperand(RLI.MPI, MachineMemOperand::MOLoad, 4, RLI.Alignment, RLI.AAInfo, RLI.Ranges); SDValue Ops[] = { RLI.Chain, RLI.Ptr }; Bits = DAG.getMemIntrinsicNode(PPCISD::LFIWZX, dl, DAG.getVTList(MVT::f64, MVT::Other), Ops, MVT::i32, MMO); spliceIntoChain(RLI.ResChain, Bits.getValue(1), DAG); } else if (((Subtarget.hasLFIWAX() && SINT.getOpcode() == ISD::SIGN_EXTEND) || (Subtarget.hasFPCVT() && SINT.getOpcode() == ISD::ZERO_EXTEND)) && SINT.getOperand(0).getValueType() == MVT::i32) { MachineFrameInfo &MFI = MF.getFrameInfo(); EVT PtrVT = getPointerTy(DAG.getDataLayout()); int FrameIdx = MFI.CreateStackObject(4, 4, false); SDValue FIdx = DAG.getFrameIndex(FrameIdx, PtrVT); SDValue Store = DAG.getStore(DAG.getEntryNode(), dl, SINT.getOperand(0), FIdx, MachinePointerInfo::getFixedStack( DAG.getMachineFunction(), FrameIdx)); assert(cast(Store)->getMemoryVT() == MVT::i32 && "Expected an i32 store"); RLI.Ptr = FIdx; RLI.Chain = Store; RLI.MPI = MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FrameIdx); RLI.Alignment = Align(4); MachineMemOperand *MMO = MF.getMachineMemOperand(RLI.MPI, MachineMemOperand::MOLoad, 4, RLI.Alignment, RLI.AAInfo, RLI.Ranges); SDValue Ops[] = { RLI.Chain, RLI.Ptr }; Bits = DAG.getMemIntrinsicNode(SINT.getOpcode() == ISD::ZERO_EXTEND ? PPCISD::LFIWZX : PPCISD::LFIWAX, dl, DAG.getVTList(MVT::f64, MVT::Other), Ops, MVT::i32, MMO); } else Bits = DAG.getNode(ISD::BITCAST, dl, MVT::f64, SINT); SDValue FP = DAG.getNode(FCFOp, dl, FCFTy, Bits); if (Op.getValueType() == MVT::f32 && !Subtarget.hasFPCVT()) FP = DAG.getNode(ISD::FP_ROUND, dl, MVT::f32, FP, DAG.getIntPtrConstant(0, dl)); return FP; } assert(Op.getOperand(0).getValueType() == MVT::i32 && "Unhandled INT_TO_FP type in custom expander!"); // Since we only generate this in 64-bit mode, we can take advantage of // 64-bit registers. In particular, sign extend the input value into the // 64-bit register with extsw, store the WHOLE 64-bit value into the stack // then lfd it and fcfid it. MachineFunction &MF = DAG.getMachineFunction(); MachineFrameInfo &MFI = MF.getFrameInfo(); EVT PtrVT = getPointerTy(MF.getDataLayout()); SDValue Ld; if (Subtarget.hasLFIWAX() || Subtarget.hasFPCVT()) { ReuseLoadInfo RLI; bool ReusingLoad; if (!(ReusingLoad = canReuseLoadAddress(Op.getOperand(0), MVT::i32, RLI, DAG))) { int FrameIdx = MFI.CreateStackObject(4, 4, false); SDValue FIdx = DAG.getFrameIndex(FrameIdx, PtrVT); SDValue Store = DAG.getStore(DAG.getEntryNode(), dl, Op.getOperand(0), FIdx, MachinePointerInfo::getFixedStack( DAG.getMachineFunction(), FrameIdx)); assert(cast(Store)->getMemoryVT() == MVT::i32 && "Expected an i32 store"); RLI.Ptr = FIdx; RLI.Chain = Store; RLI.MPI = MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FrameIdx); RLI.Alignment = Align(4); } MachineMemOperand *MMO = MF.getMachineMemOperand(RLI.MPI, MachineMemOperand::MOLoad, 4, RLI.Alignment, RLI.AAInfo, RLI.Ranges); SDValue Ops[] = { RLI.Chain, RLI.Ptr }; Ld = DAG.getMemIntrinsicNode(Op.getOpcode() == ISD::UINT_TO_FP ? PPCISD::LFIWZX : PPCISD::LFIWAX, dl, DAG.getVTList(MVT::f64, MVT::Other), Ops, MVT::i32, MMO); if (ReusingLoad) spliceIntoChain(RLI.ResChain, Ld.getValue(1), DAG); } else { assert(Subtarget.isPPC64() && "i32->FP without LFIWAX supported only on PPC64"); int FrameIdx = MFI.CreateStackObject(8, 8, false); SDValue FIdx = DAG.getFrameIndex(FrameIdx, PtrVT); SDValue Ext64 = DAG.getNode(ISD::SIGN_EXTEND, dl, MVT::i64, Op.getOperand(0)); // STD the extended value into the stack slot. SDValue Store = DAG.getStore( DAG.getEntryNode(), dl, Ext64, FIdx, MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FrameIdx)); // Load the value as a double. Ld = DAG.getLoad( MVT::f64, dl, Store, FIdx, MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FrameIdx)); } // FCFID it and return it. SDValue FP = DAG.getNode(FCFOp, dl, FCFTy, Ld); if (Op.getValueType() == MVT::f32 && !Subtarget.hasFPCVT()) FP = DAG.getNode(ISD::FP_ROUND, dl, MVT::f32, FP, DAG.getIntPtrConstant(0, dl)); return FP; } SDValue PPCTargetLowering::LowerFLT_ROUNDS_(SDValue Op, SelectionDAG &DAG) const { SDLoc dl(Op); /* The rounding mode is in bits 30:31 of FPSR, and has the following settings: 00 Round to nearest 01 Round to 0 10 Round to +inf 11 Round to -inf FLT_ROUNDS, on the other hand, expects the following: -1 Undefined 0 Round to 0 1 Round to nearest 2 Round to +inf 3 Round to -inf To perform the conversion, we do: ((FPSCR & 0x3) ^ ((~FPSCR & 0x3) >> 1)) */ MachineFunction &MF = DAG.getMachineFunction(); EVT VT = Op.getValueType(); EVT PtrVT = getPointerTy(MF.getDataLayout()); // Save FP Control Word to register SDValue Chain = Op.getOperand(0); SDValue MFFS = DAG.getNode(PPCISD::MFFS, dl, {MVT::f64, MVT::Other}, Chain); Chain = MFFS.getValue(1); // Save FP register to stack slot int SSFI = MF.getFrameInfo().CreateStackObject(8, 8, false); SDValue StackSlot = DAG.getFrameIndex(SSFI, PtrVT); Chain = DAG.getStore(Chain, dl, MFFS, StackSlot, MachinePointerInfo()); // Load FP Control Word from low 32 bits of stack slot. SDValue Four = DAG.getConstant(4, dl, PtrVT); SDValue Addr = DAG.getNode(ISD::ADD, dl, PtrVT, StackSlot, Four); SDValue CWD = DAG.getLoad(MVT::i32, dl, Chain, Addr, MachinePointerInfo()); Chain = CWD.getValue(1); // Transform as necessary SDValue CWD1 = DAG.getNode(ISD::AND, dl, MVT::i32, CWD, DAG.getConstant(3, dl, MVT::i32)); SDValue CWD2 = DAG.getNode(ISD::SRL, dl, MVT::i32, DAG.getNode(ISD::AND, dl, MVT::i32, DAG.getNode(ISD::XOR, dl, MVT::i32, CWD, DAG.getConstant(3, dl, MVT::i32)), DAG.getConstant(3, dl, MVT::i32)), DAG.getConstant(1, dl, MVT::i32)); SDValue RetVal = DAG.getNode(ISD::XOR, dl, MVT::i32, CWD1, CWD2); RetVal = DAG.getNode((VT.getSizeInBits() < 16 ? ISD::TRUNCATE : ISD::ZERO_EXTEND), dl, VT, RetVal); return DAG.getMergeValues({RetVal, Chain}, dl); } SDValue PPCTargetLowering::LowerSHL_PARTS(SDValue Op, SelectionDAG &DAG) const { EVT VT = Op.getValueType(); unsigned BitWidth = VT.getSizeInBits(); SDLoc dl(Op); assert(Op.getNumOperands() == 3 && VT == Op.getOperand(1).getValueType() && "Unexpected SHL!"); // Expand into a bunch of logical ops. Note that these ops // depend on the PPC behavior for oversized shift amounts. SDValue Lo = Op.getOperand(0); SDValue Hi = Op.getOperand(1); SDValue Amt = Op.getOperand(2); EVT AmtVT = Amt.getValueType(); SDValue Tmp1 = DAG.getNode(ISD::SUB, dl, AmtVT, DAG.getConstant(BitWidth, dl, AmtVT), Amt); SDValue Tmp2 = DAG.getNode(PPCISD::SHL, dl, VT, Hi, Amt); SDValue Tmp3 = DAG.getNode(PPCISD::SRL, dl, VT, Lo, Tmp1); SDValue Tmp4 = DAG.getNode(ISD::OR , dl, VT, Tmp2, Tmp3); SDValue Tmp5 = DAG.getNode(ISD::ADD, dl, AmtVT, Amt, DAG.getConstant(-BitWidth, dl, AmtVT)); SDValue Tmp6 = DAG.getNode(PPCISD::SHL, dl, VT, Lo, Tmp5); SDValue OutHi = DAG.getNode(ISD::OR, dl, VT, Tmp4, Tmp6); SDValue OutLo = DAG.getNode(PPCISD::SHL, dl, VT, Lo, Amt); SDValue OutOps[] = { OutLo, OutHi }; return DAG.getMergeValues(OutOps, dl); } SDValue PPCTargetLowering::LowerSRL_PARTS(SDValue Op, SelectionDAG &DAG) const { EVT VT = Op.getValueType(); SDLoc dl(Op); unsigned BitWidth = VT.getSizeInBits(); assert(Op.getNumOperands() == 3 && VT == Op.getOperand(1).getValueType() && "Unexpected SRL!"); // Expand into a bunch of logical ops. Note that these ops // depend on the PPC behavior for oversized shift amounts. SDValue Lo = Op.getOperand(0); SDValue Hi = Op.getOperand(1); SDValue Amt = Op.getOperand(2); EVT AmtVT = Amt.getValueType(); SDValue Tmp1 = DAG.getNode(ISD::SUB, dl, AmtVT, DAG.getConstant(BitWidth, dl, AmtVT), Amt); SDValue Tmp2 = DAG.getNode(PPCISD::SRL, dl, VT, Lo, Amt); SDValue Tmp3 = DAG.getNode(PPCISD::SHL, dl, VT, Hi, Tmp1); SDValue Tmp4 = DAG.getNode(ISD::OR, dl, VT, Tmp2, Tmp3); SDValue Tmp5 = DAG.getNode(ISD::ADD, dl, AmtVT, Amt, DAG.getConstant(-BitWidth, dl, AmtVT)); SDValue Tmp6 = DAG.getNode(PPCISD::SRL, dl, VT, Hi, Tmp5); SDValue OutLo = DAG.getNode(ISD::OR, dl, VT, Tmp4, Tmp6); SDValue OutHi = DAG.getNode(PPCISD::SRL, dl, VT, Hi, Amt); SDValue OutOps[] = { OutLo, OutHi }; return DAG.getMergeValues(OutOps, dl); } SDValue PPCTargetLowering::LowerSRA_PARTS(SDValue Op, SelectionDAG &DAG) const { SDLoc dl(Op); EVT VT = Op.getValueType(); unsigned BitWidth = VT.getSizeInBits(); assert(Op.getNumOperands() == 3 && VT == Op.getOperand(1).getValueType() && "Unexpected SRA!"); // Expand into a bunch of logical ops, followed by a select_cc. SDValue Lo = Op.getOperand(0); SDValue Hi = Op.getOperand(1); SDValue Amt = Op.getOperand(2); EVT AmtVT = Amt.getValueType(); SDValue Tmp1 = DAG.getNode(ISD::SUB, dl, AmtVT, DAG.getConstant(BitWidth, dl, AmtVT), Amt); SDValue Tmp2 = DAG.getNode(PPCISD::SRL, dl, VT, Lo, Amt); SDValue Tmp3 = DAG.getNode(PPCISD::SHL, dl, VT, Hi, Tmp1); SDValue Tmp4 = DAG.getNode(ISD::OR, dl, VT, Tmp2, Tmp3); SDValue Tmp5 = DAG.getNode(ISD::ADD, dl, AmtVT, Amt, DAG.getConstant(-BitWidth, dl, AmtVT)); SDValue Tmp6 = DAG.getNode(PPCISD::SRA, dl, VT, Hi, Tmp5); SDValue OutHi = DAG.getNode(PPCISD::SRA, dl, VT, Hi, Amt); SDValue OutLo = DAG.getSelectCC(dl, Tmp5, DAG.getConstant(0, dl, AmtVT), Tmp4, Tmp6, ISD::SETLE); SDValue OutOps[] = { OutLo, OutHi }; return DAG.getMergeValues(OutOps, dl); } //===----------------------------------------------------------------------===// // Vector related lowering. // /// BuildSplatI - Build a canonical splati of Val with an element size of /// SplatSize. Cast the result to VT. static SDValue BuildSplatI(int Val, unsigned SplatSize, EVT VT, SelectionDAG &DAG, const SDLoc &dl) { static const MVT VTys[] = { // canonical VT to use for each size. MVT::v16i8, MVT::v8i16, MVT::Other, MVT::v4i32 }; EVT ReqVT = VT != MVT::Other ? VT : VTys[SplatSize-1]; // Force vspltis[hw] -1 to vspltisb -1 to canonicalize. if (Val == -1) SplatSize = 1; EVT CanonicalVT = VTys[SplatSize-1]; // Build a canonical splat for this value. return DAG.getBitcast(ReqVT, DAG.getConstant(Val, dl, CanonicalVT)); } /// BuildIntrinsicOp - Return a unary operator intrinsic node with the /// specified intrinsic ID. static SDValue BuildIntrinsicOp(unsigned IID, SDValue Op, SelectionDAG &DAG, const SDLoc &dl, EVT DestVT = MVT::Other) { if (DestVT == MVT::Other) DestVT = Op.getValueType(); return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, DestVT, DAG.getConstant(IID, dl, MVT::i32), Op); } /// BuildIntrinsicOp - Return a binary operator intrinsic node with the /// specified intrinsic ID. static SDValue BuildIntrinsicOp(unsigned IID, SDValue LHS, SDValue RHS, SelectionDAG &DAG, const SDLoc &dl, EVT DestVT = MVT::Other) { if (DestVT == MVT::Other) DestVT = LHS.getValueType(); return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, DestVT, DAG.getConstant(IID, dl, MVT::i32), LHS, RHS); } /// BuildIntrinsicOp - Return a ternary operator intrinsic node with the /// specified intrinsic ID. static SDValue BuildIntrinsicOp(unsigned IID, SDValue Op0, SDValue Op1, SDValue Op2, SelectionDAG &DAG, const SDLoc &dl, EVT DestVT = MVT::Other) { if (DestVT == MVT::Other) DestVT = Op0.getValueType(); return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, DestVT, DAG.getConstant(IID, dl, MVT::i32), Op0, Op1, Op2); } /// BuildVSLDOI - Return a VECTOR_SHUFFLE that is a vsldoi of the specified /// amount. The result has the specified value type. static SDValue BuildVSLDOI(SDValue LHS, SDValue RHS, unsigned Amt, EVT VT, SelectionDAG &DAG, const SDLoc &dl) { // Force LHS/RHS to be the right type. LHS = DAG.getNode(ISD::BITCAST, dl, MVT::v16i8, LHS); RHS = DAG.getNode(ISD::BITCAST, dl, MVT::v16i8, RHS); int Ops[16]; for (unsigned i = 0; i != 16; ++i) Ops[i] = i + Amt; SDValue T = DAG.getVectorShuffle(MVT::v16i8, dl, LHS, RHS, Ops); return DAG.getNode(ISD::BITCAST, dl, VT, T); } /// Do we have an efficient pattern in a .td file for this node? /// /// \param V - pointer to the BuildVectorSDNode being matched /// \param HasDirectMove - does this subtarget have VSR <-> GPR direct moves? /// /// There are some patterns where it is beneficial to keep a BUILD_VECTOR /// node as a BUILD_VECTOR node rather than expanding it. The patterns where /// the opposite is true (expansion is beneficial) are: /// - The node builds a vector out of integers that are not 32 or 64-bits /// - The node builds a vector out of constants /// - The node is a "load-and-splat" /// In all other cases, we will choose to keep the BUILD_VECTOR. static bool haveEfficientBuildVectorPattern(BuildVectorSDNode *V, bool HasDirectMove, bool HasP8Vector) { EVT VecVT = V->getValueType(0); bool RightType = VecVT == MVT::v2f64 || (HasP8Vector && VecVT == MVT::v4f32) || (HasDirectMove && (VecVT == MVT::v2i64 || VecVT == MVT::v4i32)); if (!RightType) return false; bool IsSplat = true; bool IsLoad = false; SDValue Op0 = V->getOperand(0); // This function is called in a block that confirms the node is not a constant // splat. So a constant BUILD_VECTOR here means the vector is built out of // different constants. if (V->isConstant()) return false; for (int i = 0, e = V->getNumOperands(); i < e; ++i) { if (V->getOperand(i).isUndef()) return false; // We want to expand nodes that represent load-and-splat even if the // loaded value is a floating point truncation or conversion to int. if (V->getOperand(i).getOpcode() == ISD::LOAD || (V->getOperand(i).getOpcode() == ISD::FP_ROUND && V->getOperand(i).getOperand(0).getOpcode() == ISD::LOAD) || (V->getOperand(i).getOpcode() == ISD::FP_TO_SINT && V->getOperand(i).getOperand(0).getOpcode() == ISD::LOAD) || (V->getOperand(i).getOpcode() == ISD::FP_TO_UINT && V->getOperand(i).getOperand(0).getOpcode() == ISD::LOAD)) IsLoad = true; // If the operands are different or the input is not a load and has more // uses than just this BV node, then it isn't a splat. if (V->getOperand(i) != Op0 || (!IsLoad && !V->isOnlyUserOf(V->getOperand(i).getNode()))) IsSplat = false; } return !(IsSplat && IsLoad); } // Lower BITCAST(f128, (build_pair i64, i64)) to BUILD_FP128. SDValue PPCTargetLowering::LowerBITCAST(SDValue Op, SelectionDAG &DAG) const { SDLoc dl(Op); SDValue Op0 = Op->getOperand(0); if (!EnableQuadPrecision || (Op.getValueType() != MVT::f128 ) || (Op0.getOpcode() != ISD::BUILD_PAIR) || (Op0.getOperand(0).getValueType() != MVT::i64) || (Op0.getOperand(1).getValueType() != MVT::i64)) return SDValue(); return DAG.getNode(PPCISD::BUILD_FP128, dl, MVT::f128, Op0.getOperand(0), Op0.getOperand(1)); } static const SDValue *getNormalLoadInput(const SDValue &Op) { const SDValue *InputLoad = &Op; if (InputLoad->getOpcode() == ISD::BITCAST) InputLoad = &InputLoad->getOperand(0); if (InputLoad->getOpcode() == ISD::SCALAR_TO_VECTOR) InputLoad = &InputLoad->getOperand(0); if (InputLoad->getOpcode() != ISD::LOAD) return nullptr; LoadSDNode *LD = cast(*InputLoad); return ISD::isNormalLoad(LD) ? InputLoad : nullptr; } // If this is a case we can't handle, return null and let the default // expansion code take care of it. If we CAN select this case, and if it // selects to a single instruction, return Op. Otherwise, if we can codegen // this case more efficiently than a constant pool load, lower it to the // sequence of ops that should be used. SDValue PPCTargetLowering::LowerBUILD_VECTOR(SDValue Op, SelectionDAG &DAG) const { SDLoc dl(Op); BuildVectorSDNode *BVN = dyn_cast(Op.getNode()); assert(BVN && "Expected a BuildVectorSDNode in LowerBUILD_VECTOR"); if (Subtarget.hasQPX() && Op.getValueType() == MVT::v4i1) { // We first build an i32 vector, load it into a QPX register, // then convert it to a floating-point vector and compare it // to a zero vector to get the boolean result. MachineFrameInfo &MFI = DAG.getMachineFunction().getFrameInfo(); int FrameIdx = MFI.CreateStackObject(16, 16, false); MachinePointerInfo PtrInfo = MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FrameIdx); EVT PtrVT = getPointerTy(DAG.getDataLayout()); SDValue FIdx = DAG.getFrameIndex(FrameIdx, PtrVT); assert(BVN->getNumOperands() == 4 && "BUILD_VECTOR for v4i1 does not have 4 operands"); bool IsConst = true; for (unsigned i = 0; i < 4; ++i) { if (BVN->getOperand(i).isUndef()) continue; if (!isa(BVN->getOperand(i))) { IsConst = false; break; } } if (IsConst) { Constant *One = ConstantFP::get(Type::getFloatTy(*DAG.getContext()), 1.0); Constant *NegOne = ConstantFP::get(Type::getFloatTy(*DAG.getContext()), -1.0); Constant *CV[4]; for (unsigned i = 0; i < 4; ++i) { if (BVN->getOperand(i).isUndef()) CV[i] = UndefValue::get(Type::getFloatTy(*DAG.getContext())); else if (isNullConstant(BVN->getOperand(i))) CV[i] = NegOne; else CV[i] = One; } Constant *CP = ConstantVector::get(CV); SDValue CPIdx = DAG.getConstantPool(CP, getPointerTy(DAG.getDataLayout()), 16 /* alignment */); SDValue Ops[] = {DAG.getEntryNode(), CPIdx}; SDVTList VTs = DAG.getVTList({MVT::v4i1, /*chain*/ MVT::Other}); return DAG.getMemIntrinsicNode( PPCISD::QVLFSb, dl, VTs, Ops, MVT::v4f32, MachinePointerInfo::getConstantPool(DAG.getMachineFunction())); } SmallVector Stores; for (unsigned i = 0; i < 4; ++i) { if (BVN->getOperand(i).isUndef()) continue; unsigned Offset = 4*i; SDValue Idx = DAG.getConstant(Offset, dl, FIdx.getValueType()); Idx = DAG.getNode(ISD::ADD, dl, FIdx.getValueType(), FIdx, Idx); unsigned StoreSize = BVN->getOperand(i).getValueType().getStoreSize(); if (StoreSize > 4) { Stores.push_back( DAG.getTruncStore(DAG.getEntryNode(), dl, BVN->getOperand(i), Idx, PtrInfo.getWithOffset(Offset), MVT::i32)); } else { SDValue StoreValue = BVN->getOperand(i); if (StoreSize < 4) StoreValue = DAG.getNode(ISD::ANY_EXTEND, dl, MVT::i32, StoreValue); Stores.push_back(DAG.getStore(DAG.getEntryNode(), dl, StoreValue, Idx, PtrInfo.getWithOffset(Offset))); } } SDValue StoreChain; if (!Stores.empty()) StoreChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Stores); else StoreChain = DAG.getEntryNode(); // Now load from v4i32 into the QPX register; this will extend it to // v4i64 but not yet convert it to a floating point. Nevertheless, this // is typed as v4f64 because the QPX register integer states are not // explicitly represented. SDValue Ops[] = {StoreChain, DAG.getConstant(Intrinsic::ppc_qpx_qvlfiwz, dl, MVT::i32), FIdx}; SDVTList VTs = DAG.getVTList({MVT::v4f64, /*chain*/ MVT::Other}); SDValue LoadedVect = DAG.getMemIntrinsicNode(ISD::INTRINSIC_W_CHAIN, dl, VTs, Ops, MVT::v4i32, PtrInfo); LoadedVect = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, MVT::v4f64, DAG.getConstant(Intrinsic::ppc_qpx_qvfcfidu, dl, MVT::i32), LoadedVect); SDValue FPZeros = DAG.getConstantFP(0.0, dl, MVT::v4f64); return DAG.getSetCC(dl, MVT::v4i1, LoadedVect, FPZeros, ISD::SETEQ); } // All other QPX vectors are handled by generic code. if (Subtarget.hasQPX()) return SDValue(); // Check if this is a splat of a constant value. APInt APSplatBits, APSplatUndef; unsigned SplatBitSize; bool HasAnyUndefs; if (! BVN->isConstantSplat(APSplatBits, APSplatUndef, SplatBitSize, HasAnyUndefs, 0, !Subtarget.isLittleEndian()) || SplatBitSize > 32) { const SDValue *InputLoad = getNormalLoadInput(Op.getOperand(0)); // Handle load-and-splat patterns as we have instructions that will do this // in one go. if (InputLoad && DAG.isSplatValue(Op, true)) { LoadSDNode *LD = cast(*InputLoad); // We have handling for 4 and 8 byte elements. unsigned ElementSize = LD->getMemoryVT().getScalarSizeInBits(); // Checking for a single use of this load, we have to check for vector // width (128 bits) / ElementSize uses (since each operand of the // BUILD_VECTOR is a separate use of the value. if (InputLoad->getNode()->hasNUsesOfValue(128 / ElementSize, 0) && ((Subtarget.hasVSX() && ElementSize == 64) || (Subtarget.hasP9Vector() && ElementSize == 32))) { SDValue Ops[] = { LD->getChain(), // Chain LD->getBasePtr(), // Ptr DAG.getValueType(Op.getValueType()) // VT }; return DAG.getMemIntrinsicNode(PPCISD::LD_SPLAT, dl, DAG.getVTList(Op.getValueType(), MVT::Other), Ops, LD->getMemoryVT(), LD->getMemOperand()); } } // BUILD_VECTOR nodes that are not constant splats of up to 32-bits can be // lowered to VSX instructions under certain conditions. // Without VSX, there is no pattern more efficient than expanding the node. if (Subtarget.hasVSX() && haveEfficientBuildVectorPattern(BVN, Subtarget.hasDirectMove(), Subtarget.hasP8Vector())) return Op; return SDValue(); } unsigned SplatBits = APSplatBits.getZExtValue(); unsigned SplatUndef = APSplatUndef.getZExtValue(); unsigned SplatSize = SplatBitSize / 8; // First, handle single instruction cases. // All zeros? if (SplatBits == 0) { // Canonicalize all zero vectors to be v4i32. if (Op.getValueType() != MVT::v4i32 || HasAnyUndefs) { SDValue Z = DAG.getConstant(0, dl, MVT::v4i32); Op = DAG.getNode(ISD::BITCAST, dl, Op.getValueType(), Z); } return Op; } // We have XXSPLTIB for constant splats one byte wide // FIXME: SplatBits is an unsigned int being cast to an int while passing it // as an argument to BuildSplatiI. Given SplatSize == 1 it is okay here. if (Subtarget.hasP9Vector() && SplatSize == 1) return BuildSplatI(SplatBits, SplatSize, Op.getValueType(), DAG, dl); // If the sign extended value is in the range [-16,15], use VSPLTI[bhw]. int32_t SextVal= (int32_t(SplatBits << (32-SplatBitSize)) >> (32-SplatBitSize)); if (SextVal >= -16 && SextVal <= 15) return BuildSplatI(SextVal, SplatSize, Op.getValueType(), DAG, dl); // Two instruction sequences. // If this value is in the range [-32,30] and is even, use: // VSPLTI[bhw](val/2) + VSPLTI[bhw](val/2) // If this value is in the range [17,31] and is odd, use: // VSPLTI[bhw](val-16) - VSPLTI[bhw](-16) // If this value is in the range [-31,-17] and is odd, use: // VSPLTI[bhw](val+16) + VSPLTI[bhw](-16) // Note the last two are three-instruction sequences. if (SextVal >= -32 && SextVal <= 31) { // To avoid having these optimizations undone by constant folding, // we convert to a pseudo that will be expanded later into one of // the above forms. SDValue Elt = DAG.getConstant(SextVal, dl, MVT::i32); EVT VT = (SplatSize == 1 ? MVT::v16i8 : (SplatSize == 2 ? MVT::v8i16 : MVT::v4i32)); SDValue EltSize = DAG.getConstant(SplatSize, dl, MVT::i32); SDValue RetVal = DAG.getNode(PPCISD::VADD_SPLAT, dl, VT, Elt, EltSize); if (VT == Op.getValueType()) return RetVal; else return DAG.getNode(ISD::BITCAST, dl, Op.getValueType(), RetVal); } // If this is 0x8000_0000 x 4, turn into vspltisw + vslw. If it is // 0x7FFF_FFFF x 4, turn it into not(0x8000_0000). This is important // for fneg/fabs. if (SplatSize == 4 && SplatBits == (0x7FFFFFFF&~SplatUndef)) { // Make -1 and vspltisw -1: SDValue OnesV = BuildSplatI(-1, 4, MVT::v4i32, DAG, dl); // Make the VSLW intrinsic, computing 0x8000_0000. SDValue Res = BuildIntrinsicOp(Intrinsic::ppc_altivec_vslw, OnesV, OnesV, DAG, dl); // xor by OnesV to invert it. Res = DAG.getNode(ISD::XOR, dl, MVT::v4i32, Res, OnesV); return DAG.getNode(ISD::BITCAST, dl, Op.getValueType(), Res); } // Check to see if this is a wide variety of vsplti*, binop self cases. static const signed char SplatCsts[] = { -1, 1, -2, 2, -3, 3, -4, 4, -5, 5, -6, 6, -7, 7, -8, 8, -9, 9, -10, 10, -11, 11, -12, 12, -13, 13, 14, -14, 15, -15, -16 }; for (unsigned idx = 0; idx < array_lengthof(SplatCsts); ++idx) { // Indirect through the SplatCsts array so that we favor 'vsplti -1' for // cases which are ambiguous (e.g. formation of 0x8000_0000). 'vsplti -1' int i = SplatCsts[idx]; // Figure out what shift amount will be used by altivec if shifted by i in // this splat size. unsigned TypeShiftAmt = i & (SplatBitSize-1); // vsplti + shl self. if (SextVal == (int)((unsigned)i << TypeShiftAmt)) { SDValue Res = BuildSplatI(i, SplatSize, MVT::Other, DAG, dl); static const unsigned IIDs[] = { // Intrinsic to use for each size. Intrinsic::ppc_altivec_vslb, Intrinsic::ppc_altivec_vslh, 0, Intrinsic::ppc_altivec_vslw }; Res = BuildIntrinsicOp(IIDs[SplatSize-1], Res, Res, DAG, dl); return DAG.getNode(ISD::BITCAST, dl, Op.getValueType(), Res); } // vsplti + srl self. if (SextVal == (int)((unsigned)i >> TypeShiftAmt)) { SDValue Res = BuildSplatI(i, SplatSize, MVT::Other, DAG, dl); static const unsigned IIDs[] = { // Intrinsic to use for each size. Intrinsic::ppc_altivec_vsrb, Intrinsic::ppc_altivec_vsrh, 0, Intrinsic::ppc_altivec_vsrw }; Res = BuildIntrinsicOp(IIDs[SplatSize-1], Res, Res, DAG, dl); return DAG.getNode(ISD::BITCAST, dl, Op.getValueType(), Res); } // vsplti + sra self. if (SextVal == (int)((unsigned)i >> TypeShiftAmt)) { SDValue Res = BuildSplatI(i, SplatSize, MVT::Other, DAG, dl); static const unsigned IIDs[] = { // Intrinsic to use for each size. Intrinsic::ppc_altivec_vsrab, Intrinsic::ppc_altivec_vsrah, 0, Intrinsic::ppc_altivec_vsraw }; Res = BuildIntrinsicOp(IIDs[SplatSize-1], Res, Res, DAG, dl); return DAG.getNode(ISD::BITCAST, dl, Op.getValueType(), Res); } // vsplti + rol self. if (SextVal == (int)(((unsigned)i << TypeShiftAmt) | ((unsigned)i >> (SplatBitSize-TypeShiftAmt)))) { SDValue Res = BuildSplatI(i, SplatSize, MVT::Other, DAG, dl); static const unsigned IIDs[] = { // Intrinsic to use for each size. Intrinsic::ppc_altivec_vrlb, Intrinsic::ppc_altivec_vrlh, 0, Intrinsic::ppc_altivec_vrlw }; Res = BuildIntrinsicOp(IIDs[SplatSize-1], Res, Res, DAG, dl); return DAG.getNode(ISD::BITCAST, dl, Op.getValueType(), Res); } // t = vsplti c, result = vsldoi t, t, 1 if (SextVal == (int)(((unsigned)i << 8) | (i < 0 ? 0xFF : 0))) { SDValue T = BuildSplatI(i, SplatSize, MVT::v16i8, DAG, dl); unsigned Amt = Subtarget.isLittleEndian() ? 15 : 1; return BuildVSLDOI(T, T, Amt, Op.getValueType(), DAG, dl); } // t = vsplti c, result = vsldoi t, t, 2 if (SextVal == (int)(((unsigned)i << 16) | (i < 0 ? 0xFFFF : 0))) { SDValue T = BuildSplatI(i, SplatSize, MVT::v16i8, DAG, dl); unsigned Amt = Subtarget.isLittleEndian() ? 14 : 2; return BuildVSLDOI(T, T, Amt, Op.getValueType(), DAG, dl); } // t = vsplti c, result = vsldoi t, t, 3 if (SextVal == (int)(((unsigned)i << 24) | (i < 0 ? 0xFFFFFF : 0))) { SDValue T = BuildSplatI(i, SplatSize, MVT::v16i8, DAG, dl); unsigned Amt = Subtarget.isLittleEndian() ? 13 : 3; return BuildVSLDOI(T, T, Amt, Op.getValueType(), DAG, dl); } } return SDValue(); } /// GeneratePerfectShuffle - Given an entry in the perfect-shuffle table, emit /// the specified operations to build the shuffle. static SDValue GeneratePerfectShuffle(unsigned PFEntry, SDValue LHS, SDValue RHS, SelectionDAG &DAG, const SDLoc &dl) { unsigned OpNum = (PFEntry >> 26) & 0x0F; unsigned LHSID = (PFEntry >> 13) & ((1 << 13)-1); unsigned RHSID = (PFEntry >> 0) & ((1 << 13)-1); enum { OP_COPY = 0, // Copy, used for things like to say it is <0,1,2,3> OP_VMRGHW, OP_VMRGLW, OP_VSPLTISW0, OP_VSPLTISW1, OP_VSPLTISW2, OP_VSPLTISW3, OP_VSLDOI4, OP_VSLDOI8, OP_VSLDOI12 }; if (OpNum == OP_COPY) { if (LHSID == (1*9+2)*9+3) return LHS; assert(LHSID == ((4*9+5)*9+6)*9+7 && "Illegal OP_COPY!"); return RHS; } SDValue OpLHS, OpRHS; OpLHS = GeneratePerfectShuffle(PerfectShuffleTable[LHSID], LHS, RHS, DAG, dl); OpRHS = GeneratePerfectShuffle(PerfectShuffleTable[RHSID], LHS, RHS, DAG, dl); int ShufIdxs[16]; switch (OpNum) { default: llvm_unreachable("Unknown i32 permute!"); case OP_VMRGHW: ShufIdxs[ 0] = 0; ShufIdxs[ 1] = 1; ShufIdxs[ 2] = 2; ShufIdxs[ 3] = 3; ShufIdxs[ 4] = 16; ShufIdxs[ 5] = 17; ShufIdxs[ 6] = 18; ShufIdxs[ 7] = 19; ShufIdxs[ 8] = 4; ShufIdxs[ 9] = 5; ShufIdxs[10] = 6; ShufIdxs[11] = 7; ShufIdxs[12] = 20; ShufIdxs[13] = 21; ShufIdxs[14] = 22; ShufIdxs[15] = 23; break; case OP_VMRGLW: ShufIdxs[ 0] = 8; ShufIdxs[ 1] = 9; ShufIdxs[ 2] = 10; ShufIdxs[ 3] = 11; ShufIdxs[ 4] = 24; ShufIdxs[ 5] = 25; ShufIdxs[ 6] = 26; ShufIdxs[ 7] = 27; ShufIdxs[ 8] = 12; ShufIdxs[ 9] = 13; ShufIdxs[10] = 14; ShufIdxs[11] = 15; ShufIdxs[12] = 28; ShufIdxs[13] = 29; ShufIdxs[14] = 30; ShufIdxs[15] = 31; break; case OP_VSPLTISW0: for (unsigned i = 0; i != 16; ++i) ShufIdxs[i] = (i&3)+0; break; case OP_VSPLTISW1: for (unsigned i = 0; i != 16; ++i) ShufIdxs[i] = (i&3)+4; break; case OP_VSPLTISW2: for (unsigned i = 0; i != 16; ++i) ShufIdxs[i] = (i&3)+8; break; case OP_VSPLTISW3: for (unsigned i = 0; i != 16; ++i) ShufIdxs[i] = (i&3)+12; break; case OP_VSLDOI4: return BuildVSLDOI(OpLHS, OpRHS, 4, OpLHS.getValueType(), DAG, dl); case OP_VSLDOI8: return BuildVSLDOI(OpLHS, OpRHS, 8, OpLHS.getValueType(), DAG, dl); case OP_VSLDOI12: return BuildVSLDOI(OpLHS, OpRHS, 12, OpLHS.getValueType(), DAG, dl); } EVT VT = OpLHS.getValueType(); OpLHS = DAG.getNode(ISD::BITCAST, dl, MVT::v16i8, OpLHS); OpRHS = DAG.getNode(ISD::BITCAST, dl, MVT::v16i8, OpRHS); SDValue T = DAG.getVectorShuffle(MVT::v16i8, dl, OpLHS, OpRHS, ShufIdxs); return DAG.getNode(ISD::BITCAST, dl, VT, T); } /// lowerToVINSERTB - Return the SDValue if this VECTOR_SHUFFLE can be handled /// by the VINSERTB instruction introduced in ISA 3.0, else just return default /// SDValue. SDValue PPCTargetLowering::lowerToVINSERTB(ShuffleVectorSDNode *N, SelectionDAG &DAG) const { const unsigned BytesInVector = 16; bool IsLE = Subtarget.isLittleEndian(); SDLoc dl(N); SDValue V1 = N->getOperand(0); SDValue V2 = N->getOperand(1); unsigned ShiftElts = 0, InsertAtByte = 0; bool Swap = false; // Shifts required to get the byte we want at element 7. unsigned LittleEndianShifts[] = {8, 7, 6, 5, 4, 3, 2, 1, 0, 15, 14, 13, 12, 11, 10, 9}; unsigned BigEndianShifts[] = {9, 10, 11, 12, 13, 14, 15, 0, 1, 2, 3, 4, 5, 6, 7, 8}; ArrayRef Mask = N->getMask(); int OriginalOrder[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15}; // For each mask element, find out if we're just inserting something // from V2 into V1 or vice versa. // Possible permutations inserting an element from V2 into V1: // X, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 // 0, X, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 // ... // 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, X // Inserting from V1 into V2 will be similar, except mask range will be // [16,31]. bool FoundCandidate = false; // If both vector operands for the shuffle are the same vector, the mask // will contain only elements from the first one and the second one will be // undef. unsigned VINSERTBSrcElem = IsLE ? 8 : 7; // Go through the mask of half-words to find an element that's being moved // from one vector to the other. for (unsigned i = 0; i < BytesInVector; ++i) { unsigned CurrentElement = Mask[i]; // If 2nd operand is undefined, we should only look for element 7 in the // Mask. if (V2.isUndef() && CurrentElement != VINSERTBSrcElem) continue; bool OtherElementsInOrder = true; // Examine the other elements in the Mask to see if they're in original // order. for (unsigned j = 0; j < BytesInVector; ++j) { if (j == i) continue; // If CurrentElement is from V1 [0,15], then we the rest of the Mask to be // from V2 [16,31] and vice versa. Unless the 2nd operand is undefined, // in which we always assume we're always picking from the 1st operand. int MaskOffset = (!V2.isUndef() && CurrentElement < BytesInVector) ? BytesInVector : 0; if (Mask[j] != OriginalOrder[j] + MaskOffset) { OtherElementsInOrder = false; break; } } // If other elements are in original order, we record the number of shifts // we need to get the element we want into element 7. Also record which byte // in the vector we should insert into. if (OtherElementsInOrder) { // If 2nd operand is undefined, we assume no shifts and no swapping. if (V2.isUndef()) { ShiftElts = 0; Swap = false; } else { // Only need the last 4-bits for shifts because operands will be swapped if CurrentElement is >= 2^4. ShiftElts = IsLE ? LittleEndianShifts[CurrentElement & 0xF] : BigEndianShifts[CurrentElement & 0xF]; Swap = CurrentElement < BytesInVector; } InsertAtByte = IsLE ? BytesInVector - (i + 1) : i; FoundCandidate = true; break; } } if (!FoundCandidate) return SDValue(); // Candidate found, construct the proper SDAG sequence with VINSERTB, // optionally with VECSHL if shift is required. if (Swap) std::swap(V1, V2); if (V2.isUndef()) V2 = V1; if (ShiftElts) { SDValue Shl = DAG.getNode(PPCISD::VECSHL, dl, MVT::v16i8, V2, V2, DAG.getConstant(ShiftElts, dl, MVT::i32)); return DAG.getNode(PPCISD::VECINSERT, dl, MVT::v16i8, V1, Shl, DAG.getConstant(InsertAtByte, dl, MVT::i32)); } return DAG.getNode(PPCISD::VECINSERT, dl, MVT::v16i8, V1, V2, DAG.getConstant(InsertAtByte, dl, MVT::i32)); } /// lowerToVINSERTH - Return the SDValue if this VECTOR_SHUFFLE can be handled /// by the VINSERTH instruction introduced in ISA 3.0, else just return default /// SDValue. SDValue PPCTargetLowering::lowerToVINSERTH(ShuffleVectorSDNode *N, SelectionDAG &DAG) const { const unsigned NumHalfWords = 8; const unsigned BytesInVector = NumHalfWords * 2; // Check that the shuffle is on half-words. if (!isNByteElemShuffleMask(N, 2, 1)) return SDValue(); bool IsLE = Subtarget.isLittleEndian(); SDLoc dl(N); SDValue V1 = N->getOperand(0); SDValue V2 = N->getOperand(1); unsigned ShiftElts = 0, InsertAtByte = 0; bool Swap = false; // Shifts required to get the half-word we want at element 3. unsigned LittleEndianShifts[] = {4, 3, 2, 1, 0, 7, 6, 5}; unsigned BigEndianShifts[] = {5, 6, 7, 0, 1, 2, 3, 4}; uint32_t Mask = 0; uint32_t OriginalOrderLow = 0x1234567; uint32_t OriginalOrderHigh = 0x89ABCDEF; // Now we look at mask elements 0,2,4,6,8,10,12,14. Pack the mask into a // 32-bit space, only need 4-bit nibbles per element. for (unsigned i = 0; i < NumHalfWords; ++i) { unsigned MaskShift = (NumHalfWords - 1 - i) * 4; Mask |= ((uint32_t)(N->getMaskElt(i * 2) / 2) << MaskShift); } // For each mask element, find out if we're just inserting something // from V2 into V1 or vice versa. Possible permutations inserting an element // from V2 into V1: // X, 1, 2, 3, 4, 5, 6, 7 // 0, X, 2, 3, 4, 5, 6, 7 // 0, 1, X, 3, 4, 5, 6, 7 // 0, 1, 2, X, 4, 5, 6, 7 // 0, 1, 2, 3, X, 5, 6, 7 // 0, 1, 2, 3, 4, X, 6, 7 // 0, 1, 2, 3, 4, 5, X, 7 // 0, 1, 2, 3, 4, 5, 6, X // Inserting from V1 into V2 will be similar, except mask range will be [8,15]. bool FoundCandidate = false; // Go through the mask of half-words to find an element that's being moved // from one vector to the other. for (unsigned i = 0; i < NumHalfWords; ++i) { unsigned MaskShift = (NumHalfWords - 1 - i) * 4; uint32_t MaskOneElt = (Mask >> MaskShift) & 0xF; uint32_t MaskOtherElts = ~(0xF << MaskShift); uint32_t TargetOrder = 0x0; // If both vector operands for the shuffle are the same vector, the mask // will contain only elements from the first one and the second one will be // undef. if (V2.isUndef()) { ShiftElts = 0; unsigned VINSERTHSrcElem = IsLE ? 4 : 3; TargetOrder = OriginalOrderLow; Swap = false; // Skip if not the correct element or mask of other elements don't equal // to our expected order. if (MaskOneElt == VINSERTHSrcElem && (Mask & MaskOtherElts) == (TargetOrder & MaskOtherElts)) { InsertAtByte = IsLE ? BytesInVector - (i + 1) * 2 : i * 2; FoundCandidate = true; break; } } else { // If both operands are defined. // Target order is [8,15] if the current mask is between [0,7]. TargetOrder = (MaskOneElt < NumHalfWords) ? OriginalOrderHigh : OriginalOrderLow; // Skip if mask of other elements don't equal our expected order. if ((Mask & MaskOtherElts) == (TargetOrder & MaskOtherElts)) { // We only need the last 3 bits for the number of shifts. ShiftElts = IsLE ? LittleEndianShifts[MaskOneElt & 0x7] : BigEndianShifts[MaskOneElt & 0x7]; InsertAtByte = IsLE ? BytesInVector - (i + 1) * 2 : i * 2; Swap = MaskOneElt < NumHalfWords; FoundCandidate = true; break; } } } if (!FoundCandidate) return SDValue(); // Candidate found, construct the proper SDAG sequence with VINSERTH, // optionally with VECSHL if shift is required. if (Swap) std::swap(V1, V2); if (V2.isUndef()) V2 = V1; SDValue Conv1 = DAG.getNode(ISD::BITCAST, dl, MVT::v8i16, V1); if (ShiftElts) { // Double ShiftElts because we're left shifting on v16i8 type. SDValue Shl = DAG.getNode(PPCISD::VECSHL, dl, MVT::v16i8, V2, V2, DAG.getConstant(2 * ShiftElts, dl, MVT::i32)); SDValue Conv2 = DAG.getNode(ISD::BITCAST, dl, MVT::v8i16, Shl); SDValue Ins = DAG.getNode(PPCISD::VECINSERT, dl, MVT::v8i16, Conv1, Conv2, DAG.getConstant(InsertAtByte, dl, MVT::i32)); return DAG.getNode(ISD::BITCAST, dl, MVT::v16i8, Ins); } SDValue Conv2 = DAG.getNode(ISD::BITCAST, dl, MVT::v8i16, V2); SDValue Ins = DAG.getNode(PPCISD::VECINSERT, dl, MVT::v8i16, Conv1, Conv2, DAG.getConstant(InsertAtByte, dl, MVT::i32)); return DAG.getNode(ISD::BITCAST, dl, MVT::v16i8, Ins); } /// LowerVECTOR_SHUFFLE - Return the code we lower for VECTOR_SHUFFLE. If this /// is a shuffle we can handle in a single instruction, return it. Otherwise, /// return the code it can be lowered into. Worst case, it can always be /// lowered into a vperm. SDValue PPCTargetLowering::LowerVECTOR_SHUFFLE(SDValue Op, SelectionDAG &DAG) const { SDLoc dl(Op); SDValue V1 = Op.getOperand(0); SDValue V2 = Op.getOperand(1); ShuffleVectorSDNode *SVOp = cast(Op); EVT VT = Op.getValueType(); bool isLittleEndian = Subtarget.isLittleEndian(); unsigned ShiftElts, InsertAtByte; bool Swap = false; // If this is a load-and-splat, we can do that with a single instruction // in some cases. However if the load has multiple uses, we don't want to // combine it because that will just produce multiple loads. const SDValue *InputLoad = getNormalLoadInput(V1); if (InputLoad && Subtarget.hasVSX() && V2.isUndef() && (PPC::isSplatShuffleMask(SVOp, 4) || PPC::isSplatShuffleMask(SVOp, 8)) && InputLoad->hasOneUse()) { bool IsFourByte = PPC::isSplatShuffleMask(SVOp, 4); int SplatIdx = PPC::getSplatIdxForPPCMnemonics(SVOp, IsFourByte ? 4 : 8, DAG); LoadSDNode *LD = cast(*InputLoad); // For 4-byte load-and-splat, we need Power9. if ((IsFourByte && Subtarget.hasP9Vector()) || !IsFourByte) { uint64_t Offset = 0; if (IsFourByte) Offset = isLittleEndian ? (3 - SplatIdx) * 4 : SplatIdx * 4; else Offset = isLittleEndian ? (1 - SplatIdx) * 8 : SplatIdx * 8; SDValue BasePtr = LD->getBasePtr(); if (Offset != 0) BasePtr = DAG.getNode(ISD::ADD, dl, getPointerTy(DAG.getDataLayout()), BasePtr, DAG.getIntPtrConstant(Offset, dl)); SDValue Ops[] = { LD->getChain(), // Chain BasePtr, // BasePtr DAG.getValueType(Op.getValueType()) // VT }; SDVTList VTL = DAG.getVTList(IsFourByte ? MVT::v4i32 : MVT::v2i64, MVT::Other); SDValue LdSplt = DAG.getMemIntrinsicNode(PPCISD::LD_SPLAT, dl, VTL, Ops, LD->getMemoryVT(), LD->getMemOperand()); if (LdSplt.getValueType() != SVOp->getValueType(0)) LdSplt = DAG.getBitcast(SVOp->getValueType(0), LdSplt); return LdSplt; } } if (Subtarget.hasP9Vector() && PPC::isXXINSERTWMask(SVOp, ShiftElts, InsertAtByte, Swap, isLittleEndian)) { if (Swap) std::swap(V1, V2); SDValue Conv1 = DAG.getNode(ISD::BITCAST, dl, MVT::v4i32, V1); SDValue Conv2 = DAG.getNode(ISD::BITCAST, dl, MVT::v4i32, V2); if (ShiftElts) { SDValue Shl = DAG.getNode(PPCISD::VECSHL, dl, MVT::v4i32, Conv2, Conv2, DAG.getConstant(ShiftElts, dl, MVT::i32)); SDValue Ins = DAG.getNode(PPCISD::VECINSERT, dl, MVT::v4i32, Conv1, Shl, DAG.getConstant(InsertAtByte, dl, MVT::i32)); return DAG.getNode(ISD::BITCAST, dl, MVT::v16i8, Ins); } SDValue Ins = DAG.getNode(PPCISD::VECINSERT, dl, MVT::v4i32, Conv1, Conv2, DAG.getConstant(InsertAtByte, dl, MVT::i32)); return DAG.getNode(ISD::BITCAST, dl, MVT::v16i8, Ins); } if (Subtarget.hasP9Altivec()) { SDValue NewISDNode; if ((NewISDNode = lowerToVINSERTH(SVOp, DAG))) return NewISDNode; if ((NewISDNode = lowerToVINSERTB(SVOp, DAG))) return NewISDNode; } if (Subtarget.hasVSX() && PPC::isXXSLDWIShuffleMask(SVOp, ShiftElts, Swap, isLittleEndian)) { if (Swap) std::swap(V1, V2); SDValue Conv1 = DAG.getNode(ISD::BITCAST, dl, MVT::v4i32, V1); SDValue Conv2 = DAG.getNode(ISD::BITCAST, dl, MVT::v4i32, V2.isUndef() ? V1 : V2); SDValue Shl = DAG.getNode(PPCISD::VECSHL, dl, MVT::v4i32, Conv1, Conv2, DAG.getConstant(ShiftElts, dl, MVT::i32)); return DAG.getNode(ISD::BITCAST, dl, MVT::v16i8, Shl); } if (Subtarget.hasVSX() && PPC::isXXPERMDIShuffleMask(SVOp, ShiftElts, Swap, isLittleEndian)) { if (Swap) std::swap(V1, V2); SDValue Conv1 = DAG.getNode(ISD::BITCAST, dl, MVT::v2i64, V1); SDValue Conv2 = DAG.getNode(ISD::BITCAST, dl, MVT::v2i64, V2.isUndef() ? V1 : V2); SDValue PermDI = DAG.getNode(PPCISD::XXPERMDI, dl, MVT::v2i64, Conv1, Conv2, DAG.getConstant(ShiftElts, dl, MVT::i32)); return DAG.getNode(ISD::BITCAST, dl, MVT::v16i8, PermDI); } if (Subtarget.hasP9Vector()) { if (PPC::isXXBRHShuffleMask(SVOp)) { SDValue Conv = DAG.getNode(ISD::BITCAST, dl, MVT::v8i16, V1); SDValue ReveHWord = DAG.getNode(ISD::BSWAP, dl, MVT::v8i16, Conv); return DAG.getNode(ISD::BITCAST, dl, MVT::v16i8, ReveHWord); } else if (PPC::isXXBRWShuffleMask(SVOp)) { SDValue Conv = DAG.getNode(ISD::BITCAST, dl, MVT::v4i32, V1); SDValue ReveWord = DAG.getNode(ISD::BSWAP, dl, MVT::v4i32, Conv); return DAG.getNode(ISD::BITCAST, dl, MVT::v16i8, ReveWord); } else if (PPC::isXXBRDShuffleMask(SVOp)) { SDValue Conv = DAG.getNode(ISD::BITCAST, dl, MVT::v2i64, V1); SDValue ReveDWord = DAG.getNode(ISD::BSWAP, dl, MVT::v2i64, Conv); return DAG.getNode(ISD::BITCAST, dl, MVT::v16i8, ReveDWord); } else if (PPC::isXXBRQShuffleMask(SVOp)) { SDValue Conv = DAG.getNode(ISD::BITCAST, dl, MVT::v1i128, V1); SDValue ReveQWord = DAG.getNode(ISD::BSWAP, dl, MVT::v1i128, Conv); return DAG.getNode(ISD::BITCAST, dl, MVT::v16i8, ReveQWord); } } if (Subtarget.hasVSX()) { if (V2.isUndef() && PPC::isSplatShuffleMask(SVOp, 4)) { int SplatIdx = PPC::getSplatIdxForPPCMnemonics(SVOp, 4, DAG); SDValue Conv = DAG.getNode(ISD::BITCAST, dl, MVT::v4i32, V1); SDValue Splat = DAG.getNode(PPCISD::XXSPLT, dl, MVT::v4i32, Conv, DAG.getConstant(SplatIdx, dl, MVT::i32)); return DAG.getNode(ISD::BITCAST, dl, MVT::v16i8, Splat); } // Left shifts of 8 bytes are actually swaps. Convert accordingly. if (V2.isUndef() && PPC::isVSLDOIShuffleMask(SVOp, 1, DAG) == 8) { SDValue Conv = DAG.getNode(ISD::BITCAST, dl, MVT::v2f64, V1); SDValue Swap = DAG.getNode(PPCISD::SWAP_NO_CHAIN, dl, MVT::v2f64, Conv); return DAG.getNode(ISD::BITCAST, dl, MVT::v16i8, Swap); } } if (Subtarget.hasQPX()) { if (VT.getVectorNumElements() != 4) return SDValue(); if (V2.isUndef()) V2 = V1; int AlignIdx = PPC::isQVALIGNIShuffleMask(SVOp); if (AlignIdx != -1) { return DAG.getNode(PPCISD::QVALIGNI, dl, VT, V1, V2, DAG.getConstant(AlignIdx, dl, MVT::i32)); } else if (SVOp->isSplat()) { int SplatIdx = SVOp->getSplatIndex(); if (SplatIdx >= 4) { std::swap(V1, V2); SplatIdx -= 4; } return DAG.getNode(PPCISD::QVESPLATI, dl, VT, V1, DAG.getConstant(SplatIdx, dl, MVT::i32)); } // Lower this into a qvgpci/qvfperm pair. // Compute the qvgpci literal unsigned idx = 0; for (unsigned i = 0; i < 4; ++i) { int m = SVOp->getMaskElt(i); unsigned mm = m >= 0 ? (unsigned) m : i; idx |= mm << (3-i)*3; } SDValue V3 = DAG.getNode(PPCISD::QVGPCI, dl, MVT::v4f64, DAG.getConstant(idx, dl, MVT::i32)); return DAG.getNode(PPCISD::QVFPERM, dl, VT, V1, V2, V3); } // Cases that are handled by instructions that take permute immediates // (such as vsplt*) should be left as VECTOR_SHUFFLE nodes so they can be // selected by the instruction selector. if (V2.isUndef()) { if (PPC::isSplatShuffleMask(SVOp, 1) || PPC::isSplatShuffleMask(SVOp, 2) || PPC::isSplatShuffleMask(SVOp, 4) || PPC::isVPKUWUMShuffleMask(SVOp, 1, DAG) || PPC::isVPKUHUMShuffleMask(SVOp, 1, DAG) || PPC::isVSLDOIShuffleMask(SVOp, 1, DAG) != -1 || PPC::isVMRGLShuffleMask(SVOp, 1, 1, DAG) || PPC::isVMRGLShuffleMask(SVOp, 2, 1, DAG) || PPC::isVMRGLShuffleMask(SVOp, 4, 1, DAG) || PPC::isVMRGHShuffleMask(SVOp, 1, 1, DAG) || PPC::isVMRGHShuffleMask(SVOp, 2, 1, DAG) || PPC::isVMRGHShuffleMask(SVOp, 4, 1, DAG) || (Subtarget.hasP8Altivec() && ( PPC::isVPKUDUMShuffleMask(SVOp, 1, DAG) || PPC::isVMRGEOShuffleMask(SVOp, true, 1, DAG) || PPC::isVMRGEOShuffleMask(SVOp, false, 1, DAG)))) { return Op; } } // Altivec has a variety of "shuffle immediates" that take two vector inputs // and produce a fixed permutation. If any of these match, do not lower to // VPERM. unsigned int ShuffleKind = isLittleEndian ? 2 : 0; if (PPC::isVPKUWUMShuffleMask(SVOp, ShuffleKind, DAG) || PPC::isVPKUHUMShuffleMask(SVOp, ShuffleKind, DAG) || PPC::isVSLDOIShuffleMask(SVOp, ShuffleKind, DAG) != -1 || PPC::isVMRGLShuffleMask(SVOp, 1, ShuffleKind, DAG) || PPC::isVMRGLShuffleMask(SVOp, 2, ShuffleKind, DAG) || PPC::isVMRGLShuffleMask(SVOp, 4, ShuffleKind, DAG) || PPC::isVMRGHShuffleMask(SVOp, 1, ShuffleKind, DAG) || PPC::isVMRGHShuffleMask(SVOp, 2, ShuffleKind, DAG) || PPC::isVMRGHShuffleMask(SVOp, 4, ShuffleKind, DAG) || (Subtarget.hasP8Altivec() && ( PPC::isVPKUDUMShuffleMask(SVOp, ShuffleKind, DAG) || PPC::isVMRGEOShuffleMask(SVOp, true, ShuffleKind, DAG) || PPC::isVMRGEOShuffleMask(SVOp, false, ShuffleKind, DAG)))) return Op; // Check to see if this is a shuffle of 4-byte values. If so, we can use our // perfect shuffle table to emit an optimal matching sequence. ArrayRef PermMask = SVOp->getMask(); unsigned PFIndexes[4]; bool isFourElementShuffle = true; for (unsigned i = 0; i != 4 && isFourElementShuffle; ++i) { // Element number unsigned EltNo = 8; // Start out undef. for (unsigned j = 0; j != 4; ++j) { // Intra-element byte. if (PermMask[i*4+j] < 0) continue; // Undef, ignore it. unsigned ByteSource = PermMask[i*4+j]; if ((ByteSource & 3) != j) { isFourElementShuffle = false; break; } if (EltNo == 8) { EltNo = ByteSource/4; } else if (EltNo != ByteSource/4) { isFourElementShuffle = false; break; } } PFIndexes[i] = EltNo; } // If this shuffle can be expressed as a shuffle of 4-byte elements, use the // perfect shuffle vector to determine if it is cost effective to do this as // discrete instructions, or whether we should use a vperm. // For now, we skip this for little endian until such time as we have a // little-endian perfect shuffle table. if (isFourElementShuffle && !isLittleEndian) { // Compute the index in the perfect shuffle table. unsigned PFTableIndex = PFIndexes[0]*9*9*9+PFIndexes[1]*9*9+PFIndexes[2]*9+PFIndexes[3]; unsigned PFEntry = PerfectShuffleTable[PFTableIndex]; unsigned Cost = (PFEntry >> 30); // Determining when to avoid vperm is tricky. Many things affect the cost // of vperm, particularly how many times the perm mask needs to be computed. // For example, if the perm mask can be hoisted out of a loop or is already // used (perhaps because there are multiple permutes with the same shuffle // mask?) the vperm has a cost of 1. OTOH, hoisting the permute mask out of // the loop requires an extra register. // // As a compromise, we only emit discrete instructions if the shuffle can be // generated in 3 or fewer operations. When we have loop information // available, if this block is within a loop, we should avoid using vperm // for 3-operation perms and use a constant pool load instead. if (Cost < 3) return GeneratePerfectShuffle(PFEntry, V1, V2, DAG, dl); } // Lower this to a VPERM(V1, V2, V3) expression, where V3 is a constant // vector that will get spilled to the constant pool. if (V2.isUndef()) V2 = V1; // The SHUFFLE_VECTOR mask is almost exactly what we want for vperm, except // that it is in input element units, not in bytes. Convert now. // For little endian, the order of the input vectors is reversed, and // the permutation mask is complemented with respect to 31. This is // necessary to produce proper semantics with the big-endian-biased vperm // instruction. EVT EltVT = V1.getValueType().getVectorElementType(); unsigned BytesPerElement = EltVT.getSizeInBits()/8; SmallVector ResultMask; for (unsigned i = 0, e = VT.getVectorNumElements(); i != e; ++i) { unsigned SrcElt = PermMask[i] < 0 ? 0 : PermMask[i]; for (unsigned j = 0; j != BytesPerElement; ++j) if (isLittleEndian) ResultMask.push_back(DAG.getConstant(31 - (SrcElt*BytesPerElement + j), dl, MVT::i32)); else ResultMask.push_back(DAG.getConstant(SrcElt*BytesPerElement + j, dl, MVT::i32)); } SDValue VPermMask = DAG.getBuildVector(MVT::v16i8, dl, ResultMask); if (isLittleEndian) return DAG.getNode(PPCISD::VPERM, dl, V1.getValueType(), V2, V1, VPermMask); else return DAG.getNode(PPCISD::VPERM, dl, V1.getValueType(), V1, V2, VPermMask); } /// getVectorCompareInfo - Given an intrinsic, return false if it is not a /// vector comparison. If it is, return true and fill in Opc/isDot with /// information about the intrinsic. static bool getVectorCompareInfo(SDValue Intrin, int &CompareOpc, bool &isDot, const PPCSubtarget &Subtarget) { unsigned IntrinsicID = cast(Intrin.getOperand(0))->getZExtValue(); CompareOpc = -1; isDot = false; switch (IntrinsicID) { default: return false; // Comparison predicates. case Intrinsic::ppc_altivec_vcmpbfp_p: CompareOpc = 966; isDot = true; break; case Intrinsic::ppc_altivec_vcmpeqfp_p: CompareOpc = 198; isDot = true; break; case Intrinsic::ppc_altivec_vcmpequb_p: CompareOpc = 6; isDot = true; break; case Intrinsic::ppc_altivec_vcmpequh_p: CompareOpc = 70; isDot = true; break; case Intrinsic::ppc_altivec_vcmpequw_p: CompareOpc = 134; isDot = true; break; case Intrinsic::ppc_altivec_vcmpequd_p: if (Subtarget.hasP8Altivec()) { CompareOpc = 199; isDot = true; } else return false; break; case Intrinsic::ppc_altivec_vcmpneb_p: case Intrinsic::ppc_altivec_vcmpneh_p: case Intrinsic::ppc_altivec_vcmpnew_p: case Intrinsic::ppc_altivec_vcmpnezb_p: case Intrinsic::ppc_altivec_vcmpnezh_p: case Intrinsic::ppc_altivec_vcmpnezw_p: if (Subtarget.hasP9Altivec()) { switch (IntrinsicID) { default: llvm_unreachable("Unknown comparison intrinsic."); case Intrinsic::ppc_altivec_vcmpneb_p: CompareOpc = 7; break; case Intrinsic::ppc_altivec_vcmpneh_p: CompareOpc = 71; break; case Intrinsic::ppc_altivec_vcmpnew_p: CompareOpc = 135; break; case Intrinsic::ppc_altivec_vcmpnezb_p: CompareOpc = 263; break; case Intrinsic::ppc_altivec_vcmpnezh_p: CompareOpc = 327; break; case Intrinsic::ppc_altivec_vcmpnezw_p: CompareOpc = 391; break; } isDot = true; } else return false; break; case Intrinsic::ppc_altivec_vcmpgefp_p: CompareOpc = 454; isDot = true; break; case Intrinsic::ppc_altivec_vcmpgtfp_p: CompareOpc = 710; isDot = true; break; case Intrinsic::ppc_altivec_vcmpgtsb_p: CompareOpc = 774; isDot = true; break; case Intrinsic::ppc_altivec_vcmpgtsh_p: CompareOpc = 838; isDot = true; break; case Intrinsic::ppc_altivec_vcmpgtsw_p: CompareOpc = 902; isDot = true; break; case Intrinsic::ppc_altivec_vcmpgtsd_p: if (Subtarget.hasP8Altivec()) { CompareOpc = 967; isDot = true; } else return false; break; case Intrinsic::ppc_altivec_vcmpgtub_p: CompareOpc = 518; isDot = true; break; case Intrinsic::ppc_altivec_vcmpgtuh_p: CompareOpc = 582; isDot = true; break; case Intrinsic::ppc_altivec_vcmpgtuw_p: CompareOpc = 646; isDot = true; break; case Intrinsic::ppc_altivec_vcmpgtud_p: if (Subtarget.hasP8Altivec()) { CompareOpc = 711; isDot = true; } else return false; break; // VSX predicate comparisons use the same infrastructure case Intrinsic::ppc_vsx_xvcmpeqdp_p: case Intrinsic::ppc_vsx_xvcmpgedp_p: case Intrinsic::ppc_vsx_xvcmpgtdp_p: case Intrinsic::ppc_vsx_xvcmpeqsp_p: case Intrinsic::ppc_vsx_xvcmpgesp_p: case Intrinsic::ppc_vsx_xvcmpgtsp_p: if (Subtarget.hasVSX()) { switch (IntrinsicID) { case Intrinsic::ppc_vsx_xvcmpeqdp_p: CompareOpc = 99; break; case Intrinsic::ppc_vsx_xvcmpgedp_p: CompareOpc = 115; break; case Intrinsic::ppc_vsx_xvcmpgtdp_p: CompareOpc = 107; break; case Intrinsic::ppc_vsx_xvcmpeqsp_p: CompareOpc = 67; break; case Intrinsic::ppc_vsx_xvcmpgesp_p: CompareOpc = 83; break; case Intrinsic::ppc_vsx_xvcmpgtsp_p: CompareOpc = 75; break; } isDot = true; } else return false; break; // Normal Comparisons. case Intrinsic::ppc_altivec_vcmpbfp: CompareOpc = 966; break; case Intrinsic::ppc_altivec_vcmpeqfp: CompareOpc = 198; break; case Intrinsic::ppc_altivec_vcmpequb: CompareOpc = 6; break; case Intrinsic::ppc_altivec_vcmpequh: CompareOpc = 70; break; case Intrinsic::ppc_altivec_vcmpequw: CompareOpc = 134; break; case Intrinsic::ppc_altivec_vcmpequd: if (Subtarget.hasP8Altivec()) CompareOpc = 199; else return false; break; case Intrinsic::ppc_altivec_vcmpneb: case Intrinsic::ppc_altivec_vcmpneh: case Intrinsic::ppc_altivec_vcmpnew: case Intrinsic::ppc_altivec_vcmpnezb: case Intrinsic::ppc_altivec_vcmpnezh: case Intrinsic::ppc_altivec_vcmpnezw: if (Subtarget.hasP9Altivec()) switch (IntrinsicID) { default: llvm_unreachable("Unknown comparison intrinsic."); case Intrinsic::ppc_altivec_vcmpneb: CompareOpc = 7; break; case Intrinsic::ppc_altivec_vcmpneh: CompareOpc = 71; break; case Intrinsic::ppc_altivec_vcmpnew: CompareOpc = 135; break; case Intrinsic::ppc_altivec_vcmpnezb: CompareOpc = 263; break; case Intrinsic::ppc_altivec_vcmpnezh: CompareOpc = 327; break; case Intrinsic::ppc_altivec_vcmpnezw: CompareOpc = 391; break; } else return false; break; case Intrinsic::ppc_altivec_vcmpgefp: CompareOpc = 454; break; case Intrinsic::ppc_altivec_vcmpgtfp: CompareOpc = 710; break; case Intrinsic::ppc_altivec_vcmpgtsb: CompareOpc = 774; break; case Intrinsic::ppc_altivec_vcmpgtsh: CompareOpc = 838; break; case Intrinsic::ppc_altivec_vcmpgtsw: CompareOpc = 902; break; case Intrinsic::ppc_altivec_vcmpgtsd: if (Subtarget.hasP8Altivec()) CompareOpc = 967; else return false; break; case Intrinsic::ppc_altivec_vcmpgtub: CompareOpc = 518; break; case Intrinsic::ppc_altivec_vcmpgtuh: CompareOpc = 582; break; case Intrinsic::ppc_altivec_vcmpgtuw: CompareOpc = 646; break; case Intrinsic::ppc_altivec_vcmpgtud: if (Subtarget.hasP8Altivec()) CompareOpc = 711; else return false; break; } return true; } /// LowerINTRINSIC_WO_CHAIN - If this is an intrinsic that we want to custom /// lower, do it, otherwise return null. SDValue PPCTargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op, SelectionDAG &DAG) const { unsigned IntrinsicID = cast(Op.getOperand(0))->getZExtValue(); SDLoc dl(Op); if (IntrinsicID == Intrinsic::thread_pointer) { // Reads the thread pointer register, used for __builtin_thread_pointer. if (Subtarget.isPPC64()) return DAG.getRegister(PPC::X13, MVT::i64); return DAG.getRegister(PPC::R2, MVT::i32); } // If this is a lowered altivec predicate compare, CompareOpc is set to the // opcode number of the comparison. int CompareOpc; bool isDot; if (!getVectorCompareInfo(Op, CompareOpc, isDot, Subtarget)) return SDValue(); // Don't custom lower most intrinsics. // If this is a non-dot comparison, make the VCMP node and we are done. if (!isDot) { SDValue Tmp = DAG.getNode(PPCISD::VCMP, dl, Op.getOperand(2).getValueType(), Op.getOperand(1), Op.getOperand(2), DAG.getConstant(CompareOpc, dl, MVT::i32)); return DAG.getNode(ISD::BITCAST, dl, Op.getValueType(), Tmp); } // Create the PPCISD altivec 'dot' comparison node. SDValue Ops[] = { Op.getOperand(2), // LHS Op.getOperand(3), // RHS DAG.getConstant(CompareOpc, dl, MVT::i32) }; EVT VTs[] = { Op.getOperand(2).getValueType(), MVT::Glue }; SDValue CompNode = DAG.getNode(PPCISD::VCMPo, dl, VTs, Ops); // Now that we have the comparison, emit a copy from the CR to a GPR. // This is flagged to the above dot comparison. SDValue Flags = DAG.getNode(PPCISD::MFOCRF, dl, MVT::i32, DAG.getRegister(PPC::CR6, MVT::i32), CompNode.getValue(1)); // Unpack the result based on how the target uses it. unsigned BitNo; // Bit # of CR6. bool InvertBit; // Invert result? switch (cast(Op.getOperand(1))->getZExtValue()) { default: // Can't happen, don't crash on invalid number though. case 0: // Return the value of the EQ bit of CR6. BitNo = 0; InvertBit = false; break; case 1: // Return the inverted value of the EQ bit of CR6. BitNo = 0; InvertBit = true; break; case 2: // Return the value of the LT bit of CR6. BitNo = 2; InvertBit = false; break; case 3: // Return the inverted value of the LT bit of CR6. BitNo = 2; InvertBit = true; break; } // Shift the bit into the low position. Flags = DAG.getNode(ISD::SRL, dl, MVT::i32, Flags, DAG.getConstant(8 - (3 - BitNo), dl, MVT::i32)); // Isolate the bit. Flags = DAG.getNode(ISD::AND, dl, MVT::i32, Flags, DAG.getConstant(1, dl, MVT::i32)); // If we are supposed to, toggle the bit. if (InvertBit) Flags = DAG.getNode(ISD::XOR, dl, MVT::i32, Flags, DAG.getConstant(1, dl, MVT::i32)); return Flags; } SDValue PPCTargetLowering::LowerINTRINSIC_VOID(SDValue Op, SelectionDAG &DAG) const { // SelectionDAGBuilder::visitTargetIntrinsic may insert one extra chain to // the beginning of the argument list. int ArgStart = isa(Op.getOperand(0)) ? 0 : 1; SDLoc DL(Op); switch (cast(Op.getOperand(ArgStart))->getZExtValue()) { case Intrinsic::ppc_cfence: { assert(ArgStart == 1 && "llvm.ppc.cfence must carry a chain argument."); assert(Subtarget.isPPC64() && "Only 64-bit is supported for now."); return SDValue(DAG.getMachineNode(PPC::CFENCE8, DL, MVT::Other, DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, Op.getOperand(ArgStart + 1)), Op.getOperand(0)), 0); } default: break; } return SDValue(); } SDValue PPCTargetLowering::LowerREM(SDValue Op, SelectionDAG &DAG) const { // Check for a DIV with the same operands as this REM. for (auto UI : Op.getOperand(1)->uses()) { if ((Op.getOpcode() == ISD::SREM && UI->getOpcode() == ISD::SDIV) || (Op.getOpcode() == ISD::UREM && UI->getOpcode() == ISD::UDIV)) if (UI->getOperand(0) == Op.getOperand(0) && UI->getOperand(1) == Op.getOperand(1)) return SDValue(); } return Op; } // Lower scalar BSWAP64 to xxbrd. SDValue PPCTargetLowering::LowerBSWAP(SDValue Op, SelectionDAG &DAG) const { SDLoc dl(Op); // MTVSRDD Op = DAG.getNode(ISD::BUILD_VECTOR, dl, MVT::v2i64, Op.getOperand(0), Op.getOperand(0)); // XXBRD Op = DAG.getNode(ISD::BSWAP, dl, MVT::v2i64, Op); // MFVSRD int VectorIndex = 0; if (Subtarget.isLittleEndian()) VectorIndex = 1; Op = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i64, Op, DAG.getTargetConstant(VectorIndex, dl, MVT::i32)); return Op; } // ATOMIC_CMP_SWAP for i8/i16 needs to zero-extend its input since it will be // compared to a value that is atomically loaded (atomic loads zero-extend). SDValue PPCTargetLowering::LowerATOMIC_CMP_SWAP(SDValue Op, SelectionDAG &DAG) const { assert(Op.getOpcode() == ISD::ATOMIC_CMP_SWAP && "Expecting an atomic compare-and-swap here."); SDLoc dl(Op); auto *AtomicNode = cast(Op.getNode()); EVT MemVT = AtomicNode->getMemoryVT(); if (MemVT.getSizeInBits() >= 32) return Op; SDValue CmpOp = Op.getOperand(2); // If this is already correctly zero-extended, leave it alone. auto HighBits = APInt::getHighBitsSet(32, 32 - MemVT.getSizeInBits()); if (DAG.MaskedValueIsZero(CmpOp, HighBits)) return Op; // Clear the high bits of the compare operand. unsigned MaskVal = (1 << MemVT.getSizeInBits()) - 1; SDValue NewCmpOp = DAG.getNode(ISD::AND, dl, MVT::i32, CmpOp, DAG.getConstant(MaskVal, dl, MVT::i32)); // Replace the existing compare operand with the properly zero-extended one. SmallVector Ops; for (int i = 0, e = AtomicNode->getNumOperands(); i < e; i++) Ops.push_back(AtomicNode->getOperand(i)); Ops[2] = NewCmpOp; MachineMemOperand *MMO = AtomicNode->getMemOperand(); SDVTList Tys = DAG.getVTList(MVT::i32, MVT::Other); auto NodeTy = (MemVT == MVT::i8) ? PPCISD::ATOMIC_CMP_SWAP_8 : PPCISD::ATOMIC_CMP_SWAP_16; return DAG.getMemIntrinsicNode(NodeTy, dl, Tys, Ops, MemVT, MMO); } SDValue PPCTargetLowering::LowerSCALAR_TO_VECTOR(SDValue Op, SelectionDAG &DAG) const { SDLoc dl(Op); // Create a stack slot that is 16-byte aligned. MachineFrameInfo &MFI = DAG.getMachineFunction().getFrameInfo(); int FrameIdx = MFI.CreateStackObject(16, 16, false); EVT PtrVT = getPointerTy(DAG.getDataLayout()); SDValue FIdx = DAG.getFrameIndex(FrameIdx, PtrVT); // Store the input value into Value#0 of the stack slot. SDValue Store = DAG.getStore(DAG.getEntryNode(), dl, Op.getOperand(0), FIdx, MachinePointerInfo()); // Load it out. return DAG.getLoad(Op.getValueType(), dl, Store, FIdx, MachinePointerInfo()); } SDValue PPCTargetLowering::LowerINSERT_VECTOR_ELT(SDValue Op, SelectionDAG &DAG) const { assert(Op.getOpcode() == ISD::INSERT_VECTOR_ELT && "Should only be called for ISD::INSERT_VECTOR_ELT"); ConstantSDNode *C = dyn_cast(Op.getOperand(2)); // We have legal lowering for constant indices but not for variable ones. if (!C) return SDValue(); EVT VT = Op.getValueType(); SDLoc dl(Op); SDValue V1 = Op.getOperand(0); SDValue V2 = Op.getOperand(1); // We can use MTVSRZ + VECINSERT for v8i16 and v16i8 types. if (VT == MVT::v8i16 || VT == MVT::v16i8) { SDValue Mtvsrz = DAG.getNode(PPCISD::MTVSRZ, dl, VT, V2); unsigned BytesInEachElement = VT.getVectorElementType().getSizeInBits() / 8; unsigned InsertAtElement = C->getZExtValue(); unsigned InsertAtByte = InsertAtElement * BytesInEachElement; if (Subtarget.isLittleEndian()) { InsertAtByte = (16 - BytesInEachElement) - InsertAtByte; } return DAG.getNode(PPCISD::VECINSERT, dl, VT, V1, Mtvsrz, DAG.getConstant(InsertAtByte, dl, MVT::i32)); } return Op; } SDValue PPCTargetLowering::LowerEXTRACT_VECTOR_ELT(SDValue Op, SelectionDAG &DAG) const { SDLoc dl(Op); SDNode *N = Op.getNode(); assert(N->getOperand(0).getValueType() == MVT::v4i1 && "Unknown extract_vector_elt type"); SDValue Value = N->getOperand(0); // The first part of this is like the store lowering except that we don't // need to track the chain. // The values are now known to be -1 (false) or 1 (true). To convert this // into 0 (false) and 1 (true), add 1 and then divide by 2 (multiply by 0.5). // This can be done with an fma and the 0.5 constant: (V+1.0)*0.5 = 0.5*V+0.5 Value = DAG.getNode(PPCISD::QBFLT, dl, MVT::v4f64, Value); // FIXME: We can make this an f32 vector, but the BUILD_VECTOR code needs to // understand how to form the extending load. SDValue FPHalfs = DAG.getConstantFP(0.5, dl, MVT::v4f64); Value = DAG.getNode(ISD::FMA, dl, MVT::v4f64, Value, FPHalfs, FPHalfs); // Now convert to an integer and store. Value = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, MVT::v4f64, DAG.getConstant(Intrinsic::ppc_qpx_qvfctiwu, dl, MVT::i32), Value); MachineFrameInfo &MFI = DAG.getMachineFunction().getFrameInfo(); int FrameIdx = MFI.CreateStackObject(16, 16, false); MachinePointerInfo PtrInfo = MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FrameIdx); EVT PtrVT = getPointerTy(DAG.getDataLayout()); SDValue FIdx = DAG.getFrameIndex(FrameIdx, PtrVT); SDValue StoreChain = DAG.getEntryNode(); SDValue Ops[] = {StoreChain, DAG.getConstant(Intrinsic::ppc_qpx_qvstfiw, dl, MVT::i32), Value, FIdx}; SDVTList VTs = DAG.getVTList(/*chain*/ MVT::Other); StoreChain = DAG.getMemIntrinsicNode(ISD::INTRINSIC_VOID, dl, VTs, Ops, MVT::v4i32, PtrInfo); // Extract the value requested. unsigned Offset = 4*cast(N->getOperand(1))->getZExtValue(); SDValue Idx = DAG.getConstant(Offset, dl, FIdx.getValueType()); Idx = DAG.getNode(ISD::ADD, dl, FIdx.getValueType(), FIdx, Idx); SDValue IntVal = DAG.getLoad(MVT::i32, dl, StoreChain, Idx, PtrInfo.getWithOffset(Offset)); if (!Subtarget.useCRBits()) return IntVal; return DAG.getNode(ISD::TRUNCATE, dl, MVT::i1, IntVal); } /// Lowering for QPX v4i1 loads SDValue PPCTargetLowering::LowerVectorLoad(SDValue Op, SelectionDAG &DAG) const { SDLoc dl(Op); LoadSDNode *LN = cast(Op.getNode()); SDValue LoadChain = LN->getChain(); SDValue BasePtr = LN->getBasePtr(); if (Op.getValueType() == MVT::v4f64 || Op.getValueType() == MVT::v4f32) { EVT MemVT = LN->getMemoryVT(); unsigned Alignment = LN->getAlignment(); // If this load is properly aligned, then it is legal. if (Alignment >= MemVT.getStoreSize()) return Op; EVT ScalarVT = Op.getValueType().getScalarType(), ScalarMemVT = MemVT.getScalarType(); unsigned Stride = ScalarMemVT.getStoreSize(); SDValue Vals[4], LoadChains[4]; for (unsigned Idx = 0; Idx < 4; ++Idx) { SDValue Load; if (ScalarVT != ScalarMemVT) Load = DAG.getExtLoad(LN->getExtensionType(), dl, ScalarVT, LoadChain, BasePtr, LN->getPointerInfo().getWithOffset(Idx * Stride), ScalarMemVT, MinAlign(Alignment, Idx * Stride), LN->getMemOperand()->getFlags(), LN->getAAInfo()); else Load = DAG.getLoad(ScalarVT, dl, LoadChain, BasePtr, LN->getPointerInfo().getWithOffset(Idx * Stride), MinAlign(Alignment, Idx * Stride), LN->getMemOperand()->getFlags(), LN->getAAInfo()); if (Idx == 0 && LN->isIndexed()) { assert(LN->getAddressingMode() == ISD::PRE_INC && "Unknown addressing mode on vector load"); Load = DAG.getIndexedLoad(Load, dl, BasePtr, LN->getOffset(), LN->getAddressingMode()); } Vals[Idx] = Load; LoadChains[Idx] = Load.getValue(1); BasePtr = DAG.getNode(ISD::ADD, dl, BasePtr.getValueType(), BasePtr, DAG.getConstant(Stride, dl, BasePtr.getValueType())); } SDValue TF = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, LoadChains); SDValue Value = DAG.getBuildVector(Op.getValueType(), dl, Vals); if (LN->isIndexed()) { SDValue RetOps[] = { Value, Vals[0].getValue(1), TF }; return DAG.getMergeValues(RetOps, dl); } SDValue RetOps[] = { Value, TF }; return DAG.getMergeValues(RetOps, dl); } assert(Op.getValueType() == MVT::v4i1 && "Unknown load to lower"); assert(LN->isUnindexed() && "Indexed v4i1 loads are not supported"); // To lower v4i1 from a byte array, we load the byte elements of the // vector and then reuse the BUILD_VECTOR logic. SDValue VectElmts[4], VectElmtChains[4]; for (unsigned i = 0; i < 4; ++i) { SDValue Idx = DAG.getConstant(i, dl, BasePtr.getValueType()); Idx = DAG.getNode(ISD::ADD, dl, BasePtr.getValueType(), BasePtr, Idx); VectElmts[i] = DAG.getExtLoad( ISD::EXTLOAD, dl, MVT::i32, LoadChain, Idx, LN->getPointerInfo().getWithOffset(i), MVT::i8, /* Alignment = */ 1, LN->getMemOperand()->getFlags(), LN->getAAInfo()); VectElmtChains[i] = VectElmts[i].getValue(1); } LoadChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, VectElmtChains); SDValue Value = DAG.getBuildVector(MVT::v4i1, dl, VectElmts); SDValue RVals[] = { Value, LoadChain }; return DAG.getMergeValues(RVals, dl); } /// Lowering for QPX v4i1 stores SDValue PPCTargetLowering::LowerVectorStore(SDValue Op, SelectionDAG &DAG) const { SDLoc dl(Op); StoreSDNode *SN = cast(Op.getNode()); SDValue StoreChain = SN->getChain(); SDValue BasePtr = SN->getBasePtr(); SDValue Value = SN->getValue(); if (Value.getValueType() == MVT::v4f64 || Value.getValueType() == MVT::v4f32) { EVT MemVT = SN->getMemoryVT(); unsigned Alignment = SN->getAlignment(); // If this store is properly aligned, then it is legal. if (Alignment >= MemVT.getStoreSize()) return Op; EVT ScalarVT = Value.getValueType().getScalarType(), ScalarMemVT = MemVT.getScalarType(); unsigned Stride = ScalarMemVT.getStoreSize(); SDValue Stores[4]; for (unsigned Idx = 0; Idx < 4; ++Idx) { SDValue Ex = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, ScalarVT, Value, DAG.getVectorIdxConstant(Idx, dl)); SDValue Store; if (ScalarVT != ScalarMemVT) Store = DAG.getTruncStore(StoreChain, dl, Ex, BasePtr, SN->getPointerInfo().getWithOffset(Idx * Stride), ScalarMemVT, MinAlign(Alignment, Idx * Stride), SN->getMemOperand()->getFlags(), SN->getAAInfo()); else Store = DAG.getStore(StoreChain, dl, Ex, BasePtr, SN->getPointerInfo().getWithOffset(Idx * Stride), MinAlign(Alignment, Idx * Stride), SN->getMemOperand()->getFlags(), SN->getAAInfo()); if (Idx == 0 && SN->isIndexed()) { assert(SN->getAddressingMode() == ISD::PRE_INC && "Unknown addressing mode on vector store"); Store = DAG.getIndexedStore(Store, dl, BasePtr, SN->getOffset(), SN->getAddressingMode()); } BasePtr = DAG.getNode(ISD::ADD, dl, BasePtr.getValueType(), BasePtr, DAG.getConstant(Stride, dl, BasePtr.getValueType())); Stores[Idx] = Store; } SDValue TF = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Stores); if (SN->isIndexed()) { SDValue RetOps[] = { TF, Stores[0].getValue(1) }; return DAG.getMergeValues(RetOps, dl); } return TF; } assert(SN->isUnindexed() && "Indexed v4i1 stores are not supported"); assert(Value.getValueType() == MVT::v4i1 && "Unknown store to lower"); // The values are now known to be -1 (false) or 1 (true). To convert this // into 0 (false) and 1 (true), add 1 and then divide by 2 (multiply by 0.5). // This can be done with an fma and the 0.5 constant: (V+1.0)*0.5 = 0.5*V+0.5 Value = DAG.getNode(PPCISD::QBFLT, dl, MVT::v4f64, Value); // FIXME: We can make this an f32 vector, but the BUILD_VECTOR code needs to // understand how to form the extending load. SDValue FPHalfs = DAG.getConstantFP(0.5, dl, MVT::v4f64); Value = DAG.getNode(ISD::FMA, dl, MVT::v4f64, Value, FPHalfs, FPHalfs); // Now convert to an integer and store. Value = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, MVT::v4f64, DAG.getConstant(Intrinsic::ppc_qpx_qvfctiwu, dl, MVT::i32), Value); MachineFrameInfo &MFI = DAG.getMachineFunction().getFrameInfo(); int FrameIdx = MFI.CreateStackObject(16, 16, false); MachinePointerInfo PtrInfo = MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FrameIdx); EVT PtrVT = getPointerTy(DAG.getDataLayout()); SDValue FIdx = DAG.getFrameIndex(FrameIdx, PtrVT); SDValue Ops[] = {StoreChain, DAG.getConstant(Intrinsic::ppc_qpx_qvstfiw, dl, MVT::i32), Value, FIdx}; SDVTList VTs = DAG.getVTList(/*chain*/ MVT::Other); StoreChain = DAG.getMemIntrinsicNode(ISD::INTRINSIC_VOID, dl, VTs, Ops, MVT::v4i32, PtrInfo); // Move data into the byte array. SDValue Loads[4], LoadChains[4]; for (unsigned i = 0; i < 4; ++i) { unsigned Offset = 4*i; SDValue Idx = DAG.getConstant(Offset, dl, FIdx.getValueType()); Idx = DAG.getNode(ISD::ADD, dl, FIdx.getValueType(), FIdx, Idx); Loads[i] = DAG.getLoad(MVT::i32, dl, StoreChain, Idx, PtrInfo.getWithOffset(Offset)); LoadChains[i] = Loads[i].getValue(1); } StoreChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, LoadChains); SDValue Stores[4]; for (unsigned i = 0; i < 4; ++i) { SDValue Idx = DAG.getConstant(i, dl, BasePtr.getValueType()); Idx = DAG.getNode(ISD::ADD, dl, BasePtr.getValueType(), BasePtr, Idx); Stores[i] = DAG.getTruncStore( StoreChain, dl, Loads[i], Idx, SN->getPointerInfo().getWithOffset(i), MVT::i8, /* Alignment = */ 1, SN->getMemOperand()->getFlags(), SN->getAAInfo()); } StoreChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Stores); return StoreChain; } SDValue PPCTargetLowering::LowerMUL(SDValue Op, SelectionDAG &DAG) const { SDLoc dl(Op); if (Op.getValueType() == MVT::v4i32) { SDValue LHS = Op.getOperand(0), RHS = Op.getOperand(1); SDValue Zero = BuildSplatI( 0, 1, MVT::v4i32, DAG, dl); SDValue Neg16 = BuildSplatI(-16, 4, MVT::v4i32, DAG, dl);//+16 as shift amt. SDValue RHSSwap = // = vrlw RHS, 16 BuildIntrinsicOp(Intrinsic::ppc_altivec_vrlw, RHS, Neg16, DAG, dl); // Shrinkify inputs to v8i16. LHS = DAG.getNode(ISD::BITCAST, dl, MVT::v8i16, LHS); RHS = DAG.getNode(ISD::BITCAST, dl, MVT::v8i16, RHS); RHSSwap = DAG.getNode(ISD::BITCAST, dl, MVT::v8i16, RHSSwap); // Low parts multiplied together, generating 32-bit results (we ignore the // top parts). SDValue LoProd = BuildIntrinsicOp(Intrinsic::ppc_altivec_vmulouh, LHS, RHS, DAG, dl, MVT::v4i32); SDValue HiProd = BuildIntrinsicOp(Intrinsic::ppc_altivec_vmsumuhm, LHS, RHSSwap, Zero, DAG, dl, MVT::v4i32); // Shift the high parts up 16 bits. HiProd = BuildIntrinsicOp(Intrinsic::ppc_altivec_vslw, HiProd, Neg16, DAG, dl); return DAG.getNode(ISD::ADD, dl, MVT::v4i32, LoProd, HiProd); } else if (Op.getValueType() == MVT::v16i8) { SDValue LHS = Op.getOperand(0), RHS = Op.getOperand(1); bool isLittleEndian = Subtarget.isLittleEndian(); // Multiply the even 8-bit parts, producing 16-bit sums. SDValue EvenParts = BuildIntrinsicOp(Intrinsic::ppc_altivec_vmuleub, LHS, RHS, DAG, dl, MVT::v8i16); EvenParts = DAG.getNode(ISD::BITCAST, dl, MVT::v16i8, EvenParts); // Multiply the odd 8-bit parts, producing 16-bit sums. SDValue OddParts = BuildIntrinsicOp(Intrinsic::ppc_altivec_vmuloub, LHS, RHS, DAG, dl, MVT::v8i16); OddParts = DAG.getNode(ISD::BITCAST, dl, MVT::v16i8, OddParts); // Merge the results together. Because vmuleub and vmuloub are // instructions with a big-endian bias, we must reverse the // element numbering and reverse the meaning of "odd" and "even" // when generating little endian code. int Ops[16]; for (unsigned i = 0; i != 8; ++i) { if (isLittleEndian) { Ops[i*2 ] = 2*i; Ops[i*2+1] = 2*i+16; } else { Ops[i*2 ] = 2*i+1; Ops[i*2+1] = 2*i+1+16; } } if (isLittleEndian) return DAG.getVectorShuffle(MVT::v16i8, dl, OddParts, EvenParts, Ops); else return DAG.getVectorShuffle(MVT::v16i8, dl, EvenParts, OddParts, Ops); } else { llvm_unreachable("Unknown mul to lower!"); } } SDValue PPCTargetLowering::LowerABS(SDValue Op, SelectionDAG &DAG) const { assert(Op.getOpcode() == ISD::ABS && "Should only be called for ISD::ABS"); EVT VT = Op.getValueType(); assert(VT.isVector() && "Only set vector abs as custom, scalar abs shouldn't reach here!"); assert((VT == MVT::v2i64 || VT == MVT::v4i32 || VT == MVT::v8i16 || VT == MVT::v16i8) && "Unexpected vector element type!"); assert((VT != MVT::v2i64 || Subtarget.hasP8Altivec()) && "Current subtarget doesn't support smax v2i64!"); // For vector abs, it can be lowered to: // abs x // ==> // y = -x // smax(x, y) SDLoc dl(Op); SDValue X = Op.getOperand(0); SDValue Zero = DAG.getConstant(0, dl, VT); SDValue Y = DAG.getNode(ISD::SUB, dl, VT, Zero, X); // SMAX patch https://reviews.llvm.org/D47332 // hasn't landed yet, so use intrinsic first here. // TODO: Should use SMAX directly once SMAX patch landed Intrinsic::ID BifID = Intrinsic::ppc_altivec_vmaxsw; if (VT == MVT::v2i64) BifID = Intrinsic::ppc_altivec_vmaxsd; else if (VT == MVT::v8i16) BifID = Intrinsic::ppc_altivec_vmaxsh; else if (VT == MVT::v16i8) BifID = Intrinsic::ppc_altivec_vmaxsb; return BuildIntrinsicOp(BifID, X, Y, DAG, dl, VT); } // Custom lowering for fpext vf32 to v2f64 SDValue PPCTargetLowering::LowerFP_EXTEND(SDValue Op, SelectionDAG &DAG) const { assert(Op.getOpcode() == ISD::FP_EXTEND && "Should only be called for ISD::FP_EXTEND"); // We only want to custom lower an extend from v2f32 to v2f64. if (Op.getValueType() != MVT::v2f64 || Op.getOperand(0).getValueType() != MVT::v2f32) return SDValue(); SDLoc dl(Op); SDValue Op0 = Op.getOperand(0); switch (Op0.getOpcode()) { default: return SDValue(); case ISD::EXTRACT_SUBVECTOR: { assert(Op0.getNumOperands() == 2 && isa(Op0->getOperand(1)) && "Node should have 2 operands with second one being a constant!"); if (Op0.getOperand(0).getValueType() != MVT::v4f32) return SDValue(); // Custom lower is only done for high or low doubleword. int Idx = cast(Op0.getOperand(1))->getZExtValue(); if (Idx % 2 != 0) return SDValue(); // Since input is v4f32, at this point Idx is either 0 or 2. // Shift to get the doubleword position we want. int DWord = Idx >> 1; // High and low word positions are different on little endian. if (Subtarget.isLittleEndian()) DWord ^= 0x1; return DAG.getNode(PPCISD::FP_EXTEND_HALF, dl, MVT::v2f64, Op0.getOperand(0), DAG.getConstant(DWord, dl, MVT::i32)); } case ISD::FADD: case ISD::FMUL: case ISD::FSUB: { SDValue NewLoad[2]; for (unsigned i = 0, ie = Op0.getNumOperands(); i != ie; ++i) { // Ensure both input are loads. SDValue LdOp = Op0.getOperand(i); if (LdOp.getOpcode() != ISD::LOAD) return SDValue(); // Generate new load node. LoadSDNode *LD = cast(LdOp); SDValue LoadOps[] = {LD->getChain(), LD->getBasePtr()}; NewLoad[i] = DAG.getMemIntrinsicNode( PPCISD::LD_VSX_LH, dl, DAG.getVTList(MVT::v4f32, MVT::Other), LoadOps, LD->getMemoryVT(), LD->getMemOperand()); } SDValue NewOp = DAG.getNode(Op0.getOpcode(), SDLoc(Op0), MVT::v4f32, NewLoad[0], NewLoad[1], Op0.getNode()->getFlags()); return DAG.getNode(PPCISD::FP_EXTEND_HALF, dl, MVT::v2f64, NewOp, DAG.getConstant(0, dl, MVT::i32)); } case ISD::LOAD: { LoadSDNode *LD = cast(Op0); SDValue LoadOps[] = {LD->getChain(), LD->getBasePtr()}; SDValue NewLd = DAG.getMemIntrinsicNode( PPCISD::LD_VSX_LH, dl, DAG.getVTList(MVT::v4f32, MVT::Other), LoadOps, LD->getMemoryVT(), LD->getMemOperand()); return DAG.getNode(PPCISD::FP_EXTEND_HALF, dl, MVT::v2f64, NewLd, DAG.getConstant(0, dl, MVT::i32)); } } llvm_unreachable("ERROR:Should return for all cases within swtich."); } /// LowerOperation - Provide custom lowering hooks for some operations. /// SDValue PPCTargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const { switch (Op.getOpcode()) { default: llvm_unreachable("Wasn't expecting to be able to lower this!"); case ISD::ConstantPool: return LowerConstantPool(Op, DAG); case ISD::BlockAddress: return LowerBlockAddress(Op, DAG); case ISD::GlobalAddress: return LowerGlobalAddress(Op, DAG); case ISD::GlobalTLSAddress: return LowerGlobalTLSAddress(Op, DAG); case ISD::JumpTable: return LowerJumpTable(Op, DAG); case ISD::SETCC: return LowerSETCC(Op, DAG); case ISD::INIT_TRAMPOLINE: return LowerINIT_TRAMPOLINE(Op, DAG); case ISD::ADJUST_TRAMPOLINE: return LowerADJUST_TRAMPOLINE(Op, DAG); // Variable argument lowering. case ISD::VASTART: return LowerVASTART(Op, DAG); case ISD::VAARG: return LowerVAARG(Op, DAG); case ISD::VACOPY: return LowerVACOPY(Op, DAG); case ISD::STACKRESTORE: return LowerSTACKRESTORE(Op, DAG); case ISD::DYNAMIC_STACKALLOC: return LowerDYNAMIC_STACKALLOC(Op, DAG); case ISD::GET_DYNAMIC_AREA_OFFSET: return LowerGET_DYNAMIC_AREA_OFFSET(Op, DAG); // Exception handling lowering. case ISD::EH_DWARF_CFA: return LowerEH_DWARF_CFA(Op, DAG); case ISD::EH_SJLJ_SETJMP: return lowerEH_SJLJ_SETJMP(Op, DAG); case ISD::EH_SJLJ_LONGJMP: return lowerEH_SJLJ_LONGJMP(Op, DAG); case ISD::LOAD: return LowerLOAD(Op, DAG); case ISD::STORE: return LowerSTORE(Op, DAG); case ISD::TRUNCATE: return LowerTRUNCATE(Op, DAG); case ISD::SELECT_CC: return LowerSELECT_CC(Op, DAG); case ISD::FP_TO_UINT: case ISD::FP_TO_SINT: return LowerFP_TO_INT(Op, DAG, SDLoc(Op)); case ISD::UINT_TO_FP: case ISD::SINT_TO_FP: return LowerINT_TO_FP(Op, DAG); case ISD::FLT_ROUNDS_: return LowerFLT_ROUNDS_(Op, DAG); // Lower 64-bit shifts. case ISD::SHL_PARTS: return LowerSHL_PARTS(Op, DAG); case ISD::SRL_PARTS: return LowerSRL_PARTS(Op, DAG); case ISD::SRA_PARTS: return LowerSRA_PARTS(Op, DAG); // Vector-related lowering. case ISD::BUILD_VECTOR: return LowerBUILD_VECTOR(Op, DAG); case ISD::VECTOR_SHUFFLE: return LowerVECTOR_SHUFFLE(Op, DAG); case ISD::INTRINSIC_WO_CHAIN: return LowerINTRINSIC_WO_CHAIN(Op, DAG); case ISD::SCALAR_TO_VECTOR: return LowerSCALAR_TO_VECTOR(Op, DAG); case ISD::EXTRACT_VECTOR_ELT: return LowerEXTRACT_VECTOR_ELT(Op, DAG); case ISD::INSERT_VECTOR_ELT: return LowerINSERT_VECTOR_ELT(Op, DAG); case ISD::MUL: return LowerMUL(Op, DAG); case ISD::ABS: return LowerABS(Op, DAG); case ISD::FP_EXTEND: return LowerFP_EXTEND(Op, DAG); // For counter-based loop handling. case ISD::INTRINSIC_W_CHAIN: return SDValue(); case ISD::BITCAST: return LowerBITCAST(Op, DAG); // Frame & Return address. case ISD::RETURNADDR: return LowerRETURNADDR(Op, DAG); case ISD::FRAMEADDR: return LowerFRAMEADDR(Op, DAG); case ISD::INTRINSIC_VOID: return LowerINTRINSIC_VOID(Op, DAG); case ISD::SREM: case ISD::UREM: return LowerREM(Op, DAG); case ISD::BSWAP: return LowerBSWAP(Op, DAG); case ISD::ATOMIC_CMP_SWAP: return LowerATOMIC_CMP_SWAP(Op, DAG); } } void PPCTargetLowering::ReplaceNodeResults(SDNode *N, SmallVectorImpl&Results, SelectionDAG &DAG) const { SDLoc dl(N); switch (N->getOpcode()) { default: llvm_unreachable("Do not know how to custom type legalize this operation!"); case ISD::READCYCLECOUNTER: { SDVTList VTs = DAG.getVTList(MVT::i32, MVT::i32, MVT::Other); SDValue RTB = DAG.getNode(PPCISD::READ_TIME_BASE, dl, VTs, N->getOperand(0)); Results.push_back( DAG.getNode(ISD::BUILD_PAIR, dl, MVT::i64, RTB, RTB.getValue(1))); Results.push_back(RTB.getValue(2)); break; } case ISD::INTRINSIC_W_CHAIN: { if (cast(N->getOperand(1))->getZExtValue() != Intrinsic::loop_decrement) break; assert(N->getValueType(0) == MVT::i1 && "Unexpected result type for CTR decrement intrinsic"); EVT SVT = getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), N->getValueType(0)); SDVTList VTs = DAG.getVTList(SVT, MVT::Other); SDValue NewInt = DAG.getNode(N->getOpcode(), dl, VTs, N->getOperand(0), N->getOperand(1)); Results.push_back(DAG.getNode(ISD::TRUNCATE, dl, MVT::i1, NewInt)); Results.push_back(NewInt.getValue(1)); break; } case ISD::VAARG: { if (!Subtarget.isSVR4ABI() || Subtarget.isPPC64()) return; EVT VT = N->getValueType(0); if (VT == MVT::i64) { SDValue NewNode = LowerVAARG(SDValue(N, 1), DAG); Results.push_back(NewNode); Results.push_back(NewNode.getValue(1)); } return; } case ISD::FP_TO_SINT: case ISD::FP_TO_UINT: // LowerFP_TO_INT() can only handle f32 and f64. if (N->getOperand(0).getValueType() == MVT::ppcf128) return; Results.push_back(LowerFP_TO_INT(SDValue(N, 0), DAG, dl)); return; case ISD::TRUNCATE: { EVT TrgVT = N->getValueType(0); EVT OpVT = N->getOperand(0).getValueType(); if (TrgVT.isVector() && isOperationCustom(N->getOpcode(), TrgVT) && OpVT.getSizeInBits() <= 128 && isPowerOf2_32(OpVT.getVectorElementType().getSizeInBits())) Results.push_back(LowerTRUNCATEVector(SDValue(N, 0), DAG)); return; } case ISD::BITCAST: // Don't handle bitcast here. return; } } //===----------------------------------------------------------------------===// // Other Lowering Code //===----------------------------------------------------------------------===// static Instruction* callIntrinsic(IRBuilder<> &Builder, Intrinsic::ID Id) { Module *M = Builder.GetInsertBlock()->getParent()->getParent(); Function *Func = Intrinsic::getDeclaration(M, Id); return Builder.CreateCall(Func, {}); } // The mappings for emitLeading/TrailingFence is taken from // http://www.cl.cam.ac.uk/~pes20/cpp/cpp0xmappings.html Instruction *PPCTargetLowering::emitLeadingFence(IRBuilder<> &Builder, Instruction *Inst, AtomicOrdering Ord) const { if (Ord == AtomicOrdering::SequentiallyConsistent) return callIntrinsic(Builder, Intrinsic::ppc_sync); if (isReleaseOrStronger(Ord)) return callIntrinsic(Builder, Intrinsic::ppc_lwsync); return nullptr; } Instruction *PPCTargetLowering::emitTrailingFence(IRBuilder<> &Builder, Instruction *Inst, AtomicOrdering Ord) const { if (Inst->hasAtomicLoad() && isAcquireOrStronger(Ord)) { // See http://www.cl.cam.ac.uk/~pes20/cpp/cpp0xmappings.html and // http://www.rdrop.com/users/paulmck/scalability/paper/N2745r.2011.03.04a.html // and http://www.cl.cam.ac.uk/~pes20/cppppc/ for justification. if (isa(Inst) && Subtarget.isPPC64()) return Builder.CreateCall( Intrinsic::getDeclaration( Builder.GetInsertBlock()->getParent()->getParent(), Intrinsic::ppc_cfence, {Inst->getType()}), {Inst}); // FIXME: Can use isync for rmw operation. return callIntrinsic(Builder, Intrinsic::ppc_lwsync); } return nullptr; } MachineBasicBlock * PPCTargetLowering::EmitAtomicBinary(MachineInstr &MI, MachineBasicBlock *BB, unsigned AtomicSize, unsigned BinOpcode, unsigned CmpOpcode, unsigned CmpPred) const { // This also handles ATOMIC_SWAP, indicated by BinOpcode==0. const TargetInstrInfo *TII = Subtarget.getInstrInfo(); auto LoadMnemonic = PPC::LDARX; auto StoreMnemonic = PPC::STDCX; switch (AtomicSize) { default: llvm_unreachable("Unexpected size of atomic entity"); case 1: LoadMnemonic = PPC::LBARX; StoreMnemonic = PPC::STBCX; assert(Subtarget.hasPartwordAtomics() && "Call this only with size >=4"); break; case 2: LoadMnemonic = PPC::LHARX; StoreMnemonic = PPC::STHCX; assert(Subtarget.hasPartwordAtomics() && "Call this only with size >=4"); break; case 4: LoadMnemonic = PPC::LWARX; StoreMnemonic = PPC::STWCX; break; case 8: LoadMnemonic = PPC::LDARX; StoreMnemonic = PPC::STDCX; break; } const BasicBlock *LLVM_BB = BB->getBasicBlock(); MachineFunction *F = BB->getParent(); MachineFunction::iterator It = ++BB->getIterator(); Register dest = MI.getOperand(0).getReg(); Register ptrA = MI.getOperand(1).getReg(); Register ptrB = MI.getOperand(2).getReg(); Register incr = MI.getOperand(3).getReg(); DebugLoc dl = MI.getDebugLoc(); MachineBasicBlock *loopMBB = F->CreateMachineBasicBlock(LLVM_BB); MachineBasicBlock *loop2MBB = CmpOpcode ? F->CreateMachineBasicBlock(LLVM_BB) : nullptr; MachineBasicBlock *exitMBB = F->CreateMachineBasicBlock(LLVM_BB); F->insert(It, loopMBB); if (CmpOpcode) F->insert(It, loop2MBB); F->insert(It, exitMBB); exitMBB->splice(exitMBB->begin(), BB, std::next(MachineBasicBlock::iterator(MI)), BB->end()); exitMBB->transferSuccessorsAndUpdatePHIs(BB); MachineRegisterInfo &RegInfo = F->getRegInfo(); Register TmpReg = (!BinOpcode) ? incr : RegInfo.createVirtualRegister( AtomicSize == 8 ? &PPC::G8RCRegClass : &PPC::GPRCRegClass); // thisMBB: // ... // fallthrough --> loopMBB BB->addSuccessor(loopMBB); // loopMBB: // l[wd]arx dest, ptr // add r0, dest, incr // st[wd]cx. r0, ptr // bne- loopMBB // fallthrough --> exitMBB // For max/min... // loopMBB: // l[wd]arx dest, ptr // cmpl?[wd] incr, dest // bgt exitMBB // loop2MBB: // st[wd]cx. dest, ptr // bne- loopMBB // fallthrough --> exitMBB BB = loopMBB; BuildMI(BB, dl, TII->get(LoadMnemonic), dest) .addReg(ptrA).addReg(ptrB); if (BinOpcode) BuildMI(BB, dl, TII->get(BinOpcode), TmpReg).addReg(incr).addReg(dest); if (CmpOpcode) { // Signed comparisons of byte or halfword values must be sign-extended. if (CmpOpcode == PPC::CMPW && AtomicSize < 4) { Register ExtReg = RegInfo.createVirtualRegister(&PPC::GPRCRegClass); BuildMI(BB, dl, TII->get(AtomicSize == 1 ? PPC::EXTSB : PPC::EXTSH), ExtReg).addReg(dest); BuildMI(BB, dl, TII->get(CmpOpcode), PPC::CR0) .addReg(incr).addReg(ExtReg); } else BuildMI(BB, dl, TII->get(CmpOpcode), PPC::CR0) .addReg(incr).addReg(dest); BuildMI(BB, dl, TII->get(PPC::BCC)) .addImm(CmpPred).addReg(PPC::CR0).addMBB(exitMBB); BB->addSuccessor(loop2MBB); BB->addSuccessor(exitMBB); BB = loop2MBB; } BuildMI(BB, dl, TII->get(StoreMnemonic)) .addReg(TmpReg).addReg(ptrA).addReg(ptrB); BuildMI(BB, dl, TII->get(PPC::BCC)) .addImm(PPC::PRED_NE).addReg(PPC::CR0).addMBB(loopMBB); BB->addSuccessor(loopMBB); BB->addSuccessor(exitMBB); // exitMBB: // ... BB = exitMBB; return BB; } MachineBasicBlock *PPCTargetLowering::EmitPartwordAtomicBinary( MachineInstr &MI, MachineBasicBlock *BB, bool is8bit, // operation unsigned BinOpcode, unsigned CmpOpcode, unsigned CmpPred) const { // If we support part-word atomic mnemonics, just use them if (Subtarget.hasPartwordAtomics()) return EmitAtomicBinary(MI, BB, is8bit ? 1 : 2, BinOpcode, CmpOpcode, CmpPred); // This also handles ATOMIC_SWAP, indicated by BinOpcode==0. const TargetInstrInfo *TII = Subtarget.getInstrInfo(); // In 64 bit mode we have to use 64 bits for addresses, even though the // lwarx/stwcx are 32 bits. With the 32-bit atomics we can use address // registers without caring whether they're 32 or 64, but here we're // doing actual arithmetic on the addresses. bool is64bit = Subtarget.isPPC64(); bool isLittleEndian = Subtarget.isLittleEndian(); unsigned ZeroReg = is64bit ? PPC::ZERO8 : PPC::ZERO; const BasicBlock *LLVM_BB = BB->getBasicBlock(); MachineFunction *F = BB->getParent(); MachineFunction::iterator It = ++BB->getIterator(); Register dest = MI.getOperand(0).getReg(); Register ptrA = MI.getOperand(1).getReg(); Register ptrB = MI.getOperand(2).getReg(); Register incr = MI.getOperand(3).getReg(); DebugLoc dl = MI.getDebugLoc(); MachineBasicBlock *loopMBB = F->CreateMachineBasicBlock(LLVM_BB); MachineBasicBlock *loop2MBB = CmpOpcode ? F->CreateMachineBasicBlock(LLVM_BB) : nullptr; MachineBasicBlock *exitMBB = F->CreateMachineBasicBlock(LLVM_BB); F->insert(It, loopMBB); if (CmpOpcode) F->insert(It, loop2MBB); F->insert(It, exitMBB); exitMBB->splice(exitMBB->begin(), BB, std::next(MachineBasicBlock::iterator(MI)), BB->end()); exitMBB->transferSuccessorsAndUpdatePHIs(BB); MachineRegisterInfo &RegInfo = F->getRegInfo(); const TargetRegisterClass *RC = is64bit ? &PPC::G8RCRegClass : &PPC::GPRCRegClass; const TargetRegisterClass *GPRC = &PPC::GPRCRegClass; Register PtrReg = RegInfo.createVirtualRegister(RC); Register Shift1Reg = RegInfo.createVirtualRegister(GPRC); Register ShiftReg = isLittleEndian ? Shift1Reg : RegInfo.createVirtualRegister(GPRC); Register Incr2Reg = RegInfo.createVirtualRegister(GPRC); Register MaskReg = RegInfo.createVirtualRegister(GPRC); Register Mask2Reg = RegInfo.createVirtualRegister(GPRC); Register Mask3Reg = RegInfo.createVirtualRegister(GPRC); Register Tmp2Reg = RegInfo.createVirtualRegister(GPRC); Register Tmp3Reg = RegInfo.createVirtualRegister(GPRC); Register Tmp4Reg = RegInfo.createVirtualRegister(GPRC); Register TmpDestReg = RegInfo.createVirtualRegister(GPRC); Register Ptr1Reg; Register TmpReg = (!BinOpcode) ? Incr2Reg : RegInfo.createVirtualRegister(GPRC); // thisMBB: // ... // fallthrough --> loopMBB BB->addSuccessor(loopMBB); // The 4-byte load must be aligned, while a char or short may be // anywhere in the word. Hence all this nasty bookkeeping code. // add ptr1, ptrA, ptrB [copy if ptrA==0] // rlwinm shift1, ptr1, 3, 27, 28 [3, 27, 27] // xori shift, shift1, 24 [16] // rlwinm ptr, ptr1, 0, 0, 29 // slw incr2, incr, shift // li mask2, 255 [li mask3, 0; ori mask2, mask3, 65535] // slw mask, mask2, shift // loopMBB: // lwarx tmpDest, ptr // add tmp, tmpDest, incr2 // andc tmp2, tmpDest, mask // and tmp3, tmp, mask // or tmp4, tmp3, tmp2 // stwcx. tmp4, ptr // bne- loopMBB // fallthrough --> exitMBB // srw dest, tmpDest, shift if (ptrA != ZeroReg) { Ptr1Reg = RegInfo.createVirtualRegister(RC); BuildMI(BB, dl, TII->get(is64bit ? PPC::ADD8 : PPC::ADD4), Ptr1Reg) .addReg(ptrA) .addReg(ptrB); } else { Ptr1Reg = ptrB; } // We need use 32-bit subregister to avoid mismatch register class in 64-bit // mode. BuildMI(BB, dl, TII->get(PPC::RLWINM), Shift1Reg) .addReg(Ptr1Reg, 0, is64bit ? PPC::sub_32 : 0) .addImm(3) .addImm(27) .addImm(is8bit ? 28 : 27); if (!isLittleEndian) BuildMI(BB, dl, TII->get(PPC::XORI), ShiftReg) .addReg(Shift1Reg) .addImm(is8bit ? 24 : 16); if (is64bit) BuildMI(BB, dl, TII->get(PPC::RLDICR), PtrReg) .addReg(Ptr1Reg) .addImm(0) .addImm(61); else BuildMI(BB, dl, TII->get(PPC::RLWINM), PtrReg) .addReg(Ptr1Reg) .addImm(0) .addImm(0) .addImm(29); BuildMI(BB, dl, TII->get(PPC::SLW), Incr2Reg).addReg(incr).addReg(ShiftReg); if (is8bit) BuildMI(BB, dl, TII->get(PPC::LI), Mask2Reg).addImm(255); else { BuildMI(BB, dl, TII->get(PPC::LI), Mask3Reg).addImm(0); BuildMI(BB, dl, TII->get(PPC::ORI), Mask2Reg) .addReg(Mask3Reg) .addImm(65535); } BuildMI(BB, dl, TII->get(PPC::SLW), MaskReg) .addReg(Mask2Reg) .addReg(ShiftReg); BB = loopMBB; BuildMI(BB, dl, TII->get(PPC::LWARX), TmpDestReg) .addReg(ZeroReg) .addReg(PtrReg); if (BinOpcode) BuildMI(BB, dl, TII->get(BinOpcode), TmpReg) .addReg(Incr2Reg) .addReg(TmpDestReg); BuildMI(BB, dl, TII->get(PPC::ANDC), Tmp2Reg) .addReg(TmpDestReg) .addReg(MaskReg); BuildMI(BB, dl, TII->get(PPC::AND), Tmp3Reg).addReg(TmpReg).addReg(MaskReg); if (CmpOpcode) { // For unsigned comparisons, we can directly compare the shifted values. // For signed comparisons we shift and sign extend. Register SReg = RegInfo.createVirtualRegister(GPRC); BuildMI(BB, dl, TII->get(PPC::AND), SReg) .addReg(TmpDestReg) .addReg(MaskReg); unsigned ValueReg = SReg; unsigned CmpReg = Incr2Reg; if (CmpOpcode == PPC::CMPW) { ValueReg = RegInfo.createVirtualRegister(GPRC); BuildMI(BB, dl, TII->get(PPC::SRW), ValueReg) .addReg(SReg) .addReg(ShiftReg); Register ValueSReg = RegInfo.createVirtualRegister(GPRC); BuildMI(BB, dl, TII->get(is8bit ? PPC::EXTSB : PPC::EXTSH), ValueSReg) .addReg(ValueReg); ValueReg = ValueSReg; CmpReg = incr; } BuildMI(BB, dl, TII->get(CmpOpcode), PPC::CR0) .addReg(CmpReg) .addReg(ValueReg); BuildMI(BB, dl, TII->get(PPC::BCC)) .addImm(CmpPred) .addReg(PPC::CR0) .addMBB(exitMBB); BB->addSuccessor(loop2MBB); BB->addSuccessor(exitMBB); BB = loop2MBB; } BuildMI(BB, dl, TII->get(PPC::OR), Tmp4Reg).addReg(Tmp3Reg).addReg(Tmp2Reg); BuildMI(BB, dl, TII->get(PPC::STWCX)) .addReg(Tmp4Reg) .addReg(ZeroReg) .addReg(PtrReg); BuildMI(BB, dl, TII->get(PPC::BCC)) .addImm(PPC::PRED_NE) .addReg(PPC::CR0) .addMBB(loopMBB); BB->addSuccessor(loopMBB); BB->addSuccessor(exitMBB); // exitMBB: // ... BB = exitMBB; BuildMI(*BB, BB->begin(), dl, TII->get(PPC::SRW), dest) .addReg(TmpDestReg) .addReg(ShiftReg); return BB; } llvm::MachineBasicBlock * PPCTargetLowering::emitEHSjLjSetJmp(MachineInstr &MI, MachineBasicBlock *MBB) const { DebugLoc DL = MI.getDebugLoc(); const TargetInstrInfo *TII = Subtarget.getInstrInfo(); const PPCRegisterInfo *TRI = Subtarget.getRegisterInfo(); MachineFunction *MF = MBB->getParent(); MachineRegisterInfo &MRI = MF->getRegInfo(); const BasicBlock *BB = MBB->getBasicBlock(); MachineFunction::iterator I = ++MBB->getIterator(); Register DstReg = MI.getOperand(0).getReg(); const TargetRegisterClass *RC = MRI.getRegClass(DstReg); assert(TRI->isTypeLegalForClass(*RC, MVT::i32) && "Invalid destination!"); Register mainDstReg = MRI.createVirtualRegister(RC); Register restoreDstReg = MRI.createVirtualRegister(RC); MVT PVT = getPointerTy(MF->getDataLayout()); assert((PVT == MVT::i64 || PVT == MVT::i32) && "Invalid Pointer Size!"); // For v = setjmp(buf), we generate // // thisMBB: // SjLjSetup mainMBB // bl mainMBB // v_restore = 1 // b sinkMBB // // mainMBB: // buf[LabelOffset] = LR // v_main = 0 // // sinkMBB: // v = phi(main, restore) // MachineBasicBlock *thisMBB = MBB; MachineBasicBlock *mainMBB = MF->CreateMachineBasicBlock(BB); MachineBasicBlock *sinkMBB = MF->CreateMachineBasicBlock(BB); MF->insert(I, mainMBB); MF->insert(I, sinkMBB); MachineInstrBuilder MIB; // Transfer the remainder of BB and its successor edges to sinkMBB. sinkMBB->splice(sinkMBB->begin(), MBB, std::next(MachineBasicBlock::iterator(MI)), MBB->end()); sinkMBB->transferSuccessorsAndUpdatePHIs(MBB); // Note that the structure of the jmp_buf used here is not compatible // with that used by libc, and is not designed to be. Specifically, it // stores only those 'reserved' registers that LLVM does not otherwise // understand how to spill. Also, by convention, by the time this // intrinsic is called, Clang has already stored the frame address in the // first slot of the buffer and stack address in the third. Following the // X86 target code, we'll store the jump address in the second slot. We also // need to save the TOC pointer (R2) to handle jumps between shared // libraries, and that will be stored in the fourth slot. The thread // identifier (R13) is not affected. // thisMBB: const int64_t LabelOffset = 1 * PVT.getStoreSize(); const int64_t TOCOffset = 3 * PVT.getStoreSize(); const int64_t BPOffset = 4 * PVT.getStoreSize(); // Prepare IP either in reg. const TargetRegisterClass *PtrRC = getRegClassFor(PVT); Register LabelReg = MRI.createVirtualRegister(PtrRC); Register BufReg = MI.getOperand(1).getReg(); if (Subtarget.is64BitELFABI()) { setUsesTOCBasePtr(*MBB->getParent()); MIB = BuildMI(*thisMBB, MI, DL, TII->get(PPC::STD)) .addReg(PPC::X2) .addImm(TOCOffset) .addReg(BufReg) .cloneMemRefs(MI); } // Naked functions never have a base pointer, and so we use r1. For all // other functions, this decision must be delayed until during PEI. unsigned BaseReg; if (MF->getFunction().hasFnAttribute(Attribute::Naked)) BaseReg = Subtarget.isPPC64() ? PPC::X1 : PPC::R1; else BaseReg = Subtarget.isPPC64() ? PPC::BP8 : PPC::BP; MIB = BuildMI(*thisMBB, MI, DL, TII->get(Subtarget.isPPC64() ? PPC::STD : PPC::STW)) .addReg(BaseReg) .addImm(BPOffset) .addReg(BufReg) .cloneMemRefs(MI); // Setup MIB = BuildMI(*thisMBB, MI, DL, TII->get(PPC::BCLalways)).addMBB(mainMBB); MIB.addRegMask(TRI->getNoPreservedMask()); BuildMI(*thisMBB, MI, DL, TII->get(PPC::LI), restoreDstReg).addImm(1); MIB = BuildMI(*thisMBB, MI, DL, TII->get(PPC::EH_SjLj_Setup)) .addMBB(mainMBB); MIB = BuildMI(*thisMBB, MI, DL, TII->get(PPC::B)).addMBB(sinkMBB); thisMBB->addSuccessor(mainMBB, BranchProbability::getZero()); thisMBB->addSuccessor(sinkMBB, BranchProbability::getOne()); // mainMBB: // mainDstReg = 0 MIB = BuildMI(mainMBB, DL, TII->get(Subtarget.isPPC64() ? PPC::MFLR8 : PPC::MFLR), LabelReg); // Store IP if (Subtarget.isPPC64()) { MIB = BuildMI(mainMBB, DL, TII->get(PPC::STD)) .addReg(LabelReg) .addImm(LabelOffset) .addReg(BufReg); } else { MIB = BuildMI(mainMBB, DL, TII->get(PPC::STW)) .addReg(LabelReg) .addImm(LabelOffset) .addReg(BufReg); } MIB.cloneMemRefs(MI); BuildMI(mainMBB, DL, TII->get(PPC::LI), mainDstReg).addImm(0); mainMBB->addSuccessor(sinkMBB); // sinkMBB: BuildMI(*sinkMBB, sinkMBB->begin(), DL, TII->get(PPC::PHI), DstReg) .addReg(mainDstReg).addMBB(mainMBB) .addReg(restoreDstReg).addMBB(thisMBB); MI.eraseFromParent(); return sinkMBB; } MachineBasicBlock * PPCTargetLowering::emitEHSjLjLongJmp(MachineInstr &MI, MachineBasicBlock *MBB) const { DebugLoc DL = MI.getDebugLoc(); const TargetInstrInfo *TII = Subtarget.getInstrInfo(); MachineFunction *MF = MBB->getParent(); MachineRegisterInfo &MRI = MF->getRegInfo(); MVT PVT = getPointerTy(MF->getDataLayout()); assert((PVT == MVT::i64 || PVT == MVT::i32) && "Invalid Pointer Size!"); const TargetRegisterClass *RC = (PVT == MVT::i64) ? &PPC::G8RCRegClass : &PPC::GPRCRegClass; Register Tmp = MRI.createVirtualRegister(RC); // Since FP is only updated here but NOT referenced, it's treated as GPR. unsigned FP = (PVT == MVT::i64) ? PPC::X31 : PPC::R31; unsigned SP = (PVT == MVT::i64) ? PPC::X1 : PPC::R1; unsigned BP = (PVT == MVT::i64) ? PPC::X30 : (Subtarget.isSVR4ABI() && isPositionIndependent() ? PPC::R29 : PPC::R30); MachineInstrBuilder MIB; const int64_t LabelOffset = 1 * PVT.getStoreSize(); const int64_t SPOffset = 2 * PVT.getStoreSize(); const int64_t TOCOffset = 3 * PVT.getStoreSize(); const int64_t BPOffset = 4 * PVT.getStoreSize(); Register BufReg = MI.getOperand(0).getReg(); // Reload FP (the jumped-to function may not have had a // frame pointer, and if so, then its r31 will be restored // as necessary). if (PVT == MVT::i64) { MIB = BuildMI(*MBB, MI, DL, TII->get(PPC::LD), FP) .addImm(0) .addReg(BufReg); } else { MIB = BuildMI(*MBB, MI, DL, TII->get(PPC::LWZ), FP) .addImm(0) .addReg(BufReg); } MIB.cloneMemRefs(MI); // Reload IP if (PVT == MVT::i64) { MIB = BuildMI(*MBB, MI, DL, TII->get(PPC::LD), Tmp) .addImm(LabelOffset) .addReg(BufReg); } else { MIB = BuildMI(*MBB, MI, DL, TII->get(PPC::LWZ), Tmp) .addImm(LabelOffset) .addReg(BufReg); } MIB.cloneMemRefs(MI); // Reload SP if (PVT == MVT::i64) { MIB = BuildMI(*MBB, MI, DL, TII->get(PPC::LD), SP) .addImm(SPOffset) .addReg(BufReg); } else { MIB = BuildMI(*MBB, MI, DL, TII->get(PPC::LWZ), SP) .addImm(SPOffset) .addReg(BufReg); } MIB.cloneMemRefs(MI); // Reload BP if (PVT == MVT::i64) { MIB = BuildMI(*MBB, MI, DL, TII->get(PPC::LD), BP) .addImm(BPOffset) .addReg(BufReg); } else { MIB = BuildMI(*MBB, MI, DL, TII->get(PPC::LWZ), BP) .addImm(BPOffset) .addReg(BufReg); } MIB.cloneMemRefs(MI); // Reload TOC if (PVT == MVT::i64 && Subtarget.isSVR4ABI()) { setUsesTOCBasePtr(*MBB->getParent()); MIB = BuildMI(*MBB, MI, DL, TII->get(PPC::LD), PPC::X2) .addImm(TOCOffset) .addReg(BufReg) .cloneMemRefs(MI); } // Jump BuildMI(*MBB, MI, DL, TII->get(PVT == MVT::i64 ? PPC::MTCTR8 : PPC::MTCTR)).addReg(Tmp); BuildMI(*MBB, MI, DL, TII->get(PVT == MVT::i64 ? PPC::BCTR8 : PPC::BCTR)); MI.eraseFromParent(); return MBB; } MachineBasicBlock * PPCTargetLowering::EmitInstrWithCustomInserter(MachineInstr &MI, MachineBasicBlock *BB) const { if (MI.getOpcode() == TargetOpcode::STACKMAP || MI.getOpcode() == TargetOpcode::PATCHPOINT) { if (Subtarget.is64BitELFABI() && - MI.getOpcode() == TargetOpcode::PATCHPOINT) { + MI.getOpcode() == TargetOpcode::PATCHPOINT && + !Subtarget.isUsingPCRelativeCalls()) { // Call lowering should have added an r2 operand to indicate a dependence // on the TOC base pointer value. It can't however, because there is no // way to mark the dependence as implicit there, and so the stackmap code // will confuse it with a regular operand. Instead, add the dependence // here. MI.addOperand(MachineOperand::CreateReg(PPC::X2, false, true)); } return emitPatchPoint(MI, BB); } if (MI.getOpcode() == PPC::EH_SjLj_SetJmp32 || MI.getOpcode() == PPC::EH_SjLj_SetJmp64) { return emitEHSjLjSetJmp(MI, BB); } else if (MI.getOpcode() == PPC::EH_SjLj_LongJmp32 || MI.getOpcode() == PPC::EH_SjLj_LongJmp64) { return emitEHSjLjLongJmp(MI, BB); } const TargetInstrInfo *TII = Subtarget.getInstrInfo(); // To "insert" these instructions we actually have to insert their // control-flow patterns. const BasicBlock *LLVM_BB = BB->getBasicBlock(); MachineFunction::iterator It = ++BB->getIterator(); MachineFunction *F = BB->getParent(); if (MI.getOpcode() == PPC::SELECT_CC_I4 || MI.getOpcode() == PPC::SELECT_CC_I8 || MI.getOpcode() == PPC::SELECT_I4 || MI.getOpcode() == PPC::SELECT_I8) { SmallVector Cond; if (MI.getOpcode() == PPC::SELECT_CC_I4 || MI.getOpcode() == PPC::SELECT_CC_I8) Cond.push_back(MI.getOperand(4)); else Cond.push_back(MachineOperand::CreateImm(PPC::PRED_BIT_SET)); Cond.push_back(MI.getOperand(1)); DebugLoc dl = MI.getDebugLoc(); TII->insertSelect(*BB, MI, dl, MI.getOperand(0).getReg(), Cond, MI.getOperand(2).getReg(), MI.getOperand(3).getReg()); } else if (MI.getOpcode() == PPC::SELECT_CC_F4 || MI.getOpcode() == PPC::SELECT_CC_F8 || MI.getOpcode() == PPC::SELECT_CC_F16 || MI.getOpcode() == PPC::SELECT_CC_QFRC || MI.getOpcode() == PPC::SELECT_CC_QSRC || MI.getOpcode() == PPC::SELECT_CC_QBRC || MI.getOpcode() == PPC::SELECT_CC_VRRC || MI.getOpcode() == PPC::SELECT_CC_VSFRC || MI.getOpcode() == PPC::SELECT_CC_VSSRC || MI.getOpcode() == PPC::SELECT_CC_VSRC || MI.getOpcode() == PPC::SELECT_CC_SPE4 || MI.getOpcode() == PPC::SELECT_CC_SPE || MI.getOpcode() == PPC::SELECT_F4 || MI.getOpcode() == PPC::SELECT_F8 || MI.getOpcode() == PPC::SELECT_F16 || MI.getOpcode() == PPC::SELECT_QFRC || MI.getOpcode() == PPC::SELECT_QSRC || MI.getOpcode() == PPC::SELECT_QBRC || MI.getOpcode() == PPC::SELECT_SPE || MI.getOpcode() == PPC::SELECT_SPE4 || MI.getOpcode() == PPC::SELECT_VRRC || MI.getOpcode() == PPC::SELECT_VSFRC || MI.getOpcode() == PPC::SELECT_VSSRC || MI.getOpcode() == PPC::SELECT_VSRC) { // The incoming instruction knows the destination vreg to set, the // condition code register to branch on, the true/false values to // select between, and a branch opcode to use. // thisMBB: // ... // TrueVal = ... // cmpTY ccX, r1, r2 // bCC copy1MBB // fallthrough --> copy0MBB MachineBasicBlock *thisMBB = BB; MachineBasicBlock *copy0MBB = F->CreateMachineBasicBlock(LLVM_BB); MachineBasicBlock *sinkMBB = F->CreateMachineBasicBlock(LLVM_BB); DebugLoc dl = MI.getDebugLoc(); F->insert(It, copy0MBB); F->insert(It, sinkMBB); // Transfer the remainder of BB and its successor edges to sinkMBB. sinkMBB->splice(sinkMBB->begin(), BB, std::next(MachineBasicBlock::iterator(MI)), BB->end()); sinkMBB->transferSuccessorsAndUpdatePHIs(BB); // Next, add the true and fallthrough blocks as its successors. BB->addSuccessor(copy0MBB); BB->addSuccessor(sinkMBB); if (MI.getOpcode() == PPC::SELECT_I4 || MI.getOpcode() == PPC::SELECT_I8 || MI.getOpcode() == PPC::SELECT_F4 || MI.getOpcode() == PPC::SELECT_F8 || MI.getOpcode() == PPC::SELECT_F16 || MI.getOpcode() == PPC::SELECT_SPE4 || MI.getOpcode() == PPC::SELECT_SPE || MI.getOpcode() == PPC::SELECT_QFRC || MI.getOpcode() == PPC::SELECT_QSRC || MI.getOpcode() == PPC::SELECT_QBRC || MI.getOpcode() == PPC::SELECT_VRRC || MI.getOpcode() == PPC::SELECT_VSFRC || MI.getOpcode() == PPC::SELECT_VSSRC || MI.getOpcode() == PPC::SELECT_VSRC) { BuildMI(BB, dl, TII->get(PPC::BC)) .addReg(MI.getOperand(1).getReg()) .addMBB(sinkMBB); } else { unsigned SelectPred = MI.getOperand(4).getImm(); BuildMI(BB, dl, TII->get(PPC::BCC)) .addImm(SelectPred) .addReg(MI.getOperand(1).getReg()) .addMBB(sinkMBB); } // copy0MBB: // %FalseValue = ... // # fallthrough to sinkMBB BB = copy0MBB; // Update machine-CFG edges BB->addSuccessor(sinkMBB); // sinkMBB: // %Result = phi [ %FalseValue, copy0MBB ], [ %TrueValue, thisMBB ] // ... BB = sinkMBB; BuildMI(*BB, BB->begin(), dl, TII->get(PPC::PHI), MI.getOperand(0).getReg()) .addReg(MI.getOperand(3).getReg()) .addMBB(copy0MBB) .addReg(MI.getOperand(2).getReg()) .addMBB(thisMBB); } else if (MI.getOpcode() == PPC::ReadTB) { // To read the 64-bit time-base register on a 32-bit target, we read the // two halves. Should the counter have wrapped while it was being read, we // need to try again. // ... // readLoop: // mfspr Rx,TBU # load from TBU // mfspr Ry,TB # load from TB // mfspr Rz,TBU # load from TBU // cmpw crX,Rx,Rz # check if 'old'='new' // bne readLoop # branch if they're not equal // ... MachineBasicBlock *readMBB = F->CreateMachineBasicBlock(LLVM_BB); MachineBasicBlock *sinkMBB = F->CreateMachineBasicBlock(LLVM_BB); DebugLoc dl = MI.getDebugLoc(); F->insert(It, readMBB); F->insert(It, sinkMBB); // Transfer the remainder of BB and its successor edges to sinkMBB. sinkMBB->splice(sinkMBB->begin(), BB, std::next(MachineBasicBlock::iterator(MI)), BB->end()); sinkMBB->transferSuccessorsAndUpdatePHIs(BB); BB->addSuccessor(readMBB); BB = readMBB; MachineRegisterInfo &RegInfo = F->getRegInfo(); Register ReadAgainReg = RegInfo.createVirtualRegister(&PPC::GPRCRegClass); Register LoReg = MI.getOperand(0).getReg(); Register HiReg = MI.getOperand(1).getReg(); BuildMI(BB, dl, TII->get(PPC::MFSPR), HiReg).addImm(269); BuildMI(BB, dl, TII->get(PPC::MFSPR), LoReg).addImm(268); BuildMI(BB, dl, TII->get(PPC::MFSPR), ReadAgainReg).addImm(269); Register CmpReg = RegInfo.createVirtualRegister(&PPC::CRRCRegClass); BuildMI(BB, dl, TII->get(PPC::CMPW), CmpReg) .addReg(HiReg) .addReg(ReadAgainReg); BuildMI(BB, dl, TII->get(PPC::BCC)) .addImm(PPC::PRED_NE) .addReg(CmpReg) .addMBB(readMBB); BB->addSuccessor(readMBB); BB->addSuccessor(sinkMBB); } else if (MI.getOpcode() == PPC::ATOMIC_LOAD_ADD_I8) BB = EmitPartwordAtomicBinary(MI, BB, true, PPC::ADD4); else if (MI.getOpcode() == PPC::ATOMIC_LOAD_ADD_I16) BB = EmitPartwordAtomicBinary(MI, BB, false, PPC::ADD4); else if (MI.getOpcode() == PPC::ATOMIC_LOAD_ADD_I32) BB = EmitAtomicBinary(MI, BB, 4, PPC::ADD4); else if (MI.getOpcode() == PPC::ATOMIC_LOAD_ADD_I64) BB = EmitAtomicBinary(MI, BB, 8, PPC::ADD8); else if (MI.getOpcode() == PPC::ATOMIC_LOAD_AND_I8) BB = EmitPartwordAtomicBinary(MI, BB, true, PPC::AND); else if (MI.getOpcode() == PPC::ATOMIC_LOAD_AND_I16) BB = EmitPartwordAtomicBinary(MI, BB, false, PPC::AND); else if (MI.getOpcode() == PPC::ATOMIC_LOAD_AND_I32) BB = EmitAtomicBinary(MI, BB, 4, PPC::AND); else if (MI.getOpcode() == PPC::ATOMIC_LOAD_AND_I64) BB = EmitAtomicBinary(MI, BB, 8, PPC::AND8); else if (MI.getOpcode() == PPC::ATOMIC_LOAD_OR_I8) BB = EmitPartwordAtomicBinary(MI, BB, true, PPC::OR); else if (MI.getOpcode() == PPC::ATOMIC_LOAD_OR_I16) BB = EmitPartwordAtomicBinary(MI, BB, false, PPC::OR); else if (MI.getOpcode() == PPC::ATOMIC_LOAD_OR_I32) BB = EmitAtomicBinary(MI, BB, 4, PPC::OR); else if (MI.getOpcode() == PPC::ATOMIC_LOAD_OR_I64) BB = EmitAtomicBinary(MI, BB, 8, PPC::OR8); else if (MI.getOpcode() == PPC::ATOMIC_LOAD_XOR_I8) BB = EmitPartwordAtomicBinary(MI, BB, true, PPC::XOR); else if (MI.getOpcode() == PPC::ATOMIC_LOAD_XOR_I16) BB = EmitPartwordAtomicBinary(MI, BB, false, PPC::XOR); else if (MI.getOpcode() == PPC::ATOMIC_LOAD_XOR_I32) BB = EmitAtomicBinary(MI, BB, 4, PPC::XOR); else if (MI.getOpcode() == PPC::ATOMIC_LOAD_XOR_I64) BB = EmitAtomicBinary(MI, BB, 8, PPC::XOR8); else if (MI.getOpcode() == PPC::ATOMIC_LOAD_NAND_I8) BB = EmitPartwordAtomicBinary(MI, BB, true, PPC::NAND); else if (MI.getOpcode() == PPC::ATOMIC_LOAD_NAND_I16) BB = EmitPartwordAtomicBinary(MI, BB, false, PPC::NAND); else if (MI.getOpcode() == PPC::ATOMIC_LOAD_NAND_I32) BB = EmitAtomicBinary(MI, BB, 4, PPC::NAND); else if (MI.getOpcode() == PPC::ATOMIC_LOAD_NAND_I64) BB = EmitAtomicBinary(MI, BB, 8, PPC::NAND8); else if (MI.getOpcode() == PPC::ATOMIC_LOAD_SUB_I8) BB = EmitPartwordAtomicBinary(MI, BB, true, PPC::SUBF); else if (MI.getOpcode() == PPC::ATOMIC_LOAD_SUB_I16) BB = EmitPartwordAtomicBinary(MI, BB, false, PPC::SUBF); else if (MI.getOpcode() == PPC::ATOMIC_LOAD_SUB_I32) BB = EmitAtomicBinary(MI, BB, 4, PPC::SUBF); else if (MI.getOpcode() == PPC::ATOMIC_LOAD_SUB_I64) BB = EmitAtomicBinary(MI, BB, 8, PPC::SUBF8); else if (MI.getOpcode() == PPC::ATOMIC_LOAD_MIN_I8) BB = EmitPartwordAtomicBinary(MI, BB, true, 0, PPC::CMPW, PPC::PRED_GE); else if (MI.getOpcode() == PPC::ATOMIC_LOAD_MIN_I16) BB = EmitPartwordAtomicBinary(MI, BB, false, 0, PPC::CMPW, PPC::PRED_GE); else if (MI.getOpcode() == PPC::ATOMIC_LOAD_MIN_I32) BB = EmitAtomicBinary(MI, BB, 4, 0, PPC::CMPW, PPC::PRED_GE); else if (MI.getOpcode() == PPC::ATOMIC_LOAD_MIN_I64) BB = EmitAtomicBinary(MI, BB, 8, 0, PPC::CMPD, PPC::PRED_GE); else if (MI.getOpcode() == PPC::ATOMIC_LOAD_MAX_I8) BB = EmitPartwordAtomicBinary(MI, BB, true, 0, PPC::CMPW, PPC::PRED_LE); else if (MI.getOpcode() == PPC::ATOMIC_LOAD_MAX_I16) BB = EmitPartwordAtomicBinary(MI, BB, false, 0, PPC::CMPW, PPC::PRED_LE); else if (MI.getOpcode() == PPC::ATOMIC_LOAD_MAX_I32) BB = EmitAtomicBinary(MI, BB, 4, 0, PPC::CMPW, PPC::PRED_LE); else if (MI.getOpcode() == PPC::ATOMIC_LOAD_MAX_I64) BB = EmitAtomicBinary(MI, BB, 8, 0, PPC::CMPD, PPC::PRED_LE); else if (MI.getOpcode() == PPC::ATOMIC_LOAD_UMIN_I8) BB = EmitPartwordAtomicBinary(MI, BB, true, 0, PPC::CMPLW, PPC::PRED_GE); else if (MI.getOpcode() == PPC::ATOMIC_LOAD_UMIN_I16) BB = EmitPartwordAtomicBinary(MI, BB, false, 0, PPC::CMPLW, PPC::PRED_GE); else if (MI.getOpcode() == PPC::ATOMIC_LOAD_UMIN_I32) BB = EmitAtomicBinary(MI, BB, 4, 0, PPC::CMPLW, PPC::PRED_GE); else if (MI.getOpcode() == PPC::ATOMIC_LOAD_UMIN_I64) BB = EmitAtomicBinary(MI, BB, 8, 0, PPC::CMPLD, PPC::PRED_GE); else if (MI.getOpcode() == PPC::ATOMIC_LOAD_UMAX_I8) BB = EmitPartwordAtomicBinary(MI, BB, true, 0, PPC::CMPLW, PPC::PRED_LE); else if (MI.getOpcode() == PPC::ATOMIC_LOAD_UMAX_I16) BB = EmitPartwordAtomicBinary(MI, BB, false, 0, PPC::CMPLW, PPC::PRED_LE); else if (MI.getOpcode() == PPC::ATOMIC_LOAD_UMAX_I32) BB = EmitAtomicBinary(MI, BB, 4, 0, PPC::CMPLW, PPC::PRED_LE); else if (MI.getOpcode() == PPC::ATOMIC_LOAD_UMAX_I64) BB = EmitAtomicBinary(MI, BB, 8, 0, PPC::CMPLD, PPC::PRED_LE); else if (MI.getOpcode() == PPC::ATOMIC_SWAP_I8) BB = EmitPartwordAtomicBinary(MI, BB, true, 0); else if (MI.getOpcode() == PPC::ATOMIC_SWAP_I16) BB = EmitPartwordAtomicBinary(MI, BB, false, 0); else if (MI.getOpcode() == PPC::ATOMIC_SWAP_I32) BB = EmitAtomicBinary(MI, BB, 4, 0); else if (MI.getOpcode() == PPC::ATOMIC_SWAP_I64) BB = EmitAtomicBinary(MI, BB, 8, 0); else if (MI.getOpcode() == PPC::ATOMIC_CMP_SWAP_I32 || MI.getOpcode() == PPC::ATOMIC_CMP_SWAP_I64 || (Subtarget.hasPartwordAtomics() && MI.getOpcode() == PPC::ATOMIC_CMP_SWAP_I8) || (Subtarget.hasPartwordAtomics() && MI.getOpcode() == PPC::ATOMIC_CMP_SWAP_I16)) { bool is64bit = MI.getOpcode() == PPC::ATOMIC_CMP_SWAP_I64; auto LoadMnemonic = PPC::LDARX; auto StoreMnemonic = PPC::STDCX; switch (MI.getOpcode()) { default: llvm_unreachable("Compare and swap of unknown size"); case PPC::ATOMIC_CMP_SWAP_I8: LoadMnemonic = PPC::LBARX; StoreMnemonic = PPC::STBCX; assert(Subtarget.hasPartwordAtomics() && "No support partword atomics."); break; case PPC::ATOMIC_CMP_SWAP_I16: LoadMnemonic = PPC::LHARX; StoreMnemonic = PPC::STHCX; assert(Subtarget.hasPartwordAtomics() && "No support partword atomics."); break; case PPC::ATOMIC_CMP_SWAP_I32: LoadMnemonic = PPC::LWARX; StoreMnemonic = PPC::STWCX; break; case PPC::ATOMIC_CMP_SWAP_I64: LoadMnemonic = PPC::LDARX; StoreMnemonic = PPC::STDCX; break; } Register dest = MI.getOperand(0).getReg(); Register ptrA = MI.getOperand(1).getReg(); Register ptrB = MI.getOperand(2).getReg(); Register oldval = MI.getOperand(3).getReg(); Register newval = MI.getOperand(4).getReg(); DebugLoc dl = MI.getDebugLoc(); MachineBasicBlock *loop1MBB = F->CreateMachineBasicBlock(LLVM_BB); MachineBasicBlock *loop2MBB = F->CreateMachineBasicBlock(LLVM_BB); MachineBasicBlock *midMBB = F->CreateMachineBasicBlock(LLVM_BB); MachineBasicBlock *exitMBB = F->CreateMachineBasicBlock(LLVM_BB); F->insert(It, loop1MBB); F->insert(It, loop2MBB); F->insert(It, midMBB); F->insert(It, exitMBB); exitMBB->splice(exitMBB->begin(), BB, std::next(MachineBasicBlock::iterator(MI)), BB->end()); exitMBB->transferSuccessorsAndUpdatePHIs(BB); // thisMBB: // ... // fallthrough --> loopMBB BB->addSuccessor(loop1MBB); // loop1MBB: // l[bhwd]arx dest, ptr // cmp[wd] dest, oldval // bne- midMBB // loop2MBB: // st[bhwd]cx. newval, ptr // bne- loopMBB // b exitBB // midMBB: // st[bhwd]cx. dest, ptr // exitBB: BB = loop1MBB; BuildMI(BB, dl, TII->get(LoadMnemonic), dest).addReg(ptrA).addReg(ptrB); BuildMI(BB, dl, TII->get(is64bit ? PPC::CMPD : PPC::CMPW), PPC::CR0) .addReg(oldval) .addReg(dest); BuildMI(BB, dl, TII->get(PPC::BCC)) .addImm(PPC::PRED_NE) .addReg(PPC::CR0) .addMBB(midMBB); BB->addSuccessor(loop2MBB); BB->addSuccessor(midMBB); BB = loop2MBB; BuildMI(BB, dl, TII->get(StoreMnemonic)) .addReg(newval) .addReg(ptrA) .addReg(ptrB); BuildMI(BB, dl, TII->get(PPC::BCC)) .addImm(PPC::PRED_NE) .addReg(PPC::CR0) .addMBB(loop1MBB); BuildMI(BB, dl, TII->get(PPC::B)).addMBB(exitMBB); BB->addSuccessor(loop1MBB); BB->addSuccessor(exitMBB); BB = midMBB; BuildMI(BB, dl, TII->get(StoreMnemonic)) .addReg(dest) .addReg(ptrA) .addReg(ptrB); BB->addSuccessor(exitMBB); // exitMBB: // ... BB = exitMBB; } else if (MI.getOpcode() == PPC::ATOMIC_CMP_SWAP_I8 || MI.getOpcode() == PPC::ATOMIC_CMP_SWAP_I16) { // We must use 64-bit registers for addresses when targeting 64-bit, // since we're actually doing arithmetic on them. Other registers // can be 32-bit. bool is64bit = Subtarget.isPPC64(); bool isLittleEndian = Subtarget.isLittleEndian(); bool is8bit = MI.getOpcode() == PPC::ATOMIC_CMP_SWAP_I8; Register dest = MI.getOperand(0).getReg(); Register ptrA = MI.getOperand(1).getReg(); Register ptrB = MI.getOperand(2).getReg(); Register oldval = MI.getOperand(3).getReg(); Register newval = MI.getOperand(4).getReg(); DebugLoc dl = MI.getDebugLoc(); MachineBasicBlock *loop1MBB = F->CreateMachineBasicBlock(LLVM_BB); MachineBasicBlock *loop2MBB = F->CreateMachineBasicBlock(LLVM_BB); MachineBasicBlock *midMBB = F->CreateMachineBasicBlock(LLVM_BB); MachineBasicBlock *exitMBB = F->CreateMachineBasicBlock(LLVM_BB); F->insert(It, loop1MBB); F->insert(It, loop2MBB); F->insert(It, midMBB); F->insert(It, exitMBB); exitMBB->splice(exitMBB->begin(), BB, std::next(MachineBasicBlock::iterator(MI)), BB->end()); exitMBB->transferSuccessorsAndUpdatePHIs(BB); MachineRegisterInfo &RegInfo = F->getRegInfo(); const TargetRegisterClass *RC = is64bit ? &PPC::G8RCRegClass : &PPC::GPRCRegClass; const TargetRegisterClass *GPRC = &PPC::GPRCRegClass; Register PtrReg = RegInfo.createVirtualRegister(RC); Register Shift1Reg = RegInfo.createVirtualRegister(GPRC); Register ShiftReg = isLittleEndian ? Shift1Reg : RegInfo.createVirtualRegister(GPRC); Register NewVal2Reg = RegInfo.createVirtualRegister(GPRC); Register NewVal3Reg = RegInfo.createVirtualRegister(GPRC); Register OldVal2Reg = RegInfo.createVirtualRegister(GPRC); Register OldVal3Reg = RegInfo.createVirtualRegister(GPRC); Register MaskReg = RegInfo.createVirtualRegister(GPRC); Register Mask2Reg = RegInfo.createVirtualRegister(GPRC); Register Mask3Reg = RegInfo.createVirtualRegister(GPRC); Register Tmp2Reg = RegInfo.createVirtualRegister(GPRC); Register Tmp4Reg = RegInfo.createVirtualRegister(GPRC); Register TmpDestReg = RegInfo.createVirtualRegister(GPRC); Register Ptr1Reg; Register TmpReg = RegInfo.createVirtualRegister(GPRC); Register ZeroReg = is64bit ? PPC::ZERO8 : PPC::ZERO; // thisMBB: // ... // fallthrough --> loopMBB BB->addSuccessor(loop1MBB); // The 4-byte load must be aligned, while a char or short may be // anywhere in the word. Hence all this nasty bookkeeping code. // add ptr1, ptrA, ptrB [copy if ptrA==0] // rlwinm shift1, ptr1, 3, 27, 28 [3, 27, 27] // xori shift, shift1, 24 [16] // rlwinm ptr, ptr1, 0, 0, 29 // slw newval2, newval, shift // slw oldval2, oldval,shift // li mask2, 255 [li mask3, 0; ori mask2, mask3, 65535] // slw mask, mask2, shift // and newval3, newval2, mask // and oldval3, oldval2, mask // loop1MBB: // lwarx tmpDest, ptr // and tmp, tmpDest, mask // cmpw tmp, oldval3 // bne- midMBB // loop2MBB: // andc tmp2, tmpDest, mask // or tmp4, tmp2, newval3 // stwcx. tmp4, ptr // bne- loop1MBB // b exitBB // midMBB: // stwcx. tmpDest, ptr // exitBB: // srw dest, tmpDest, shift if (ptrA != ZeroReg) { Ptr1Reg = RegInfo.createVirtualRegister(RC); BuildMI(BB, dl, TII->get(is64bit ? PPC::ADD8 : PPC::ADD4), Ptr1Reg) .addReg(ptrA) .addReg(ptrB); } else { Ptr1Reg = ptrB; } // We need use 32-bit subregister to avoid mismatch register class in 64-bit // mode. BuildMI(BB, dl, TII->get(PPC::RLWINM), Shift1Reg) .addReg(Ptr1Reg, 0, is64bit ? PPC::sub_32 : 0) .addImm(3) .addImm(27) .addImm(is8bit ? 28 : 27); if (!isLittleEndian) BuildMI(BB, dl, TII->get(PPC::XORI), ShiftReg) .addReg(Shift1Reg) .addImm(is8bit ? 24 : 16); if (is64bit) BuildMI(BB, dl, TII->get(PPC::RLDICR), PtrReg) .addReg(Ptr1Reg) .addImm(0) .addImm(61); else BuildMI(BB, dl, TII->get(PPC::RLWINM), PtrReg) .addReg(Ptr1Reg) .addImm(0) .addImm(0) .addImm(29); BuildMI(BB, dl, TII->get(PPC::SLW), NewVal2Reg) .addReg(newval) .addReg(ShiftReg); BuildMI(BB, dl, TII->get(PPC::SLW), OldVal2Reg) .addReg(oldval) .addReg(ShiftReg); if (is8bit) BuildMI(BB, dl, TII->get(PPC::LI), Mask2Reg).addImm(255); else { BuildMI(BB, dl, TII->get(PPC::LI), Mask3Reg).addImm(0); BuildMI(BB, dl, TII->get(PPC::ORI), Mask2Reg) .addReg(Mask3Reg) .addImm(65535); } BuildMI(BB, dl, TII->get(PPC::SLW), MaskReg) .addReg(Mask2Reg) .addReg(ShiftReg); BuildMI(BB, dl, TII->get(PPC::AND), NewVal3Reg) .addReg(NewVal2Reg) .addReg(MaskReg); BuildMI(BB, dl, TII->get(PPC::AND), OldVal3Reg) .addReg(OldVal2Reg) .addReg(MaskReg); BB = loop1MBB; BuildMI(BB, dl, TII->get(PPC::LWARX), TmpDestReg) .addReg(ZeroReg) .addReg(PtrReg); BuildMI(BB, dl, TII->get(PPC::AND), TmpReg) .addReg(TmpDestReg) .addReg(MaskReg); BuildMI(BB, dl, TII->get(PPC::CMPW), PPC::CR0) .addReg(TmpReg) .addReg(OldVal3Reg); BuildMI(BB, dl, TII->get(PPC::BCC)) .addImm(PPC::PRED_NE) .addReg(PPC::CR0) .addMBB(midMBB); BB->addSuccessor(loop2MBB); BB->addSuccessor(midMBB); BB = loop2MBB; BuildMI(BB, dl, TII->get(PPC::ANDC), Tmp2Reg) .addReg(TmpDestReg) .addReg(MaskReg); BuildMI(BB, dl, TII->get(PPC::OR), Tmp4Reg) .addReg(Tmp2Reg) .addReg(NewVal3Reg); BuildMI(BB, dl, TII->get(PPC::STWCX)) .addReg(Tmp4Reg) .addReg(ZeroReg) .addReg(PtrReg); BuildMI(BB, dl, TII->get(PPC::BCC)) .addImm(PPC::PRED_NE) .addReg(PPC::CR0) .addMBB(loop1MBB); BuildMI(BB, dl, TII->get(PPC::B)).addMBB(exitMBB); BB->addSuccessor(loop1MBB); BB->addSuccessor(exitMBB); BB = midMBB; BuildMI(BB, dl, TII->get(PPC::STWCX)) .addReg(TmpDestReg) .addReg(ZeroReg) .addReg(PtrReg); BB->addSuccessor(exitMBB); // exitMBB: // ... BB = exitMBB; BuildMI(*BB, BB->begin(), dl, TII->get(PPC::SRW), dest) .addReg(TmpReg) .addReg(ShiftReg); } else if (MI.getOpcode() == PPC::FADDrtz) { // This pseudo performs an FADD with rounding mode temporarily forced // to round-to-zero. We emit this via custom inserter since the FPSCR // is not modeled at the SelectionDAG level. Register Dest = MI.getOperand(0).getReg(); Register Src1 = MI.getOperand(1).getReg(); Register Src2 = MI.getOperand(2).getReg(); DebugLoc dl = MI.getDebugLoc(); MachineRegisterInfo &RegInfo = F->getRegInfo(); Register MFFSReg = RegInfo.createVirtualRegister(&PPC::F8RCRegClass); // Save FPSCR value. BuildMI(*BB, MI, dl, TII->get(PPC::MFFS), MFFSReg); // Set rounding mode to round-to-zero. BuildMI(*BB, MI, dl, TII->get(PPC::MTFSB1)).addImm(31); BuildMI(*BB, MI, dl, TII->get(PPC::MTFSB0)).addImm(30); // Perform addition. BuildMI(*BB, MI, dl, TII->get(PPC::FADD), Dest).addReg(Src1).addReg(Src2); // Restore FPSCR value. BuildMI(*BB, MI, dl, TII->get(PPC::MTFSFb)).addImm(1).addReg(MFFSReg); } else if (MI.getOpcode() == PPC::ANDI_rec_1_EQ_BIT || MI.getOpcode() == PPC::ANDI_rec_1_GT_BIT || MI.getOpcode() == PPC::ANDI_rec_1_EQ_BIT8 || MI.getOpcode() == PPC::ANDI_rec_1_GT_BIT8) { unsigned Opcode = (MI.getOpcode() == PPC::ANDI_rec_1_EQ_BIT8 || MI.getOpcode() == PPC::ANDI_rec_1_GT_BIT8) ? PPC::ANDI8_rec : PPC::ANDI_rec; bool IsEQ = (MI.getOpcode() == PPC::ANDI_rec_1_EQ_BIT || MI.getOpcode() == PPC::ANDI_rec_1_EQ_BIT8); MachineRegisterInfo &RegInfo = F->getRegInfo(); Register Dest = RegInfo.createVirtualRegister( Opcode == PPC::ANDI_rec ? &PPC::GPRCRegClass : &PPC::G8RCRegClass); DebugLoc Dl = MI.getDebugLoc(); BuildMI(*BB, MI, Dl, TII->get(Opcode), Dest) .addReg(MI.getOperand(1).getReg()) .addImm(1); BuildMI(*BB, MI, Dl, TII->get(TargetOpcode::COPY), MI.getOperand(0).getReg()) .addReg(IsEQ ? PPC::CR0EQ : PPC::CR0GT); } else if (MI.getOpcode() == PPC::TCHECK_RET) { DebugLoc Dl = MI.getDebugLoc(); MachineRegisterInfo &RegInfo = F->getRegInfo(); Register CRReg = RegInfo.createVirtualRegister(&PPC::CRRCRegClass); BuildMI(*BB, MI, Dl, TII->get(PPC::TCHECK), CRReg); BuildMI(*BB, MI, Dl, TII->get(TargetOpcode::COPY), MI.getOperand(0).getReg()) .addReg(CRReg); } else if (MI.getOpcode() == PPC::TBEGIN_RET) { DebugLoc Dl = MI.getDebugLoc(); unsigned Imm = MI.getOperand(1).getImm(); BuildMI(*BB, MI, Dl, TII->get(PPC::TBEGIN)).addImm(Imm); BuildMI(*BB, MI, Dl, TII->get(TargetOpcode::COPY), MI.getOperand(0).getReg()) .addReg(PPC::CR0EQ); } else if (MI.getOpcode() == PPC::SETRNDi) { DebugLoc dl = MI.getDebugLoc(); Register OldFPSCRReg = MI.getOperand(0).getReg(); // Save FPSCR value. BuildMI(*BB, MI, dl, TII->get(PPC::MFFS), OldFPSCRReg); // The floating point rounding mode is in the bits 62:63 of FPCSR, and has // the following settings: // 00 Round to nearest // 01 Round to 0 // 10 Round to +inf // 11 Round to -inf // When the operand is immediate, using the two least significant bits of // the immediate to set the bits 62:63 of FPSCR. unsigned Mode = MI.getOperand(1).getImm(); BuildMI(*BB, MI, dl, TII->get((Mode & 1) ? PPC::MTFSB1 : PPC::MTFSB0)) .addImm(31); BuildMI(*BB, MI, dl, TII->get((Mode & 2) ? PPC::MTFSB1 : PPC::MTFSB0)) .addImm(30); } else if (MI.getOpcode() == PPC::SETRND) { DebugLoc dl = MI.getDebugLoc(); // Copy register from F8RCRegClass::SrcReg to G8RCRegClass::DestReg // or copy register from G8RCRegClass::SrcReg to F8RCRegClass::DestReg. // If the target doesn't have DirectMove, we should use stack to do the // conversion, because the target doesn't have the instructions like mtvsrd // or mfvsrd to do this conversion directly. auto copyRegFromG8RCOrF8RC = [&] (unsigned DestReg, unsigned SrcReg) { if (Subtarget.hasDirectMove()) { BuildMI(*BB, MI, dl, TII->get(TargetOpcode::COPY), DestReg) .addReg(SrcReg); } else { // Use stack to do the register copy. unsigned StoreOp = PPC::STD, LoadOp = PPC::LFD; MachineRegisterInfo &RegInfo = F->getRegInfo(); const TargetRegisterClass *RC = RegInfo.getRegClass(SrcReg); if (RC == &PPC::F8RCRegClass) { // Copy register from F8RCRegClass to G8RCRegclass. assert((RegInfo.getRegClass(DestReg) == &PPC::G8RCRegClass) && "Unsupported RegClass."); StoreOp = PPC::STFD; LoadOp = PPC::LD; } else { // Copy register from G8RCRegClass to F8RCRegclass. assert((RegInfo.getRegClass(SrcReg) == &PPC::G8RCRegClass) && (RegInfo.getRegClass(DestReg) == &PPC::F8RCRegClass) && "Unsupported RegClass."); } MachineFrameInfo &MFI = F->getFrameInfo(); int FrameIdx = MFI.CreateStackObject(8, 8, false); MachineMemOperand *MMOStore = F->getMachineMemOperand( MachinePointerInfo::getFixedStack(*F, FrameIdx, 0), MachineMemOperand::MOStore, MFI.getObjectSize(FrameIdx), MFI.getObjectAlign(FrameIdx)); // Store the SrcReg into the stack. BuildMI(*BB, MI, dl, TII->get(StoreOp)) .addReg(SrcReg) .addImm(0) .addFrameIndex(FrameIdx) .addMemOperand(MMOStore); MachineMemOperand *MMOLoad = F->getMachineMemOperand( MachinePointerInfo::getFixedStack(*F, FrameIdx, 0), MachineMemOperand::MOLoad, MFI.getObjectSize(FrameIdx), MFI.getObjectAlign(FrameIdx)); // Load from the stack where SrcReg is stored, and save to DestReg, // so we have done the RegClass conversion from RegClass::SrcReg to // RegClass::DestReg. BuildMI(*BB, MI, dl, TII->get(LoadOp), DestReg) .addImm(0) .addFrameIndex(FrameIdx) .addMemOperand(MMOLoad); } }; Register OldFPSCRReg = MI.getOperand(0).getReg(); // Save FPSCR value. BuildMI(*BB, MI, dl, TII->get(PPC::MFFS), OldFPSCRReg); // When the operand is gprc register, use two least significant bits of the // register and mtfsf instruction to set the bits 62:63 of FPSCR. // // copy OldFPSCRTmpReg, OldFPSCRReg // (INSERT_SUBREG ExtSrcReg, (IMPLICIT_DEF ImDefReg), SrcOp, 1) // rldimi NewFPSCRTmpReg, ExtSrcReg, OldFPSCRReg, 0, 62 // copy NewFPSCRReg, NewFPSCRTmpReg // mtfsf 255, NewFPSCRReg MachineOperand SrcOp = MI.getOperand(1); MachineRegisterInfo &RegInfo = F->getRegInfo(); Register OldFPSCRTmpReg = RegInfo.createVirtualRegister(&PPC::G8RCRegClass); copyRegFromG8RCOrF8RC(OldFPSCRTmpReg, OldFPSCRReg); Register ImDefReg = RegInfo.createVirtualRegister(&PPC::G8RCRegClass); Register ExtSrcReg = RegInfo.createVirtualRegister(&PPC::G8RCRegClass); // The first operand of INSERT_SUBREG should be a register which has // subregisters, we only care about its RegClass, so we should use an // IMPLICIT_DEF register. BuildMI(*BB, MI, dl, TII->get(TargetOpcode::IMPLICIT_DEF), ImDefReg); BuildMI(*BB, MI, dl, TII->get(PPC::INSERT_SUBREG), ExtSrcReg) .addReg(ImDefReg) .add(SrcOp) .addImm(1); Register NewFPSCRTmpReg = RegInfo.createVirtualRegister(&PPC::G8RCRegClass); BuildMI(*BB, MI, dl, TII->get(PPC::RLDIMI), NewFPSCRTmpReg) .addReg(OldFPSCRTmpReg) .addReg(ExtSrcReg) .addImm(0) .addImm(62); Register NewFPSCRReg = RegInfo.createVirtualRegister(&PPC::F8RCRegClass); copyRegFromG8RCOrF8RC(NewFPSCRReg, NewFPSCRTmpReg); // The mask 255 means that put the 32:63 bits of NewFPSCRReg to the 32:63 // bits of FPSCR. BuildMI(*BB, MI, dl, TII->get(PPC::MTFSF)) .addImm(255) .addReg(NewFPSCRReg) .addImm(0) .addImm(0); } else { llvm_unreachable("Unexpected instr type to insert"); } MI.eraseFromParent(); // The pseudo instruction is gone now. return BB; } //===----------------------------------------------------------------------===// // Target Optimization Hooks //===----------------------------------------------------------------------===// static int getEstimateRefinementSteps(EVT VT, const PPCSubtarget &Subtarget) { // For the estimates, convergence is quadratic, so we essentially double the // number of digits correct after every iteration. For both FRE and FRSQRTE, // the minimum architected relative accuracy is 2^-5. When hasRecipPrec(), // this is 2^-14. IEEE float has 23 digits and double has 52 digits. int RefinementSteps = Subtarget.hasRecipPrec() ? 1 : 3; if (VT.getScalarType() == MVT::f64) RefinementSteps++; return RefinementSteps; } SDValue PPCTargetLowering::getSqrtEstimate(SDValue Operand, SelectionDAG &DAG, int Enabled, int &RefinementSteps, bool &UseOneConstNR, bool Reciprocal) const { EVT VT = Operand.getValueType(); if ((VT == MVT::f32 && Subtarget.hasFRSQRTES()) || (VT == MVT::f64 && Subtarget.hasFRSQRTE()) || (VT == MVT::v4f32 && Subtarget.hasAltivec()) || (VT == MVT::v2f64 && Subtarget.hasVSX()) || (VT == MVT::v4f32 && Subtarget.hasQPX()) || (VT == MVT::v4f64 && Subtarget.hasQPX())) { if (RefinementSteps == ReciprocalEstimate::Unspecified) RefinementSteps = getEstimateRefinementSteps(VT, Subtarget); // The Newton-Raphson computation with a single constant does not provide // enough accuracy on some CPUs. UseOneConstNR = !Subtarget.needsTwoConstNR(); return DAG.getNode(PPCISD::FRSQRTE, SDLoc(Operand), VT, Operand); } return SDValue(); } SDValue PPCTargetLowering::getRecipEstimate(SDValue Operand, SelectionDAG &DAG, int Enabled, int &RefinementSteps) const { EVT VT = Operand.getValueType(); if ((VT == MVT::f32 && Subtarget.hasFRES()) || (VT == MVT::f64 && Subtarget.hasFRE()) || (VT == MVT::v4f32 && Subtarget.hasAltivec()) || (VT == MVT::v2f64 && Subtarget.hasVSX()) || (VT == MVT::v4f32 && Subtarget.hasQPX()) || (VT == MVT::v4f64 && Subtarget.hasQPX())) { if (RefinementSteps == ReciprocalEstimate::Unspecified) RefinementSteps = getEstimateRefinementSteps(VT, Subtarget); return DAG.getNode(PPCISD::FRE, SDLoc(Operand), VT, Operand); } return SDValue(); } unsigned PPCTargetLowering::combineRepeatedFPDivisors() const { // Note: This functionality is used only when unsafe-fp-math is enabled, and // on cores with reciprocal estimates (which are used when unsafe-fp-math is // enabled for division), this functionality is redundant with the default // combiner logic (once the division -> reciprocal/multiply transformation // has taken place). As a result, this matters more for older cores than for // newer ones. // Combine multiple FDIVs with the same divisor into multiple FMULs by the // reciprocal if there are two or more FDIVs (for embedded cores with only // one FP pipeline) for three or more FDIVs (for generic OOO cores). switch (Subtarget.getCPUDirective()) { default: return 3; case PPC::DIR_440: case PPC::DIR_A2: case PPC::DIR_E500: case PPC::DIR_E500mc: case PPC::DIR_E5500: return 2; } } // isConsecutiveLSLoc needs to work even if all adds have not yet been // collapsed, and so we need to look through chains of them. static void getBaseWithConstantOffset(SDValue Loc, SDValue &Base, int64_t& Offset, SelectionDAG &DAG) { if (DAG.isBaseWithConstantOffset(Loc)) { Base = Loc.getOperand(0); Offset += cast(Loc.getOperand(1))->getSExtValue(); // The base might itself be a base plus an offset, and if so, accumulate // that as well. getBaseWithConstantOffset(Loc.getOperand(0), Base, Offset, DAG); } } static bool isConsecutiveLSLoc(SDValue Loc, EVT VT, LSBaseSDNode *Base, unsigned Bytes, int Dist, SelectionDAG &DAG) { if (VT.getSizeInBits() / 8 != Bytes) return false; SDValue BaseLoc = Base->getBasePtr(); if (Loc.getOpcode() == ISD::FrameIndex) { if (BaseLoc.getOpcode() != ISD::FrameIndex) return false; const MachineFrameInfo &MFI = DAG.getMachineFunction().getFrameInfo(); int FI = cast(Loc)->getIndex(); int BFI = cast(BaseLoc)->getIndex(); int FS = MFI.getObjectSize(FI); int BFS = MFI.getObjectSize(BFI); if (FS != BFS || FS != (int)Bytes) return false; return MFI.getObjectOffset(FI) == (MFI.getObjectOffset(BFI) + Dist*Bytes); } SDValue Base1 = Loc, Base2 = BaseLoc; int64_t Offset1 = 0, Offset2 = 0; getBaseWithConstantOffset(Loc, Base1, Offset1, DAG); getBaseWithConstantOffset(BaseLoc, Base2, Offset2, DAG); if (Base1 == Base2 && Offset1 == (Offset2 + Dist * Bytes)) return true; const TargetLowering &TLI = DAG.getTargetLoweringInfo(); const GlobalValue *GV1 = nullptr; const GlobalValue *GV2 = nullptr; Offset1 = 0; Offset2 = 0; bool isGA1 = TLI.isGAPlusOffset(Loc.getNode(), GV1, Offset1); bool isGA2 = TLI.isGAPlusOffset(BaseLoc.getNode(), GV2, Offset2); if (isGA1 && isGA2 && GV1 == GV2) return Offset1 == (Offset2 + Dist*Bytes); return false; } // Like SelectionDAG::isConsecutiveLoad, but also works for stores, and does // not enforce equality of the chain operands. static bool isConsecutiveLS(SDNode *N, LSBaseSDNode *Base, unsigned Bytes, int Dist, SelectionDAG &DAG) { if (LSBaseSDNode *LS = dyn_cast(N)) { EVT VT = LS->getMemoryVT(); SDValue Loc = LS->getBasePtr(); return isConsecutiveLSLoc(Loc, VT, Base, Bytes, Dist, DAG); } if (N->getOpcode() == ISD::INTRINSIC_W_CHAIN) { EVT VT; switch (cast(N->getOperand(1))->getZExtValue()) { default: return false; case Intrinsic::ppc_qpx_qvlfd: case Intrinsic::ppc_qpx_qvlfda: VT = MVT::v4f64; break; case Intrinsic::ppc_qpx_qvlfs: case Intrinsic::ppc_qpx_qvlfsa: VT = MVT::v4f32; break; case Intrinsic::ppc_qpx_qvlfcd: case Intrinsic::ppc_qpx_qvlfcda: VT = MVT::v2f64; break; case Intrinsic::ppc_qpx_qvlfcs: case Intrinsic::ppc_qpx_qvlfcsa: VT = MVT::v2f32; break; case Intrinsic::ppc_qpx_qvlfiwa: case Intrinsic::ppc_qpx_qvlfiwz: case Intrinsic::ppc_altivec_lvx: case Intrinsic::ppc_altivec_lvxl: case Intrinsic::ppc_vsx_lxvw4x: case Intrinsic::ppc_vsx_lxvw4x_be: VT = MVT::v4i32; break; case Intrinsic::ppc_vsx_lxvd2x: case Intrinsic::ppc_vsx_lxvd2x_be: VT = MVT::v2f64; break; case Intrinsic::ppc_altivec_lvebx: VT = MVT::i8; break; case Intrinsic::ppc_altivec_lvehx: VT = MVT::i16; break; case Intrinsic::ppc_altivec_lvewx: VT = MVT::i32; break; } return isConsecutiveLSLoc(N->getOperand(2), VT, Base, Bytes, Dist, DAG); } if (N->getOpcode() == ISD::INTRINSIC_VOID) { EVT VT; switch (cast(N->getOperand(1))->getZExtValue()) { default: return false; case Intrinsic::ppc_qpx_qvstfd: case Intrinsic::ppc_qpx_qvstfda: VT = MVT::v4f64; break; case Intrinsic::ppc_qpx_qvstfs: case Intrinsic::ppc_qpx_qvstfsa: VT = MVT::v4f32; break; case Intrinsic::ppc_qpx_qvstfcd: case Intrinsic::ppc_qpx_qvstfcda: VT = MVT::v2f64; break; case Intrinsic::ppc_qpx_qvstfcs: case Intrinsic::ppc_qpx_qvstfcsa: VT = MVT::v2f32; break; case Intrinsic::ppc_qpx_qvstfiw: case Intrinsic::ppc_qpx_qvstfiwa: case Intrinsic::ppc_altivec_stvx: case Intrinsic::ppc_altivec_stvxl: case Intrinsic::ppc_vsx_stxvw4x: VT = MVT::v4i32; break; case Intrinsic::ppc_vsx_stxvd2x: VT = MVT::v2f64; break; case Intrinsic::ppc_vsx_stxvw4x_be: VT = MVT::v4i32; break; case Intrinsic::ppc_vsx_stxvd2x_be: VT = MVT::v2f64; break; case Intrinsic::ppc_altivec_stvebx: VT = MVT::i8; break; case Intrinsic::ppc_altivec_stvehx: VT = MVT::i16; break; case Intrinsic::ppc_altivec_stvewx: VT = MVT::i32; break; } return isConsecutiveLSLoc(N->getOperand(3), VT, Base, Bytes, Dist, DAG); } return false; } // Return true is there is a nearyby consecutive load to the one provided // (regardless of alignment). We search up and down the chain, looking though // token factors and other loads (but nothing else). As a result, a true result // indicates that it is safe to create a new consecutive load adjacent to the // load provided. static bool findConsecutiveLoad(LoadSDNode *LD, SelectionDAG &DAG) { SDValue Chain = LD->getChain(); EVT VT = LD->getMemoryVT(); SmallSet LoadRoots; SmallVector Queue(1, Chain.getNode()); SmallSet Visited; // First, search up the chain, branching to follow all token-factor operands. // If we find a consecutive load, then we're done, otherwise, record all // nodes just above the top-level loads and token factors. while (!Queue.empty()) { SDNode *ChainNext = Queue.pop_back_val(); if (!Visited.insert(ChainNext).second) continue; if (MemSDNode *ChainLD = dyn_cast(ChainNext)) { if (isConsecutiveLS(ChainLD, LD, VT.getStoreSize(), 1, DAG)) return true; if (!Visited.count(ChainLD->getChain().getNode())) Queue.push_back(ChainLD->getChain().getNode()); } else if (ChainNext->getOpcode() == ISD::TokenFactor) { for (const SDUse &O : ChainNext->ops()) if (!Visited.count(O.getNode())) Queue.push_back(O.getNode()); } else LoadRoots.insert(ChainNext); } // Second, search down the chain, starting from the top-level nodes recorded // in the first phase. These top-level nodes are the nodes just above all // loads and token factors. Starting with their uses, recursively look though // all loads (just the chain uses) and token factors to find a consecutive // load. Visited.clear(); Queue.clear(); for (SmallSet::iterator I = LoadRoots.begin(), IE = LoadRoots.end(); I != IE; ++I) { Queue.push_back(*I); while (!Queue.empty()) { SDNode *LoadRoot = Queue.pop_back_val(); if (!Visited.insert(LoadRoot).second) continue; if (MemSDNode *ChainLD = dyn_cast(LoadRoot)) if (isConsecutiveLS(ChainLD, LD, VT.getStoreSize(), 1, DAG)) return true; for (SDNode::use_iterator UI = LoadRoot->use_begin(), UE = LoadRoot->use_end(); UI != UE; ++UI) if (((isa(*UI) && cast(*UI)->getChain().getNode() == LoadRoot) || UI->getOpcode() == ISD::TokenFactor) && !Visited.count(*UI)) Queue.push_back(*UI); } } return false; } /// This function is called when we have proved that a SETCC node can be replaced /// by subtraction (and other supporting instructions) so that the result of /// comparison is kept in a GPR instead of CR. This function is purely for /// codegen purposes and has some flags to guide the codegen process. static SDValue generateEquivalentSub(SDNode *N, int Size, bool Complement, bool Swap, SDLoc &DL, SelectionDAG &DAG) { assert(N->getOpcode() == ISD::SETCC && "ISD::SETCC Expected."); // Zero extend the operands to the largest legal integer. Originally, they // must be of a strictly smaller size. auto Op0 = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i64, N->getOperand(0), DAG.getConstant(Size, DL, MVT::i32)); auto Op1 = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i64, N->getOperand(1), DAG.getConstant(Size, DL, MVT::i32)); // Swap if needed. Depends on the condition code. if (Swap) std::swap(Op0, Op1); // Subtract extended integers. auto SubNode = DAG.getNode(ISD::SUB, DL, MVT::i64, Op0, Op1); // Move the sign bit to the least significant position and zero out the rest. // Now the least significant bit carries the result of original comparison. auto Shifted = DAG.getNode(ISD::SRL, DL, MVT::i64, SubNode, DAG.getConstant(Size - 1, DL, MVT::i32)); auto Final = Shifted; // Complement the result if needed. Based on the condition code. if (Complement) Final = DAG.getNode(ISD::XOR, DL, MVT::i64, Shifted, DAG.getConstant(1, DL, MVT::i64)); return DAG.getNode(ISD::TRUNCATE, DL, MVT::i1, Final); } SDValue PPCTargetLowering::ConvertSETCCToSubtract(SDNode *N, DAGCombinerInfo &DCI) const { assert(N->getOpcode() == ISD::SETCC && "ISD::SETCC Expected."); SelectionDAG &DAG = DCI.DAG; SDLoc DL(N); // Size of integers being compared has a critical role in the following // analysis, so we prefer to do this when all types are legal. if (!DCI.isAfterLegalizeDAG()) return SDValue(); // If all users of SETCC extend its value to a legal integer type // then we replace SETCC with a subtraction for (SDNode::use_iterator UI = N->use_begin(), UE = N->use_end(); UI != UE; ++UI) { if (UI->getOpcode() != ISD::ZERO_EXTEND) return SDValue(); } ISD::CondCode CC = cast(N->getOperand(2))->get(); auto OpSize = N->getOperand(0).getValueSizeInBits(); unsigned Size = DAG.getDataLayout().getLargestLegalIntTypeSizeInBits(); if (OpSize < Size) { switch (CC) { default: break; case ISD::SETULT: return generateEquivalentSub(N, Size, false, false, DL, DAG); case ISD::SETULE: return generateEquivalentSub(N, Size, true, true, DL, DAG); case ISD::SETUGT: return generateEquivalentSub(N, Size, false, true, DL, DAG); case ISD::SETUGE: return generateEquivalentSub(N, Size, true, false, DL, DAG); } } return SDValue(); } SDValue PPCTargetLowering::DAGCombineTruncBoolExt(SDNode *N, DAGCombinerInfo &DCI) const { SelectionDAG &DAG = DCI.DAG; SDLoc dl(N); assert(Subtarget.useCRBits() && "Expecting to be tracking CR bits"); // If we're tracking CR bits, we need to be careful that we don't have: // trunc(binary-ops(zext(x), zext(y))) // or // trunc(binary-ops(binary-ops(zext(x), zext(y)), ...) // such that we're unnecessarily moving things into GPRs when it would be // better to keep them in CR bits. // Note that trunc here can be an actual i1 trunc, or can be the effective // truncation that comes from a setcc or select_cc. if (N->getOpcode() == ISD::TRUNCATE && N->getValueType(0) != MVT::i1) return SDValue(); if (N->getOperand(0).getValueType() != MVT::i32 && N->getOperand(0).getValueType() != MVT::i64) return SDValue(); if (N->getOpcode() == ISD::SETCC || N->getOpcode() == ISD::SELECT_CC) { // If we're looking at a comparison, then we need to make sure that the // high bits (all except for the first) don't matter the result. ISD::CondCode CC = cast(N->getOperand( N->getOpcode() == ISD::SETCC ? 2 : 4))->get(); unsigned OpBits = N->getOperand(0).getValueSizeInBits(); if (ISD::isSignedIntSetCC(CC)) { if (DAG.ComputeNumSignBits(N->getOperand(0)) != OpBits || DAG.ComputeNumSignBits(N->getOperand(1)) != OpBits) return SDValue(); } else if (ISD::isUnsignedIntSetCC(CC)) { if (!DAG.MaskedValueIsZero(N->getOperand(0), APInt::getHighBitsSet(OpBits, OpBits-1)) || !DAG.MaskedValueIsZero(N->getOperand(1), APInt::getHighBitsSet(OpBits, OpBits-1))) return (N->getOpcode() == ISD::SETCC ? ConvertSETCCToSubtract(N, DCI) : SDValue()); } else { // This is neither a signed nor an unsigned comparison, just make sure // that the high bits are equal. KnownBits Op1Known = DAG.computeKnownBits(N->getOperand(0)); KnownBits Op2Known = DAG.computeKnownBits(N->getOperand(1)); // We don't really care about what is known about the first bit (if // anything), so clear it in all masks prior to comparing them. Op1Known.Zero.clearBit(0); Op1Known.One.clearBit(0); Op2Known.Zero.clearBit(0); Op2Known.One.clearBit(0); if (Op1Known.Zero != Op2Known.Zero || Op1Known.One != Op2Known.One) return SDValue(); } } // We now know that the higher-order bits are irrelevant, we just need to // make sure that all of the intermediate operations are bit operations, and // all inputs are extensions. if (N->getOperand(0).getOpcode() != ISD::AND && N->getOperand(0).getOpcode() != ISD::OR && N->getOperand(0).getOpcode() != ISD::XOR && N->getOperand(0).getOpcode() != ISD::SELECT && N->getOperand(0).getOpcode() != ISD::SELECT_CC && N->getOperand(0).getOpcode() != ISD::TRUNCATE && N->getOperand(0).getOpcode() != ISD::SIGN_EXTEND && N->getOperand(0).getOpcode() != ISD::ZERO_EXTEND && N->getOperand(0).getOpcode() != ISD::ANY_EXTEND) return SDValue(); if ((N->getOpcode() == ISD::SETCC || N->getOpcode() == ISD::SELECT_CC) && N->getOperand(1).getOpcode() != ISD::AND && N->getOperand(1).getOpcode() != ISD::OR && N->getOperand(1).getOpcode() != ISD::XOR && N->getOperand(1).getOpcode() != ISD::SELECT && N->getOperand(1).getOpcode() != ISD::SELECT_CC && N->getOperand(1).getOpcode() != ISD::TRUNCATE && N->getOperand(1).getOpcode() != ISD::SIGN_EXTEND && N->getOperand(1).getOpcode() != ISD::ZERO_EXTEND && N->getOperand(1).getOpcode() != ISD::ANY_EXTEND) return SDValue(); SmallVector Inputs; SmallVector BinOps, PromOps; SmallPtrSet Visited; for (unsigned i = 0; i < 2; ++i) { if (((N->getOperand(i).getOpcode() == ISD::SIGN_EXTEND || N->getOperand(i).getOpcode() == ISD::ZERO_EXTEND || N->getOperand(i).getOpcode() == ISD::ANY_EXTEND) && N->getOperand(i).getOperand(0).getValueType() == MVT::i1) || isa(N->getOperand(i))) Inputs.push_back(N->getOperand(i)); else BinOps.push_back(N->getOperand(i)); if (N->getOpcode() == ISD::TRUNCATE) break; } // Visit all inputs, collect all binary operations (and, or, xor and // select) that are all fed by extensions. while (!BinOps.empty()) { SDValue BinOp = BinOps.back(); BinOps.pop_back(); if (!Visited.insert(BinOp.getNode()).second) continue; PromOps.push_back(BinOp); for (unsigned i = 0, ie = BinOp.getNumOperands(); i != ie; ++i) { // The condition of the select is not promoted. if (BinOp.getOpcode() == ISD::SELECT && i == 0) continue; if (BinOp.getOpcode() == ISD::SELECT_CC && i != 2 && i != 3) continue; if (((BinOp.getOperand(i).getOpcode() == ISD::SIGN_EXTEND || BinOp.getOperand(i).getOpcode() == ISD::ZERO_EXTEND || BinOp.getOperand(i).getOpcode() == ISD::ANY_EXTEND) && BinOp.getOperand(i).getOperand(0).getValueType() == MVT::i1) || isa(BinOp.getOperand(i))) { Inputs.push_back(BinOp.getOperand(i)); } else if (BinOp.getOperand(i).getOpcode() == ISD::AND || BinOp.getOperand(i).getOpcode() == ISD::OR || BinOp.getOperand(i).getOpcode() == ISD::XOR || BinOp.getOperand(i).getOpcode() == ISD::SELECT || BinOp.getOperand(i).getOpcode() == ISD::SELECT_CC || BinOp.getOperand(i).getOpcode() == ISD::TRUNCATE || BinOp.getOperand(i).getOpcode() == ISD::SIGN_EXTEND || BinOp.getOperand(i).getOpcode() == ISD::ZERO_EXTEND || BinOp.getOperand(i).getOpcode() == ISD::ANY_EXTEND) { BinOps.push_back(BinOp.getOperand(i)); } else { // We have an input that is not an extension or another binary // operation; we'll abort this transformation. return SDValue(); } } } // Make sure that this is a self-contained cluster of operations (which // is not quite the same thing as saying that everything has only one // use). for (unsigned i = 0, ie = Inputs.size(); i != ie; ++i) { if (isa(Inputs[i])) continue; for (SDNode::use_iterator UI = Inputs[i].getNode()->use_begin(), UE = Inputs[i].getNode()->use_end(); UI != UE; ++UI) { SDNode *User = *UI; if (User != N && !Visited.count(User)) return SDValue(); // Make sure that we're not going to promote the non-output-value // operand(s) or SELECT or SELECT_CC. // FIXME: Although we could sometimes handle this, and it does occur in // practice that one of the condition inputs to the select is also one of // the outputs, we currently can't deal with this. if (User->getOpcode() == ISD::SELECT) { if (User->getOperand(0) == Inputs[i]) return SDValue(); } else if (User->getOpcode() == ISD::SELECT_CC) { if (User->getOperand(0) == Inputs[i] || User->getOperand(1) == Inputs[i]) return SDValue(); } } } for (unsigned i = 0, ie = PromOps.size(); i != ie; ++i) { for (SDNode::use_iterator UI = PromOps[i].getNode()->use_begin(), UE = PromOps[i].getNode()->use_end(); UI != UE; ++UI) { SDNode *User = *UI; if (User != N && !Visited.count(User)) return SDValue(); // Make sure that we're not going to promote the non-output-value // operand(s) or SELECT or SELECT_CC. // FIXME: Although we could sometimes handle this, and it does occur in // practice that one of the condition inputs to the select is also one of // the outputs, we currently can't deal with this. if (User->getOpcode() == ISD::SELECT) { if (User->getOperand(0) == PromOps[i]) return SDValue(); } else if (User->getOpcode() == ISD::SELECT_CC) { if (User->getOperand(0) == PromOps[i] || User->getOperand(1) == PromOps[i]) return SDValue(); } } } // Replace all inputs with the extension operand. for (unsigned i = 0, ie = Inputs.size(); i != ie; ++i) { // Constants may have users outside the cluster of to-be-promoted nodes, // and so we need to replace those as we do the promotions. if (isa(Inputs[i])) continue; else DAG.ReplaceAllUsesOfValueWith(Inputs[i], Inputs[i].getOperand(0)); } std::list PromOpHandles; for (auto &PromOp : PromOps) PromOpHandles.emplace_back(PromOp); // Replace all operations (these are all the same, but have a different // (i1) return type). DAG.getNode will validate that the types of // a binary operator match, so go through the list in reverse so that // we've likely promoted both operands first. Any intermediate truncations or // extensions disappear. while (!PromOpHandles.empty()) { SDValue PromOp = PromOpHandles.back().getValue(); PromOpHandles.pop_back(); if (PromOp.getOpcode() == ISD::TRUNCATE || PromOp.getOpcode() == ISD::SIGN_EXTEND || PromOp.getOpcode() == ISD::ZERO_EXTEND || PromOp.getOpcode() == ISD::ANY_EXTEND) { if (!isa(PromOp.getOperand(0)) && PromOp.getOperand(0).getValueType() != MVT::i1) { // The operand is not yet ready (see comment below). PromOpHandles.emplace_front(PromOp); continue; } SDValue RepValue = PromOp.getOperand(0); if (isa(RepValue)) RepValue = DAG.getNode(ISD::TRUNCATE, dl, MVT::i1, RepValue); DAG.ReplaceAllUsesOfValueWith(PromOp, RepValue); continue; } unsigned C; switch (PromOp.getOpcode()) { default: C = 0; break; case ISD::SELECT: C = 1; break; case ISD::SELECT_CC: C = 2; break; } if ((!isa(PromOp.getOperand(C)) && PromOp.getOperand(C).getValueType() != MVT::i1) || (!isa(PromOp.getOperand(C+1)) && PromOp.getOperand(C+1).getValueType() != MVT::i1)) { // The to-be-promoted operands of this node have not yet been // promoted (this should be rare because we're going through the // list backward, but if one of the operands has several users in // this cluster of to-be-promoted nodes, it is possible). PromOpHandles.emplace_front(PromOp); continue; } SmallVector Ops(PromOp.getNode()->op_begin(), PromOp.getNode()->op_end()); // If there are any constant inputs, make sure they're replaced now. for (unsigned i = 0; i < 2; ++i) if (isa(Ops[C+i])) Ops[C+i] = DAG.getNode(ISD::TRUNCATE, dl, MVT::i1, Ops[C+i]); DAG.ReplaceAllUsesOfValueWith(PromOp, DAG.getNode(PromOp.getOpcode(), dl, MVT::i1, Ops)); } // Now we're left with the initial truncation itself. if (N->getOpcode() == ISD::TRUNCATE) return N->getOperand(0); // Otherwise, this is a comparison. The operands to be compared have just // changed type (to i1), but everything else is the same. return SDValue(N, 0); } SDValue PPCTargetLowering::DAGCombineExtBoolTrunc(SDNode *N, DAGCombinerInfo &DCI) const { SelectionDAG &DAG = DCI.DAG; SDLoc dl(N); // If we're tracking CR bits, we need to be careful that we don't have: // zext(binary-ops(trunc(x), trunc(y))) // or // zext(binary-ops(binary-ops(trunc(x), trunc(y)), ...) // such that we're unnecessarily moving things into CR bits that can more // efficiently stay in GPRs. Note that if we're not certain that the high // bits are set as required by the final extension, we still may need to do // some masking to get the proper behavior. // This same functionality is important on PPC64 when dealing with // 32-to-64-bit extensions; these occur often when 32-bit values are used as // the return values of functions. Because it is so similar, it is handled // here as well. if (N->getValueType(0) != MVT::i32 && N->getValueType(0) != MVT::i64) return SDValue(); if (!((N->getOperand(0).getValueType() == MVT::i1 && Subtarget.useCRBits()) || (N->getOperand(0).getValueType() == MVT::i32 && Subtarget.isPPC64()))) return SDValue(); if (N->getOperand(0).getOpcode() != ISD::AND && N->getOperand(0).getOpcode() != ISD::OR && N->getOperand(0).getOpcode() != ISD::XOR && N->getOperand(0).getOpcode() != ISD::SELECT && N->getOperand(0).getOpcode() != ISD::SELECT_CC) return SDValue(); SmallVector Inputs; SmallVector BinOps(1, N->getOperand(0)), PromOps; SmallPtrSet Visited; // Visit all inputs, collect all binary operations (and, or, xor and // select) that are all fed by truncations. while (!BinOps.empty()) { SDValue BinOp = BinOps.back(); BinOps.pop_back(); if (!Visited.insert(BinOp.getNode()).second) continue; PromOps.push_back(BinOp); for (unsigned i = 0, ie = BinOp.getNumOperands(); i != ie; ++i) { // The condition of the select is not promoted. if (BinOp.getOpcode() == ISD::SELECT && i == 0) continue; if (BinOp.getOpcode() == ISD::SELECT_CC && i != 2 && i != 3) continue; if (BinOp.getOperand(i).getOpcode() == ISD::TRUNCATE || isa(BinOp.getOperand(i))) { Inputs.push_back(BinOp.getOperand(i)); } else if (BinOp.getOperand(i).getOpcode() == ISD::AND || BinOp.getOperand(i).getOpcode() == ISD::OR || BinOp.getOperand(i).getOpcode() == ISD::XOR || BinOp.getOperand(i).getOpcode() == ISD::SELECT || BinOp.getOperand(i).getOpcode() == ISD::SELECT_CC) { BinOps.push_back(BinOp.getOperand(i)); } else { // We have an input that is not a truncation or another binary // operation; we'll abort this transformation. return SDValue(); } } } // The operands of a select that must be truncated when the select is // promoted because the operand is actually part of the to-be-promoted set. DenseMap SelectTruncOp[2]; // Make sure that this is a self-contained cluster of operations (which // is not quite the same thing as saying that everything has only one // use). for (unsigned i = 0, ie = Inputs.size(); i != ie; ++i) { if (isa(Inputs[i])) continue; for (SDNode::use_iterator UI = Inputs[i].getNode()->use_begin(), UE = Inputs[i].getNode()->use_end(); UI != UE; ++UI) { SDNode *User = *UI; if (User != N && !Visited.count(User)) return SDValue(); // If we're going to promote the non-output-value operand(s) or SELECT or // SELECT_CC, record them for truncation. if (User->getOpcode() == ISD::SELECT) { if (User->getOperand(0) == Inputs[i]) SelectTruncOp[0].insert(std::make_pair(User, User->getOperand(0).getValueType())); } else if (User->getOpcode() == ISD::SELECT_CC) { if (User->getOperand(0) == Inputs[i]) SelectTruncOp[0].insert(std::make_pair(User, User->getOperand(0).getValueType())); if (User->getOperand(1) == Inputs[i]) SelectTruncOp[1].insert(std::make_pair(User, User->getOperand(1).getValueType())); } } } for (unsigned i = 0, ie = PromOps.size(); i != ie; ++i) { for (SDNode::use_iterator UI = PromOps[i].getNode()->use_begin(), UE = PromOps[i].getNode()->use_end(); UI != UE; ++UI) { SDNode *User = *UI; if (User != N && !Visited.count(User)) return SDValue(); // If we're going to promote the non-output-value operand(s) or SELECT or // SELECT_CC, record them for truncation. if (User->getOpcode() == ISD::SELECT) { if (User->getOperand(0) == PromOps[i]) SelectTruncOp[0].insert(std::make_pair(User, User->getOperand(0).getValueType())); } else if (User->getOpcode() == ISD::SELECT_CC) { if (User->getOperand(0) == PromOps[i]) SelectTruncOp[0].insert(std::make_pair(User, User->getOperand(0).getValueType())); if (User->getOperand(1) == PromOps[i]) SelectTruncOp[1].insert(std::make_pair(User, User->getOperand(1).getValueType())); } } } unsigned PromBits = N->getOperand(0).getValueSizeInBits(); bool ReallyNeedsExt = false; if (N->getOpcode() != ISD::ANY_EXTEND) { // If all of the inputs are not already sign/zero extended, then // we'll still need to do that at the end. for (unsigned i = 0, ie = Inputs.size(); i != ie; ++i) { if (isa(Inputs[i])) continue; unsigned OpBits = Inputs[i].getOperand(0).getValueSizeInBits(); assert(PromBits < OpBits && "Truncation not to a smaller bit count?"); if ((N->getOpcode() == ISD::ZERO_EXTEND && !DAG.MaskedValueIsZero(Inputs[i].getOperand(0), APInt::getHighBitsSet(OpBits, OpBits-PromBits))) || (N->getOpcode() == ISD::SIGN_EXTEND && DAG.ComputeNumSignBits(Inputs[i].getOperand(0)) < (OpBits-(PromBits-1)))) { ReallyNeedsExt = true; break; } } } // Replace all inputs, either with the truncation operand, or a // truncation or extension to the final output type. for (unsigned i = 0, ie = Inputs.size(); i != ie; ++i) { // Constant inputs need to be replaced with the to-be-promoted nodes that // use them because they might have users outside of the cluster of // promoted nodes. if (isa(Inputs[i])) continue; SDValue InSrc = Inputs[i].getOperand(0); if (Inputs[i].getValueType() == N->getValueType(0)) DAG.ReplaceAllUsesOfValueWith(Inputs[i], InSrc); else if (N->getOpcode() == ISD::SIGN_EXTEND) DAG.ReplaceAllUsesOfValueWith(Inputs[i], DAG.getSExtOrTrunc(InSrc, dl, N->getValueType(0))); else if (N->getOpcode() == ISD::ZERO_EXTEND) DAG.ReplaceAllUsesOfValueWith(Inputs[i], DAG.getZExtOrTrunc(InSrc, dl, N->getValueType(0))); else DAG.ReplaceAllUsesOfValueWith(Inputs[i], DAG.getAnyExtOrTrunc(InSrc, dl, N->getValueType(0))); } std::list PromOpHandles; for (auto &PromOp : PromOps) PromOpHandles.emplace_back(PromOp); // Replace all operations (these are all the same, but have a different // (promoted) return type). DAG.getNode will validate that the types of // a binary operator match, so go through the list in reverse so that // we've likely promoted both operands first. while (!PromOpHandles.empty()) { SDValue PromOp = PromOpHandles.back().getValue(); PromOpHandles.pop_back(); unsigned C; switch (PromOp.getOpcode()) { default: C = 0; break; case ISD::SELECT: C = 1; break; case ISD::SELECT_CC: C = 2; break; } if ((!isa(PromOp.getOperand(C)) && PromOp.getOperand(C).getValueType() != N->getValueType(0)) || (!isa(PromOp.getOperand(C+1)) && PromOp.getOperand(C+1).getValueType() != N->getValueType(0))) { // The to-be-promoted operands of this node have not yet been // promoted (this should be rare because we're going through the // list backward, but if one of the operands has several users in // this cluster of to-be-promoted nodes, it is possible). PromOpHandles.emplace_front(PromOp); continue; } // For SELECT and SELECT_CC nodes, we do a similar check for any // to-be-promoted comparison inputs. if (PromOp.getOpcode() == ISD::SELECT || PromOp.getOpcode() == ISD::SELECT_CC) { if ((SelectTruncOp[0].count(PromOp.getNode()) && PromOp.getOperand(0).getValueType() != N->getValueType(0)) || (SelectTruncOp[1].count(PromOp.getNode()) && PromOp.getOperand(1).getValueType() != N->getValueType(0))) { PromOpHandles.emplace_front(PromOp); continue; } } SmallVector Ops(PromOp.getNode()->op_begin(), PromOp.getNode()->op_end()); // If this node has constant inputs, then they'll need to be promoted here. for (unsigned i = 0; i < 2; ++i) { if (!isa(Ops[C+i])) continue; if (Ops[C+i].getValueType() == N->getValueType(0)) continue; if (N->getOpcode() == ISD::SIGN_EXTEND) Ops[C+i] = DAG.getSExtOrTrunc(Ops[C+i], dl, N->getValueType(0)); else if (N->getOpcode() == ISD::ZERO_EXTEND) Ops[C+i] = DAG.getZExtOrTrunc(Ops[C+i], dl, N->getValueType(0)); else Ops[C+i] = DAG.getAnyExtOrTrunc(Ops[C+i], dl, N->getValueType(0)); } // If we've promoted the comparison inputs of a SELECT or SELECT_CC, // truncate them again to the original value type. if (PromOp.getOpcode() == ISD::SELECT || PromOp.getOpcode() == ISD::SELECT_CC) { auto SI0 = SelectTruncOp[0].find(PromOp.getNode()); if (SI0 != SelectTruncOp[0].end()) Ops[0] = DAG.getNode(ISD::TRUNCATE, dl, SI0->second, Ops[0]); auto SI1 = SelectTruncOp[1].find(PromOp.getNode()); if (SI1 != SelectTruncOp[1].end()) Ops[1] = DAG.getNode(ISD::TRUNCATE, dl, SI1->second, Ops[1]); } DAG.ReplaceAllUsesOfValueWith(PromOp, DAG.getNode(PromOp.getOpcode(), dl, N->getValueType(0), Ops)); } // Now we're left with the initial extension itself. if (!ReallyNeedsExt) return N->getOperand(0); // To zero extend, just mask off everything except for the first bit (in the // i1 case). if (N->getOpcode() == ISD::ZERO_EXTEND) return DAG.getNode(ISD::AND, dl, N->getValueType(0), N->getOperand(0), DAG.getConstant(APInt::getLowBitsSet( N->getValueSizeInBits(0), PromBits), dl, N->getValueType(0))); assert(N->getOpcode() == ISD::SIGN_EXTEND && "Invalid extension type"); EVT ShiftAmountTy = getShiftAmountTy(N->getValueType(0), DAG.getDataLayout()); SDValue ShiftCst = DAG.getConstant(N->getValueSizeInBits(0) - PromBits, dl, ShiftAmountTy); return DAG.getNode( ISD::SRA, dl, N->getValueType(0), DAG.getNode(ISD::SHL, dl, N->getValueType(0), N->getOperand(0), ShiftCst), ShiftCst); } SDValue PPCTargetLowering::combineSetCC(SDNode *N, DAGCombinerInfo &DCI) const { assert(N->getOpcode() == ISD::SETCC && "Should be called with a SETCC node"); ISD::CondCode CC = cast(N->getOperand(2))->get(); if (CC == ISD::SETNE || CC == ISD::SETEQ) { SDValue LHS = N->getOperand(0); SDValue RHS = N->getOperand(1); // If there is a '0 - y' pattern, canonicalize the pattern to the RHS. if (LHS.getOpcode() == ISD::SUB && isNullConstant(LHS.getOperand(0)) && LHS.hasOneUse()) std::swap(LHS, RHS); // x == 0-y --> x+y == 0 // x != 0-y --> x+y != 0 if (RHS.getOpcode() == ISD::SUB && isNullConstant(RHS.getOperand(0)) && RHS.hasOneUse()) { SDLoc DL(N); SelectionDAG &DAG = DCI.DAG; EVT VT = N->getValueType(0); EVT OpVT = LHS.getValueType(); SDValue Add = DAG.getNode(ISD::ADD, DL, OpVT, LHS, RHS.getOperand(1)); return DAG.getSetCC(DL, VT, Add, DAG.getConstant(0, DL, OpVT), CC); } } return DAGCombineTruncBoolExt(N, DCI); } // Is this an extending load from an f32 to an f64? static bool isFPExtLoad(SDValue Op) { if (LoadSDNode *LD = dyn_cast(Op.getNode())) return LD->getExtensionType() == ISD::EXTLOAD && Op.getValueType() == MVT::f64; return false; } /// Reduces the number of fp-to-int conversion when building a vector. /// /// If this vector is built out of floating to integer conversions, /// transform it to a vector built out of floating point values followed by a /// single floating to integer conversion of the vector. /// Namely (build_vector (fptosi $A), (fptosi $B), ...) /// becomes (fptosi (build_vector ($A, $B, ...))) SDValue PPCTargetLowering:: combineElementTruncationToVectorTruncation(SDNode *N, DAGCombinerInfo &DCI) const { assert(N->getOpcode() == ISD::BUILD_VECTOR && "Should be called with a BUILD_VECTOR node"); SelectionDAG &DAG = DCI.DAG; SDLoc dl(N); SDValue FirstInput = N->getOperand(0); assert(FirstInput.getOpcode() == PPCISD::MFVSR && "The input operand must be an fp-to-int conversion."); // This combine happens after legalization so the fp_to_[su]i nodes are // already converted to PPCSISD nodes. unsigned FirstConversion = FirstInput.getOperand(0).getOpcode(); if (FirstConversion == PPCISD::FCTIDZ || FirstConversion == PPCISD::FCTIDUZ || FirstConversion == PPCISD::FCTIWZ || FirstConversion == PPCISD::FCTIWUZ) { bool IsSplat = true; bool Is32Bit = FirstConversion == PPCISD::FCTIWZ || FirstConversion == PPCISD::FCTIWUZ; EVT SrcVT = FirstInput.getOperand(0).getValueType(); SmallVector Ops; EVT TargetVT = N->getValueType(0); for (int i = 0, e = N->getNumOperands(); i < e; ++i) { SDValue NextOp = N->getOperand(i); if (NextOp.getOpcode() != PPCISD::MFVSR) return SDValue(); unsigned NextConversion = NextOp.getOperand(0).getOpcode(); if (NextConversion != FirstConversion) return SDValue(); // If we are converting to 32-bit integers, we need to add an FP_ROUND. // This is not valid if the input was originally double precision. It is // also not profitable to do unless this is an extending load in which // case doing this combine will allow us to combine consecutive loads. if (Is32Bit && !isFPExtLoad(NextOp.getOperand(0).getOperand(0))) return SDValue(); if (N->getOperand(i) != FirstInput) IsSplat = false; } // If this is a splat, we leave it as-is since there will be only a single // fp-to-int conversion followed by a splat of the integer. This is better // for 32-bit and smaller ints and neutral for 64-bit ints. if (IsSplat) return SDValue(); // Now that we know we have the right type of node, get its operands for (int i = 0, e = N->getNumOperands(); i < e; ++i) { SDValue In = N->getOperand(i).getOperand(0); if (Is32Bit) { // For 32-bit values, we need to add an FP_ROUND node (if we made it // here, we know that all inputs are extending loads so this is safe). if (In.isUndef()) Ops.push_back(DAG.getUNDEF(SrcVT)); else { SDValue Trunc = DAG.getNode(ISD::FP_ROUND, dl, MVT::f32, In.getOperand(0), DAG.getIntPtrConstant(1, dl)); Ops.push_back(Trunc); } } else Ops.push_back(In.isUndef() ? DAG.getUNDEF(SrcVT) : In.getOperand(0)); } unsigned Opcode; if (FirstConversion == PPCISD::FCTIDZ || FirstConversion == PPCISD::FCTIWZ) Opcode = ISD::FP_TO_SINT; else Opcode = ISD::FP_TO_UINT; EVT NewVT = TargetVT == MVT::v2i64 ? MVT::v2f64 : MVT::v4f32; SDValue BV = DAG.getBuildVector(NewVT, dl, Ops); return DAG.getNode(Opcode, dl, TargetVT, BV); } return SDValue(); } /// Reduce the number of loads when building a vector. /// /// Building a vector out of multiple loads can be converted to a load /// of the vector type if the loads are consecutive. If the loads are /// consecutive but in descending order, a shuffle is added at the end /// to reorder the vector. static SDValue combineBVOfConsecutiveLoads(SDNode *N, SelectionDAG &DAG) { assert(N->getOpcode() == ISD::BUILD_VECTOR && "Should be called with a BUILD_VECTOR node"); SDLoc dl(N); // Return early for non byte-sized type, as they can't be consecutive. if (!N->getValueType(0).getVectorElementType().isByteSized()) return SDValue(); bool InputsAreConsecutiveLoads = true; bool InputsAreReverseConsecutive = true; unsigned ElemSize = N->getValueType(0).getScalarType().getStoreSize(); SDValue FirstInput = N->getOperand(0); bool IsRoundOfExtLoad = false; if (FirstInput.getOpcode() == ISD::FP_ROUND && FirstInput.getOperand(0).getOpcode() == ISD::LOAD) { LoadSDNode *LD = dyn_cast(FirstInput.getOperand(0)); IsRoundOfExtLoad = LD->getExtensionType() == ISD::EXTLOAD; } // Not a build vector of (possibly fp_rounded) loads. if ((!IsRoundOfExtLoad && FirstInput.getOpcode() != ISD::LOAD) || N->getNumOperands() == 1) return SDValue(); for (int i = 1, e = N->getNumOperands(); i < e; ++i) { // If any inputs are fp_round(extload), they all must be. if (IsRoundOfExtLoad && N->getOperand(i).getOpcode() != ISD::FP_ROUND) return SDValue(); SDValue NextInput = IsRoundOfExtLoad ? N->getOperand(i).getOperand(0) : N->getOperand(i); if (NextInput.getOpcode() != ISD::LOAD) return SDValue(); SDValue PreviousInput = IsRoundOfExtLoad ? N->getOperand(i-1).getOperand(0) : N->getOperand(i-1); LoadSDNode *LD1 = dyn_cast(PreviousInput); LoadSDNode *LD2 = dyn_cast(NextInput); // If any inputs are fp_round(extload), they all must be. if (IsRoundOfExtLoad && LD2->getExtensionType() != ISD::EXTLOAD) return SDValue(); if (!isConsecutiveLS(LD2, LD1, ElemSize, 1, DAG)) InputsAreConsecutiveLoads = false; if (!isConsecutiveLS(LD1, LD2, ElemSize, 1, DAG)) InputsAreReverseConsecutive = false; // Exit early if the loads are neither consecutive nor reverse consecutive. if (!InputsAreConsecutiveLoads && !InputsAreReverseConsecutive) return SDValue(); } assert(!(InputsAreConsecutiveLoads && InputsAreReverseConsecutive) && "The loads cannot be both consecutive and reverse consecutive."); SDValue FirstLoadOp = IsRoundOfExtLoad ? FirstInput.getOperand(0) : FirstInput; SDValue LastLoadOp = IsRoundOfExtLoad ? N->getOperand(N->getNumOperands()-1).getOperand(0) : N->getOperand(N->getNumOperands()-1); LoadSDNode *LD1 = dyn_cast(FirstLoadOp); LoadSDNode *LDL = dyn_cast(LastLoadOp); if (InputsAreConsecutiveLoads) { assert(LD1 && "Input needs to be a LoadSDNode."); return DAG.getLoad(N->getValueType(0), dl, LD1->getChain(), LD1->getBasePtr(), LD1->getPointerInfo(), LD1->getAlignment()); } if (InputsAreReverseConsecutive) { assert(LDL && "Input needs to be a LoadSDNode."); SDValue Load = DAG.getLoad(N->getValueType(0), dl, LDL->getChain(), LDL->getBasePtr(), LDL->getPointerInfo(), LDL->getAlignment()); SmallVector Ops; for (int i = N->getNumOperands() - 1; i >= 0; i--) Ops.push_back(i); return DAG.getVectorShuffle(N->getValueType(0), dl, Load, DAG.getUNDEF(N->getValueType(0)), Ops); } return SDValue(); } // This function adds the required vector_shuffle needed to get // the elements of the vector extract in the correct position // as specified by the CorrectElems encoding. static SDValue addShuffleForVecExtend(SDNode *N, SelectionDAG &DAG, SDValue Input, uint64_t Elems, uint64_t CorrectElems) { SDLoc dl(N); unsigned NumElems = Input.getValueType().getVectorNumElements(); SmallVector ShuffleMask(NumElems, -1); // Knowing the element indices being extracted from the original // vector and the order in which they're being inserted, just put // them at element indices required for the instruction. for (unsigned i = 0; i < N->getNumOperands(); i++) { if (DAG.getDataLayout().isLittleEndian()) ShuffleMask[CorrectElems & 0xF] = Elems & 0xF; else ShuffleMask[(CorrectElems & 0xF0) >> 4] = (Elems & 0xF0) >> 4; CorrectElems = CorrectElems >> 8; Elems = Elems >> 8; } SDValue Shuffle = DAG.getVectorShuffle(Input.getValueType(), dl, Input, DAG.getUNDEF(Input.getValueType()), ShuffleMask); EVT VT = N->getValueType(0); SDValue Conv = DAG.getBitcast(VT, Shuffle); EVT ExtVT = EVT::getVectorVT(*DAG.getContext(), Input.getValueType().getVectorElementType(), VT.getVectorNumElements()); return DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, VT, Conv, DAG.getValueType(ExtVT)); } // Look for build vector patterns where input operands come from sign // extended vector_extract elements of specific indices. If the correct indices // aren't used, add a vector shuffle to fix up the indices and create // SIGN_EXTEND_INREG node which selects the vector sign extend instructions // during instruction selection. static SDValue combineBVOfVecSExt(SDNode *N, SelectionDAG &DAG) { // This array encodes the indices that the vector sign extend instructions // extract from when extending from one type to another for both BE and LE. // The right nibble of each byte corresponds to the LE incides. // and the left nibble of each byte corresponds to the BE incides. // For example: 0x3074B8FC byte->word // For LE: the allowed indices are: 0x0,0x4,0x8,0xC // For BE: the allowed indices are: 0x3,0x7,0xB,0xF // For example: 0x000070F8 byte->double word // For LE: the allowed indices are: 0x0,0x8 // For BE: the allowed indices are: 0x7,0xF uint64_t TargetElems[] = { 0x3074B8FC, // b->w 0x000070F8, // b->d 0x10325476, // h->w 0x00003074, // h->d 0x00001032, // w->d }; uint64_t Elems = 0; int Index; SDValue Input; auto isSExtOfVecExtract = [&](SDValue Op) -> bool { if (!Op) return false; if (Op.getOpcode() != ISD::SIGN_EXTEND && Op.getOpcode() != ISD::SIGN_EXTEND_INREG) return false; // A SIGN_EXTEND_INREG might be fed by an ANY_EXTEND to produce a value // of the right width. SDValue Extract = Op.getOperand(0); if (Extract.getOpcode() == ISD::ANY_EXTEND) Extract = Extract.getOperand(0); if (Extract.getOpcode() != ISD::EXTRACT_VECTOR_ELT) return false; ConstantSDNode *ExtOp = dyn_cast(Extract.getOperand(1)); if (!ExtOp) return false; Index = ExtOp->getZExtValue(); if (Input && Input != Extract.getOperand(0)) return false; if (!Input) Input = Extract.getOperand(0); Elems = Elems << 8; Index = DAG.getDataLayout().isLittleEndian() ? Index : Index << 4; Elems |= Index; return true; }; // If the build vector operands aren't sign extended vector extracts, // of the same input vector, then return. for (unsigned i = 0; i < N->getNumOperands(); i++) { if (!isSExtOfVecExtract(N->getOperand(i))) { return SDValue(); } } // If the vector extract indicies are not correct, add the appropriate // vector_shuffle. int TgtElemArrayIdx; int InputSize = Input.getValueType().getScalarSizeInBits(); int OutputSize = N->getValueType(0).getScalarSizeInBits(); if (InputSize + OutputSize == 40) TgtElemArrayIdx = 0; else if (InputSize + OutputSize == 72) TgtElemArrayIdx = 1; else if (InputSize + OutputSize == 48) TgtElemArrayIdx = 2; else if (InputSize + OutputSize == 80) TgtElemArrayIdx = 3; else if (InputSize + OutputSize == 96) TgtElemArrayIdx = 4; else return SDValue(); uint64_t CorrectElems = TargetElems[TgtElemArrayIdx]; CorrectElems = DAG.getDataLayout().isLittleEndian() ? CorrectElems & 0x0F0F0F0F0F0F0F0F : CorrectElems & 0xF0F0F0F0F0F0F0F0; if (Elems != CorrectElems) { return addShuffleForVecExtend(N, DAG, Input, Elems, CorrectElems); } // Regular lowering will catch cases where a shuffle is not needed. return SDValue(); } SDValue PPCTargetLowering::DAGCombineBuildVector(SDNode *N, DAGCombinerInfo &DCI) const { assert(N->getOpcode() == ISD::BUILD_VECTOR && "Should be called with a BUILD_VECTOR node"); SelectionDAG &DAG = DCI.DAG; SDLoc dl(N); if (!Subtarget.hasVSX()) return SDValue(); // The target independent DAG combiner will leave a build_vector of // float-to-int conversions intact. We can generate MUCH better code for // a float-to-int conversion of a vector of floats. SDValue FirstInput = N->getOperand(0); if (FirstInput.getOpcode() == PPCISD::MFVSR) { SDValue Reduced = combineElementTruncationToVectorTruncation(N, DCI); if (Reduced) return Reduced; } // If we're building a vector out of consecutive loads, just load that // vector type. SDValue Reduced = combineBVOfConsecutiveLoads(N, DAG); if (Reduced) return Reduced; // If we're building a vector out of extended elements from another vector // we have P9 vector integer extend instructions. The code assumes legal // input types (i.e. it can't handle things like v4i16) so do not run before // legalization. if (Subtarget.hasP9Altivec() && !DCI.isBeforeLegalize()) { Reduced = combineBVOfVecSExt(N, DAG); if (Reduced) return Reduced; } if (N->getValueType(0) != MVT::v2f64) return SDValue(); // Looking for: // (build_vector ([su]int_to_fp (extractelt 0)), [su]int_to_fp (extractelt 1)) if (FirstInput.getOpcode() != ISD::SINT_TO_FP && FirstInput.getOpcode() != ISD::UINT_TO_FP) return SDValue(); if (N->getOperand(1).getOpcode() != ISD::SINT_TO_FP && N->getOperand(1).getOpcode() != ISD::UINT_TO_FP) return SDValue(); if (FirstInput.getOpcode() != N->getOperand(1).getOpcode()) return SDValue(); SDValue Ext1 = FirstInput.getOperand(0); SDValue Ext2 = N->getOperand(1).getOperand(0); if(Ext1.getOpcode() != ISD::EXTRACT_VECTOR_ELT || Ext2.getOpcode() != ISD::EXTRACT_VECTOR_ELT) return SDValue(); ConstantSDNode *Ext1Op = dyn_cast(Ext1.getOperand(1)); ConstantSDNode *Ext2Op = dyn_cast(Ext2.getOperand(1)); if (!Ext1Op || !Ext2Op) return SDValue(); if (Ext1.getOperand(0).getValueType() != MVT::v4i32 || Ext1.getOperand(0) != Ext2.getOperand(0)) return SDValue(); int FirstElem = Ext1Op->getZExtValue(); int SecondElem = Ext2Op->getZExtValue(); int SubvecIdx; if (FirstElem == 0 && SecondElem == 1) SubvecIdx = Subtarget.isLittleEndian() ? 1 : 0; else if (FirstElem == 2 && SecondElem == 3) SubvecIdx = Subtarget.isLittleEndian() ? 0 : 1; else return SDValue(); SDValue SrcVec = Ext1.getOperand(0); auto NodeType = (N->getOperand(1).getOpcode() == ISD::SINT_TO_FP) ? PPCISD::SINT_VEC_TO_FP : PPCISD::UINT_VEC_TO_FP; return DAG.getNode(NodeType, dl, MVT::v2f64, SrcVec, DAG.getIntPtrConstant(SubvecIdx, dl)); } SDValue PPCTargetLowering::combineFPToIntToFP(SDNode *N, DAGCombinerInfo &DCI) const { assert((N->getOpcode() == ISD::SINT_TO_FP || N->getOpcode() == ISD::UINT_TO_FP) && "Need an int -> FP conversion node here"); if (useSoftFloat() || !Subtarget.has64BitSupport()) return SDValue(); SelectionDAG &DAG = DCI.DAG; SDLoc dl(N); SDValue Op(N, 0); // Don't handle ppc_fp128 here or conversions that are out-of-range capable // from the hardware. if (Op.getValueType() != MVT::f32 && Op.getValueType() != MVT::f64) return SDValue(); if (Op.getOperand(0).getValueType().getSimpleVT() <= MVT(MVT::i1) || Op.getOperand(0).getValueType().getSimpleVT() > MVT(MVT::i64)) return SDValue(); SDValue FirstOperand(Op.getOperand(0)); bool SubWordLoad = FirstOperand.getOpcode() == ISD::LOAD && (FirstOperand.getValueType() == MVT::i8 || FirstOperand.getValueType() == MVT::i16); if (Subtarget.hasP9Vector() && Subtarget.hasP9Altivec() && SubWordLoad) { bool Signed = N->getOpcode() == ISD::SINT_TO_FP; bool DstDouble = Op.getValueType() == MVT::f64; unsigned ConvOp = Signed ? (DstDouble ? PPCISD::FCFID : PPCISD::FCFIDS) : (DstDouble ? PPCISD::FCFIDU : PPCISD::FCFIDUS); SDValue WidthConst = DAG.getIntPtrConstant(FirstOperand.getValueType() == MVT::i8 ? 1 : 2, dl, false); LoadSDNode *LDN = cast(FirstOperand.getNode()); SDValue Ops[] = { LDN->getChain(), LDN->getBasePtr(), WidthConst }; SDValue Ld = DAG.getMemIntrinsicNode(PPCISD::LXSIZX, dl, DAG.getVTList(MVT::f64, MVT::Other), Ops, MVT::i8, LDN->getMemOperand()); // For signed conversion, we need to sign-extend the value in the VSR if (Signed) { SDValue ExtOps[] = { Ld, WidthConst }; SDValue Ext = DAG.getNode(PPCISD::VEXTS, dl, MVT::f64, ExtOps); return DAG.getNode(ConvOp, dl, DstDouble ? MVT::f64 : MVT::f32, Ext); } else return DAG.getNode(ConvOp, dl, DstDouble ? MVT::f64 : MVT::f32, Ld); } // For i32 intermediate values, unfortunately, the conversion functions // leave the upper 32 bits of the value are undefined. Within the set of // scalar instructions, we have no method for zero- or sign-extending the // value. Thus, we cannot handle i32 intermediate values here. if (Op.getOperand(0).getValueType() == MVT::i32) return SDValue(); assert((Op.getOpcode() == ISD::SINT_TO_FP || Subtarget.hasFPCVT()) && "UINT_TO_FP is supported only with FPCVT"); // If we have FCFIDS, then use it when converting to single-precision. // Otherwise, convert to double-precision and then round. unsigned FCFOp = (Subtarget.hasFPCVT() && Op.getValueType() == MVT::f32) ? (Op.getOpcode() == ISD::UINT_TO_FP ? PPCISD::FCFIDUS : PPCISD::FCFIDS) : (Op.getOpcode() == ISD::UINT_TO_FP ? PPCISD::FCFIDU : PPCISD::FCFID); MVT FCFTy = (Subtarget.hasFPCVT() && Op.getValueType() == MVT::f32) ? MVT::f32 : MVT::f64; // If we're converting from a float, to an int, and back to a float again, // then we don't need the store/load pair at all. if ((Op.getOperand(0).getOpcode() == ISD::FP_TO_UINT && Subtarget.hasFPCVT()) || (Op.getOperand(0).getOpcode() == ISD::FP_TO_SINT)) { SDValue Src = Op.getOperand(0).getOperand(0); if (Src.getValueType() == MVT::f32) { Src = DAG.getNode(ISD::FP_EXTEND, dl, MVT::f64, Src); DCI.AddToWorklist(Src.getNode()); } else if (Src.getValueType() != MVT::f64) { // Make sure that we don't pick up a ppc_fp128 source value. return SDValue(); } unsigned FCTOp = Op.getOperand(0).getOpcode() == ISD::FP_TO_SINT ? PPCISD::FCTIDZ : PPCISD::FCTIDUZ; SDValue Tmp = DAG.getNode(FCTOp, dl, MVT::f64, Src); SDValue FP = DAG.getNode(FCFOp, dl, FCFTy, Tmp); if (Op.getValueType() == MVT::f32 && !Subtarget.hasFPCVT()) { FP = DAG.getNode(ISD::FP_ROUND, dl, MVT::f32, FP, DAG.getIntPtrConstant(0, dl)); DCI.AddToWorklist(FP.getNode()); } return FP; } return SDValue(); } // expandVSXLoadForLE - Convert VSX loads (which may be intrinsics for // builtins) into loads with swaps. SDValue PPCTargetLowering::expandVSXLoadForLE(SDNode *N, DAGCombinerInfo &DCI) const { SelectionDAG &DAG = DCI.DAG; SDLoc dl(N); SDValue Chain; SDValue Base; MachineMemOperand *MMO; switch (N->getOpcode()) { default: llvm_unreachable("Unexpected opcode for little endian VSX load"); case ISD::LOAD: { LoadSDNode *LD = cast(N); Chain = LD->getChain(); Base = LD->getBasePtr(); MMO = LD->getMemOperand(); // If the MMO suggests this isn't a load of a full vector, leave // things alone. For a built-in, we have to make the change for // correctness, so if there is a size problem that will be a bug. if (MMO->getSize() < 16) return SDValue(); break; } case ISD::INTRINSIC_W_CHAIN: { MemIntrinsicSDNode *Intrin = cast(N); Chain = Intrin->getChain(); // Similarly to the store case below, Intrin->getBasePtr() doesn't get // us what we want. Get operand 2 instead. Base = Intrin->getOperand(2); MMO = Intrin->getMemOperand(); break; } } MVT VecTy = N->getValueType(0).getSimpleVT(); // Do not expand to PPCISD::LXVD2X + PPCISD::XXSWAPD when the load is // aligned and the type is a vector with elements up to 4 bytes if (Subtarget.needsSwapsForVSXMemOps() && MMO->getAlign() >= Align(16) && VecTy.getScalarSizeInBits() <= 32) { return SDValue(); } SDValue LoadOps[] = { Chain, Base }; SDValue Load = DAG.getMemIntrinsicNode(PPCISD::LXVD2X, dl, DAG.getVTList(MVT::v2f64, MVT::Other), LoadOps, MVT::v2f64, MMO); DCI.AddToWorklist(Load.getNode()); Chain = Load.getValue(1); SDValue Swap = DAG.getNode( PPCISD::XXSWAPD, dl, DAG.getVTList(MVT::v2f64, MVT::Other), Chain, Load); DCI.AddToWorklist(Swap.getNode()); // Add a bitcast if the resulting load type doesn't match v2f64. if (VecTy != MVT::v2f64) { SDValue N = DAG.getNode(ISD::BITCAST, dl, VecTy, Swap); DCI.AddToWorklist(N.getNode()); // Package {bitcast value, swap's chain} to match Load's shape. return DAG.getNode(ISD::MERGE_VALUES, dl, DAG.getVTList(VecTy, MVT::Other), N, Swap.getValue(1)); } return Swap; } // expandVSXStoreForLE - Convert VSX stores (which may be intrinsics for // builtins) into stores with swaps. SDValue PPCTargetLowering::expandVSXStoreForLE(SDNode *N, DAGCombinerInfo &DCI) const { SelectionDAG &DAG = DCI.DAG; SDLoc dl(N); SDValue Chain; SDValue Base; unsigned SrcOpnd; MachineMemOperand *MMO; switch (N->getOpcode()) { default: llvm_unreachable("Unexpected opcode for little endian VSX store"); case ISD::STORE: { StoreSDNode *ST = cast(N); Chain = ST->getChain(); Base = ST->getBasePtr(); MMO = ST->getMemOperand(); SrcOpnd = 1; // If the MMO suggests this isn't a store of a full vector, leave // things alone. For a built-in, we have to make the change for // correctness, so if there is a size problem that will be a bug. if (MMO->getSize() < 16) return SDValue(); break; } case ISD::INTRINSIC_VOID: { MemIntrinsicSDNode *Intrin = cast(N); Chain = Intrin->getChain(); // Intrin->getBasePtr() oddly does not get what we want. Base = Intrin->getOperand(3); MMO = Intrin->getMemOperand(); SrcOpnd = 2; break; } } SDValue Src = N->getOperand(SrcOpnd); MVT VecTy = Src.getValueType().getSimpleVT(); // Do not expand to PPCISD::XXSWAPD and PPCISD::STXVD2X when the load is // aligned and the type is a vector with elements up to 4 bytes if (Subtarget.needsSwapsForVSXMemOps() && MMO->getAlign() >= Align(16) && VecTy.getScalarSizeInBits() <= 32) { return SDValue(); } // All stores are done as v2f64 and possible bit cast. if (VecTy != MVT::v2f64) { Src = DAG.getNode(ISD::BITCAST, dl, MVT::v2f64, Src); DCI.AddToWorklist(Src.getNode()); } SDValue Swap = DAG.getNode(PPCISD::XXSWAPD, dl, DAG.getVTList(MVT::v2f64, MVT::Other), Chain, Src); DCI.AddToWorklist(Swap.getNode()); Chain = Swap.getValue(1); SDValue StoreOps[] = { Chain, Swap, Base }; SDValue Store = DAG.getMemIntrinsicNode(PPCISD::STXVD2X, dl, DAG.getVTList(MVT::Other), StoreOps, VecTy, MMO); DCI.AddToWorklist(Store.getNode()); return Store; } // Handle DAG combine for STORE (FP_TO_INT F). SDValue PPCTargetLowering::combineStoreFPToInt(SDNode *N, DAGCombinerInfo &DCI) const { SelectionDAG &DAG = DCI.DAG; SDLoc dl(N); unsigned Opcode = N->getOperand(1).getOpcode(); assert((Opcode == ISD::FP_TO_SINT || Opcode == ISD::FP_TO_UINT) && "Not a FP_TO_INT Instruction!"); SDValue Val = N->getOperand(1).getOperand(0); EVT Op1VT = N->getOperand(1).getValueType(); EVT ResVT = Val.getValueType(); // Floating point types smaller than 32 bits are not legal on Power. if (ResVT.getScalarSizeInBits() < 32) return SDValue(); // Only perform combine for conversion to i64/i32 or power9 i16/i8. bool ValidTypeForStoreFltAsInt = (Op1VT == MVT::i32 || Op1VT == MVT::i64 || (Subtarget.hasP9Vector() && (Op1VT == MVT::i16 || Op1VT == MVT::i8))); if (ResVT == MVT::ppcf128 || !Subtarget.hasP8Vector() || cast(N)->isTruncatingStore() || !ValidTypeForStoreFltAsInt) return SDValue(); // Extend f32 values to f64 if (ResVT.getScalarSizeInBits() == 32) { Val = DAG.getNode(ISD::FP_EXTEND, dl, MVT::f64, Val); DCI.AddToWorklist(Val.getNode()); } // Set signed or unsigned conversion opcode. unsigned ConvOpcode = (Opcode == ISD::FP_TO_SINT) ? PPCISD::FP_TO_SINT_IN_VSR : PPCISD::FP_TO_UINT_IN_VSR; Val = DAG.getNode(ConvOpcode, dl, ResVT == MVT::f128 ? MVT::f128 : MVT::f64, Val); DCI.AddToWorklist(Val.getNode()); // Set number of bytes being converted. unsigned ByteSize = Op1VT.getScalarSizeInBits() / 8; SDValue Ops[] = { N->getOperand(0), Val, N->getOperand(2), DAG.getIntPtrConstant(ByteSize, dl, false), DAG.getValueType(Op1VT) }; Val = DAG.getMemIntrinsicNode(PPCISD::ST_VSR_SCAL_INT, dl, DAG.getVTList(MVT::Other), Ops, cast(N)->getMemoryVT(), cast(N)->getMemOperand()); DCI.AddToWorklist(Val.getNode()); return Val; } SDValue PPCTargetLowering::combineVReverseMemOP(ShuffleVectorSDNode *SVN, LSBaseSDNode *LSBase, DAGCombinerInfo &DCI) const { assert((ISD::isNormalLoad(LSBase) || ISD::isNormalStore(LSBase)) && "Not a reverse memop pattern!"); auto IsElementReverse = [](const ShuffleVectorSDNode *SVN) -> bool { auto Mask = SVN->getMask(); int i = 0; auto I = Mask.rbegin(); auto E = Mask.rend(); for (; I != E; ++I) { if (*I != i) return false; i++; } return true; }; SelectionDAG &DAG = DCI.DAG; EVT VT = SVN->getValueType(0); if (!isTypeLegal(VT) || !Subtarget.isLittleEndian() || !Subtarget.hasVSX()) return SDValue(); // Before P9, we have PPCVSXSwapRemoval pass to hack the element order. // See comment in PPCVSXSwapRemoval.cpp. // It is conflict with PPCVSXSwapRemoval opt. So we don't do it. if (!Subtarget.hasP9Vector()) return SDValue(); if(!IsElementReverse(SVN)) return SDValue(); if (LSBase->getOpcode() == ISD::LOAD) { SDLoc dl(SVN); SDValue LoadOps[] = {LSBase->getChain(), LSBase->getBasePtr()}; return DAG.getMemIntrinsicNode( PPCISD::LOAD_VEC_BE, dl, DAG.getVTList(VT, MVT::Other), LoadOps, LSBase->getMemoryVT(), LSBase->getMemOperand()); } if (LSBase->getOpcode() == ISD::STORE) { SDLoc dl(LSBase); SDValue StoreOps[] = {LSBase->getChain(), SVN->getOperand(0), LSBase->getBasePtr()}; return DAG.getMemIntrinsicNode( PPCISD::STORE_VEC_BE, dl, DAG.getVTList(MVT::Other), StoreOps, LSBase->getMemoryVT(), LSBase->getMemOperand()); } llvm_unreachable("Expected a load or store node here"); } SDValue PPCTargetLowering::PerformDAGCombine(SDNode *N, DAGCombinerInfo &DCI) const { SelectionDAG &DAG = DCI.DAG; SDLoc dl(N); switch (N->getOpcode()) { default: break; case ISD::ADD: return combineADD(N, DCI); case ISD::SHL: return combineSHL(N, DCI); case ISD::SRA: return combineSRA(N, DCI); case ISD::SRL: return combineSRL(N, DCI); case ISD::MUL: return combineMUL(N, DCI); case PPCISD::SHL: if (isNullConstant(N->getOperand(0))) // 0 << V -> 0. return N->getOperand(0); break; case PPCISD::SRL: if (isNullConstant(N->getOperand(0))) // 0 >>u V -> 0. return N->getOperand(0); break; case PPCISD::SRA: if (ConstantSDNode *C = dyn_cast(N->getOperand(0))) { if (C->isNullValue() || // 0 >>s V -> 0. C->isAllOnesValue()) // -1 >>s V -> -1. return N->getOperand(0); } break; case ISD::SIGN_EXTEND: case ISD::ZERO_EXTEND: case ISD::ANY_EXTEND: return DAGCombineExtBoolTrunc(N, DCI); case ISD::TRUNCATE: return combineTRUNCATE(N, DCI); case ISD::SETCC: if (SDValue CSCC = combineSetCC(N, DCI)) return CSCC; LLVM_FALLTHROUGH; case ISD::SELECT_CC: return DAGCombineTruncBoolExt(N, DCI); case ISD::SINT_TO_FP: case ISD::UINT_TO_FP: return combineFPToIntToFP(N, DCI); case ISD::VECTOR_SHUFFLE: if (ISD::isNormalLoad(N->getOperand(0).getNode())) { LSBaseSDNode* LSBase = cast(N->getOperand(0)); return combineVReverseMemOP(cast(N), LSBase, DCI); } break; case ISD::STORE: { EVT Op1VT = N->getOperand(1).getValueType(); unsigned Opcode = N->getOperand(1).getOpcode(); if (Opcode == ISD::FP_TO_SINT || Opcode == ISD::FP_TO_UINT) { SDValue Val= combineStoreFPToInt(N, DCI); if (Val) return Val; } if (Opcode == ISD::VECTOR_SHUFFLE && ISD::isNormalStore(N)) { ShuffleVectorSDNode *SVN = cast(N->getOperand(1)); SDValue Val= combineVReverseMemOP(SVN, cast(N), DCI); if (Val) return Val; } // Turn STORE (BSWAP) -> sthbrx/stwbrx. if (cast(N)->isUnindexed() && Opcode == ISD::BSWAP && N->getOperand(1).getNode()->hasOneUse() && (Op1VT == MVT::i32 || Op1VT == MVT::i16 || (Subtarget.hasLDBRX() && Subtarget.isPPC64() && Op1VT == MVT::i64))) { // STBRX can only handle simple types and it makes no sense to store less // two bytes in byte-reversed order. EVT mVT = cast(N)->getMemoryVT(); if (mVT.isExtended() || mVT.getSizeInBits() < 16) break; SDValue BSwapOp = N->getOperand(1).getOperand(0); // Do an any-extend to 32-bits if this is a half-word input. if (BSwapOp.getValueType() == MVT::i16) BSwapOp = DAG.getNode(ISD::ANY_EXTEND, dl, MVT::i32, BSwapOp); // If the type of BSWAP operand is wider than stored memory width // it need to be shifted to the right side before STBRX. if (Op1VT.bitsGT(mVT)) { int Shift = Op1VT.getSizeInBits() - mVT.getSizeInBits(); BSwapOp = DAG.getNode(ISD::SRL, dl, Op1VT, BSwapOp, DAG.getConstant(Shift, dl, MVT::i32)); // Need to truncate if this is a bswap of i64 stored as i32/i16. if (Op1VT == MVT::i64) BSwapOp = DAG.getNode(ISD::TRUNCATE, dl, MVT::i32, BSwapOp); } SDValue Ops[] = { N->getOperand(0), BSwapOp, N->getOperand(2), DAG.getValueType(mVT) }; return DAG.getMemIntrinsicNode(PPCISD::STBRX, dl, DAG.getVTList(MVT::Other), Ops, cast(N)->getMemoryVT(), cast(N)->getMemOperand()); } // STORE Constant:i32<0> -> STORE Constant:i64<0> // So it can increase the chance of CSE constant construction. if (Subtarget.isPPC64() && !DCI.isBeforeLegalize() && isa(N->getOperand(1)) && Op1VT == MVT::i32) { // Need to sign-extended to 64-bits to handle negative values. EVT MemVT = cast(N)->getMemoryVT(); uint64_t Val64 = SignExtend64(N->getConstantOperandVal(1), MemVT.getSizeInBits()); SDValue Const64 = DAG.getConstant(Val64, dl, MVT::i64); // DAG.getTruncStore() can't be used here because it doesn't accept // the general (base + offset) addressing mode. // So we use UpdateNodeOperands and setTruncatingStore instead. DAG.UpdateNodeOperands(N, N->getOperand(0), Const64, N->getOperand(2), N->getOperand(3)); cast(N)->setTruncatingStore(true); return SDValue(N, 0); } // For little endian, VSX stores require generating xxswapd/lxvd2x. // Not needed on ISA 3.0 based CPUs since we have a non-permuting store. if (Op1VT.isSimple()) { MVT StoreVT = Op1VT.getSimpleVT(); if (Subtarget.needsSwapsForVSXMemOps() && (StoreVT == MVT::v2f64 || StoreVT == MVT::v2i64 || StoreVT == MVT::v4f32 || StoreVT == MVT::v4i32)) return expandVSXStoreForLE(N, DCI); } break; } case ISD::LOAD: { LoadSDNode *LD = cast(N); EVT VT = LD->getValueType(0); // For little endian, VSX loads require generating lxvd2x/xxswapd. // Not needed on ISA 3.0 based CPUs since we have a non-permuting load. if (VT.isSimple()) { MVT LoadVT = VT.getSimpleVT(); if (Subtarget.needsSwapsForVSXMemOps() && (LoadVT == MVT::v2f64 || LoadVT == MVT::v2i64 || LoadVT == MVT::v4f32 || LoadVT == MVT::v4i32)) return expandVSXLoadForLE(N, DCI); } // We sometimes end up with a 64-bit integer load, from which we extract // two single-precision floating-point numbers. This happens with // std::complex, and other similar structures, because of the way we // canonicalize structure copies. However, if we lack direct moves, // then the final bitcasts from the extracted integer values to the // floating-point numbers turn into store/load pairs. Even with direct moves, // just loading the two floating-point numbers is likely better. auto ReplaceTwoFloatLoad = [&]() { if (VT != MVT::i64) return false; if (LD->getExtensionType() != ISD::NON_EXTLOAD || LD->isVolatile()) return false; // We're looking for a sequence like this: // t13: i64,ch = load t0, t6, undef:i64 // t16: i64 = srl t13, Constant:i32<32> // t17: i32 = truncate t16 // t18: f32 = bitcast t17 // t19: i32 = truncate t13 // t20: f32 = bitcast t19 if (!LD->hasNUsesOfValue(2, 0)) return false; auto UI = LD->use_begin(); while (UI.getUse().getResNo() != 0) ++UI; SDNode *Trunc = *UI++; while (UI.getUse().getResNo() != 0) ++UI; SDNode *RightShift = *UI; if (Trunc->getOpcode() != ISD::TRUNCATE) std::swap(Trunc, RightShift); if (Trunc->getOpcode() != ISD::TRUNCATE || Trunc->getValueType(0) != MVT::i32 || !Trunc->hasOneUse()) return false; if (RightShift->getOpcode() != ISD::SRL || !isa(RightShift->getOperand(1)) || RightShift->getConstantOperandVal(1) != 32 || !RightShift->hasOneUse()) return false; SDNode *Trunc2 = *RightShift->use_begin(); if (Trunc2->getOpcode() != ISD::TRUNCATE || Trunc2->getValueType(0) != MVT::i32 || !Trunc2->hasOneUse()) return false; SDNode *Bitcast = *Trunc->use_begin(); SDNode *Bitcast2 = *Trunc2->use_begin(); if (Bitcast->getOpcode() != ISD::BITCAST || Bitcast->getValueType(0) != MVT::f32) return false; if (Bitcast2->getOpcode() != ISD::BITCAST || Bitcast2->getValueType(0) != MVT::f32) return false; if (Subtarget.isLittleEndian()) std::swap(Bitcast, Bitcast2); // Bitcast has the second float (in memory-layout order) and Bitcast2 // has the first one. SDValue BasePtr = LD->getBasePtr(); if (LD->isIndexed()) { assert(LD->getAddressingMode() == ISD::PRE_INC && "Non-pre-inc AM on PPC?"); BasePtr = DAG.getNode(ISD::ADD, dl, BasePtr.getValueType(), BasePtr, LD->getOffset()); } auto MMOFlags = LD->getMemOperand()->getFlags() & ~MachineMemOperand::MOVolatile; SDValue FloatLoad = DAG.getLoad(MVT::f32, dl, LD->getChain(), BasePtr, LD->getPointerInfo(), LD->getAlignment(), MMOFlags, LD->getAAInfo()); SDValue AddPtr = DAG.getNode(ISD::ADD, dl, BasePtr.getValueType(), BasePtr, DAG.getIntPtrConstant(4, dl)); SDValue FloatLoad2 = DAG.getLoad( MVT::f32, dl, SDValue(FloatLoad.getNode(), 1), AddPtr, LD->getPointerInfo().getWithOffset(4), MinAlign(LD->getAlignment(), 4), MMOFlags, LD->getAAInfo()); if (LD->isIndexed()) { // Note that DAGCombine should re-form any pre-increment load(s) from // what is produced here if that makes sense. DAG.ReplaceAllUsesOfValueWith(SDValue(LD, 1), BasePtr); } DCI.CombineTo(Bitcast2, FloatLoad); DCI.CombineTo(Bitcast, FloatLoad2); DAG.ReplaceAllUsesOfValueWith(SDValue(LD, LD->isIndexed() ? 2 : 1), SDValue(FloatLoad2.getNode(), 1)); return true; }; if (ReplaceTwoFloatLoad()) return SDValue(N, 0); EVT MemVT = LD->getMemoryVT(); Type *Ty = MemVT.getTypeForEVT(*DAG.getContext()); unsigned ABIAlignment = DAG.getDataLayout().getABITypeAlignment(Ty); Type *STy = MemVT.getScalarType().getTypeForEVT(*DAG.getContext()); unsigned ScalarABIAlignment = DAG.getDataLayout().getABITypeAlignment(STy); if (LD->isUnindexed() && VT.isVector() && ((Subtarget.hasAltivec() && ISD::isNON_EXTLoad(N) && // P8 and later hardware should just use LOAD. !Subtarget.hasP8Vector() && (VT == MVT::v16i8 || VT == MVT::v8i16 || VT == MVT::v4i32 || VT == MVT::v4f32)) || (Subtarget.hasQPX() && (VT == MVT::v4f64 || VT == MVT::v4f32) && LD->getAlignment() >= ScalarABIAlignment)) && LD->getAlignment() < ABIAlignment) { // This is a type-legal unaligned Altivec or QPX load. SDValue Chain = LD->getChain(); SDValue Ptr = LD->getBasePtr(); bool isLittleEndian = Subtarget.isLittleEndian(); // This implements the loading of unaligned vectors as described in // the venerable Apple Velocity Engine overview. Specifically: // https://developer.apple.com/hardwaredrivers/ve/alignment.html // https://developer.apple.com/hardwaredrivers/ve/code_optimization.html // // The general idea is to expand a sequence of one or more unaligned // loads into an alignment-based permutation-control instruction (lvsl // or lvsr), a series of regular vector loads (which always truncate // their input address to an aligned address), and a series of // permutations. The results of these permutations are the requested // loaded values. The trick is that the last "extra" load is not taken // from the address you might suspect (sizeof(vector) bytes after the // last requested load), but rather sizeof(vector) - 1 bytes after the // last requested vector. The point of this is to avoid a page fault if // the base address happened to be aligned. This works because if the // base address is aligned, then adding less than a full vector length // will cause the last vector in the sequence to be (re)loaded. // Otherwise, the next vector will be fetched as you might suspect was // necessary. // We might be able to reuse the permutation generation from // a different base address offset from this one by an aligned amount. // The INTRINSIC_WO_CHAIN DAG combine will attempt to perform this // optimization later. Intrinsic::ID Intr, IntrLD, IntrPerm; MVT PermCntlTy, PermTy, LDTy; if (Subtarget.hasAltivec()) { Intr = isLittleEndian ? Intrinsic::ppc_altivec_lvsr : Intrinsic::ppc_altivec_lvsl; IntrLD = Intrinsic::ppc_altivec_lvx; IntrPerm = Intrinsic::ppc_altivec_vperm; PermCntlTy = MVT::v16i8; PermTy = MVT::v4i32; LDTy = MVT::v4i32; } else { Intr = MemVT == MVT::v4f64 ? Intrinsic::ppc_qpx_qvlpcld : Intrinsic::ppc_qpx_qvlpcls; IntrLD = MemVT == MVT::v4f64 ? Intrinsic::ppc_qpx_qvlfd : Intrinsic::ppc_qpx_qvlfs; IntrPerm = Intrinsic::ppc_qpx_qvfperm; PermCntlTy = MVT::v4f64; PermTy = MVT::v4f64; LDTy = MemVT.getSimpleVT(); } SDValue PermCntl = BuildIntrinsicOp(Intr, Ptr, DAG, dl, PermCntlTy); // Create the new MMO for the new base load. It is like the original MMO, // but represents an area in memory almost twice the vector size centered // on the original address. If the address is unaligned, we might start // reading up to (sizeof(vector)-1) bytes below the address of the // original unaligned load. MachineFunction &MF = DAG.getMachineFunction(); MachineMemOperand *BaseMMO = MF.getMachineMemOperand(LD->getMemOperand(), -(long)MemVT.getStoreSize()+1, 2*MemVT.getStoreSize()-1); // Create the new base load. SDValue LDXIntID = DAG.getTargetConstant(IntrLD, dl, getPointerTy(MF.getDataLayout())); SDValue BaseLoadOps[] = { Chain, LDXIntID, Ptr }; SDValue BaseLoad = DAG.getMemIntrinsicNode(ISD::INTRINSIC_W_CHAIN, dl, DAG.getVTList(PermTy, MVT::Other), BaseLoadOps, LDTy, BaseMMO); // Note that the value of IncOffset (which is provided to the next // load's pointer info offset value, and thus used to calculate the // alignment), and the value of IncValue (which is actually used to // increment the pointer value) are different! This is because we // require the next load to appear to be aligned, even though it // is actually offset from the base pointer by a lesser amount. int IncOffset = VT.getSizeInBits() / 8; int IncValue = IncOffset; // Walk (both up and down) the chain looking for another load at the real // (aligned) offset (the alignment of the other load does not matter in // this case). If found, then do not use the offset reduction trick, as // that will prevent the loads from being later combined (as they would // otherwise be duplicates). if (!findConsecutiveLoad(LD, DAG)) --IncValue; SDValue Increment = DAG.getConstant(IncValue, dl, getPointerTy(MF.getDataLayout())); Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr, Increment); MachineMemOperand *ExtraMMO = MF.getMachineMemOperand(LD->getMemOperand(), 1, 2*MemVT.getStoreSize()-1); SDValue ExtraLoadOps[] = { Chain, LDXIntID, Ptr }; SDValue ExtraLoad = DAG.getMemIntrinsicNode(ISD::INTRINSIC_W_CHAIN, dl, DAG.getVTList(PermTy, MVT::Other), ExtraLoadOps, LDTy, ExtraMMO); SDValue TF = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, BaseLoad.getValue(1), ExtraLoad.getValue(1)); // Because vperm has a big-endian bias, we must reverse the order // of the input vectors and complement the permute control vector // when generating little endian code. We have already handled the // latter by using lvsr instead of lvsl, so just reverse BaseLoad // and ExtraLoad here. SDValue Perm; if (isLittleEndian) Perm = BuildIntrinsicOp(IntrPerm, ExtraLoad, BaseLoad, PermCntl, DAG, dl); else Perm = BuildIntrinsicOp(IntrPerm, BaseLoad, ExtraLoad, PermCntl, DAG, dl); if (VT != PermTy) Perm = Subtarget.hasAltivec() ? DAG.getNode(ISD::BITCAST, dl, VT, Perm) : DAG.getNode(ISD::FP_ROUND, dl, VT, Perm, // QPX DAG.getTargetConstant(1, dl, MVT::i64)); // second argument is 1 because this rounding // is always exact. // The output of the permutation is our loaded result, the TokenFactor is // our new chain. DCI.CombineTo(N, Perm, TF); return SDValue(N, 0); } } break; case ISD::INTRINSIC_WO_CHAIN: { bool isLittleEndian = Subtarget.isLittleEndian(); unsigned IID = cast(N->getOperand(0))->getZExtValue(); Intrinsic::ID Intr = (isLittleEndian ? Intrinsic::ppc_altivec_lvsr : Intrinsic::ppc_altivec_lvsl); if ((IID == Intr || IID == Intrinsic::ppc_qpx_qvlpcld || IID == Intrinsic::ppc_qpx_qvlpcls) && N->getOperand(1)->getOpcode() == ISD::ADD) { SDValue Add = N->getOperand(1); int Bits = IID == Intrinsic::ppc_qpx_qvlpcld ? 5 /* 32 byte alignment */ : 4 /* 16 byte alignment */; if (DAG.MaskedValueIsZero(Add->getOperand(1), APInt::getAllOnesValue(Bits /* alignment */) .zext(Add.getScalarValueSizeInBits()))) { SDNode *BasePtr = Add->getOperand(0).getNode(); for (SDNode::use_iterator UI = BasePtr->use_begin(), UE = BasePtr->use_end(); UI != UE; ++UI) { if (UI->getOpcode() == ISD::INTRINSIC_WO_CHAIN && cast(UI->getOperand(0))->getZExtValue() == IID) { // We've found another LVSL/LVSR, and this address is an aligned // multiple of that one. The results will be the same, so use the // one we've just found instead. return SDValue(*UI, 0); } } } if (isa(Add->getOperand(1))) { SDNode *BasePtr = Add->getOperand(0).getNode(); for (SDNode::use_iterator UI = BasePtr->use_begin(), UE = BasePtr->use_end(); UI != UE; ++UI) { if (UI->getOpcode() == ISD::ADD && isa(UI->getOperand(1)) && (cast(Add->getOperand(1))->getZExtValue() - cast(UI->getOperand(1))->getZExtValue()) % (1ULL << Bits) == 0) { SDNode *OtherAdd = *UI; for (SDNode::use_iterator VI = OtherAdd->use_begin(), VE = OtherAdd->use_end(); VI != VE; ++VI) { if (VI->getOpcode() == ISD::INTRINSIC_WO_CHAIN && cast(VI->getOperand(0))->getZExtValue() == IID) { return SDValue(*VI, 0); } } } } } } // Combine vmaxsw/h/b(a, a's negation) to abs(a) // Expose the vabsduw/h/b opportunity for down stream if (!DCI.isAfterLegalizeDAG() && Subtarget.hasP9Altivec() && (IID == Intrinsic::ppc_altivec_vmaxsw || IID == Intrinsic::ppc_altivec_vmaxsh || IID == Intrinsic::ppc_altivec_vmaxsb)) { SDValue V1 = N->getOperand(1); SDValue V2 = N->getOperand(2); if ((V1.getSimpleValueType() == MVT::v4i32 || V1.getSimpleValueType() == MVT::v8i16 || V1.getSimpleValueType() == MVT::v16i8) && V1.getSimpleValueType() == V2.getSimpleValueType()) { // (0-a, a) if (V1.getOpcode() == ISD::SUB && ISD::isBuildVectorAllZeros(V1.getOperand(0).getNode()) && V1.getOperand(1) == V2) { return DAG.getNode(ISD::ABS, dl, V2.getValueType(), V2); } // (a, 0-a) if (V2.getOpcode() == ISD::SUB && ISD::isBuildVectorAllZeros(V2.getOperand(0).getNode()) && V2.getOperand(1) == V1) { return DAG.getNode(ISD::ABS, dl, V1.getValueType(), V1); } // (x-y, y-x) if (V1.getOpcode() == ISD::SUB && V2.getOpcode() == ISD::SUB && V1.getOperand(0) == V2.getOperand(1) && V1.getOperand(1) == V2.getOperand(0)) { return DAG.getNode(ISD::ABS, dl, V1.getValueType(), V1); } } } } break; case ISD::INTRINSIC_W_CHAIN: // For little endian, VSX loads require generating lxvd2x/xxswapd. // Not needed on ISA 3.0 based CPUs since we have a non-permuting load. if (Subtarget.needsSwapsForVSXMemOps()) { switch (cast(N->getOperand(1))->getZExtValue()) { default: break; case Intrinsic::ppc_vsx_lxvw4x: case Intrinsic::ppc_vsx_lxvd2x: return expandVSXLoadForLE(N, DCI); } } break; case ISD::INTRINSIC_VOID: // For little endian, VSX stores require generating xxswapd/stxvd2x. // Not needed on ISA 3.0 based CPUs since we have a non-permuting store. if (Subtarget.needsSwapsForVSXMemOps()) { switch (cast(N->getOperand(1))->getZExtValue()) { default: break; case Intrinsic::ppc_vsx_stxvw4x: case Intrinsic::ppc_vsx_stxvd2x: return expandVSXStoreForLE(N, DCI); } } break; case ISD::BSWAP: // Turn BSWAP (LOAD) -> lhbrx/lwbrx. if (ISD::isNON_EXTLoad(N->getOperand(0).getNode()) && N->getOperand(0).hasOneUse() && (N->getValueType(0) == MVT::i32 || N->getValueType(0) == MVT::i16 || (Subtarget.hasLDBRX() && Subtarget.isPPC64() && N->getValueType(0) == MVT::i64))) { SDValue Load = N->getOperand(0); LoadSDNode *LD = cast(Load); // Create the byte-swapping load. SDValue Ops[] = { LD->getChain(), // Chain LD->getBasePtr(), // Ptr DAG.getValueType(N->getValueType(0)) // VT }; SDValue BSLoad = DAG.getMemIntrinsicNode(PPCISD::LBRX, dl, DAG.getVTList(N->getValueType(0) == MVT::i64 ? MVT::i64 : MVT::i32, MVT::Other), Ops, LD->getMemoryVT(), LD->getMemOperand()); // If this is an i16 load, insert the truncate. SDValue ResVal = BSLoad; if (N->getValueType(0) == MVT::i16) ResVal = DAG.getNode(ISD::TRUNCATE, dl, MVT::i16, BSLoad); // First, combine the bswap away. This makes the value produced by the // load dead. DCI.CombineTo(N, ResVal); // Next, combine the load away, we give it a bogus result value but a real // chain result. The result value is dead because the bswap is dead. DCI.CombineTo(Load.getNode(), ResVal, BSLoad.getValue(1)); // Return N so it doesn't get rechecked! return SDValue(N, 0); } break; case PPCISD::VCMP: // If a VCMPo node already exists with exactly the same operands as this // node, use its result instead of this node (VCMPo computes both a CR6 and // a normal output). // if (!N->getOperand(0).hasOneUse() && !N->getOperand(1).hasOneUse() && !N->getOperand(2).hasOneUse()) { // Scan all of the users of the LHS, looking for VCMPo's that match. SDNode *VCMPoNode = nullptr; SDNode *LHSN = N->getOperand(0).getNode(); for (SDNode::use_iterator UI = LHSN->use_begin(), E = LHSN->use_end(); UI != E; ++UI) if (UI->getOpcode() == PPCISD::VCMPo && UI->getOperand(1) == N->getOperand(1) && UI->getOperand(2) == N->getOperand(2) && UI->getOperand(0) == N->getOperand(0)) { VCMPoNode = *UI; break; } // If there is no VCMPo node, or if the flag value has a single use, don't // transform this. if (!VCMPoNode || VCMPoNode->hasNUsesOfValue(0, 1)) break; // Look at the (necessarily single) use of the flag value. If it has a // chain, this transformation is more complex. Note that multiple things // could use the value result, which we should ignore. SDNode *FlagUser = nullptr; for (SDNode::use_iterator UI = VCMPoNode->use_begin(); FlagUser == nullptr; ++UI) { assert(UI != VCMPoNode->use_end() && "Didn't find user!"); SDNode *User = *UI; for (unsigned i = 0, e = User->getNumOperands(); i != e; ++i) { if (User->getOperand(i) == SDValue(VCMPoNode, 1)) { FlagUser = User; break; } } } // If the user is a MFOCRF instruction, we know this is safe. // Otherwise we give up for right now. if (FlagUser->getOpcode() == PPCISD::MFOCRF) return SDValue(VCMPoNode, 0); } break; case ISD::BRCOND: { SDValue Cond = N->getOperand(1); SDValue Target = N->getOperand(2); if (Cond.getOpcode() == ISD::INTRINSIC_W_CHAIN && cast(Cond.getOperand(1))->getZExtValue() == Intrinsic::loop_decrement) { // We now need to make the intrinsic dead (it cannot be instruction // selected). DAG.ReplaceAllUsesOfValueWith(Cond.getValue(1), Cond.getOperand(0)); assert(Cond.getNode()->hasOneUse() && "Counter decrement has more than one use"); return DAG.getNode(PPCISD::BDNZ, dl, MVT::Other, N->getOperand(0), Target); } } break; case ISD::BR_CC: { // If this is a branch on an altivec predicate comparison, lower this so // that we don't have to do a MFOCRF: instead, branch directly on CR6. This // lowering is done pre-legalize, because the legalizer lowers the predicate // compare down to code that is difficult to reassemble. ISD::CondCode CC = cast(N->getOperand(1))->get(); SDValue LHS = N->getOperand(2), RHS = N->getOperand(3); // Sometimes the promoted value of the intrinsic is ANDed by some non-zero // value. If so, pass-through the AND to get to the intrinsic. if (LHS.getOpcode() == ISD::AND && LHS.getOperand(0).getOpcode() == ISD::INTRINSIC_W_CHAIN && cast(LHS.getOperand(0).getOperand(1))->getZExtValue() == Intrinsic::loop_decrement && isa(LHS.getOperand(1)) && !isNullConstant(LHS.getOperand(1))) LHS = LHS.getOperand(0); if (LHS.getOpcode() == ISD::INTRINSIC_W_CHAIN && cast(LHS.getOperand(1))->getZExtValue() == Intrinsic::loop_decrement && isa(RHS)) { assert((CC == ISD::SETEQ || CC == ISD::SETNE) && "Counter decrement comparison is not EQ or NE"); unsigned Val = cast(RHS)->getZExtValue(); bool isBDNZ = (CC == ISD::SETEQ && Val) || (CC == ISD::SETNE && !Val); // We now need to make the intrinsic dead (it cannot be instruction // selected). DAG.ReplaceAllUsesOfValueWith(LHS.getValue(1), LHS.getOperand(0)); assert(LHS.getNode()->hasOneUse() && "Counter decrement has more than one use"); return DAG.getNode(isBDNZ ? PPCISD::BDNZ : PPCISD::BDZ, dl, MVT::Other, N->getOperand(0), N->getOperand(4)); } int CompareOpc; bool isDot; if (LHS.getOpcode() == ISD::INTRINSIC_WO_CHAIN && isa(RHS) && (CC == ISD::SETEQ || CC == ISD::SETNE) && getVectorCompareInfo(LHS, CompareOpc, isDot, Subtarget)) { assert(isDot && "Can't compare against a vector result!"); // If this is a comparison against something other than 0/1, then we know // that the condition is never/always true. unsigned Val = cast(RHS)->getZExtValue(); if (Val != 0 && Val != 1) { if (CC == ISD::SETEQ) // Cond never true, remove branch. return N->getOperand(0); // Always !=, turn it into an unconditional branch. return DAG.getNode(ISD::BR, dl, MVT::Other, N->getOperand(0), N->getOperand(4)); } bool BranchOnWhenPredTrue = (CC == ISD::SETEQ) ^ (Val == 0); // Create the PPCISD altivec 'dot' comparison node. SDValue Ops[] = { LHS.getOperand(2), // LHS of compare LHS.getOperand(3), // RHS of compare DAG.getConstant(CompareOpc, dl, MVT::i32) }; EVT VTs[] = { LHS.getOperand(2).getValueType(), MVT::Glue }; SDValue CompNode = DAG.getNode(PPCISD::VCMPo, dl, VTs, Ops); // Unpack the result based on how the target uses it. PPC::Predicate CompOpc; switch (cast(LHS.getOperand(1))->getZExtValue()) { default: // Can't happen, don't crash on invalid number though. case 0: // Branch on the value of the EQ bit of CR6. CompOpc = BranchOnWhenPredTrue ? PPC::PRED_EQ : PPC::PRED_NE; break; case 1: // Branch on the inverted value of the EQ bit of CR6. CompOpc = BranchOnWhenPredTrue ? PPC::PRED_NE : PPC::PRED_EQ; break; case 2: // Branch on the value of the LT bit of CR6. CompOpc = BranchOnWhenPredTrue ? PPC::PRED_LT : PPC::PRED_GE; break; case 3: // Branch on the inverted value of the LT bit of CR6. CompOpc = BranchOnWhenPredTrue ? PPC::PRED_GE : PPC::PRED_LT; break; } return DAG.getNode(PPCISD::COND_BRANCH, dl, MVT::Other, N->getOperand(0), DAG.getConstant(CompOpc, dl, MVT::i32), DAG.getRegister(PPC::CR6, MVT::i32), N->getOperand(4), CompNode.getValue(1)); } break; } case ISD::BUILD_VECTOR: return DAGCombineBuildVector(N, DCI); case ISD::ABS: return combineABS(N, DCI); case ISD::VSELECT: return combineVSelect(N, DCI); } return SDValue(); } SDValue PPCTargetLowering::BuildSDIVPow2(SDNode *N, const APInt &Divisor, SelectionDAG &DAG, SmallVectorImpl &Created) const { // fold (sdiv X, pow2) EVT VT = N->getValueType(0); if (VT == MVT::i64 && !Subtarget.isPPC64()) return SDValue(); if ((VT != MVT::i32 && VT != MVT::i64) || !(Divisor.isPowerOf2() || (-Divisor).isPowerOf2())) return SDValue(); SDLoc DL(N); SDValue N0 = N->getOperand(0); bool IsNegPow2 = (-Divisor).isPowerOf2(); unsigned Lg2 = (IsNegPow2 ? -Divisor : Divisor).countTrailingZeros(); SDValue ShiftAmt = DAG.getConstant(Lg2, DL, VT); SDValue Op = DAG.getNode(PPCISD::SRA_ADDZE, DL, VT, N0, ShiftAmt); Created.push_back(Op.getNode()); if (IsNegPow2) { Op = DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT), Op); Created.push_back(Op.getNode()); } return Op; } //===----------------------------------------------------------------------===// // Inline Assembly Support //===----------------------------------------------------------------------===// void PPCTargetLowering::computeKnownBitsForTargetNode(const SDValue Op, KnownBits &Known, const APInt &DemandedElts, const SelectionDAG &DAG, unsigned Depth) const { Known.resetAll(); switch (Op.getOpcode()) { default: break; case PPCISD::LBRX: { // lhbrx is known to have the top bits cleared out. if (cast(Op.getOperand(2))->getVT() == MVT::i16) Known.Zero = 0xFFFF0000; break; } case ISD::INTRINSIC_WO_CHAIN: { switch (cast(Op.getOperand(0))->getZExtValue()) { default: break; case Intrinsic::ppc_altivec_vcmpbfp_p: case Intrinsic::ppc_altivec_vcmpeqfp_p: case Intrinsic::ppc_altivec_vcmpequb_p: case Intrinsic::ppc_altivec_vcmpequh_p: case Intrinsic::ppc_altivec_vcmpequw_p: case Intrinsic::ppc_altivec_vcmpequd_p: case Intrinsic::ppc_altivec_vcmpgefp_p: case Intrinsic::ppc_altivec_vcmpgtfp_p: case Intrinsic::ppc_altivec_vcmpgtsb_p: case Intrinsic::ppc_altivec_vcmpgtsh_p: case Intrinsic::ppc_altivec_vcmpgtsw_p: case Intrinsic::ppc_altivec_vcmpgtsd_p: case Intrinsic::ppc_altivec_vcmpgtub_p: case Intrinsic::ppc_altivec_vcmpgtuh_p: case Intrinsic::ppc_altivec_vcmpgtuw_p: case Intrinsic::ppc_altivec_vcmpgtud_p: Known.Zero = ~1U; // All bits but the low one are known to be zero. break; } } } } Align PPCTargetLowering::getPrefLoopAlignment(MachineLoop *ML) const { switch (Subtarget.getCPUDirective()) { default: break; case PPC::DIR_970: case PPC::DIR_PWR4: case PPC::DIR_PWR5: case PPC::DIR_PWR5X: case PPC::DIR_PWR6: case PPC::DIR_PWR6X: case PPC::DIR_PWR7: case PPC::DIR_PWR8: case PPC::DIR_PWR9: case PPC::DIR_PWR_FUTURE: { if (!ML) break; if (!DisableInnermostLoopAlign32) { // If the nested loop is an innermost loop, prefer to a 32-byte alignment, // so that we can decrease cache misses and branch-prediction misses. // Actual alignment of the loop will depend on the hotness check and other // logic in alignBlocks. if (ML->getLoopDepth() > 1 && ML->getSubLoops().empty()) return Align(32); } const PPCInstrInfo *TII = Subtarget.getInstrInfo(); // For small loops (between 5 and 8 instructions), align to a 32-byte // boundary so that the entire loop fits in one instruction-cache line. uint64_t LoopSize = 0; for (auto I = ML->block_begin(), IE = ML->block_end(); I != IE; ++I) for (auto J = (*I)->begin(), JE = (*I)->end(); J != JE; ++J) { LoopSize += TII->getInstSizeInBytes(*J); if (LoopSize > 32) break; } if (LoopSize > 16 && LoopSize <= 32) return Align(32); break; } } return TargetLowering::getPrefLoopAlignment(ML); } /// getConstraintType - Given a constraint, return the type of /// constraint it is for this target. PPCTargetLowering::ConstraintType PPCTargetLowering::getConstraintType(StringRef Constraint) const { if (Constraint.size() == 1) { switch (Constraint[0]) { default: break; case 'b': case 'r': case 'f': case 'd': case 'v': case 'y': return C_RegisterClass; case 'Z': // FIXME: While Z does indicate a memory constraint, it specifically // indicates an r+r address (used in conjunction with the 'y' modifier // in the replacement string). Currently, we're forcing the base // register to be r0 in the asm printer (which is interpreted as zero) // and forming the complete address in the second register. This is // suboptimal. return C_Memory; } } else if (Constraint == "wc") { // individual CR bits. return C_RegisterClass; } else if (Constraint == "wa" || Constraint == "wd" || Constraint == "wf" || Constraint == "ws" || Constraint == "wi" || Constraint == "ww") { return C_RegisterClass; // VSX registers. } return TargetLowering::getConstraintType(Constraint); } /// Examine constraint type and operand type and determine a weight value. /// This object must already have been set up with the operand type /// and the current alternative constraint selected. TargetLowering::ConstraintWeight PPCTargetLowering::getSingleConstraintMatchWeight( AsmOperandInfo &info, const char *constraint) const { ConstraintWeight weight = CW_Invalid; Value *CallOperandVal = info.CallOperandVal; // If we don't have a value, we can't do a match, // but allow it at the lowest weight. if (!CallOperandVal) return CW_Default; Type *type = CallOperandVal->getType(); // Look at the constraint type. if (StringRef(constraint) == "wc" && type->isIntegerTy(1)) return CW_Register; // an individual CR bit. else if ((StringRef(constraint) == "wa" || StringRef(constraint) == "wd" || StringRef(constraint) == "wf") && type->isVectorTy()) return CW_Register; else if (StringRef(constraint) == "wi" && type->isIntegerTy(64)) return CW_Register; // just hold 64-bit integers data. else if (StringRef(constraint) == "ws" && type->isDoubleTy()) return CW_Register; else if (StringRef(constraint) == "ww" && type->isFloatTy()) return CW_Register; switch (*constraint) { default: weight = TargetLowering::getSingleConstraintMatchWeight(info, constraint); break; case 'b': if (type->isIntegerTy()) weight = CW_Register; break; case 'f': if (type->isFloatTy()) weight = CW_Register; break; case 'd': if (type->isDoubleTy()) weight = CW_Register; break; case 'v': if (type->isVectorTy()) weight = CW_Register; break; case 'y': weight = CW_Register; break; case 'Z': weight = CW_Memory; break; } return weight; } std::pair PPCTargetLowering::getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI, StringRef Constraint, MVT VT) const { if (Constraint.size() == 1) { // GCC RS6000 Constraint Letters switch (Constraint[0]) { case 'b': // R1-R31 if (VT == MVT::i64 && Subtarget.isPPC64()) return std::make_pair(0U, &PPC::G8RC_NOX0RegClass); return std::make_pair(0U, &PPC::GPRC_NOR0RegClass); case 'r': // R0-R31 if (VT == MVT::i64 && Subtarget.isPPC64()) return std::make_pair(0U, &PPC::G8RCRegClass); return std::make_pair(0U, &PPC::GPRCRegClass); // 'd' and 'f' constraints are both defined to be "the floating point // registers", where one is for 32-bit and the other for 64-bit. We don't // really care overly much here so just give them all the same reg classes. case 'd': case 'f': if (Subtarget.hasSPE()) { if (VT == MVT::f32 || VT == MVT::i32) return std::make_pair(0U, &PPC::GPRCRegClass); if (VT == MVT::f64 || VT == MVT::i64) return std::make_pair(0U, &PPC::SPERCRegClass); } else { if (VT == MVT::f32 || VT == MVT::i32) return std::make_pair(0U, &PPC::F4RCRegClass); if (VT == MVT::f64 || VT == MVT::i64) return std::make_pair(0U, &PPC::F8RCRegClass); if (VT == MVT::v4f64 && Subtarget.hasQPX()) return std::make_pair(0U, &PPC::QFRCRegClass); if (VT == MVT::v4f32 && Subtarget.hasQPX()) return std::make_pair(0U, &PPC::QSRCRegClass); } break; case 'v': if (VT == MVT::v4f64 && Subtarget.hasQPX()) return std::make_pair(0U, &PPC::QFRCRegClass); if (VT == MVT::v4f32 && Subtarget.hasQPX()) return std::make_pair(0U, &PPC::QSRCRegClass); if (Subtarget.hasAltivec()) return std::make_pair(0U, &PPC::VRRCRegClass); break; case 'y': // crrc return std::make_pair(0U, &PPC::CRRCRegClass); } } else if (Constraint == "wc" && Subtarget.useCRBits()) { // An individual CR bit. return std::make_pair(0U, &PPC::CRBITRCRegClass); } else if ((Constraint == "wa" || Constraint == "wd" || Constraint == "wf" || Constraint == "wi") && Subtarget.hasVSX()) { return std::make_pair(0U, &PPC::VSRCRegClass); } else if ((Constraint == "ws" || Constraint == "ww") && Subtarget.hasVSX()) { if (VT == MVT::f32 && Subtarget.hasP8Vector()) return std::make_pair(0U, &PPC::VSSRCRegClass); else return std::make_pair(0U, &PPC::VSFRCRegClass); } // If we name a VSX register, we can't defer to the base class because it // will not recognize the correct register (their names will be VSL{0-31} // and V{0-31} so they won't match). So we match them here. if (Constraint.size() > 3 && Constraint[1] == 'v' && Constraint[2] == 's') { int VSNum = atoi(Constraint.data() + 3); assert(VSNum >= 0 && VSNum <= 63 && "Attempted to access a vsr out of range"); if (VSNum < 32) return std::make_pair(PPC::VSL0 + VSNum, &PPC::VSRCRegClass); return std::make_pair(PPC::V0 + VSNum - 32, &PPC::VSRCRegClass); } std::pair R = TargetLowering::getRegForInlineAsmConstraint(TRI, Constraint, VT); // r[0-9]+ are used, on PPC64, to refer to the corresponding 64-bit registers // (which we call X[0-9]+). If a 64-bit value has been requested, and a // 32-bit GPR has been selected, then 'upgrade' it to the 64-bit parent // register. // FIXME: If TargetLowering::getRegForInlineAsmConstraint could somehow use // the AsmName field from *RegisterInfo.td, then this would not be necessary. if (R.first && VT == MVT::i64 && Subtarget.isPPC64() && PPC::GPRCRegClass.contains(R.first)) return std::make_pair(TRI->getMatchingSuperReg(R.first, PPC::sub_32, &PPC::G8RCRegClass), &PPC::G8RCRegClass); // GCC accepts 'cc' as an alias for 'cr0', and we need to do the same. if (!R.second && StringRef("{cc}").equals_lower(Constraint)) { R.first = PPC::CR0; R.second = &PPC::CRRCRegClass; } return R; } /// LowerAsmOperandForConstraint - Lower the specified operand into the Ops /// vector. If it is invalid, don't add anything to Ops. void PPCTargetLowering::LowerAsmOperandForConstraint(SDValue Op, std::string &Constraint, std::vector&Ops, SelectionDAG &DAG) const { SDValue Result; // Only support length 1 constraints. if (Constraint.length() > 1) return; char Letter = Constraint[0]; switch (Letter) { default: break; case 'I': case 'J': case 'K': case 'L': case 'M': case 'N': case 'O': case 'P': { ConstantSDNode *CST = dyn_cast(Op); if (!CST) return; // Must be an immediate to match. SDLoc dl(Op); int64_t Value = CST->getSExtValue(); EVT TCVT = MVT::i64; // All constants taken to be 64 bits so that negative // numbers are printed as such. switch (Letter) { default: llvm_unreachable("Unknown constraint letter!"); case 'I': // "I" is a signed 16-bit constant. if (isInt<16>(Value)) Result = DAG.getTargetConstant(Value, dl, TCVT); break; case 'J': // "J" is a constant with only the high-order 16 bits nonzero. if (isShiftedUInt<16, 16>(Value)) Result = DAG.getTargetConstant(Value, dl, TCVT); break; case 'L': // "L" is a signed 16-bit constant shifted left 16 bits. if (isShiftedInt<16, 16>(Value)) Result = DAG.getTargetConstant(Value, dl, TCVT); break; case 'K': // "K" is a constant with only the low-order 16 bits nonzero. if (isUInt<16>(Value)) Result = DAG.getTargetConstant(Value, dl, TCVT); break; case 'M': // "M" is a constant that is greater than 31. if (Value > 31) Result = DAG.getTargetConstant(Value, dl, TCVT); break; case 'N': // "N" is a positive constant that is an exact power of two. if (Value > 0 && isPowerOf2_64(Value)) Result = DAG.getTargetConstant(Value, dl, TCVT); break; case 'O': // "O" is the constant zero. if (Value == 0) Result = DAG.getTargetConstant(Value, dl, TCVT); break; case 'P': // "P" is a constant whose negation is a signed 16-bit constant. if (isInt<16>(-Value)) Result = DAG.getTargetConstant(Value, dl, TCVT); break; } break; } } if (Result.getNode()) { Ops.push_back(Result); return; } // Handle standard constraint letters. TargetLowering::LowerAsmOperandForConstraint(Op, Constraint, Ops, DAG); } // isLegalAddressingMode - Return true if the addressing mode represented // by AM is legal for this target, for a load/store of the specified type. bool PPCTargetLowering::isLegalAddressingMode(const DataLayout &DL, const AddrMode &AM, Type *Ty, unsigned AS, Instruction *I) const { // PPC does not allow r+i addressing modes for vectors! if (Ty->isVectorTy() && AM.BaseOffs != 0) return false; // PPC allows a sign-extended 16-bit immediate field. if (AM.BaseOffs <= -(1LL << 16) || AM.BaseOffs >= (1LL << 16)-1) return false; // No global is ever allowed as a base. if (AM.BaseGV) return false; // PPC only support r+r, switch (AM.Scale) { case 0: // "r+i" or just "i", depending on HasBaseReg. break; case 1: if (AM.HasBaseReg && AM.BaseOffs) // "r+r+i" is not allowed. return false; // Otherwise we have r+r or r+i. break; case 2: if (AM.HasBaseReg || AM.BaseOffs) // 2*r+r or 2*r+i is not allowed. return false; // Allow 2*r as r+r. break; default: // No other scales are supported. return false; } return true; } SDValue PPCTargetLowering::LowerRETURNADDR(SDValue Op, SelectionDAG &DAG) const { MachineFunction &MF = DAG.getMachineFunction(); MachineFrameInfo &MFI = MF.getFrameInfo(); MFI.setReturnAddressIsTaken(true); if (verifyReturnAddressArgumentIsConstant(Op, DAG)) return SDValue(); SDLoc dl(Op); unsigned Depth = cast(Op.getOperand(0))->getZExtValue(); // Make sure the function does not optimize away the store of the RA to // the stack. PPCFunctionInfo *FuncInfo = MF.getInfo(); FuncInfo->setLRStoreRequired(); bool isPPC64 = Subtarget.isPPC64(); auto PtrVT = getPointerTy(MF.getDataLayout()); if (Depth > 0) { SDValue FrameAddr = LowerFRAMEADDR(Op, DAG); SDValue Offset = DAG.getConstant(Subtarget.getFrameLowering()->getReturnSaveOffset(), dl, isPPC64 ? MVT::i64 : MVT::i32); return DAG.getLoad(PtrVT, dl, DAG.getEntryNode(), DAG.getNode(ISD::ADD, dl, PtrVT, FrameAddr, Offset), MachinePointerInfo()); } // Just load the return address off the stack. SDValue RetAddrFI = getReturnAddrFrameIndex(DAG); return DAG.getLoad(PtrVT, dl, DAG.getEntryNode(), RetAddrFI, MachinePointerInfo()); } SDValue PPCTargetLowering::LowerFRAMEADDR(SDValue Op, SelectionDAG &DAG) const { SDLoc dl(Op); unsigned Depth = cast(Op.getOperand(0))->getZExtValue(); MachineFunction &MF = DAG.getMachineFunction(); MachineFrameInfo &MFI = MF.getFrameInfo(); MFI.setFrameAddressIsTaken(true); EVT PtrVT = getPointerTy(MF.getDataLayout()); bool isPPC64 = PtrVT == MVT::i64; // Naked functions never have a frame pointer, and so we use r1. For all // other functions, this decision must be delayed until during PEI. unsigned FrameReg; if (MF.getFunction().hasFnAttribute(Attribute::Naked)) FrameReg = isPPC64 ? PPC::X1 : PPC::R1; else FrameReg = isPPC64 ? PPC::FP8 : PPC::FP; SDValue FrameAddr = DAG.getCopyFromReg(DAG.getEntryNode(), dl, FrameReg, PtrVT); while (Depth--) FrameAddr = DAG.getLoad(Op.getValueType(), dl, DAG.getEntryNode(), FrameAddr, MachinePointerInfo()); return FrameAddr; } // FIXME? Maybe this could be a TableGen attribute on some registers and // this table could be generated automatically from RegInfo. Register PPCTargetLowering::getRegisterByName(const char* RegName, LLT VT, const MachineFunction &MF) const { bool isPPC64 = Subtarget.isPPC64(); bool is64Bit = isPPC64 && VT == LLT::scalar(64); if (!is64Bit && VT != LLT::scalar(32)) report_fatal_error("Invalid register global variable type"); Register Reg = StringSwitch(RegName) .Case("r1", is64Bit ? PPC::X1 : PPC::R1) .Case("r2", isPPC64 ? Register() : PPC::R2) .Case("r13", (is64Bit ? PPC::X13 : PPC::R13)) .Default(Register()); if (Reg) return Reg; report_fatal_error("Invalid register name global variable"); } bool PPCTargetLowering::isAccessedAsGotIndirect(SDValue GA) const { // 32-bit SVR4 ABI access everything as got-indirect. if (Subtarget.is32BitELFABI()) return true; // AIX accesses everything indirectly through the TOC, which is similar to // the GOT. if (Subtarget.isAIXABI()) return true; CodeModel::Model CModel = getTargetMachine().getCodeModel(); // If it is small or large code model, module locals are accessed // indirectly by loading their address from .toc/.got. if (CModel == CodeModel::Small || CModel == CodeModel::Large) return true; // JumpTable and BlockAddress are accessed as got-indirect. if (isa(GA) || isa(GA)) return true; if (GlobalAddressSDNode *G = dyn_cast(GA)) return Subtarget.isGVIndirectSymbol(G->getGlobal()); return false; } bool PPCTargetLowering::isOffsetFoldingLegal(const GlobalAddressSDNode *GA) const { // The PowerPC target isn't yet aware of offsets. return false; } bool PPCTargetLowering::getTgtMemIntrinsic(IntrinsicInfo &Info, const CallInst &I, MachineFunction &MF, unsigned Intrinsic) const { switch (Intrinsic) { case Intrinsic::ppc_qpx_qvlfd: case Intrinsic::ppc_qpx_qvlfs: case Intrinsic::ppc_qpx_qvlfcd: case Intrinsic::ppc_qpx_qvlfcs: case Intrinsic::ppc_qpx_qvlfiwa: case Intrinsic::ppc_qpx_qvlfiwz: case Intrinsic::ppc_altivec_lvx: case Intrinsic::ppc_altivec_lvxl: case Intrinsic::ppc_altivec_lvebx: case Intrinsic::ppc_altivec_lvehx: case Intrinsic::ppc_altivec_lvewx: case Intrinsic::ppc_vsx_lxvd2x: case Intrinsic::ppc_vsx_lxvw4x: { EVT VT; switch (Intrinsic) { case Intrinsic::ppc_altivec_lvebx: VT = MVT::i8; break; case Intrinsic::ppc_altivec_lvehx: VT = MVT::i16; break; case Intrinsic::ppc_altivec_lvewx: VT = MVT::i32; break; case Intrinsic::ppc_vsx_lxvd2x: VT = MVT::v2f64; break; case Intrinsic::ppc_qpx_qvlfd: VT = MVT::v4f64; break; case Intrinsic::ppc_qpx_qvlfs: VT = MVT::v4f32; break; case Intrinsic::ppc_qpx_qvlfcd: VT = MVT::v2f64; break; case Intrinsic::ppc_qpx_qvlfcs: VT = MVT::v2f32; break; default: VT = MVT::v4i32; break; } Info.opc = ISD::INTRINSIC_W_CHAIN; Info.memVT = VT; Info.ptrVal = I.getArgOperand(0); Info.offset = -VT.getStoreSize()+1; Info.size = 2*VT.getStoreSize()-1; Info.align = Align(1); Info.flags = MachineMemOperand::MOLoad; return true; } case Intrinsic::ppc_qpx_qvlfda: case Intrinsic::ppc_qpx_qvlfsa: case Intrinsic::ppc_qpx_qvlfcda: case Intrinsic::ppc_qpx_qvlfcsa: case Intrinsic::ppc_qpx_qvlfiwaa: case Intrinsic::ppc_qpx_qvlfiwza: { EVT VT; switch (Intrinsic) { case Intrinsic::ppc_qpx_qvlfda: VT = MVT::v4f64; break; case Intrinsic::ppc_qpx_qvlfsa: VT = MVT::v4f32; break; case Intrinsic::ppc_qpx_qvlfcda: VT = MVT::v2f64; break; case Intrinsic::ppc_qpx_qvlfcsa: VT = MVT::v2f32; break; default: VT = MVT::v4i32; break; } Info.opc = ISD::INTRINSIC_W_CHAIN; Info.memVT = VT; Info.ptrVal = I.getArgOperand(0); Info.offset = 0; Info.size = VT.getStoreSize(); Info.align = Align(1); Info.flags = MachineMemOperand::MOLoad; return true; } case Intrinsic::ppc_qpx_qvstfd: case Intrinsic::ppc_qpx_qvstfs: case Intrinsic::ppc_qpx_qvstfcd: case Intrinsic::ppc_qpx_qvstfcs: case Intrinsic::ppc_qpx_qvstfiw: case Intrinsic::ppc_altivec_stvx: case Intrinsic::ppc_altivec_stvxl: case Intrinsic::ppc_altivec_stvebx: case Intrinsic::ppc_altivec_stvehx: case Intrinsic::ppc_altivec_stvewx: case Intrinsic::ppc_vsx_stxvd2x: case Intrinsic::ppc_vsx_stxvw4x: { EVT VT; switch (Intrinsic) { case Intrinsic::ppc_altivec_stvebx: VT = MVT::i8; break; case Intrinsic::ppc_altivec_stvehx: VT = MVT::i16; break; case Intrinsic::ppc_altivec_stvewx: VT = MVT::i32; break; case Intrinsic::ppc_vsx_stxvd2x: VT = MVT::v2f64; break; case Intrinsic::ppc_qpx_qvstfd: VT = MVT::v4f64; break; case Intrinsic::ppc_qpx_qvstfs: VT = MVT::v4f32; break; case Intrinsic::ppc_qpx_qvstfcd: VT = MVT::v2f64; break; case Intrinsic::ppc_qpx_qvstfcs: VT = MVT::v2f32; break; default: VT = MVT::v4i32; break; } Info.opc = ISD::INTRINSIC_VOID; Info.memVT = VT; Info.ptrVal = I.getArgOperand(1); Info.offset = -VT.getStoreSize()+1; Info.size = 2*VT.getStoreSize()-1; Info.align = Align(1); Info.flags = MachineMemOperand::MOStore; return true; } case Intrinsic::ppc_qpx_qvstfda: case Intrinsic::ppc_qpx_qvstfsa: case Intrinsic::ppc_qpx_qvstfcda: case Intrinsic::ppc_qpx_qvstfcsa: case Intrinsic::ppc_qpx_qvstfiwa: { EVT VT; switch (Intrinsic) { case Intrinsic::ppc_qpx_qvstfda: VT = MVT::v4f64; break; case Intrinsic::ppc_qpx_qvstfsa: VT = MVT::v4f32; break; case Intrinsic::ppc_qpx_qvstfcda: VT = MVT::v2f64; break; case Intrinsic::ppc_qpx_qvstfcsa: VT = MVT::v2f32; break; default: VT = MVT::v4i32; break; } Info.opc = ISD::INTRINSIC_VOID; Info.memVT = VT; Info.ptrVal = I.getArgOperand(1); Info.offset = 0; Info.size = VT.getStoreSize(); Info.align = Align(1); Info.flags = MachineMemOperand::MOStore; return true; } default: break; } return false; } /// It returns EVT::Other if the type should be determined using generic /// target-independent logic. EVT PPCTargetLowering::getOptimalMemOpType( const MemOp &Op, const AttributeList &FuncAttributes) const { if (getTargetMachine().getOptLevel() != CodeGenOpt::None) { // When expanding a memset, require at least two QPX instructions to cover // the cost of loading the value to be stored from the constant pool. if (Subtarget.hasQPX() && Op.size() >= 32 && (Op.isMemcpy() || Op.size() >= 64) && Op.isAligned(Align(32)) && !FuncAttributes.hasFnAttribute(Attribute::NoImplicitFloat)) { return MVT::v4f64; } // We should use Altivec/VSX loads and stores when available. For unaligned // addresses, unaligned VSX loads are only fast starting with the P8. if (Subtarget.hasAltivec() && Op.size() >= 16 && (Op.isAligned(Align(16)) || ((Op.isMemset() && Subtarget.hasVSX()) || Subtarget.hasP8Vector()))) return MVT::v4i32; } if (Subtarget.isPPC64()) { return MVT::i64; } return MVT::i32; } /// Returns true if it is beneficial to convert a load of a constant /// to just the constant itself. bool PPCTargetLowering::shouldConvertConstantLoadToIntImm(const APInt &Imm, Type *Ty) const { assert(Ty->isIntegerTy()); unsigned BitSize = Ty->getPrimitiveSizeInBits(); return !(BitSize == 0 || BitSize > 64); } bool PPCTargetLowering::isTruncateFree(Type *Ty1, Type *Ty2) const { if (!Ty1->isIntegerTy() || !Ty2->isIntegerTy()) return false; unsigned NumBits1 = Ty1->getPrimitiveSizeInBits(); unsigned NumBits2 = Ty2->getPrimitiveSizeInBits(); return NumBits1 == 64 && NumBits2 == 32; } bool PPCTargetLowering::isTruncateFree(EVT VT1, EVT VT2) const { if (!VT1.isInteger() || !VT2.isInteger()) return false; unsigned NumBits1 = VT1.getSizeInBits(); unsigned NumBits2 = VT2.getSizeInBits(); return NumBits1 == 64 && NumBits2 == 32; } bool PPCTargetLowering::isZExtFree(SDValue Val, EVT VT2) const { // Generally speaking, zexts are not free, but they are free when they can be // folded with other operations. if (LoadSDNode *LD = dyn_cast(Val)) { EVT MemVT = LD->getMemoryVT(); if ((MemVT == MVT::i1 || MemVT == MVT::i8 || MemVT == MVT::i16 || (Subtarget.isPPC64() && MemVT == MVT::i32)) && (LD->getExtensionType() == ISD::NON_EXTLOAD || LD->getExtensionType() == ISD::ZEXTLOAD)) return true; } // FIXME: Add other cases... // - 32-bit shifts with a zext to i64 // - zext after ctlz, bswap, etc. // - zext after and by a constant mask return TargetLowering::isZExtFree(Val, VT2); } bool PPCTargetLowering::isFPExtFree(EVT DestVT, EVT SrcVT) const { assert(DestVT.isFloatingPoint() && SrcVT.isFloatingPoint() && "invalid fpext types"); // Extending to float128 is not free. if (DestVT == MVT::f128) return false; return true; } bool PPCTargetLowering::isLegalICmpImmediate(int64_t Imm) const { return isInt<16>(Imm) || isUInt<16>(Imm); } bool PPCTargetLowering::isLegalAddImmediate(int64_t Imm) const { return isInt<16>(Imm) || isUInt<16>(Imm); } bool PPCTargetLowering::allowsMisalignedMemoryAccesses(EVT VT, unsigned, unsigned, MachineMemOperand::Flags, bool *Fast) const { if (DisablePPCUnaligned) return false; // PowerPC supports unaligned memory access for simple non-vector types. // Although accessing unaligned addresses is not as efficient as accessing // aligned addresses, it is generally more efficient than manual expansion, // and generally only traps for software emulation when crossing page // boundaries. if (!VT.isSimple()) return false; if (VT.isFloatingPoint() && !Subtarget.allowsUnalignedFPAccess()) return false; if (VT.getSimpleVT().isVector()) { if (Subtarget.hasVSX()) { if (VT != MVT::v2f64 && VT != MVT::v2i64 && VT != MVT::v4f32 && VT != MVT::v4i32) return false; } else { return false; } } if (VT == MVT::ppcf128) return false; if (Fast) *Fast = true; return true; } bool PPCTargetLowering::isFMAFasterThanFMulAndFAdd(const MachineFunction &MF, EVT VT) const { return isFMAFasterThanFMulAndFAdd( MF.getFunction(), VT.getTypeForEVT(MF.getFunction().getContext())); } bool PPCTargetLowering::isFMAFasterThanFMulAndFAdd(const Function &F, Type *Ty) const { switch (Ty->getScalarType()->getTypeID()) { case Type::FloatTyID: case Type::DoubleTyID: return true; case Type::FP128TyID: return EnableQuadPrecision && Subtarget.hasP9Vector(); default: return false; } } // Currently this is a copy from AArch64TargetLowering::isProfitableToHoist. // FIXME: add more patterns which are profitable to hoist. bool PPCTargetLowering::isProfitableToHoist(Instruction *I) const { if (I->getOpcode() != Instruction::FMul) return true; if (!I->hasOneUse()) return true; Instruction *User = I->user_back(); assert(User && "A single use instruction with no uses."); if (User->getOpcode() != Instruction::FSub && User->getOpcode() != Instruction::FAdd) return true; const TargetOptions &Options = getTargetMachine().Options; const Function *F = I->getFunction(); const DataLayout &DL = F->getParent()->getDataLayout(); Type *Ty = User->getOperand(0)->getType(); return !( isFMAFasterThanFMulAndFAdd(*F, Ty) && isOperationLegalOrCustom(ISD::FMA, getValueType(DL, Ty)) && (Options.AllowFPOpFusion == FPOpFusion::Fast || Options.UnsafeFPMath)); } const MCPhysReg * PPCTargetLowering::getScratchRegisters(CallingConv::ID) const { // LR is a callee-save register, but we must treat it as clobbered by any call // site. Hence we include LR in the scratch registers, which are in turn added // as implicit-defs for stackmaps and patchpoints. The same reasoning applies // to CTR, which is used by any indirect call. static const MCPhysReg ScratchRegs[] = { PPC::X12, PPC::LR8, PPC::CTR8, 0 }; return ScratchRegs; } unsigned PPCTargetLowering::getExceptionPointerRegister( const Constant *PersonalityFn) const { return Subtarget.isPPC64() ? PPC::X3 : PPC::R3; } unsigned PPCTargetLowering::getExceptionSelectorRegister( const Constant *PersonalityFn) const { return Subtarget.isPPC64() ? PPC::X4 : PPC::R4; } bool PPCTargetLowering::shouldExpandBuildVectorWithShuffles( EVT VT , unsigned DefinedValues) const { if (VT == MVT::v2i64) return Subtarget.hasDirectMove(); // Don't need stack ops with direct moves if (Subtarget.hasVSX() || Subtarget.hasQPX()) return true; return TargetLowering::shouldExpandBuildVectorWithShuffles(VT, DefinedValues); } Sched::Preference PPCTargetLowering::getSchedulingPreference(SDNode *N) const { if (DisableILPPref || Subtarget.enableMachineScheduler()) return TargetLowering::getSchedulingPreference(N); return Sched::ILP; } // Create a fast isel object. FastISel * PPCTargetLowering::createFastISel(FunctionLoweringInfo &FuncInfo, const TargetLibraryInfo *LibInfo) const { return PPC::createFastISel(FuncInfo, LibInfo); } void PPCTargetLowering::initializeSplitCSR(MachineBasicBlock *Entry) const { if (!Subtarget.isPPC64()) return; // Update IsSplitCSR in PPCFunctionInfo PPCFunctionInfo *PFI = Entry->getParent()->getInfo(); PFI->setIsSplitCSR(true); } void PPCTargetLowering::insertCopiesSplitCSR( MachineBasicBlock *Entry, const SmallVectorImpl &Exits) const { const PPCRegisterInfo *TRI = Subtarget.getRegisterInfo(); const MCPhysReg *IStart = TRI->getCalleeSavedRegsViaCopy(Entry->getParent()); if (!IStart) return; const TargetInstrInfo *TII = Subtarget.getInstrInfo(); MachineRegisterInfo *MRI = &Entry->getParent()->getRegInfo(); MachineBasicBlock::iterator MBBI = Entry->begin(); for (const MCPhysReg *I = IStart; *I; ++I) { const TargetRegisterClass *RC = nullptr; if (PPC::G8RCRegClass.contains(*I)) RC = &PPC::G8RCRegClass; else if (PPC::F8RCRegClass.contains(*I)) RC = &PPC::F8RCRegClass; else if (PPC::CRRCRegClass.contains(*I)) RC = &PPC::CRRCRegClass; else if (PPC::VRRCRegClass.contains(*I)) RC = &PPC::VRRCRegClass; else llvm_unreachable("Unexpected register class in CSRsViaCopy!"); Register NewVR = MRI->createVirtualRegister(RC); // Create copy from CSR to a virtual register. // FIXME: this currently does not emit CFI pseudo-instructions, it works // fine for CXX_FAST_TLS since the C++-style TLS access functions should be // nounwind. If we want to generalize this later, we may need to emit // CFI pseudo-instructions. assert(Entry->getParent()->getFunction().hasFnAttribute( Attribute::NoUnwind) && "Function should be nounwind in insertCopiesSplitCSR!"); Entry->addLiveIn(*I); BuildMI(*Entry, MBBI, DebugLoc(), TII->get(TargetOpcode::COPY), NewVR) .addReg(*I); // Insert the copy-back instructions right before the terminator. for (auto *Exit : Exits) BuildMI(*Exit, Exit->getFirstTerminator(), DebugLoc(), TII->get(TargetOpcode::COPY), *I) .addReg(NewVR); } } // Override to enable LOAD_STACK_GUARD lowering on Linux. bool PPCTargetLowering::useLoadStackGuardNode() const { if (!Subtarget.isTargetLinux()) return TargetLowering::useLoadStackGuardNode(); return true; } // Override to disable global variable loading on Linux. void PPCTargetLowering::insertSSPDeclarations(Module &M) const { if (!Subtarget.isTargetLinux()) return TargetLowering::insertSSPDeclarations(M); } bool PPCTargetLowering::isFPImmLegal(const APFloat &Imm, EVT VT, bool ForCodeSize) const { if (!VT.isSimple() || !Subtarget.hasVSX()) return false; switch(VT.getSimpleVT().SimpleTy) { default: // For FP types that are currently not supported by PPC backend, return // false. Examples: f16, f80. return false; case MVT::f32: case MVT::f64: case MVT::ppcf128: return Imm.isPosZero(); } } // For vector shift operation op, fold // (op x, (and y, ((1 << numbits(x)) - 1))) -> (target op x, y) static SDValue stripModuloOnShift(const TargetLowering &TLI, SDNode *N, SelectionDAG &DAG) { SDValue N0 = N->getOperand(0); SDValue N1 = N->getOperand(1); EVT VT = N0.getValueType(); unsigned OpSizeInBits = VT.getScalarSizeInBits(); unsigned Opcode = N->getOpcode(); unsigned TargetOpcode; switch (Opcode) { default: llvm_unreachable("Unexpected shift operation"); case ISD::SHL: TargetOpcode = PPCISD::SHL; break; case ISD::SRL: TargetOpcode = PPCISD::SRL; break; case ISD::SRA: TargetOpcode = PPCISD::SRA; break; } if (VT.isVector() && TLI.isOperationLegal(Opcode, VT) && N1->getOpcode() == ISD::AND) if (ConstantSDNode *Mask = isConstOrConstSplat(N1->getOperand(1))) if (Mask->getZExtValue() == OpSizeInBits - 1) return DAG.getNode(TargetOpcode, SDLoc(N), VT, N0, N1->getOperand(0)); return SDValue(); } SDValue PPCTargetLowering::combineSHL(SDNode *N, DAGCombinerInfo &DCI) const { if (auto Value = stripModuloOnShift(*this, N, DCI.DAG)) return Value; SDValue N0 = N->getOperand(0); ConstantSDNode *CN1 = dyn_cast(N->getOperand(1)); if (!Subtarget.isISA3_0() || N0.getOpcode() != ISD::SIGN_EXTEND || N0.getOperand(0).getValueType() != MVT::i32 || CN1 == nullptr || N->getValueType(0) != MVT::i64) return SDValue(); // We can't save an operation here if the value is already extended, and // the existing shift is easier to combine. SDValue ExtsSrc = N0.getOperand(0); if (ExtsSrc.getOpcode() == ISD::TRUNCATE && ExtsSrc.getOperand(0).getOpcode() == ISD::AssertSext) return SDValue(); SDLoc DL(N0); SDValue ShiftBy = SDValue(CN1, 0); // We want the shift amount to be i32 on the extswli, but the shift could // have an i64. if (ShiftBy.getValueType() == MVT::i64) ShiftBy = DCI.DAG.getConstant(CN1->getZExtValue(), DL, MVT::i32); return DCI.DAG.getNode(PPCISD::EXTSWSLI, DL, MVT::i64, N0->getOperand(0), ShiftBy); } SDValue PPCTargetLowering::combineSRA(SDNode *N, DAGCombinerInfo &DCI) const { if (auto Value = stripModuloOnShift(*this, N, DCI.DAG)) return Value; return SDValue(); } SDValue PPCTargetLowering::combineSRL(SDNode *N, DAGCombinerInfo &DCI) const { if (auto Value = stripModuloOnShift(*this, N, DCI.DAG)) return Value; return SDValue(); } // Transform (add X, (zext(setne Z, C))) -> (addze X, (addic (addi Z, -C), -1)) // Transform (add X, (zext(sete Z, C))) -> (addze X, (subfic (addi Z, -C), 0)) // When C is zero, the equation (addi Z, -C) can be simplified to Z // Requirement: -C in [-32768, 32767], X and Z are MVT::i64 types static SDValue combineADDToADDZE(SDNode *N, SelectionDAG &DAG, const PPCSubtarget &Subtarget) { if (!Subtarget.isPPC64()) return SDValue(); SDValue LHS = N->getOperand(0); SDValue RHS = N->getOperand(1); auto isZextOfCompareWithConstant = [](SDValue Op) { if (Op.getOpcode() != ISD::ZERO_EXTEND || !Op.hasOneUse() || Op.getValueType() != MVT::i64) return false; SDValue Cmp = Op.getOperand(0); if (Cmp.getOpcode() != ISD::SETCC || !Cmp.hasOneUse() || Cmp.getOperand(0).getValueType() != MVT::i64) return false; if (auto *Constant = dyn_cast(Cmp.getOperand(1))) { int64_t NegConstant = 0 - Constant->getSExtValue(); // Due to the limitations of the addi instruction, // -C is required to be [-32768, 32767]. return isInt<16>(NegConstant); } return false; }; bool LHSHasPattern = isZextOfCompareWithConstant(LHS); bool RHSHasPattern = isZextOfCompareWithConstant(RHS); // If there is a pattern, canonicalize a zext operand to the RHS. if (LHSHasPattern && !RHSHasPattern) std::swap(LHS, RHS); else if (!LHSHasPattern && !RHSHasPattern) return SDValue(); SDLoc DL(N); SDVTList VTs = DAG.getVTList(MVT::i64, MVT::Glue); SDValue Cmp = RHS.getOperand(0); SDValue Z = Cmp.getOperand(0); auto *Constant = dyn_cast(Cmp.getOperand(1)); assert(Constant && "Constant Should not be a null pointer."); int64_t NegConstant = 0 - Constant->getSExtValue(); switch(cast(Cmp.getOperand(2))->get()) { default: break; case ISD::SETNE: { // when C == 0 // --> addze X, (addic Z, -1).carry // / // add X, (zext(setne Z, C))-- // \ when -32768 <= -C <= 32767 && C != 0 // --> addze X, (addic (addi Z, -C), -1).carry SDValue Add = DAG.getNode(ISD::ADD, DL, MVT::i64, Z, DAG.getConstant(NegConstant, DL, MVT::i64)); SDValue AddOrZ = NegConstant != 0 ? Add : Z; SDValue Addc = DAG.getNode(ISD::ADDC, DL, DAG.getVTList(MVT::i64, MVT::Glue), AddOrZ, DAG.getConstant(-1ULL, DL, MVT::i64)); return DAG.getNode(ISD::ADDE, DL, VTs, LHS, DAG.getConstant(0, DL, MVT::i64), SDValue(Addc.getNode(), 1)); } case ISD::SETEQ: { // when C == 0 // --> addze X, (subfic Z, 0).carry // / // add X, (zext(sete Z, C))-- // \ when -32768 <= -C <= 32767 && C != 0 // --> addze X, (subfic (addi Z, -C), 0).carry SDValue Add = DAG.getNode(ISD::ADD, DL, MVT::i64, Z, DAG.getConstant(NegConstant, DL, MVT::i64)); SDValue AddOrZ = NegConstant != 0 ? Add : Z; SDValue Subc = DAG.getNode(ISD::SUBC, DL, DAG.getVTList(MVT::i64, MVT::Glue), DAG.getConstant(0, DL, MVT::i64), AddOrZ); return DAG.getNode(ISD::ADDE, DL, VTs, LHS, DAG.getConstant(0, DL, MVT::i64), SDValue(Subc.getNode(), 1)); } } return SDValue(); } SDValue PPCTargetLowering::combineADD(SDNode *N, DAGCombinerInfo &DCI) const { if (auto Value = combineADDToADDZE(N, DCI.DAG, Subtarget)) return Value; return SDValue(); } // Detect TRUNCATE operations on bitcasts of float128 values. // What we are looking for here is the situtation where we extract a subset // of bits from a 128 bit float. // This can be of two forms: // 1) BITCAST of f128 feeding TRUNCATE // 2) BITCAST of f128 feeding SRL (a shift) feeding TRUNCATE // The reason this is required is because we do not have a legal i128 type // and so we want to prevent having to store the f128 and then reload part // of it. SDValue PPCTargetLowering::combineTRUNCATE(SDNode *N, DAGCombinerInfo &DCI) const { // If we are using CRBits then try that first. if (Subtarget.useCRBits()) { // Check if CRBits did anything and return that if it did. if (SDValue CRTruncValue = DAGCombineTruncBoolExt(N, DCI)) return CRTruncValue; } SDLoc dl(N); SDValue Op0 = N->getOperand(0); // Looking for a truncate of i128 to i64. if (Op0.getValueType() != MVT::i128 || N->getValueType(0) != MVT::i64) return SDValue(); int EltToExtract = DCI.DAG.getDataLayout().isBigEndian() ? 1 : 0; // SRL feeding TRUNCATE. if (Op0.getOpcode() == ISD::SRL) { ConstantSDNode *ConstNode = dyn_cast(Op0.getOperand(1)); // The right shift has to be by 64 bits. if (!ConstNode || ConstNode->getZExtValue() != 64) return SDValue(); // Switch the element number to extract. EltToExtract = EltToExtract ? 0 : 1; // Update Op0 past the SRL. Op0 = Op0.getOperand(0); } // BITCAST feeding a TRUNCATE possibly via SRL. if (Op0.getOpcode() == ISD::BITCAST && Op0.getValueType() == MVT::i128 && Op0.getOperand(0).getValueType() == MVT::f128) { SDValue Bitcast = DCI.DAG.getBitcast(MVT::v2i64, Op0.getOperand(0)); return DCI.DAG.getNode( ISD::EXTRACT_VECTOR_ELT, dl, MVT::i64, Bitcast, DCI.DAG.getTargetConstant(EltToExtract, dl, MVT::i32)); } return SDValue(); } SDValue PPCTargetLowering::combineMUL(SDNode *N, DAGCombinerInfo &DCI) const { SelectionDAG &DAG = DCI.DAG; ConstantSDNode *ConstOpOrElement = isConstOrConstSplat(N->getOperand(1)); if (!ConstOpOrElement) return SDValue(); // An imul is usually smaller than the alternative sequence for legal type. if (DAG.getMachineFunction().getFunction().hasMinSize() && isOperationLegal(ISD::MUL, N->getValueType(0))) return SDValue(); auto IsProfitable = [this](bool IsNeg, bool IsAddOne, EVT VT) -> bool { switch (this->Subtarget.getCPUDirective()) { default: // TODO: enhance the condition for subtarget before pwr8 return false; case PPC::DIR_PWR8: // type mul add shl // scalar 4 1 1 // vector 7 2 2 return true; case PPC::DIR_PWR9: case PPC::DIR_PWR_FUTURE: // type mul add shl // scalar 5 2 2 // vector 7 2 2 // The cycle RATIO of related operations are showed as a table above. // Because mul is 5(scalar)/7(vector), add/sub/shl are all 2 for both // scalar and vector type. For 2 instrs patterns, add/sub + shl // are 4, it is always profitable; but for 3 instrs patterns // (mul x, -(2^N + 1)) => -(add (shl x, N), x), sub + add + shl are 6. // So we should only do it for vector type. return IsAddOne && IsNeg ? VT.isVector() : true; } }; EVT VT = N->getValueType(0); SDLoc DL(N); const APInt &MulAmt = ConstOpOrElement->getAPIntValue(); bool IsNeg = MulAmt.isNegative(); APInt MulAmtAbs = MulAmt.abs(); if ((MulAmtAbs - 1).isPowerOf2()) { // (mul x, 2^N + 1) => (add (shl x, N), x) // (mul x, -(2^N + 1)) => -(add (shl x, N), x) if (!IsProfitable(IsNeg, true, VT)) return SDValue(); SDValue Op0 = N->getOperand(0); SDValue Op1 = DAG.getNode(ISD::SHL, DL, VT, N->getOperand(0), DAG.getConstant((MulAmtAbs - 1).logBase2(), DL, VT)); SDValue Res = DAG.getNode(ISD::ADD, DL, VT, Op0, Op1); if (!IsNeg) return Res; return DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT), Res); } else if ((MulAmtAbs + 1).isPowerOf2()) { // (mul x, 2^N - 1) => (sub (shl x, N), x) // (mul x, -(2^N - 1)) => (sub x, (shl x, N)) if (!IsProfitable(IsNeg, false, VT)) return SDValue(); SDValue Op0 = N->getOperand(0); SDValue Op1 = DAG.getNode(ISD::SHL, DL, VT, N->getOperand(0), DAG.getConstant((MulAmtAbs + 1).logBase2(), DL, VT)); if (!IsNeg) return DAG.getNode(ISD::SUB, DL, VT, Op1, Op0); else return DAG.getNode(ISD::SUB, DL, VT, Op0, Op1); } else { return SDValue(); } } bool PPCTargetLowering::mayBeEmittedAsTailCall(const CallInst *CI) const { // Only duplicate to increase tail-calls for the 64bit SysV ABIs. if (!Subtarget.is64BitELFABI()) return false; // If not a tail call then no need to proceed. if (!CI->isTailCall()) return false; // If sibling calls have been disabled and tail-calls aren't guaranteed // there is no reason to duplicate. auto &TM = getTargetMachine(); if (!TM.Options.GuaranteedTailCallOpt && DisableSCO) return false; // Can't tail call a function called indirectly, or if it has variadic args. const Function *Callee = CI->getCalledFunction(); if (!Callee || Callee->isVarArg()) return false; // Make sure the callee and caller calling conventions are eligible for tco. const Function *Caller = CI->getParent()->getParent(); if (!areCallingConvEligibleForTCO_64SVR4(Caller->getCallingConv(), CI->getCallingConv())) return false; // If the function is local then we have a good chance at tail-calling it return getTargetMachine().shouldAssumeDSOLocal(*Caller->getParent(), Callee); } bool PPCTargetLowering::hasBitPreservingFPLogic(EVT VT) const { if (!Subtarget.hasVSX()) return false; if (Subtarget.hasP9Vector() && VT == MVT::f128) return true; return VT == MVT::f32 || VT == MVT::f64 || VT == MVT::v4f32 || VT == MVT::v2f64; } bool PPCTargetLowering:: isMaskAndCmp0FoldingBeneficial(const Instruction &AndI) const { const Value *Mask = AndI.getOperand(1); // If the mask is suitable for andi. or andis. we should sink the and. if (const ConstantInt *CI = dyn_cast(Mask)) { // Can't handle constants wider than 64-bits. if (CI->getBitWidth() > 64) return false; int64_t ConstVal = CI->getZExtValue(); return isUInt<16>(ConstVal) || (isUInt<16>(ConstVal >> 16) && !(ConstVal & 0xFFFF)); } // For non-constant masks, we can always use the record-form and. return true; } // Transform (abs (sub (zext a), (zext b))) to (vabsd a b 0) // Transform (abs (sub (zext a), (zext_invec b))) to (vabsd a b 0) // Transform (abs (sub (zext_invec a), (zext_invec b))) to (vabsd a b 0) // Transform (abs (sub (zext_invec a), (zext b))) to (vabsd a b 0) // Transform (abs (sub a, b) to (vabsd a b 1)) if a & b of type v4i32 SDValue PPCTargetLowering::combineABS(SDNode *N, DAGCombinerInfo &DCI) const { assert((N->getOpcode() == ISD::ABS) && "Need ABS node here"); assert(Subtarget.hasP9Altivec() && "Only combine this when P9 altivec supported!"); EVT VT = N->getValueType(0); if (VT != MVT::v4i32 && VT != MVT::v8i16 && VT != MVT::v16i8) return SDValue(); SelectionDAG &DAG = DCI.DAG; SDLoc dl(N); if (N->getOperand(0).getOpcode() == ISD::SUB) { // Even for signed integers, if it's known to be positive (as signed // integer) due to zero-extended inputs. unsigned SubOpcd0 = N->getOperand(0)->getOperand(0).getOpcode(); unsigned SubOpcd1 = N->getOperand(0)->getOperand(1).getOpcode(); if ((SubOpcd0 == ISD::ZERO_EXTEND || SubOpcd0 == ISD::ZERO_EXTEND_VECTOR_INREG) && (SubOpcd1 == ISD::ZERO_EXTEND || SubOpcd1 == ISD::ZERO_EXTEND_VECTOR_INREG)) { return DAG.getNode(PPCISD::VABSD, dl, N->getOperand(0).getValueType(), N->getOperand(0)->getOperand(0), N->getOperand(0)->getOperand(1), DAG.getTargetConstant(0, dl, MVT::i32)); } // For type v4i32, it can be optimized with xvnegsp + vabsduw if (N->getOperand(0).getValueType() == MVT::v4i32 && N->getOperand(0).hasOneUse()) { return DAG.getNode(PPCISD::VABSD, dl, N->getOperand(0).getValueType(), N->getOperand(0)->getOperand(0), N->getOperand(0)->getOperand(1), DAG.getTargetConstant(1, dl, MVT::i32)); } } return SDValue(); } // For type v4i32/v8ii16/v16i8, transform // from (vselect (setcc a, b, setugt), (sub a, b), (sub b, a)) to (vabsd a, b) // from (vselect (setcc a, b, setuge), (sub a, b), (sub b, a)) to (vabsd a, b) // from (vselect (setcc a, b, setult), (sub b, a), (sub a, b)) to (vabsd a, b) // from (vselect (setcc a, b, setule), (sub b, a), (sub a, b)) to (vabsd a, b) SDValue PPCTargetLowering::combineVSelect(SDNode *N, DAGCombinerInfo &DCI) const { assert((N->getOpcode() == ISD::VSELECT) && "Need VSELECT node here"); assert(Subtarget.hasP9Altivec() && "Only combine this when P9 altivec supported!"); SelectionDAG &DAG = DCI.DAG; SDLoc dl(N); SDValue Cond = N->getOperand(0); SDValue TrueOpnd = N->getOperand(1); SDValue FalseOpnd = N->getOperand(2); EVT VT = N->getOperand(1).getValueType(); if (Cond.getOpcode() != ISD::SETCC || TrueOpnd.getOpcode() != ISD::SUB || FalseOpnd.getOpcode() != ISD::SUB) return SDValue(); // ABSD only available for type v4i32/v8i16/v16i8 if (VT != MVT::v4i32 && VT != MVT::v8i16 && VT != MVT::v16i8) return SDValue(); // At least to save one more dependent computation if (!(Cond.hasOneUse() || TrueOpnd.hasOneUse() || FalseOpnd.hasOneUse())) return SDValue(); ISD::CondCode CC = cast(Cond.getOperand(2))->get(); // Can only handle unsigned comparison here switch (CC) { default: return SDValue(); case ISD::SETUGT: case ISD::SETUGE: break; case ISD::SETULT: case ISD::SETULE: std::swap(TrueOpnd, FalseOpnd); break; } SDValue CmpOpnd1 = Cond.getOperand(0); SDValue CmpOpnd2 = Cond.getOperand(1); // SETCC CmpOpnd1 CmpOpnd2 cond // TrueOpnd = CmpOpnd1 - CmpOpnd2 // FalseOpnd = CmpOpnd2 - CmpOpnd1 if (TrueOpnd.getOperand(0) == CmpOpnd1 && TrueOpnd.getOperand(1) == CmpOpnd2 && FalseOpnd.getOperand(0) == CmpOpnd2 && FalseOpnd.getOperand(1) == CmpOpnd1) { return DAG.getNode(PPCISD::VABSD, dl, N->getOperand(1).getValueType(), CmpOpnd1, CmpOpnd2, DAG.getTargetConstant(0, dl, MVT::i32)); } return SDValue(); } diff --git a/llvm/lib/Target/PowerPC/PPCISelLowering.h b/llvm/lib/Target/PowerPC/PPCISelLowering.h index 4107d2bc188e..3a1001c0e135 100644 --- a/llvm/lib/Target/PowerPC/PPCISelLowering.h +++ b/llvm/lib/Target/PowerPC/PPCISelLowering.h @@ -1,1264 +1,1266 @@ //===-- PPCISelLowering.h - PPC32 DAG Lowering Interface --------*- C++ -*-===// // // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. // See https://llvm.org/LICENSE.txt for license information. // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // // This file defines the interfaces that PPC uses to lower LLVM code into a // selection DAG. // //===----------------------------------------------------------------------===// #ifndef LLVM_LIB_TARGET_POWERPC_PPCISELLOWERING_H #define LLVM_LIB_TARGET_POWERPC_PPCISELLOWERING_H #include "PPCInstrInfo.h" #include "llvm/CodeGen/CallingConvLower.h" #include "llvm/CodeGen/MachineFunction.h" #include "llvm/CodeGen/MachineMemOperand.h" #include "llvm/CodeGen/SelectionDAG.h" #include "llvm/CodeGen/SelectionDAGNodes.h" #include "llvm/CodeGen/TargetLowering.h" #include "llvm/CodeGen/ValueTypes.h" #include "llvm/IR/Attributes.h" #include "llvm/IR/CallingConv.h" #include "llvm/IR/Function.h" #include "llvm/IR/InlineAsm.h" #include "llvm/IR/Metadata.h" #include "llvm/IR/Type.h" #include "llvm/Support/MachineValueType.h" #include namespace llvm { namespace PPCISD { // When adding a NEW PPCISD node please add it to the correct position in // the enum. The order of elements in this enum matters! // Values that are added after this entry: // STBRX = ISD::FIRST_TARGET_MEMORY_OPCODE // are considered memory opcodes and are treated differently than entries // that come before it. For example, ADD or MUL should be placed before // the ISD::FIRST_TARGET_MEMORY_OPCODE while a LOAD or STORE should come // after it. enum NodeType : unsigned { // Start the numbering where the builtin ops and target ops leave off. FIRST_NUMBER = ISD::BUILTIN_OP_END, /// FSEL - Traditional three-operand fsel node. /// FSEL, /// XSMAXCDP, XSMINCDP - C-type min/max instructions. XSMAXCDP, XSMINCDP, /// FCFID - The FCFID instruction, taking an f64 operand and producing /// and f64 value containing the FP representation of the integer that /// was temporarily in the f64 operand. FCFID, /// Newer FCFID[US] integer-to-floating-point conversion instructions for /// unsigned integers and single-precision outputs. FCFIDU, FCFIDS, FCFIDUS, /// FCTI[D,W]Z - The FCTIDZ and FCTIWZ instructions, taking an f32 or f64 /// operand, producing an f64 value containing the integer representation /// of that FP value. FCTIDZ, FCTIWZ, /// Newer FCTI[D,W]UZ floating-point-to-integer conversion instructions for /// unsigned integers with round toward zero. FCTIDUZ, FCTIWUZ, /// Floating-point-to-interger conversion instructions FP_TO_UINT_IN_VSR, FP_TO_SINT_IN_VSR, /// VEXTS, ByteWidth - takes an input in VSFRC and produces an output in /// VSFRC that is sign-extended from ByteWidth to a 64-byte integer. VEXTS, /// Reciprocal estimate instructions (unary FP ops). FRE, FRSQRTE, // VMADDFP, VNMSUBFP - The VMADDFP and VNMSUBFP instructions, taking // three v4f32 operands and producing a v4f32 result. VMADDFP, VNMSUBFP, /// VPERM - The PPC VPERM Instruction. /// VPERM, /// XXSPLT - The PPC VSX splat instructions /// XXSPLT, /// VECINSERT - The PPC vector insert instruction /// VECINSERT, /// VECSHL - The PPC vector shift left instruction /// VECSHL, /// XXPERMDI - The PPC XXPERMDI instruction /// XXPERMDI, /// The CMPB instruction (takes two operands of i32 or i64). CMPB, /// Hi/Lo - These represent the high and low 16-bit parts of a global /// address respectively. These nodes have two operands, the first of /// which must be a TargetGlobalAddress, and the second of which must be a /// Constant. Selected naively, these turn into 'lis G+C' and 'li G+C', /// though these are usually folded into other nodes. Hi, Lo, /// The following two target-specific nodes are used for calls through /// function pointers in the 64-bit SVR4 ABI. /// OPRC, CHAIN = DYNALLOC(CHAIN, NEGSIZE, FRAME_INDEX) /// This instruction is lowered in PPCRegisterInfo::eliminateFrameIndex to /// compute an allocation on the stack. DYNALLOC, /// This instruction is lowered in PPCRegisterInfo::eliminateFrameIndex to /// compute an offset from native SP to the address of the most recent /// dynamic alloca. DYNAREAOFFSET, /// GlobalBaseReg - On Darwin, this node represents the result of the mflr /// at function entry, used for PIC code. GlobalBaseReg, /// These nodes represent PPC shifts. /// /// For scalar types, only the last `n + 1` bits of the shift amounts /// are used, where n is log2(sizeof(element) * 8). See sld/slw, etc. /// for exact behaviors. /// /// For vector types, only the last n bits are used. See vsld. SRL, SRA, SHL, /// EXTSWSLI = The PPC extswsli instruction, which does an extend-sign /// word and shift left immediate. EXTSWSLI, /// The combination of sra[wd]i and addze used to implemented signed /// integer division by a power of 2. The first operand is the dividend, /// and the second is the constant shift amount (representing the /// divisor). SRA_ADDZE, /// CALL - A direct function call. /// CALL_NOP is a call with the special NOP which follows 64-bit + /// CALL_NOTOC the caller does not use the TOC. /// SVR4 calls and 32-bit/64-bit AIX calls. CALL, CALL_NOP, + CALL_NOTOC, /// CHAIN,FLAG = MTCTR(VAL, CHAIN[, INFLAG]) - Directly corresponds to a /// MTCTR instruction. MTCTR, /// CHAIN,FLAG = BCTRL(CHAIN, INFLAG) - Directly corresponds to a /// BCTRL instruction. BCTRL, /// CHAIN,FLAG = BCTRL(CHAIN, ADDR, INFLAG) - The combination of a bctrl /// instruction and the TOC reload required on 64-bit ELF, 32-bit AIX /// and 64-bit AIX. BCTRL_LOAD_TOC, /// Return with a flag operand, matched by 'blr' RET_FLAG, /// R32 = MFOCRF(CRREG, INFLAG) - Represents the MFOCRF instruction. /// This copies the bits corresponding to the specified CRREG into the /// resultant GPR. Bits corresponding to other CR regs are undefined. MFOCRF, /// Direct move from a VSX register to a GPR MFVSR, /// Direct move from a GPR to a VSX register (algebraic) MTVSRA, /// Direct move from a GPR to a VSX register (zero) MTVSRZ, /// Direct move of 2 consecutive GPR to a VSX register. BUILD_FP128, /// BUILD_SPE64 and EXTRACT_SPE are analogous to BUILD_PAIR and /// EXTRACT_ELEMENT but take f64 arguments instead of i64, as i64 is /// unsupported for this target. /// Merge 2 GPRs to a single SPE register. BUILD_SPE64, /// Extract SPE register component, second argument is high or low. EXTRACT_SPE, /// Extract a subvector from signed integer vector and convert to FP. /// It is primarily used to convert a (widened) illegal integer vector /// type to a legal floating point vector type. /// For example v2i32 -> widened to v4i32 -> v2f64 SINT_VEC_TO_FP, /// Extract a subvector from unsigned integer vector and convert to FP. /// As with SINT_VEC_TO_FP, used for converting illegal types. UINT_VEC_TO_FP, // FIXME: Remove these once the ANDI glue bug is fixed: /// i1 = ANDI_rec_1_[EQ|GT]_BIT(i32 or i64 x) - Represents the result of the /// eq or gt bit of CR0 after executing andi. x, 1. This is used to /// implement truncation of i32 or i64 to i1. ANDI_rec_1_EQ_BIT, ANDI_rec_1_GT_BIT, // READ_TIME_BASE - A read of the 64-bit time-base register on a 32-bit // target (returns (Lo, Hi)). It takes a chain operand. READ_TIME_BASE, // EH_SJLJ_SETJMP - SjLj exception handling setjmp. EH_SJLJ_SETJMP, // EH_SJLJ_LONGJMP - SjLj exception handling longjmp. EH_SJLJ_LONGJMP, /// RESVEC = VCMP(LHS, RHS, OPC) - Represents one of the altivec VCMP* /// instructions. For lack of better number, we use the opcode number /// encoding for the OPC field to identify the compare. For example, 838 /// is VCMPGTSH. VCMP, /// RESVEC, OUTFLAG = VCMPo(LHS, RHS, OPC) - Represents one of the /// altivec VCMP*o instructions. For lack of better number, we use the /// opcode number encoding for the OPC field to identify the compare. For /// example, 838 is VCMPGTSH. VCMPo, /// CHAIN = COND_BRANCH CHAIN, CRRC, OPC, DESTBB [, INFLAG] - This /// corresponds to the COND_BRANCH pseudo instruction. CRRC is the /// condition register to branch on, OPC is the branch opcode to use (e.g. /// PPC::BLE), DESTBB is the destination block to branch to, and INFLAG is /// an optional input flag argument. COND_BRANCH, /// CHAIN = BDNZ CHAIN, DESTBB - These are used to create counter-based /// loops. BDNZ, BDZ, /// F8RC = FADDRTZ F8RC, F8RC - This is an FADD done with rounding /// towards zero. Used only as part of the long double-to-int /// conversion sequence. FADDRTZ, /// F8RC = MFFS - This moves the FPSCR (not modeled) into the register. MFFS, /// TC_RETURN - A tail call return. /// operand #0 chain /// operand #1 callee (register or absolute) /// operand #2 stack adjustment /// operand #3 optional in flag TC_RETURN, /// ch, gl = CR6[UN]SET ch, inglue - Toggle CR bit 6 for SVR4 vararg calls CR6SET, CR6UNSET, /// GPRC = address of _GLOBAL_OFFSET_TABLE_. Used by initial-exec TLS /// for non-position independent code on PPC32. PPC32_GOT, /// GPRC = address of _GLOBAL_OFFSET_TABLE_. Used by general dynamic and /// local dynamic TLS and position indendepent code on PPC32. PPC32_PICGOT, /// G8RC = ADDIS_GOT_TPREL_HA %x2, Symbol - Used by the initial-exec /// TLS model, produces an ADDIS8 instruction that adds the GOT /// base to sym\@got\@tprel\@ha. ADDIS_GOT_TPREL_HA, /// G8RC = LD_GOT_TPREL_L Symbol, G8RReg - Used by the initial-exec /// TLS model, produces a LD instruction with base register G8RReg /// and offset sym\@got\@tprel\@l. This completes the addition that /// finds the offset of "sym" relative to the thread pointer. LD_GOT_TPREL_L, /// G8RC = ADD_TLS G8RReg, Symbol - Used by the initial-exec TLS /// model, produces an ADD instruction that adds the contents of /// G8RReg to the thread pointer. Symbol contains a relocation /// sym\@tls which is to be replaced by the thread pointer and /// identifies to the linker that the instruction is part of a /// TLS sequence. ADD_TLS, /// G8RC = ADDIS_TLSGD_HA %x2, Symbol - For the general-dynamic TLS /// model, produces an ADDIS8 instruction that adds the GOT base /// register to sym\@got\@tlsgd\@ha. ADDIS_TLSGD_HA, /// %x3 = ADDI_TLSGD_L G8RReg, Symbol - For the general-dynamic TLS /// model, produces an ADDI8 instruction that adds G8RReg to /// sym\@got\@tlsgd\@l and stores the result in X3. Hidden by /// ADDIS_TLSGD_L_ADDR until after register assignment. ADDI_TLSGD_L, /// %x3 = GET_TLS_ADDR %x3, Symbol - For the general-dynamic TLS /// model, produces a call to __tls_get_addr(sym\@tlsgd). Hidden by /// ADDIS_TLSGD_L_ADDR until after register assignment. GET_TLS_ADDR, /// G8RC = ADDI_TLSGD_L_ADDR G8RReg, Symbol, Symbol - Op that /// combines ADDI_TLSGD_L and GET_TLS_ADDR until expansion following /// register assignment. ADDI_TLSGD_L_ADDR, /// G8RC = ADDIS_TLSLD_HA %x2, Symbol - For the local-dynamic TLS /// model, produces an ADDIS8 instruction that adds the GOT base /// register to sym\@got\@tlsld\@ha. ADDIS_TLSLD_HA, /// %x3 = ADDI_TLSLD_L G8RReg, Symbol - For the local-dynamic TLS /// model, produces an ADDI8 instruction that adds G8RReg to /// sym\@got\@tlsld\@l and stores the result in X3. Hidden by /// ADDIS_TLSLD_L_ADDR until after register assignment. ADDI_TLSLD_L, /// %x3 = GET_TLSLD_ADDR %x3, Symbol - For the local-dynamic TLS /// model, produces a call to __tls_get_addr(sym\@tlsld). Hidden by /// ADDIS_TLSLD_L_ADDR until after register assignment. GET_TLSLD_ADDR, /// G8RC = ADDI_TLSLD_L_ADDR G8RReg, Symbol, Symbol - Op that /// combines ADDI_TLSLD_L and GET_TLSLD_ADDR until expansion /// following register assignment. ADDI_TLSLD_L_ADDR, /// G8RC = ADDIS_DTPREL_HA %x3, Symbol - For the local-dynamic TLS /// model, produces an ADDIS8 instruction that adds X3 to /// sym\@dtprel\@ha. ADDIS_DTPREL_HA, /// G8RC = ADDI_DTPREL_L G8RReg, Symbol - For the local-dynamic TLS /// model, produces an ADDI8 instruction that adds G8RReg to /// sym\@got\@dtprel\@l. ADDI_DTPREL_L, /// VRRC = VADD_SPLAT Elt, EltSize - Temporary node to be expanded /// during instruction selection to optimize a BUILD_VECTOR into /// operations on splats. This is necessary to avoid losing these /// optimizations due to constant folding. VADD_SPLAT, /// CHAIN = SC CHAIN, Imm128 - System call. The 7-bit unsigned /// operand identifies the operating system entry point. SC, /// CHAIN = CLRBHRB CHAIN - Clear branch history rolling buffer. CLRBHRB, /// GPRC, CHAIN = MFBHRBE CHAIN, Entry, Dummy - Move from branch /// history rolling buffer entry. MFBHRBE, /// CHAIN = RFEBB CHAIN, State - Return from event-based branch. RFEBB, /// VSRC, CHAIN = XXSWAPD CHAIN, VSRC - Occurs only for little /// endian. Maps to an xxswapd instruction that corrects an lxvd2x /// or stxvd2x instruction. The chain is necessary because the /// sequence replaces a load and needs to provide the same number /// of outputs. XXSWAPD, /// An SDNode for swaps that are not associated with any loads/stores /// and thereby have no chain. SWAP_NO_CHAIN, /// An SDNode for Power9 vector absolute value difference. /// operand #0 vector /// operand #1 vector /// operand #2 constant i32 0 or 1, to indicate whether needs to patch /// the most significant bit for signed i32 /// /// Power9 VABSD* instructions are designed to support unsigned integer /// vectors (byte/halfword/word), if we want to make use of them for signed /// integer vectors, we have to flip their sign bits first. To flip sign bit /// for byte/halfword integer vector would become inefficient, but for word /// integer vector, we can leverage XVNEGSP to make it efficiently. eg: /// abs(sub(a,b)) => VABSDUW(a+0x80000000, b+0x80000000) /// => VABSDUW((XVNEGSP a), (XVNEGSP b)) VABSD, /// QVFPERM = This corresponds to the QPX qvfperm instruction. QVFPERM, /// QVGPCI = This corresponds to the QPX qvgpci instruction. QVGPCI, /// QVALIGNI = This corresponds to the QPX qvaligni instruction. QVALIGNI, /// QVESPLATI = This corresponds to the QPX qvesplati instruction. QVESPLATI, /// QBFLT = Access the underlying QPX floating-point boolean /// representation. QBFLT, /// FP_EXTEND_HALF(VECTOR, IDX) - Custom extend upper (IDX=0) half or /// lower (IDX=1) half of v4f32 to v2f64. FP_EXTEND_HALF, /// CHAIN = STBRX CHAIN, GPRC, Ptr, Type - This is a /// byte-swapping store instruction. It byte-swaps the low "Type" bits of /// the GPRC input, then stores it through Ptr. Type can be either i16 or /// i32. STBRX = ISD::FIRST_TARGET_MEMORY_OPCODE, /// GPRC, CHAIN = LBRX CHAIN, Ptr, Type - This is a /// byte-swapping load instruction. It loads "Type" bits, byte swaps it, /// then puts it in the bottom bits of the GPRC. TYPE can be either i16 /// or i32. LBRX, /// STFIWX - The STFIWX instruction. The first operand is an input token /// chain, then an f64 value to store, then an address to store it to. STFIWX, /// GPRC, CHAIN = LFIWAX CHAIN, Ptr - This is a floating-point /// load which sign-extends from a 32-bit integer value into the /// destination 64-bit register. LFIWAX, /// GPRC, CHAIN = LFIWZX CHAIN, Ptr - This is a floating-point /// load which zero-extends from a 32-bit integer value into the /// destination 64-bit register. LFIWZX, /// GPRC, CHAIN = LXSIZX, CHAIN, Ptr, ByteWidth - This is a load of an /// integer smaller than 64 bits into a VSR. The integer is zero-extended. /// This can be used for converting loaded integers to floating point. LXSIZX, /// STXSIX - The STXSI[bh]X instruction. The first operand is an input /// chain, then an f64 value to store, then an address to store it to, /// followed by a byte-width for the store. STXSIX, /// VSRC, CHAIN = LXVD2X_LE CHAIN, Ptr - Occurs only for little endian. /// Maps directly to an lxvd2x instruction that will be followed by /// an xxswapd. LXVD2X, /// VSRC, CHAIN = LOAD_VEC_BE CHAIN, Ptr - Occurs only for little endian. /// Maps directly to one of lxvd2x/lxvw4x/lxvh8x/lxvb16x depending on /// the vector type to load vector in big-endian element order. LOAD_VEC_BE, /// VSRC, CHAIN = LD_VSX_LH CHAIN, Ptr - This is a floating-point load of a /// v2f32 value into the lower half of a VSR register. LD_VSX_LH, /// VSRC, CHAIN = LD_SPLAT, CHAIN, Ptr - a splatting load memory /// instructions such as LXVDSX, LXVWSX. LD_SPLAT, /// CHAIN = STXVD2X CHAIN, VSRC, Ptr - Occurs only for little endian. /// Maps directly to an stxvd2x instruction that will be preceded by /// an xxswapd. STXVD2X, /// CHAIN = STORE_VEC_BE CHAIN, VSRC, Ptr - Occurs only for little endian. /// Maps directly to one of stxvd2x/stxvw4x/stxvh8x/stxvb16x depending on /// the vector type to store vector in big-endian element order. STORE_VEC_BE, /// Store scalar integers from VSR. ST_VSR_SCAL_INT, /// QBRC, CHAIN = QVLFSb CHAIN, Ptr /// The 4xf32 load used for v4i1 constants. QVLFSb, /// ATOMIC_CMP_SWAP - the exact same as the target-independent nodes /// except they ensure that the compare input is zero-extended for /// sub-word versions because the atomic loads zero-extend. ATOMIC_CMP_SWAP_8, ATOMIC_CMP_SWAP_16, /// GPRC = TOC_ENTRY GA, TOC /// Loads the entry for GA from the TOC, where the TOC base is given by /// the last operand. TOC_ENTRY }; } // end namespace PPCISD /// Define some predicates that are used for node matching. namespace PPC { /// isVPKUHUMShuffleMask - Return true if this is the shuffle mask for a /// VPKUHUM instruction. bool isVPKUHUMShuffleMask(ShuffleVectorSDNode *N, unsigned ShuffleKind, SelectionDAG &DAG); /// isVPKUWUMShuffleMask - Return true if this is the shuffle mask for a /// VPKUWUM instruction. bool isVPKUWUMShuffleMask(ShuffleVectorSDNode *N, unsigned ShuffleKind, SelectionDAG &DAG); /// isVPKUDUMShuffleMask - Return true if this is the shuffle mask for a /// VPKUDUM instruction. bool isVPKUDUMShuffleMask(ShuffleVectorSDNode *N, unsigned ShuffleKind, SelectionDAG &DAG); /// isVMRGLShuffleMask - Return true if this is a shuffle mask suitable for /// a VRGL* instruction with the specified unit size (1,2 or 4 bytes). bool isVMRGLShuffleMask(ShuffleVectorSDNode *N, unsigned UnitSize, unsigned ShuffleKind, SelectionDAG &DAG); /// isVMRGHShuffleMask - Return true if this is a shuffle mask suitable for /// a VRGH* instruction with the specified unit size (1,2 or 4 bytes). bool isVMRGHShuffleMask(ShuffleVectorSDNode *N, unsigned UnitSize, unsigned ShuffleKind, SelectionDAG &DAG); /// isVMRGEOShuffleMask - Return true if this is a shuffle mask suitable for /// a VMRGEW or VMRGOW instruction bool isVMRGEOShuffleMask(ShuffleVectorSDNode *N, bool CheckEven, unsigned ShuffleKind, SelectionDAG &DAG); /// isXXSLDWIShuffleMask - Return true if this is a shuffle mask suitable /// for a XXSLDWI instruction. bool isXXSLDWIShuffleMask(ShuffleVectorSDNode *N, unsigned &ShiftElts, bool &Swap, bool IsLE); /// isXXBRHShuffleMask - Return true if this is a shuffle mask suitable /// for a XXBRH instruction. bool isXXBRHShuffleMask(ShuffleVectorSDNode *N); /// isXXBRWShuffleMask - Return true if this is a shuffle mask suitable /// for a XXBRW instruction. bool isXXBRWShuffleMask(ShuffleVectorSDNode *N); /// isXXBRDShuffleMask - Return true if this is a shuffle mask suitable /// for a XXBRD instruction. bool isXXBRDShuffleMask(ShuffleVectorSDNode *N); /// isXXBRQShuffleMask - Return true if this is a shuffle mask suitable /// for a XXBRQ instruction. bool isXXBRQShuffleMask(ShuffleVectorSDNode *N); /// isXXPERMDIShuffleMask - Return true if this is a shuffle mask suitable /// for a XXPERMDI instruction. bool isXXPERMDIShuffleMask(ShuffleVectorSDNode *N, unsigned &ShiftElts, bool &Swap, bool IsLE); /// isVSLDOIShuffleMask - If this is a vsldoi shuffle mask, return the /// shift amount, otherwise return -1. int isVSLDOIShuffleMask(SDNode *N, unsigned ShuffleKind, SelectionDAG &DAG); /// isSplatShuffleMask - Return true if the specified VECTOR_SHUFFLE operand /// specifies a splat of a single element that is suitable for input to /// VSPLTB/VSPLTH/VSPLTW. bool isSplatShuffleMask(ShuffleVectorSDNode *N, unsigned EltSize); /// isXXINSERTWMask - Return true if this VECTOR_SHUFFLE can be handled by /// the XXINSERTW instruction introduced in ISA 3.0. This is essentially any /// shuffle of v4f32/v4i32 vectors that just inserts one element from one /// vector into the other. This function will also set a couple of /// output parameters for how much the source vector needs to be shifted and /// what byte number needs to be specified for the instruction to put the /// element in the desired location of the target vector. bool isXXINSERTWMask(ShuffleVectorSDNode *N, unsigned &ShiftElts, unsigned &InsertAtByte, bool &Swap, bool IsLE); /// getSplatIdxForPPCMnemonics - Return the splat index as a value that is /// appropriate for PPC mnemonics (which have a big endian bias - namely /// elements are counted from the left of the vector register). unsigned getSplatIdxForPPCMnemonics(SDNode *N, unsigned EltSize, SelectionDAG &DAG); /// get_VSPLTI_elt - If this is a build_vector of constants which can be /// formed by using a vspltis[bhw] instruction of the specified element /// size, return the constant being splatted. The ByteSize field indicates /// the number of bytes of each element [124] -> [bhw]. SDValue get_VSPLTI_elt(SDNode *N, unsigned ByteSize, SelectionDAG &DAG); /// If this is a qvaligni shuffle mask, return the shift /// amount, otherwise return -1. int isQVALIGNIShuffleMask(SDNode *N); } // end namespace PPC class PPCTargetLowering : public TargetLowering { const PPCSubtarget &Subtarget; public: explicit PPCTargetLowering(const PPCTargetMachine &TM, const PPCSubtarget &STI); /// getTargetNodeName() - This method returns the name of a target specific /// DAG node. const char *getTargetNodeName(unsigned Opcode) const override; bool isSelectSupported(SelectSupportKind Kind) const override { // PowerPC does not support scalar condition selects on vectors. return (Kind != SelectSupportKind::ScalarCondVectorVal); } /// getPreferredVectorAction - The code we generate when vector types are /// legalized by promoting the integer element type is often much worse /// than code we generate if we widen the type for applicable vector types. /// The issue with promoting is that the vector is scalaraized, individual /// elements promoted and then the vector is rebuilt. So say we load a pair /// of v4i8's and shuffle them. This will turn into a mess of 8 extending /// loads, moves back into VSR's (or memory ops if we don't have moves) and /// then the VPERM for the shuffle. All in all a very slow sequence. TargetLoweringBase::LegalizeTypeAction getPreferredVectorAction(MVT VT) const override { if (VT.getScalarSizeInBits() % 8 == 0) return TypeWidenVector; return TargetLoweringBase::getPreferredVectorAction(VT); } bool useSoftFloat() const override; bool hasSPE() const; MVT getScalarShiftAmountTy(const DataLayout &, EVT) const override { return MVT::i32; } bool isCheapToSpeculateCttz() const override { return true; } bool isCheapToSpeculateCtlz() const override { return true; } bool isCtlzFast() const override { return true; } bool isEqualityCmpFoldedWithSignedCmp() const override { return false; } bool hasAndNotCompare(SDValue) const override { return true; } bool preferIncOfAddToSubOfNot(EVT VT) const override; bool convertSetCCLogicToBitwiseLogic(EVT VT) const override { return VT.isScalarInteger(); } bool supportSplitCSR(MachineFunction *MF) const override { return MF->getFunction().getCallingConv() == CallingConv::CXX_FAST_TLS && MF->getFunction().hasFnAttribute(Attribute::NoUnwind); } void initializeSplitCSR(MachineBasicBlock *Entry) const override; void insertCopiesSplitCSR( MachineBasicBlock *Entry, const SmallVectorImpl &Exits) const override; /// getSetCCResultType - Return the ISD::SETCC ValueType EVT getSetCCResultType(const DataLayout &DL, LLVMContext &Context, EVT VT) const override; /// Return true if target always beneficiates from combining into FMA for a /// given value type. This must typically return false on targets where FMA /// takes more cycles to execute than FADD. bool enableAggressiveFMAFusion(EVT VT) const override; /// getPreIndexedAddressParts - returns true by value, base pointer and /// offset pointer and addressing mode by reference if the node's address /// can be legally represented as pre-indexed load / store address. bool getPreIndexedAddressParts(SDNode *N, SDValue &Base, SDValue &Offset, ISD::MemIndexedMode &AM, SelectionDAG &DAG) const override; /// SelectAddressEVXRegReg - Given the specified addressed, check to see if /// it can be more efficiently represented as [r+imm]. bool SelectAddressEVXRegReg(SDValue N, SDValue &Base, SDValue &Index, SelectionDAG &DAG) const; /// SelectAddressRegReg - Given the specified addressed, check to see if it /// can be more efficiently represented as [r+imm]. If \p EncodingAlignment /// is non-zero, only accept displacement which is not suitable for [r+imm]. /// Returns false if it can be represented by [r+imm], which are preferred. bool SelectAddressRegReg(SDValue N, SDValue &Base, SDValue &Index, SelectionDAG &DAG, unsigned EncodingAlignment = 0) const; /// SelectAddressRegImm - Returns true if the address N can be represented /// by a base register plus a signed 16-bit displacement [r+imm], and if it /// is not better represented as reg+reg. If \p EncodingAlignment is /// non-zero, only accept displacements suitable for instruction encoding /// requirement, i.e. multiples of 4 for DS form. bool SelectAddressRegImm(SDValue N, SDValue &Disp, SDValue &Base, SelectionDAG &DAG, unsigned EncodingAlignment) const; /// SelectAddressRegRegOnly - Given the specified addressed, force it to be /// represented as an indexed [r+r] operation. bool SelectAddressRegRegOnly(SDValue N, SDValue &Base, SDValue &Index, SelectionDAG &DAG) const; Sched::Preference getSchedulingPreference(SDNode *N) const override; /// LowerOperation - Provide custom lowering hooks for some operations. /// SDValue LowerOperation(SDValue Op, SelectionDAG &DAG) const override; /// ReplaceNodeResults - Replace the results of node with an illegal result /// type with new values built out of custom code. /// void ReplaceNodeResults(SDNode *N, SmallVectorImpl&Results, SelectionDAG &DAG) const override; SDValue expandVSXLoadForLE(SDNode *N, DAGCombinerInfo &DCI) const; SDValue expandVSXStoreForLE(SDNode *N, DAGCombinerInfo &DCI) const; SDValue PerformDAGCombine(SDNode *N, DAGCombinerInfo &DCI) const override; SDValue BuildSDIVPow2(SDNode *N, const APInt &Divisor, SelectionDAG &DAG, SmallVectorImpl &Created) const override; Register getRegisterByName(const char* RegName, LLT VT, const MachineFunction &MF) const override; void computeKnownBitsForTargetNode(const SDValue Op, KnownBits &Known, const APInt &DemandedElts, const SelectionDAG &DAG, unsigned Depth = 0) const override; Align getPrefLoopAlignment(MachineLoop *ML) const override; bool shouldInsertFencesForAtomic(const Instruction *I) const override { return true; } Instruction *emitLeadingFence(IRBuilder<> &Builder, Instruction *Inst, AtomicOrdering Ord) const override; Instruction *emitTrailingFence(IRBuilder<> &Builder, Instruction *Inst, AtomicOrdering Ord) const override; MachineBasicBlock * EmitInstrWithCustomInserter(MachineInstr &MI, MachineBasicBlock *MBB) const override; MachineBasicBlock *EmitAtomicBinary(MachineInstr &MI, MachineBasicBlock *MBB, unsigned AtomicSize, unsigned BinOpcode, unsigned CmpOpcode = 0, unsigned CmpPred = 0) const; MachineBasicBlock *EmitPartwordAtomicBinary(MachineInstr &MI, MachineBasicBlock *MBB, bool is8bit, unsigned Opcode, unsigned CmpOpcode = 0, unsigned CmpPred = 0) const; MachineBasicBlock *emitEHSjLjSetJmp(MachineInstr &MI, MachineBasicBlock *MBB) const; MachineBasicBlock *emitEHSjLjLongJmp(MachineInstr &MI, MachineBasicBlock *MBB) const; ConstraintType getConstraintType(StringRef Constraint) const override; /// Examine constraint string and operand type and determine a weight value. /// The operand object must already have been set up with the operand type. ConstraintWeight getSingleConstraintMatchWeight( AsmOperandInfo &info, const char *constraint) const override; std::pair getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI, StringRef Constraint, MVT VT) const override; /// getByValTypeAlignment - Return the desired alignment for ByVal aggregate /// function arguments in the caller parameter area. This is the actual /// alignment, not its logarithm. unsigned getByValTypeAlignment(Type *Ty, const DataLayout &DL) const override; /// LowerAsmOperandForConstraint - Lower the specified operand into the Ops /// vector. If it is invalid, don't add anything to Ops. void LowerAsmOperandForConstraint(SDValue Op, std::string &Constraint, std::vector &Ops, SelectionDAG &DAG) const override; unsigned getInlineAsmMemConstraint(StringRef ConstraintCode) const override { if (ConstraintCode == "es") return InlineAsm::Constraint_es; else if (ConstraintCode == "o") return InlineAsm::Constraint_o; else if (ConstraintCode == "Q") return InlineAsm::Constraint_Q; else if (ConstraintCode == "Z") return InlineAsm::Constraint_Z; else if (ConstraintCode == "Zy") return InlineAsm::Constraint_Zy; return TargetLowering::getInlineAsmMemConstraint(ConstraintCode); } /// isLegalAddressingMode - Return true if the addressing mode represented /// by AM is legal for this target, for a load/store of the specified type. bool isLegalAddressingMode(const DataLayout &DL, const AddrMode &AM, Type *Ty, unsigned AS, Instruction *I = nullptr) const override; /// isLegalICmpImmediate - Return true if the specified immediate is legal /// icmp immediate, that is the target has icmp instructions which can /// compare a register against the immediate without having to materialize /// the immediate into a register. bool isLegalICmpImmediate(int64_t Imm) const override; /// isLegalAddImmediate - Return true if the specified immediate is legal /// add immediate, that is the target has add instructions which can /// add a register and the immediate without having to materialize /// the immediate into a register. bool isLegalAddImmediate(int64_t Imm) const override; /// isTruncateFree - Return true if it's free to truncate a value of /// type Ty1 to type Ty2. e.g. On PPC it's free to truncate a i64 value in /// register X1 to i32 by referencing its sub-register R1. bool isTruncateFree(Type *Ty1, Type *Ty2) const override; bool isTruncateFree(EVT VT1, EVT VT2) const override; bool isZExtFree(SDValue Val, EVT VT2) const override; bool isFPExtFree(EVT DestVT, EVT SrcVT) const override; /// Returns true if it is beneficial to convert a load of a constant /// to just the constant itself. bool shouldConvertConstantLoadToIntImm(const APInt &Imm, Type *Ty) const override; bool convertSelectOfConstantsToMath(EVT VT) const override { return true; } bool isDesirableToTransformToIntegerOp(unsigned Opc, EVT VT) const override { // Only handle float load/store pair because float(fpr) load/store // instruction has more cycles than integer(gpr) load/store in PPC. if (Opc != ISD::LOAD && Opc != ISD::STORE) return false; if (VT != MVT::f32 && VT != MVT::f64) return false; return true; } // Returns true if the address of the global is stored in TOC entry. bool isAccessedAsGotIndirect(SDValue N) const; bool isOffsetFoldingLegal(const GlobalAddressSDNode *GA) const override; bool getTgtMemIntrinsic(IntrinsicInfo &Info, const CallInst &I, MachineFunction &MF, unsigned Intrinsic) const override; /// It returns EVT::Other if the type should be determined using generic /// target-independent logic. EVT getOptimalMemOpType(const MemOp &Op, const AttributeList &FuncAttributes) const override; /// Is unaligned memory access allowed for the given type, and is it fast /// relative to software emulation. bool allowsMisalignedMemoryAccesses( EVT VT, unsigned AddrSpace, unsigned Align = 1, MachineMemOperand::Flags Flags = MachineMemOperand::MONone, bool *Fast = nullptr) const override; /// isFMAFasterThanFMulAndFAdd - Return true if an FMA operation is faster /// than a pair of fmul and fadd instructions. fmuladd intrinsics will be /// expanded to FMAs when this method returns true, otherwise fmuladd is /// expanded to fmul + fadd. bool isFMAFasterThanFMulAndFAdd(const MachineFunction &MF, EVT VT) const override; bool isFMAFasterThanFMulAndFAdd(const Function &F, Type *Ty) const override; /// isProfitableToHoist - Check if it is profitable to hoist instruction /// \p I to its dominator block. /// For example, it is not profitable if \p I and it's only user can form a /// FMA instruction, because Powerpc prefers FMADD. bool isProfitableToHoist(Instruction *I) const override; const MCPhysReg *getScratchRegisters(CallingConv::ID CC) const override; // Should we expand the build vector with shuffles? bool shouldExpandBuildVectorWithShuffles(EVT VT, unsigned DefinedValues) const override; /// createFastISel - This method returns a target-specific FastISel object, /// or null if the target does not support "fast" instruction selection. FastISel *createFastISel(FunctionLoweringInfo &FuncInfo, const TargetLibraryInfo *LibInfo) const override; /// Returns true if an argument of type Ty needs to be passed in a /// contiguous block of registers in calling convention CallConv. bool functionArgumentNeedsConsecutiveRegisters( Type *Ty, CallingConv::ID CallConv, bool isVarArg) const override { // We support any array type as "consecutive" block in the parameter // save area. The element type defines the alignment requirement and // whether the argument should go in GPRs, FPRs, or VRs if available. // // Note that clang uses this capability both to implement the ELFv2 // homogeneous float/vector aggregate ABI, and to avoid having to use // "byval" when passing aggregates that might fully fit in registers. return Ty->isArrayTy(); } /// If a physical register, this returns the register that receives the /// exception address on entry to an EH pad. unsigned getExceptionPointerRegister(const Constant *PersonalityFn) const override; /// If a physical register, this returns the register that receives the /// exception typeid on entry to a landing pad. unsigned getExceptionSelectorRegister(const Constant *PersonalityFn) const override; /// Override to support customized stack guard loading. bool useLoadStackGuardNode() const override; void insertSSPDeclarations(Module &M) const override; bool isFPImmLegal(const APFloat &Imm, EVT VT, bool ForCodeSize) const override; unsigned getJumpTableEncoding() const override; bool isJumpTableRelative() const override; SDValue getPICJumpTableRelocBase(SDValue Table, SelectionDAG &DAG) const override; const MCExpr *getPICJumpTableRelocBaseExpr(const MachineFunction *MF, unsigned JTI, MCContext &Ctx) const override; /// Structure that collects some common arguments that get passed around /// between the functions for call lowering. struct CallFlags { const CallingConv::ID CallConv; const bool IsTailCall : 1; const bool IsVarArg : 1; const bool IsPatchPoint : 1; const bool IsIndirect : 1; const bool HasNest : 1; CallFlags(CallingConv::ID CC, bool IsTailCall, bool IsVarArg, bool IsPatchPoint, bool IsIndirect, bool HasNest) : CallConv(CC), IsTailCall(IsTailCall), IsVarArg(IsVarArg), IsPatchPoint(IsPatchPoint), IsIndirect(IsIndirect), HasNest(HasNest) {} }; private: struct ReuseLoadInfo { SDValue Ptr; SDValue Chain; SDValue ResChain; MachinePointerInfo MPI; bool IsDereferenceable = false; bool IsInvariant = false; Align Alignment; AAMDNodes AAInfo; const MDNode *Ranges = nullptr; ReuseLoadInfo() = default; MachineMemOperand::Flags MMOFlags() const { MachineMemOperand::Flags F = MachineMemOperand::MONone; if (IsDereferenceable) F |= MachineMemOperand::MODereferenceable; if (IsInvariant) F |= MachineMemOperand::MOInvariant; return F; } }; bool isNoopAddrSpaceCast(unsigned SrcAS, unsigned DestAS) const override { // Addrspacecasts are always noops. return true; } bool canReuseLoadAddress(SDValue Op, EVT MemVT, ReuseLoadInfo &RLI, SelectionDAG &DAG, ISD::LoadExtType ET = ISD::NON_EXTLOAD) const; void spliceIntoChain(SDValue ResChain, SDValue NewResChain, SelectionDAG &DAG) const; void LowerFP_TO_INTForReuse(SDValue Op, ReuseLoadInfo &RLI, SelectionDAG &DAG, const SDLoc &dl) const; SDValue LowerFP_TO_INTDirectMove(SDValue Op, SelectionDAG &DAG, const SDLoc &dl) const; bool directMoveIsProfitable(const SDValue &Op) const; SDValue LowerINT_TO_FPDirectMove(SDValue Op, SelectionDAG &DAG, const SDLoc &dl) const; SDValue LowerINT_TO_FPVector(SDValue Op, SelectionDAG &DAG, const SDLoc &dl) const; SDValue LowerTRUNCATEVector(SDValue Op, SelectionDAG &DAG) const; SDValue getFramePointerFrameIndex(SelectionDAG & DAG) const; SDValue getReturnAddrFrameIndex(SelectionDAG & DAG) const; bool IsEligibleForTailCallOptimization(SDValue Callee, CallingConv::ID CalleeCC, bool isVarArg, const SmallVectorImpl &Ins, SelectionDAG& DAG) const; bool IsEligibleForTailCallOptimization_64SVR4( SDValue Callee, CallingConv::ID CalleeCC, ImmutableCallSite CS, bool isVarArg, const SmallVectorImpl &Outs, const SmallVectorImpl &Ins, SelectionDAG& DAG) const; SDValue EmitTailCallLoadFPAndRetAddr(SelectionDAG &DAG, int SPDiff, SDValue Chain, SDValue &LROpOut, SDValue &FPOpOut, const SDLoc &dl) const; SDValue getTOCEntry(SelectionDAG &DAG, const SDLoc &dl, SDValue GA) const; SDValue LowerRETURNADDR(SDValue Op, SelectionDAG &DAG) const; SDValue LowerFRAMEADDR(SDValue Op, SelectionDAG &DAG) const; SDValue LowerConstantPool(SDValue Op, SelectionDAG &DAG) const; SDValue LowerBlockAddress(SDValue Op, SelectionDAG &DAG) const; SDValue LowerGlobalTLSAddress(SDValue Op, SelectionDAG &DAG) const; SDValue LowerGlobalAddress(SDValue Op, SelectionDAG &DAG) const; SDValue LowerJumpTable(SDValue Op, SelectionDAG &DAG) const; SDValue LowerSETCC(SDValue Op, SelectionDAG &DAG) const; SDValue LowerINIT_TRAMPOLINE(SDValue Op, SelectionDAG &DAG) const; SDValue LowerADJUST_TRAMPOLINE(SDValue Op, SelectionDAG &DAG) const; SDValue LowerVASTART(SDValue Op, SelectionDAG &DAG) const; SDValue LowerVAARG(SDValue Op, SelectionDAG &DAG) const; SDValue LowerVACOPY(SDValue Op, SelectionDAG &DAG) const; SDValue LowerSTACKRESTORE(SDValue Op, SelectionDAG &DAG) const; SDValue LowerGET_DYNAMIC_AREA_OFFSET(SDValue Op, SelectionDAG &DAG) const; SDValue LowerDYNAMIC_STACKALLOC(SDValue Op, SelectionDAG &DAG) const; SDValue LowerEH_DWARF_CFA(SDValue Op, SelectionDAG &DAG) const; SDValue LowerLOAD(SDValue Op, SelectionDAG &DAG) const; SDValue LowerSTORE(SDValue Op, SelectionDAG &DAG) const; SDValue LowerTRUNCATE(SDValue Op, SelectionDAG &DAG) const; SDValue LowerSELECT_CC(SDValue Op, SelectionDAG &DAG) const; SDValue LowerFP_TO_INT(SDValue Op, SelectionDAG &DAG, const SDLoc &dl) const; SDValue LowerINT_TO_FP(SDValue Op, SelectionDAG &DAG) const; SDValue LowerFLT_ROUNDS_(SDValue Op, SelectionDAG &DAG) const; SDValue LowerSHL_PARTS(SDValue Op, SelectionDAG &DAG) const; SDValue LowerSRL_PARTS(SDValue Op, SelectionDAG &DAG) const; SDValue LowerSRA_PARTS(SDValue Op, SelectionDAG &DAG) const; SDValue LowerBUILD_VECTOR(SDValue Op, SelectionDAG &DAG) const; SDValue LowerVECTOR_SHUFFLE(SDValue Op, SelectionDAG &DAG) const; SDValue LowerINSERT_VECTOR_ELT(SDValue Op, SelectionDAG &DAG) const; SDValue LowerEXTRACT_VECTOR_ELT(SDValue Op, SelectionDAG &DAG) const; SDValue LowerINTRINSIC_WO_CHAIN(SDValue Op, SelectionDAG &DAG) const; SDValue LowerINTRINSIC_VOID(SDValue Op, SelectionDAG &DAG) const; SDValue LowerREM(SDValue Op, SelectionDAG &DAG) const; SDValue LowerBSWAP(SDValue Op, SelectionDAG &DAG) const; SDValue LowerATOMIC_CMP_SWAP(SDValue Op, SelectionDAG &DAG) const; SDValue LowerSCALAR_TO_VECTOR(SDValue Op, SelectionDAG &DAG) const; SDValue LowerSIGN_EXTEND_INREG(SDValue Op, SelectionDAG &DAG) const; SDValue LowerMUL(SDValue Op, SelectionDAG &DAG) const; SDValue LowerABS(SDValue Op, SelectionDAG &DAG) const; SDValue LowerFP_EXTEND(SDValue Op, SelectionDAG &DAG) const; SDValue LowerVectorLoad(SDValue Op, SelectionDAG &DAG) const; SDValue LowerVectorStore(SDValue Op, SelectionDAG &DAG) const; SDValue LowerCallResult(SDValue Chain, SDValue InFlag, CallingConv::ID CallConv, bool isVarArg, const SmallVectorImpl &Ins, const SDLoc &dl, SelectionDAG &DAG, SmallVectorImpl &InVals) const; SDValue FinishCall(CallFlags CFlags, const SDLoc &dl, SelectionDAG &DAG, SmallVector, 8> &RegsToPass, SDValue InFlag, SDValue Chain, SDValue CallSeqStart, SDValue &Callee, int SPDiff, unsigned NumBytes, const SmallVectorImpl &Ins, SmallVectorImpl &InVals, ImmutableCallSite CS) const; SDValue LowerFormalArguments(SDValue Chain, CallingConv::ID CallConv, bool isVarArg, const SmallVectorImpl &Ins, const SDLoc &dl, SelectionDAG &DAG, SmallVectorImpl &InVals) const override; SDValue LowerCall(TargetLowering::CallLoweringInfo &CLI, SmallVectorImpl &InVals) const override; bool CanLowerReturn(CallingConv::ID CallConv, MachineFunction &MF, bool isVarArg, const SmallVectorImpl &Outs, LLVMContext &Context) const override; SDValue LowerReturn(SDValue Chain, CallingConv::ID CallConv, bool isVarArg, const SmallVectorImpl &Outs, const SmallVectorImpl &OutVals, const SDLoc &dl, SelectionDAG &DAG) const override; SDValue extendArgForPPC64(ISD::ArgFlagsTy Flags, EVT ObjectVT, SelectionDAG &DAG, SDValue ArgVal, const SDLoc &dl) const; SDValue LowerFormalArguments_AIX( SDValue Chain, CallingConv::ID CallConv, bool isVarArg, const SmallVectorImpl &Ins, const SDLoc &dl, SelectionDAG &DAG, SmallVectorImpl &InVals) const; SDValue LowerFormalArguments_Darwin( SDValue Chain, CallingConv::ID CallConv, bool isVarArg, const SmallVectorImpl &Ins, const SDLoc &dl, SelectionDAG &DAG, SmallVectorImpl &InVals) const; SDValue LowerFormalArguments_64SVR4( SDValue Chain, CallingConv::ID CallConv, bool isVarArg, const SmallVectorImpl &Ins, const SDLoc &dl, SelectionDAG &DAG, SmallVectorImpl &InVals) const; SDValue LowerFormalArguments_32SVR4( SDValue Chain, CallingConv::ID CallConv, bool isVarArg, const SmallVectorImpl &Ins, const SDLoc &dl, SelectionDAG &DAG, SmallVectorImpl &InVals) const; SDValue createMemcpyOutsideCallSeq(SDValue Arg, SDValue PtrOff, SDValue CallSeqStart, ISD::ArgFlagsTy Flags, SelectionDAG &DAG, const SDLoc &dl) const; SDValue LowerCall_Darwin(SDValue Chain, SDValue Callee, CallFlags CFlags, const SmallVectorImpl &Outs, const SmallVectorImpl &OutVals, const SmallVectorImpl &Ins, const SDLoc &dl, SelectionDAG &DAG, SmallVectorImpl &InVals, ImmutableCallSite CS) const; SDValue LowerCall_64SVR4(SDValue Chain, SDValue Callee, CallFlags CFlags, const SmallVectorImpl &Outs, const SmallVectorImpl &OutVals, const SmallVectorImpl &Ins, const SDLoc &dl, SelectionDAG &DAG, SmallVectorImpl &InVals, ImmutableCallSite CS) const; SDValue LowerCall_32SVR4(SDValue Chain, SDValue Callee, CallFlags CFlags, const SmallVectorImpl &Outs, const SmallVectorImpl &OutVals, const SmallVectorImpl &Ins, const SDLoc &dl, SelectionDAG &DAG, SmallVectorImpl &InVals, ImmutableCallSite CS) const; SDValue LowerCall_AIX(SDValue Chain, SDValue Callee, CallFlags CFlags, const SmallVectorImpl &Outs, const SmallVectorImpl &OutVals, const SmallVectorImpl &Ins, const SDLoc &dl, SelectionDAG &DAG, SmallVectorImpl &InVals, ImmutableCallSite CS) const; SDValue lowerEH_SJLJ_SETJMP(SDValue Op, SelectionDAG &DAG) const; SDValue lowerEH_SJLJ_LONGJMP(SDValue Op, SelectionDAG &DAG) const; SDValue LowerBITCAST(SDValue Op, SelectionDAG &DAG) const; SDValue DAGCombineExtBoolTrunc(SDNode *N, DAGCombinerInfo &DCI) const; SDValue DAGCombineBuildVector(SDNode *N, DAGCombinerInfo &DCI) const; SDValue DAGCombineTruncBoolExt(SDNode *N, DAGCombinerInfo &DCI) const; SDValue combineStoreFPToInt(SDNode *N, DAGCombinerInfo &DCI) const; SDValue combineFPToIntToFP(SDNode *N, DAGCombinerInfo &DCI) const; SDValue combineSHL(SDNode *N, DAGCombinerInfo &DCI) const; SDValue combineSRA(SDNode *N, DAGCombinerInfo &DCI) const; SDValue combineSRL(SDNode *N, DAGCombinerInfo &DCI) const; SDValue combineMUL(SDNode *N, DAGCombinerInfo &DCI) const; SDValue combineADD(SDNode *N, DAGCombinerInfo &DCI) const; SDValue combineTRUNCATE(SDNode *N, DAGCombinerInfo &DCI) const; SDValue combineSetCC(SDNode *N, DAGCombinerInfo &DCI) const; SDValue combineABS(SDNode *N, DAGCombinerInfo &DCI) const; SDValue combineVSelect(SDNode *N, DAGCombinerInfo &DCI) const; SDValue combineVReverseMemOP(ShuffleVectorSDNode *SVN, LSBaseSDNode *LSBase, DAGCombinerInfo &DCI) const; /// ConvertSETCCToSubtract - looks at SETCC that compares ints. It replaces /// SETCC with integer subtraction when (1) there is a legal way of doing it /// (2) keeping the result of comparison in GPR has performance benefit. SDValue ConvertSETCCToSubtract(SDNode *N, DAGCombinerInfo &DCI) const; SDValue getSqrtEstimate(SDValue Operand, SelectionDAG &DAG, int Enabled, int &RefinementSteps, bool &UseOneConstNR, bool Reciprocal) const override; SDValue getRecipEstimate(SDValue Operand, SelectionDAG &DAG, int Enabled, int &RefinementSteps) const override; unsigned combineRepeatedFPDivisors() const override; SDValue combineElementTruncationToVectorTruncation(SDNode *N, DAGCombinerInfo &DCI) const; /// lowerToVINSERTH - Return the SDValue if this VECTOR_SHUFFLE can be /// handled by the VINSERTH instruction introduced in ISA 3.0. This is /// essentially any shuffle of v8i16 vectors that just inserts one element /// from one vector into the other. SDValue lowerToVINSERTH(ShuffleVectorSDNode *N, SelectionDAG &DAG) const; /// lowerToVINSERTB - Return the SDValue if this VECTOR_SHUFFLE can be /// handled by the VINSERTB instruction introduced in ISA 3.0. This is /// essentially v16i8 vector version of VINSERTH. SDValue lowerToVINSERTB(ShuffleVectorSDNode *N, SelectionDAG &DAG) const; // Return whether the call instruction can potentially be optimized to a // tail call. This will cause the optimizers to attempt to move, or // duplicate return instructions to help enable tail call optimizations. bool mayBeEmittedAsTailCall(const CallInst *CI) const override; bool hasBitPreservingFPLogic(EVT VT) const override; bool isMaskAndCmp0FoldingBeneficial(const Instruction &AndI) const override; }; // end class PPCTargetLowering namespace PPC { FastISel *createFastISel(FunctionLoweringInfo &FuncInfo, const TargetLibraryInfo *LibInfo); } // end namespace PPC bool isIntS16Immediate(SDNode *N, int16_t &Imm); bool isIntS16Immediate(SDValue Op, int16_t &Imm); } // end namespace llvm #endif // LLVM_TARGET_POWERPC_PPC32ISELLOWERING_H diff --git a/llvm/lib/Target/PowerPC/PPCInstr64Bit.td b/llvm/lib/Target/PowerPC/PPCInstr64Bit.td index 542225ca0d10..073436191dd9 100644 --- a/llvm/lib/Target/PowerPC/PPCInstr64Bit.td +++ b/llvm/lib/Target/PowerPC/PPCInstr64Bit.td @@ -1,1492 +1,1506 @@ //===-- PPCInstr64Bit.td - The PowerPC 64-bit Support ------*- tablegen -*-===// // // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. // See https://llvm.org/LICENSE.txt for license information. // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // // This file describes the PowerPC 64-bit instructions. These patterns are used // both when in ppc64 mode and when in "use 64-bit extensions in 32-bit" mode. // //===----------------------------------------------------------------------===// //===----------------------------------------------------------------------===// // 64-bit operands. // def s16imm64 : Operand { let PrintMethod = "printS16ImmOperand"; let EncoderMethod = "getImm16Encoding"; let ParserMatchClass = PPCS16ImmAsmOperand; let DecoderMethod = "decodeSImmOperand<16>"; } def u16imm64 : Operand { let PrintMethod = "printU16ImmOperand"; let EncoderMethod = "getImm16Encoding"; let ParserMatchClass = PPCU16ImmAsmOperand; let DecoderMethod = "decodeUImmOperand<16>"; } def s17imm64 : Operand { // This operand type is used for addis/lis to allow the assembler parser // to accept immediates in the range -65536..65535 for compatibility with // the GNU assembler. The operand is treated as 16-bit otherwise. let PrintMethod = "printS16ImmOperand"; let EncoderMethod = "getImm16Encoding"; let ParserMatchClass = PPCS17ImmAsmOperand; let DecoderMethod = "decodeSImmOperand<16>"; } def tocentry : Operand { let MIOperandInfo = (ops i64imm:$imm); } def tlsreg : Operand { let EncoderMethod = "getTLSRegEncoding"; let ParserMatchClass = PPCTLSRegOperand; } def tlsgd : Operand {} def tlscall : Operand { let PrintMethod = "printTLSCall"; let MIOperandInfo = (ops calltarget:$func, tlsgd:$sym); let EncoderMethod = "getTLSCallEncoding"; } //===----------------------------------------------------------------------===// // 64-bit transformation functions. // def SHL64 : SDNodeXFormgetZExtValue(), SDLoc(N)); }]>; def SRL64 : SDNodeXFormgetZExtValue() ? getI32Imm(64 - N->getZExtValue(), SDLoc(N)) : getI32Imm(0, SDLoc(N)); }]>; //===----------------------------------------------------------------------===// // Calls. // let Interpretation64Bit = 1, isCodeGenOnly = 1 in { let isTerminator = 1, isBarrier = 1, PPC970_Unit = 7 in { let isReturn = 1, isPredicable = 1, Uses = [LR8, RM] in def BLR8 : XLForm_2_ext<19, 16, 20, 0, 0, (outs), (ins), "blr", IIC_BrB, [(retflag)]>, Requires<[In64BitMode]>; let isBranch = 1, isIndirectBranch = 1, Uses = [CTR8] in { let isPredicable = 1 in def BCTR8 : XLForm_2_ext<19, 528, 20, 0, 0, (outs), (ins), "bctr", IIC_BrB, []>, Requires<[In64BitMode]>; def BCCCTR8 : XLForm_2_br<19, 528, 0, (outs), (ins pred:$cond), "b${cond:cc}ctr${cond:pm} ${cond:reg}", IIC_BrB, []>, Requires<[In64BitMode]>; def BCCTR8 : XLForm_2_br2<19, 528, 12, 0, (outs), (ins crbitrc:$bi), "bcctr 12, $bi, 0", IIC_BrB, []>, Requires<[In64BitMode]>; def BCCTR8n : XLForm_2_br2<19, 528, 4, 0, (outs), (ins crbitrc:$bi), "bcctr 4, $bi, 0", IIC_BrB, []>, Requires<[In64BitMode]>; } } let Defs = [LR8] in def MovePCtoLR8 : PPCEmitTimePseudo<(outs), (ins), "#MovePCtoLR8", []>, PPC970_Unit_BRU; let isBranch = 1, isTerminator = 1, hasCtrlDep = 1, PPC970_Unit = 7 in { let Defs = [CTR8], Uses = [CTR8] in { def BDZ8 : BForm_1<16, 18, 0, 0, (outs), (ins condbrtarget:$dst), "bdz $dst">; def BDNZ8 : BForm_1<16, 16, 0, 0, (outs), (ins condbrtarget:$dst), "bdnz $dst">; } let isReturn = 1, Defs = [CTR8], Uses = [CTR8, LR8, RM] in { def BDZLR8 : XLForm_2_ext<19, 16, 18, 0, 0, (outs), (ins), "bdzlr", IIC_BrB, []>; def BDNZLR8 : XLForm_2_ext<19, 16, 16, 0, 0, (outs), (ins), "bdnzlr", IIC_BrB, []>; } } let isCall = 1, PPC970_Unit = 7, Defs = [LR8] in { // Convenient aliases for call instructions let Uses = [RM] in { def BL8 : IForm<18, 0, 1, (outs), (ins calltarget:$func), "bl $func", IIC_BrB, []>; // See Pat patterns below. def BL8_TLS : IForm<18, 0, 1, (outs), (ins tlscall:$func), "bl $func", IIC_BrB, []>; def BLA8 : IForm<18, 1, 1, (outs), (ins abscalltarget:$func), "bla $func", IIC_BrB, [(PPCcall (i64 imm:$func))]>; } let Uses = [RM], isCodeGenOnly = 1 in { def BL8_NOP : IForm_and_DForm_4_zero<18, 0, 1, 24, (outs), (ins calltarget:$func), "bl $func\n\tnop", IIC_BrB, []>; def BL8_NOP_TLS : IForm_and_DForm_4_zero<18, 0, 1, 24, (outs), (ins tlscall:$func), "bl $func\n\tnop", IIC_BrB, []>; def BLA8_NOP : IForm_and_DForm_4_zero<18, 1, 1, 24, (outs), (ins abscalltarget:$func), "bla $func\n\tnop", IIC_BrB, [(PPCcall_nop (i64 imm:$func))]>; + let Predicates = [PCRelativeMemops] in { + // BL8_NOTOC means that the caller does not use the TOC pointer and if + // it does use R2 then it is just a caller saved register. Therefore it is + // safe to emit only the bl and not the nop for this instruction. The + // linker will not try to restore R2 after the call. + def BL8_NOTOC : IForm_and_DForm_4_zero<18, 0, 1, 24, (outs), + (ins calltarget:$func), + "bl $func", IIC_BrB, []>; + } } let Uses = [CTR8, RM] in { let isPredicable = 1 in def BCTRL8 : XLForm_2_ext<19, 528, 20, 0, 1, (outs), (ins), "bctrl", IIC_BrB, [(PPCbctrl)]>, Requires<[In64BitMode]>; let isCodeGenOnly = 1 in { def BCCCTRL8 : XLForm_2_br<19, 528, 1, (outs), (ins pred:$cond), "b${cond:cc}ctrl${cond:pm} ${cond:reg}", IIC_BrB, []>, Requires<[In64BitMode]>; def BCCTRL8 : XLForm_2_br2<19, 528, 12, 1, (outs), (ins crbitrc:$bi), "bcctrl 12, $bi, 0", IIC_BrB, []>, Requires<[In64BitMode]>; def BCCTRL8n : XLForm_2_br2<19, 528, 4, 1, (outs), (ins crbitrc:$bi), "bcctrl 4, $bi, 0", IIC_BrB, []>, Requires<[In64BitMode]>; } } } let isCall = 1, PPC970_Unit = 7, isCodeGenOnly = 1, Defs = [LR8, X2], Uses = [CTR8, RM], RST = 2 in { def BCTRL8_LDinto_toc : XLForm_2_ext_and_DSForm_1<19, 528, 20, 0, 1, 58, 0, (outs), (ins memrix:$src), "bctrl\n\tld 2, $src", IIC_BrB, [(PPCbctrl_load_toc iaddrX4:$src)]>, Requires<[In64BitMode]>; } } // Interpretation64Bit // FIXME: Duplicating this for the asm parser should be unnecessary, but the // previous definition must be marked as CodeGen only to prevent decoding // conflicts. let Interpretation64Bit = 1, isAsmParserOnly = 1 in let isCall = 1, PPC970_Unit = 7, Defs = [LR8], Uses = [RM] in def BL8_TLS_ : IForm<18, 0, 1, (outs), (ins tlscall:$func), "bl $func", IIC_BrB, []>; // Calls def : Pat<(PPCcall (i64 tglobaladdr:$dst)), (BL8 tglobaladdr:$dst)>; def : Pat<(PPCcall_nop (i64 tglobaladdr:$dst)), (BL8_NOP tglobaladdr:$dst)>; def : Pat<(PPCcall (i64 texternalsym:$dst)), (BL8 texternalsym:$dst)>; def : Pat<(PPCcall_nop (i64 texternalsym:$dst)), (BL8_NOP texternalsym:$dst)>; +def : Pat<(PPCcall_notoc (i64 tglobaladdr:$dst)), + (BL8_NOTOC tglobaladdr:$dst)>; +def : Pat<(PPCcall_notoc (i64 texternalsym:$dst)), + (BL8_NOTOC texternalsym:$dst)>; + // Calls for AIX def : Pat<(PPCcall (i64 mcsym:$dst)), (BL8 mcsym:$dst)>; def : Pat<(PPCcall_nop (i64 mcsym:$dst)), (BL8_NOP mcsym:$dst)>; // Atomic operations // FIXME: some of these might be used with constant operands. This will result // in constant materialization instructions that may be redundant. We currently // clean this up in PPCMIPeephole with calls to // PPCInstrInfo::convertToImmediateForm() but we should probably not emit them // in the first place. let Defs = [CR0] in { def ATOMIC_LOAD_ADD_I64 : PPCCustomInserterPseudo< (outs g8rc:$dst), (ins memrr:$ptr, g8rc:$incr), "#ATOMIC_LOAD_ADD_I64", [(set i64:$dst, (atomic_load_add_64 xoaddr:$ptr, i64:$incr))]>; def ATOMIC_LOAD_SUB_I64 : PPCCustomInserterPseudo< (outs g8rc:$dst), (ins memrr:$ptr, g8rc:$incr), "#ATOMIC_LOAD_SUB_I64", [(set i64:$dst, (atomic_load_sub_64 xoaddr:$ptr, i64:$incr))]>; def ATOMIC_LOAD_OR_I64 : PPCCustomInserterPseudo< (outs g8rc:$dst), (ins memrr:$ptr, g8rc:$incr), "#ATOMIC_LOAD_OR_I64", [(set i64:$dst, (atomic_load_or_64 xoaddr:$ptr, i64:$incr))]>; def ATOMIC_LOAD_XOR_I64 : PPCCustomInserterPseudo< (outs g8rc:$dst), (ins memrr:$ptr, g8rc:$incr), "#ATOMIC_LOAD_XOR_I64", [(set i64:$dst, (atomic_load_xor_64 xoaddr:$ptr, i64:$incr))]>; def ATOMIC_LOAD_AND_I64 : PPCCustomInserterPseudo< (outs g8rc:$dst), (ins memrr:$ptr, g8rc:$incr), "#ATOMIC_LOAD_AND_i64", [(set i64:$dst, (atomic_load_and_64 xoaddr:$ptr, i64:$incr))]>; def ATOMIC_LOAD_NAND_I64 : PPCCustomInserterPseudo< (outs g8rc:$dst), (ins memrr:$ptr, g8rc:$incr), "#ATOMIC_LOAD_NAND_I64", [(set i64:$dst, (atomic_load_nand_64 xoaddr:$ptr, i64:$incr))]>; def ATOMIC_LOAD_MIN_I64 : PPCCustomInserterPseudo< (outs g8rc:$dst), (ins memrr:$ptr, g8rc:$incr), "#ATOMIC_LOAD_MIN_I64", [(set i64:$dst, (atomic_load_min_64 xoaddr:$ptr, i64:$incr))]>; def ATOMIC_LOAD_MAX_I64 : PPCCustomInserterPseudo< (outs g8rc:$dst), (ins memrr:$ptr, g8rc:$incr), "#ATOMIC_LOAD_MAX_I64", [(set i64:$dst, (atomic_load_max_64 xoaddr:$ptr, i64:$incr))]>; def ATOMIC_LOAD_UMIN_I64 : PPCCustomInserterPseudo< (outs g8rc:$dst), (ins memrr:$ptr, g8rc:$incr), "#ATOMIC_LOAD_UMIN_I64", [(set i64:$dst, (atomic_load_umin_64 xoaddr:$ptr, i64:$incr))]>; def ATOMIC_LOAD_UMAX_I64 : PPCCustomInserterPseudo< (outs g8rc:$dst), (ins memrr:$ptr, g8rc:$incr), "#ATOMIC_LOAD_UMAX_I64", [(set i64:$dst, (atomic_load_umax_64 xoaddr:$ptr, i64:$incr))]>; def ATOMIC_CMP_SWAP_I64 : PPCCustomInserterPseudo< (outs g8rc:$dst), (ins memrr:$ptr, g8rc:$old, g8rc:$new), "#ATOMIC_CMP_SWAP_I64", [(set i64:$dst, (atomic_cmp_swap_64 xoaddr:$ptr, i64:$old, i64:$new))]>; def ATOMIC_SWAP_I64 : PPCCustomInserterPseudo< (outs g8rc:$dst), (ins memrr:$ptr, g8rc:$new), "#ATOMIC_SWAP_I64", [(set i64:$dst, (atomic_swap_64 xoaddr:$ptr, i64:$new))]>; } // Instructions to support atomic operations let mayLoad = 1, hasSideEffects = 0 in { def LDARX : XForm_1_memOp<31, 84, (outs g8rc:$rD), (ins memrr:$ptr), "ldarx $rD, $ptr", IIC_LdStLDARX, []>; // Instruction to support lock versions of atomics // (EH=1 - see Power ISA 2.07 Book II 4.4.2) def LDARXL : XForm_1<31, 84, (outs g8rc:$rD), (ins memrr:$ptr), "ldarx $rD, $ptr, 1", IIC_LdStLDARX, []>, isRecordForm; let hasExtraDefRegAllocReq = 1 in def LDAT : X_RD5_RS5_IM5<31, 614, (outs g8rc:$rD), (ins g8rc:$rA, u5imm:$FC), "ldat $rD, $rA, $FC", IIC_LdStLoad>, isPPC64, Requires<[IsISA3_0]>; } let Defs = [CR0], mayStore = 1, mayLoad = 0, hasSideEffects = 0 in def STDCX : XForm_1_memOp<31, 214, (outs), (ins g8rc:$rS, memrr:$dst), "stdcx. $rS, $dst", IIC_LdStSTDCX, []>, isRecordForm; let mayStore = 1, mayLoad = 0, hasSideEffects = 0 in def STDAT : X_RD5_RS5_IM5<31, 742, (outs), (ins g8rc:$rS, g8rc:$rA, u5imm:$FC), "stdat $rS, $rA, $FC", IIC_LdStStore>, isPPC64, Requires<[IsISA3_0]>; let Interpretation64Bit = 1, isCodeGenOnly = 1 in { let isCall = 1, isTerminator = 1, isReturn = 1, isBarrier = 1, Uses = [RM] in def TCRETURNdi8 :PPCEmitTimePseudo< (outs), (ins calltarget:$dst, i32imm:$offset), "#TC_RETURNd8 $dst $offset", []>; let isCall = 1, isTerminator = 1, isReturn = 1, isBarrier = 1, Uses = [RM] in def TCRETURNai8 :PPCEmitTimePseudo<(outs), (ins abscalltarget:$func, i32imm:$offset), "#TC_RETURNa8 $func $offset", [(PPCtc_return (i64 imm:$func), imm:$offset)]>; let isCall = 1, isTerminator = 1, isReturn = 1, isBarrier = 1, Uses = [RM] in def TCRETURNri8 : PPCEmitTimePseudo<(outs), (ins CTRRC8:$dst, i32imm:$offset), "#TC_RETURNr8 $dst $offset", []>; let isTerminator = 1, isBarrier = 1, PPC970_Unit = 7, isBranch = 1, isIndirectBranch = 1, isCall = 1, isReturn = 1, Uses = [CTR8, RM] in def TAILBCTR8 : XLForm_2_ext<19, 528, 20, 0, 0, (outs), (ins), "bctr", IIC_BrB, []>, Requires<[In64BitMode]>; let isBranch = 1, isTerminator = 1, hasCtrlDep = 1, PPC970_Unit = 7, isBarrier = 1, isCall = 1, isReturn = 1, Uses = [RM] in def TAILB8 : IForm<18, 0, 0, (outs), (ins calltarget:$dst), "b $dst", IIC_BrB, []>; let isBranch = 1, isTerminator = 1, hasCtrlDep = 1, PPC970_Unit = 7, isBarrier = 1, isCall = 1, isReturn = 1, Uses = [RM] in def TAILBA8 : IForm<18, 0, 0, (outs), (ins abscalltarget:$dst), "ba $dst", IIC_BrB, []>; } // Interpretation64Bit def : Pat<(PPCtc_return (i64 tglobaladdr:$dst), imm:$imm), (TCRETURNdi8 tglobaladdr:$dst, imm:$imm)>; def : Pat<(PPCtc_return (i64 texternalsym:$dst), imm:$imm), (TCRETURNdi8 texternalsym:$dst, imm:$imm)>; def : Pat<(PPCtc_return CTRRC8:$dst, imm:$imm), (TCRETURNri8 CTRRC8:$dst, imm:$imm)>; // 64-bit CR instructions let Interpretation64Bit = 1, isCodeGenOnly = 1 in { let hasSideEffects = 0 in { // mtocrf's input needs to be prepared by shifting by an amount dependent // on the cr register selected. Thus, post-ra anti-dep breaking must not // later change that register assignment. let hasExtraDefRegAllocReq = 1 in { def MTOCRF8: XFXForm_5a<31, 144, (outs crbitm:$FXM), (ins g8rc:$ST), "mtocrf $FXM, $ST", IIC_BrMCRX>, PPC970_DGroup_First, PPC970_Unit_CRU; // Similarly to mtocrf, the mask for mtcrf must be prepared in a way that // is dependent on the cr fields being set. def MTCRF8 : XFXForm_5<31, 144, (outs), (ins i32imm:$FXM, g8rc:$rS), "mtcrf $FXM, $rS", IIC_BrMCRX>, PPC970_MicroCode, PPC970_Unit_CRU; } // hasExtraDefRegAllocReq = 1 // mfocrf's input needs to be prepared by shifting by an amount dependent // on the cr register selected. Thus, post-ra anti-dep breaking must not // later change that register assignment. let hasExtraSrcRegAllocReq = 1 in { def MFOCRF8: XFXForm_5a<31, 19, (outs g8rc:$rT), (ins crbitm:$FXM), "mfocrf $rT, $FXM", IIC_SprMFCRF>, PPC970_DGroup_First, PPC970_Unit_CRU; // Similarly to mfocrf, the mask for mfcrf must be prepared in a way that // is dependent on the cr fields being copied. def MFCR8 : XFXForm_3<31, 19, (outs g8rc:$rT), (ins), "mfcr $rT", IIC_SprMFCR>, PPC970_MicroCode, PPC970_Unit_CRU; } // hasExtraSrcRegAllocReq = 1 } // hasSideEffects = 0 // While longjmp is a control-flow barrier (fallthrough isn't allowed), setjmp // is not. let hasSideEffects = 1 in { let Defs = [CTR8] in def EH_SjLj_SetJmp64 : PPCCustomInserterPseudo<(outs gprc:$dst), (ins memr:$buf), "#EH_SJLJ_SETJMP64", [(set i32:$dst, (PPCeh_sjlj_setjmp addr:$buf))]>, Requires<[In64BitMode]>; } let hasSideEffects = 1, isBarrier = 1 in { let isTerminator = 1 in def EH_SjLj_LongJmp64 : PPCCustomInserterPseudo<(outs), (ins memr:$buf), "#EH_SJLJ_LONGJMP64", [(PPCeh_sjlj_longjmp addr:$buf)]>, Requires<[In64BitMode]>; } def MFSPR8 : XFXForm_1<31, 339, (outs g8rc:$RT), (ins i32imm:$SPR), "mfspr $RT, $SPR", IIC_SprMFSPR>; def MTSPR8 : XFXForm_1<31, 467, (outs), (ins i32imm:$SPR, g8rc:$RT), "mtspr $SPR, $RT", IIC_SprMTSPR>; //===----------------------------------------------------------------------===// // 64-bit SPR manipulation instrs. let Uses = [CTR8] in { def MFCTR8 : XFXForm_1_ext<31, 339, 9, (outs g8rc:$rT), (ins), "mfctr $rT", IIC_SprMFSPR>, PPC970_DGroup_First, PPC970_Unit_FXU; } let Pattern = [(PPCmtctr i64:$rS)], Defs = [CTR8] in { def MTCTR8 : XFXForm_7_ext<31, 467, 9, (outs), (ins g8rc:$rS), "mtctr $rS", IIC_SprMTSPR>, PPC970_DGroup_First, PPC970_Unit_FXU; } let hasSideEffects = 1, Defs = [CTR8] in { let Pattern = [(int_set_loop_iterations i64:$rS)] in def MTCTR8loop : XFXForm_7_ext<31, 467, 9, (outs), (ins g8rc:$rS), "mtctr $rS", IIC_SprMTSPR>, PPC970_DGroup_First, PPC970_Unit_FXU; } let Pattern = [(set i64:$rT, readcyclecounter)] in def MFTB8 : XFXForm_1_ext<31, 339, 268, (outs g8rc:$rT), (ins), "mfspr $rT, 268", IIC_SprMFTB>, PPC970_DGroup_First, PPC970_Unit_FXU; // Note that encoding mftb using mfspr is now the preferred form, // and has been since at least ISA v2.03. The mftb instruction has // now been phased out. Using mfspr, however, is known not to work on // the POWER3. let Defs = [X1], Uses = [X1] in def DYNALLOC8 : PPCEmitTimePseudo<(outs g8rc:$result), (ins g8rc:$negsize, memri:$fpsi),"#DYNALLOC8", [(set i64:$result, (PPCdynalloc i64:$negsize, iaddr:$fpsi))]>; def DYNAREAOFFSET8 : PPCEmitTimePseudo<(outs i64imm:$result), (ins memri:$fpsi), "#DYNAREAOFFSET8", [(set i64:$result, (PPCdynareaoffset iaddr:$fpsi))]>; let hasSideEffects = 0 in { let Defs = [LR8] in { def MTLR8 : XFXForm_7_ext<31, 467, 8, (outs), (ins g8rc:$rS), "mtlr $rS", IIC_SprMTSPR>, PPC970_DGroup_First, PPC970_Unit_FXU; } let Uses = [LR8] in { def MFLR8 : XFXForm_1_ext<31, 339, 8, (outs g8rc:$rT), (ins), "mflr $rT", IIC_SprMFSPR>, PPC970_DGroup_First, PPC970_Unit_FXU; } } // Interpretation64Bit } //===----------------------------------------------------------------------===// // Fixed point instructions. // let PPC970_Unit = 1 in { // FXU Operations. let Interpretation64Bit = 1 in { let hasSideEffects = 0 in { let isCodeGenOnly = 1 in { let isReMaterializable = 1, isAsCheapAsAMove = 1, isMoveImm = 1 in { def LI8 : DForm_2_r0<14, (outs g8rc:$rD), (ins s16imm64:$imm), "li $rD, $imm", IIC_IntSimple, [(set i64:$rD, imm64SExt16:$imm)]>; def LIS8 : DForm_2_r0<15, (outs g8rc:$rD), (ins s17imm64:$imm), "lis $rD, $imm", IIC_IntSimple, [(set i64:$rD, imm16ShiftedSExt:$imm)]>; } // Logical ops. let isCommutable = 1 in { defm NAND8: XForm_6r<31, 476, (outs g8rc:$rA), (ins g8rc:$rS, g8rc:$rB), "nand", "$rA, $rS, $rB", IIC_IntSimple, [(set i64:$rA, (not (and i64:$rS, i64:$rB)))]>; defm AND8 : XForm_6r<31, 28, (outs g8rc:$rA), (ins g8rc:$rS, g8rc:$rB), "and", "$rA, $rS, $rB", IIC_IntSimple, [(set i64:$rA, (and i64:$rS, i64:$rB))]>; } // isCommutable defm ANDC8: XForm_6r<31, 60, (outs g8rc:$rA), (ins g8rc:$rS, g8rc:$rB), "andc", "$rA, $rS, $rB", IIC_IntSimple, [(set i64:$rA, (and i64:$rS, (not i64:$rB)))]>; let isCommutable = 1 in { defm OR8 : XForm_6r<31, 444, (outs g8rc:$rA), (ins g8rc:$rS, g8rc:$rB), "or", "$rA, $rS, $rB", IIC_IntSimple, [(set i64:$rA, (or i64:$rS, i64:$rB))]>; defm NOR8 : XForm_6r<31, 124, (outs g8rc:$rA), (ins g8rc:$rS, g8rc:$rB), "nor", "$rA, $rS, $rB", IIC_IntSimple, [(set i64:$rA, (not (or i64:$rS, i64:$rB)))]>; } // isCommutable defm ORC8 : XForm_6r<31, 412, (outs g8rc:$rA), (ins g8rc:$rS, g8rc:$rB), "orc", "$rA, $rS, $rB", IIC_IntSimple, [(set i64:$rA, (or i64:$rS, (not i64:$rB)))]>; let isCommutable = 1 in { defm EQV8 : XForm_6r<31, 284, (outs g8rc:$rA), (ins g8rc:$rS, g8rc:$rB), "eqv", "$rA, $rS, $rB", IIC_IntSimple, [(set i64:$rA, (not (xor i64:$rS, i64:$rB)))]>; defm XOR8 : XForm_6r<31, 316, (outs g8rc:$rA), (ins g8rc:$rS, g8rc:$rB), "xor", "$rA, $rS, $rB", IIC_IntSimple, [(set i64:$rA, (xor i64:$rS, i64:$rB))]>; } // let isCommutable = 1 // Logical ops with immediate. let Defs = [CR0] in { def ANDI8_rec : DForm_4<28, (outs g8rc:$dst), (ins g8rc:$src1, u16imm64:$src2), "andi. $dst, $src1, $src2", IIC_IntGeneral, [(set i64:$dst, (and i64:$src1, immZExt16:$src2))]>, isRecordForm; def ANDIS8_rec : DForm_4<29, (outs g8rc:$dst), (ins g8rc:$src1, u16imm64:$src2), "andis. $dst, $src1, $src2", IIC_IntGeneral, [(set i64:$dst, (and i64:$src1, imm16ShiftedZExt:$src2))]>, isRecordForm; } def ORI8 : DForm_4<24, (outs g8rc:$dst), (ins g8rc:$src1, u16imm64:$src2), "ori $dst, $src1, $src2", IIC_IntSimple, [(set i64:$dst, (or i64:$src1, immZExt16:$src2))]>; def ORIS8 : DForm_4<25, (outs g8rc:$dst), (ins g8rc:$src1, u16imm64:$src2), "oris $dst, $src1, $src2", IIC_IntSimple, [(set i64:$dst, (or i64:$src1, imm16ShiftedZExt:$src2))]>; def XORI8 : DForm_4<26, (outs g8rc:$dst), (ins g8rc:$src1, u16imm64:$src2), "xori $dst, $src1, $src2", IIC_IntSimple, [(set i64:$dst, (xor i64:$src1, immZExt16:$src2))]>; def XORIS8 : DForm_4<27, (outs g8rc:$dst), (ins g8rc:$src1, u16imm64:$src2), "xoris $dst, $src1, $src2", IIC_IntSimple, [(set i64:$dst, (xor i64:$src1, imm16ShiftedZExt:$src2))]>; let isCommutable = 1 in defm ADD8 : XOForm_1rx<31, 266, 0, (outs g8rc:$rT), (ins g8rc:$rA, g8rc:$rB), "add", "$rT, $rA, $rB", IIC_IntSimple, [(set i64:$rT, (add i64:$rA, i64:$rB))]>; // ADD8 has a special form: reg = ADD8(reg, sym@tls) for use by the // initial-exec thread-local storage model. We need to forbid r0 here - // while it works for add just fine, the linker can relax this to local-exec // addi, which won't work for r0. def ADD8TLS : XOForm_1<31, 266, 0, (outs g8rc:$rT), (ins g8rc_nox0:$rA, tlsreg:$rB), "add $rT, $rA, $rB", IIC_IntSimple, [(set i64:$rT, (add i64:$rA, tglobaltlsaddr:$rB))]>; let mayLoad = 1 in { def LBZXTLS : XForm_1<31, 87, (outs g8rc:$rD), (ins ptr_rc_nor0:$rA, tlsreg:$rB), "lbzx $rD, $rA, $rB", IIC_LdStLoad, []>; def LHZXTLS : XForm_1<31, 279, (outs g8rc:$rD), (ins ptr_rc_nor0:$rA, tlsreg:$rB), "lhzx $rD, $rA, $rB", IIC_LdStLoad, []>; def LWZXTLS : XForm_1<31, 23, (outs g8rc:$rD), (ins ptr_rc_nor0:$rA, tlsreg:$rB), "lwzx $rD, $rA, $rB", IIC_LdStLoad, []>; def LDXTLS : XForm_1<31, 21, (outs g8rc:$rD), (ins ptr_rc_nor0:$rA, tlsreg:$rB), "ldx $rD, $rA, $rB", IIC_LdStLD, []>, isPPC64; def LBZXTLS_32 : XForm_1<31, 87, (outs gprc:$rD), (ins ptr_rc_nor0:$rA, tlsreg:$rB), "lbzx $rD, $rA, $rB", IIC_LdStLoad, []>; def LHZXTLS_32 : XForm_1<31, 279, (outs gprc:$rD), (ins ptr_rc_nor0:$rA, tlsreg:$rB), "lhzx $rD, $rA, $rB", IIC_LdStLoad, []>; def LWZXTLS_32 : XForm_1<31, 23, (outs gprc:$rD), (ins ptr_rc_nor0:$rA, tlsreg:$rB), "lwzx $rD, $rA, $rB", IIC_LdStLoad, []>; } let mayStore = 1 in { def STBXTLS : XForm_8<31, 215, (outs), (ins g8rc:$rS, ptr_rc_nor0:$rA, tlsreg:$rB), "stbx $rS, $rA, $rB", IIC_LdStStore, []>, PPC970_DGroup_Cracked; def STHXTLS : XForm_8<31, 407, (outs), (ins g8rc:$rS, ptr_rc_nor0:$rA, tlsreg:$rB), "sthx $rS, $rA, $rB", IIC_LdStStore, []>, PPC970_DGroup_Cracked; def STWXTLS : XForm_8<31, 151, (outs), (ins g8rc:$rS, ptr_rc_nor0:$rA, tlsreg:$rB), "stwx $rS, $rA, $rB", IIC_LdStStore, []>, PPC970_DGroup_Cracked; def STDXTLS : XForm_8<31, 149, (outs), (ins g8rc:$rS, ptr_rc_nor0:$rA, tlsreg:$rB), "stdx $rS, $rA, $rB", IIC_LdStSTD, []>, isPPC64, PPC970_DGroup_Cracked; def STBXTLS_32 : XForm_8<31, 215, (outs), (ins gprc:$rS, ptr_rc_nor0:$rA, tlsreg:$rB), "stbx $rS, $rA, $rB", IIC_LdStStore, []>, PPC970_DGroup_Cracked; def STHXTLS_32 : XForm_8<31, 407, (outs), (ins gprc:$rS, ptr_rc_nor0:$rA, tlsreg:$rB), "sthx $rS, $rA, $rB", IIC_LdStStore, []>, PPC970_DGroup_Cracked; def STWXTLS_32 : XForm_8<31, 151, (outs), (ins gprc:$rS, ptr_rc_nor0:$rA, tlsreg:$rB), "stwx $rS, $rA, $rB", IIC_LdStStore, []>, PPC970_DGroup_Cracked; } let isCommutable = 1 in defm ADDC8 : XOForm_1rc<31, 10, 0, (outs g8rc:$rT), (ins g8rc:$rA, g8rc:$rB), "addc", "$rT, $rA, $rB", IIC_IntGeneral, [(set i64:$rT, (addc i64:$rA, i64:$rB))]>, PPC970_DGroup_Cracked; let Defs = [CARRY] in def ADDIC8 : DForm_2<12, (outs g8rc:$rD), (ins g8rc:$rA, s16imm64:$imm), "addic $rD, $rA, $imm", IIC_IntGeneral, [(set i64:$rD, (addc i64:$rA, imm64SExt16:$imm))]>; def ADDI8 : DForm_2<14, (outs g8rc:$rD), (ins g8rc_nox0:$rA, s16imm64:$imm), "addi $rD, $rA, $imm", IIC_IntSimple, [(set i64:$rD, (add i64:$rA, imm64SExt16:$imm))]>; def ADDIS8 : DForm_2<15, (outs g8rc:$rD), (ins g8rc_nox0:$rA, s17imm64:$imm), "addis $rD, $rA, $imm", IIC_IntSimple, [(set i64:$rD, (add i64:$rA, imm16ShiftedSExt:$imm))]>; let Defs = [CARRY] in { def SUBFIC8: DForm_2< 8, (outs g8rc:$rD), (ins g8rc:$rA, s16imm64:$imm), "subfic $rD, $rA, $imm", IIC_IntGeneral, [(set i64:$rD, (subc imm64SExt16:$imm, i64:$rA))]>; } defm SUBFC8 : XOForm_1rc<31, 8, 0, (outs g8rc:$rT), (ins g8rc:$rA, g8rc:$rB), "subfc", "$rT, $rA, $rB", IIC_IntGeneral, [(set i64:$rT, (subc i64:$rB, i64:$rA))]>, PPC970_DGroup_Cracked; defm SUBF8 : XOForm_1rx<31, 40, 0, (outs g8rc:$rT), (ins g8rc:$rA, g8rc:$rB), "subf", "$rT, $rA, $rB", IIC_IntGeneral, [(set i64:$rT, (sub i64:$rB, i64:$rA))]>; defm NEG8 : XOForm_3r<31, 104, 0, (outs g8rc:$rT), (ins g8rc:$rA), "neg", "$rT, $rA", IIC_IntSimple, [(set i64:$rT, (ineg i64:$rA))]>; let Uses = [CARRY] in { let isCommutable = 1 in defm ADDE8 : XOForm_1rc<31, 138, 0, (outs g8rc:$rT), (ins g8rc:$rA, g8rc:$rB), "adde", "$rT, $rA, $rB", IIC_IntGeneral, [(set i64:$rT, (adde i64:$rA, i64:$rB))]>; defm ADDME8 : XOForm_3rc<31, 234, 0, (outs g8rc:$rT), (ins g8rc:$rA), "addme", "$rT, $rA", IIC_IntGeneral, [(set i64:$rT, (adde i64:$rA, -1))]>; defm ADDZE8 : XOForm_3rc<31, 202, 0, (outs g8rc:$rT), (ins g8rc:$rA), "addze", "$rT, $rA", IIC_IntGeneral, [(set i64:$rT, (adde i64:$rA, 0))]>; defm SUBFE8 : XOForm_1rc<31, 136, 0, (outs g8rc:$rT), (ins g8rc:$rA, g8rc:$rB), "subfe", "$rT, $rA, $rB", IIC_IntGeneral, [(set i64:$rT, (sube i64:$rB, i64:$rA))]>; defm SUBFME8 : XOForm_3rc<31, 232, 0, (outs g8rc:$rT), (ins g8rc:$rA), "subfme", "$rT, $rA", IIC_IntGeneral, [(set i64:$rT, (sube -1, i64:$rA))]>; defm SUBFZE8 : XOForm_3rc<31, 200, 0, (outs g8rc:$rT), (ins g8rc:$rA), "subfze", "$rT, $rA", IIC_IntGeneral, [(set i64:$rT, (sube 0, i64:$rA))]>; } } // isCodeGenOnly // FIXME: Duplicating this for the asm parser should be unnecessary, but the // previous definition must be marked as CodeGen only to prevent decoding // conflicts. let isAsmParserOnly = 1 in { def ADD8TLS_ : XOForm_1<31, 266, 0, (outs g8rc:$rT), (ins g8rc:$rA, tlsreg:$rB), "add $rT, $rA, $rB", IIC_IntSimple, []>; let mayLoad = 1 in { def LBZXTLS_ : XForm_1<31, 87, (outs g8rc:$rD), (ins ptr_rc_nor0:$rA, tlsreg:$rB), "lbzx $rD, $rA, $rB", IIC_LdStLoad, []>; def LHZXTLS_ : XForm_1<31, 279, (outs g8rc:$rD), (ins ptr_rc_nor0:$rA, tlsreg:$rB), "lhzx $rD, $rA, $rB", IIC_LdStLoad, []>; def LWZXTLS_ : XForm_1<31, 23, (outs g8rc:$rD), (ins ptr_rc_nor0:$rA, tlsreg:$rB), "lwzx $rD, $rA, $rB", IIC_LdStLoad, []>; def LDXTLS_ : XForm_1<31, 21, (outs g8rc:$rD), (ins ptr_rc_nor0:$rA, tlsreg:$rB), "ldx $rD, $rA, $rB", IIC_LdStLD, []>, isPPC64; } let mayStore = 1 in { def STBXTLS_ : XForm_8<31, 215, (outs), (ins g8rc:$rS, ptr_rc_nor0:$rA, tlsreg:$rB), "stbx $rS, $rA, $rB", IIC_LdStStore, []>, PPC970_DGroup_Cracked; def STHXTLS_ : XForm_8<31, 407, (outs), (ins g8rc:$rS, ptr_rc_nor0:$rA, tlsreg:$rB), "sthx $rS, $rA, $rB", IIC_LdStStore, []>, PPC970_DGroup_Cracked; def STWXTLS_ : XForm_8<31, 151, (outs), (ins g8rc:$rS, ptr_rc_nor0:$rA, tlsreg:$rB), "stwx $rS, $rA, $rB", IIC_LdStStore, []>, PPC970_DGroup_Cracked; def STDXTLS_ : XForm_8<31, 149, (outs), (ins g8rc:$rS, ptr_rc_nor0:$rA, tlsreg:$rB), "stdx $rS, $rA, $rB", IIC_LdStSTD, []>, isPPC64, PPC970_DGroup_Cracked; } } let isCommutable = 1 in { defm MULHD : XOForm_1r<31, 73, 0, (outs g8rc:$rT), (ins g8rc:$rA, g8rc:$rB), "mulhd", "$rT, $rA, $rB", IIC_IntMulHW, [(set i64:$rT, (mulhs i64:$rA, i64:$rB))]>; defm MULHDU : XOForm_1r<31, 9, 0, (outs g8rc:$rT), (ins g8rc:$rA, g8rc:$rB), "mulhdu", "$rT, $rA, $rB", IIC_IntMulHWU, [(set i64:$rT, (mulhu i64:$rA, i64:$rB))]>; } // isCommutable } } // Interpretation64Bit let isCompare = 1, hasSideEffects = 0 in { def CMPD : XForm_16_ext<31, 0, (outs crrc:$crD), (ins g8rc:$rA, g8rc:$rB), "cmpd $crD, $rA, $rB", IIC_IntCompare>, isPPC64; def CMPLD : XForm_16_ext<31, 32, (outs crrc:$crD), (ins g8rc:$rA, g8rc:$rB), "cmpld $crD, $rA, $rB", IIC_IntCompare>, isPPC64; def CMPDI : DForm_5_ext<11, (outs crrc:$crD), (ins g8rc:$rA, s16imm64:$imm), "cmpdi $crD, $rA, $imm", IIC_IntCompare>, isPPC64; def CMPLDI : DForm_6_ext<10, (outs crrc:$dst), (ins g8rc:$src1, u16imm64:$src2), "cmpldi $dst, $src1, $src2", IIC_IntCompare>, isPPC64; let Interpretation64Bit = 1, isCodeGenOnly = 1 in def CMPRB8 : X_BF3_L1_RS5_RS5<31, 192, (outs crbitrc:$BF), (ins u1imm:$L, g8rc:$rA, g8rc:$rB), "cmprb $BF, $L, $rA, $rB", IIC_IntCompare, []>, Requires<[IsISA3_0]>; def CMPEQB : X_BF3_RS5_RS5<31, 224, (outs crbitrc:$BF), (ins g8rc:$rA, g8rc:$rB), "cmpeqb $BF, $rA, $rB", IIC_IntCompare, []>, Requires<[IsISA3_0]>; } let hasSideEffects = 0 in { defm SLD : XForm_6r<31, 27, (outs g8rc:$rA), (ins g8rc:$rS, gprc:$rB), "sld", "$rA, $rS, $rB", IIC_IntRotateD, [(set i64:$rA, (PPCshl i64:$rS, i32:$rB))]>, isPPC64; defm SRD : XForm_6r<31, 539, (outs g8rc:$rA), (ins g8rc:$rS, gprc:$rB), "srd", "$rA, $rS, $rB", IIC_IntRotateD, [(set i64:$rA, (PPCsrl i64:$rS, i32:$rB))]>, isPPC64; defm SRAD : XForm_6rc<31, 794, (outs g8rc:$rA), (ins g8rc:$rS, gprc:$rB), "srad", "$rA, $rS, $rB", IIC_IntRotateD, [(set i64:$rA, (PPCsra i64:$rS, i32:$rB))]>, isPPC64; let Interpretation64Bit = 1, isCodeGenOnly = 1 in { defm CNTLZW8 : XForm_11r<31, 26, (outs g8rc:$rA), (ins g8rc:$rS), "cntlzw", "$rA, $rS", IIC_IntGeneral, []>; defm CNTTZW8 : XForm_11r<31, 538, (outs g8rc:$rA), (ins g8rc:$rS), "cnttzw", "$rA, $rS", IIC_IntGeneral, []>, Requires<[IsISA3_0]>; defm EXTSB8 : XForm_11r<31, 954, (outs g8rc:$rA), (ins g8rc:$rS), "extsb", "$rA, $rS", IIC_IntSimple, [(set i64:$rA, (sext_inreg i64:$rS, i8))]>; defm EXTSH8 : XForm_11r<31, 922, (outs g8rc:$rA), (ins g8rc:$rS), "extsh", "$rA, $rS", IIC_IntSimple, [(set i64:$rA, (sext_inreg i64:$rS, i16))]>; defm SLW8 : XForm_6r<31, 24, (outs g8rc:$rA), (ins g8rc:$rS, g8rc:$rB), "slw", "$rA, $rS, $rB", IIC_IntGeneral, []>; defm SRW8 : XForm_6r<31, 536, (outs g8rc:$rA), (ins g8rc:$rS, g8rc:$rB), "srw", "$rA, $rS, $rB", IIC_IntGeneral, []>; } // Interpretation64Bit // For fast-isel: let isCodeGenOnly = 1 in { def EXTSB8_32_64 : XForm_11<31, 954, (outs g8rc:$rA), (ins gprc:$rS), "extsb $rA, $rS", IIC_IntSimple, []>, isPPC64; def EXTSH8_32_64 : XForm_11<31, 922, (outs g8rc:$rA), (ins gprc:$rS), "extsh $rA, $rS", IIC_IntSimple, []>, isPPC64; } // isCodeGenOnly for fast-isel defm EXTSW : XForm_11r<31, 986, (outs g8rc:$rA), (ins g8rc:$rS), "extsw", "$rA, $rS", IIC_IntSimple, [(set i64:$rA, (sext_inreg i64:$rS, i32))]>, isPPC64; let Interpretation64Bit = 1, isCodeGenOnly = 1 in defm EXTSW_32_64 : XForm_11r<31, 986, (outs g8rc:$rA), (ins gprc:$rS), "extsw", "$rA, $rS", IIC_IntSimple, [(set i64:$rA, (sext i32:$rS))]>, isPPC64; let isCodeGenOnly = 1 in def EXTSW_32 : XForm_11<31, 986, (outs gprc:$rA), (ins gprc:$rS), "extsw $rA, $rS", IIC_IntSimple, []>, isPPC64; defm SRADI : XSForm_1rc<31, 413, (outs g8rc:$rA), (ins g8rc:$rS, u6imm:$SH), "sradi", "$rA, $rS, $SH", IIC_IntRotateDI, [(set i64:$rA, (sra i64:$rS, (i32 imm:$SH)))]>, isPPC64; let Interpretation64Bit = 1, isCodeGenOnly = 1 in defm EXTSWSLI_32_64 : XSForm_1r<31, 445, (outs g8rc:$rA), (ins gprc:$rS, u6imm:$SH), "extswsli", "$rA, $rS, $SH", IIC_IntRotateDI, [(set i64:$rA, (PPCextswsli i32:$rS, (i32 imm:$SH)))]>, isPPC64, Requires<[IsISA3_0]>; defm EXTSWSLI : XSForm_1rc<31, 445, (outs g8rc:$rA), (ins g8rc:$rS, u6imm:$SH), "extswsli", "$rA, $rS, $SH", IIC_IntRotateDI, []>, isPPC64, Requires<[IsISA3_0]>; // For fast-isel: let isCodeGenOnly = 1, Defs = [CARRY] in def SRADI_32 : XSForm_1<31, 413, (outs gprc:$rA), (ins gprc:$rS, u6imm:$SH), "sradi $rA, $rS, $SH", IIC_IntRotateDI, []>, isPPC64; defm CNTLZD : XForm_11r<31, 58, (outs g8rc:$rA), (ins g8rc:$rS), "cntlzd", "$rA, $rS", IIC_IntGeneral, [(set i64:$rA, (ctlz i64:$rS))]>; defm CNTTZD : XForm_11r<31, 570, (outs g8rc:$rA), (ins g8rc:$rS), "cnttzd", "$rA, $rS", IIC_IntGeneral, [(set i64:$rA, (cttz i64:$rS))]>, Requires<[IsISA3_0]>; def POPCNTD : XForm_11<31, 506, (outs g8rc:$rA), (ins g8rc:$rS), "popcntd $rA, $rS", IIC_IntGeneral, [(set i64:$rA, (ctpop i64:$rS))]>; def BPERMD : XForm_6<31, 252, (outs g8rc:$rA), (ins g8rc:$rS, g8rc:$rB), "bpermd $rA, $rS, $rB", IIC_IntGeneral, [(set i64:$rA, (int_ppc_bpermd g8rc:$rS, g8rc:$rB))]>, isPPC64, Requires<[HasBPERMD]>; let isCodeGenOnly = 1, isCommutable = 1 in def CMPB8 : XForm_6<31, 508, (outs g8rc:$rA), (ins g8rc:$rS, g8rc:$rB), "cmpb $rA, $rS, $rB", IIC_IntGeneral, [(set i64:$rA, (PPCcmpb i64:$rS, i64:$rB))]>; // popcntw also does a population count on the high 32 bits (storing the // results in the high 32-bits of the output). We'll ignore that here (which is // safe because we never separately use the high part of the 64-bit registers). def POPCNTW : XForm_11<31, 378, (outs gprc:$rA), (ins gprc:$rS), "popcntw $rA, $rS", IIC_IntGeneral, [(set i32:$rA, (ctpop i32:$rS))]>; def POPCNTB : XForm_11<31, 122, (outs gprc:$rA), (ins gprc:$rS), "popcntb $rA, $rS", IIC_IntGeneral, []>; defm DIVD : XOForm_1rcr<31, 489, 0, (outs g8rc:$rT), (ins g8rc:$rA, g8rc:$rB), "divd", "$rT, $rA, $rB", IIC_IntDivD, [(set i64:$rT, (sdiv i64:$rA, i64:$rB))]>, isPPC64; defm DIVDU : XOForm_1rcr<31, 457, 0, (outs g8rc:$rT), (ins g8rc:$rA, g8rc:$rB), "divdu", "$rT, $rA, $rB", IIC_IntDivD, [(set i64:$rT, (udiv i64:$rA, i64:$rB))]>, isPPC64; defm DIVDE : XOForm_1rcr<31, 425, 0, (outs g8rc:$rT), (ins g8rc:$rA, g8rc:$rB), "divde", "$rT, $rA, $rB", IIC_IntDivD, [(set i64:$rT, (int_ppc_divde g8rc:$rA, g8rc:$rB))]>, isPPC64, Requires<[HasExtDiv]>; let Predicates = [IsISA3_0] in { def MADDHD : VAForm_1a<48, (outs g8rc :$RT), (ins g8rc:$RA, g8rc:$RB, g8rc:$RC), "maddhd $RT, $RA, $RB, $RC", IIC_IntMulHD, []>, isPPC64; def MADDHDU : VAForm_1a<49, (outs g8rc :$RT), (ins g8rc:$RA, g8rc:$RB, g8rc:$RC), "maddhdu $RT, $RA, $RB, $RC", IIC_IntMulHD, []>, isPPC64; def MADDLD : VAForm_1a<51, (outs gprc :$RT), (ins gprc:$RA, gprc:$RB, gprc:$RC), "maddld $RT, $RA, $RB, $RC", IIC_IntMulHD, [(set i32:$RT, (add_without_simm16 (mul_without_simm16 i32:$RA, i32:$RB), i32:$RC))]>, isPPC64; def SETB : XForm_44<31, 128, (outs gprc:$RT), (ins crrc:$BFA), "setb $RT, $BFA", IIC_IntGeneral>, isPPC64; let Interpretation64Bit = 1, isCodeGenOnly = 1 in { def MADDLD8 : VAForm_1a<51, (outs g8rc :$RT), (ins g8rc:$RA, g8rc:$RB, g8rc:$RC), "maddld $RT, $RA, $RB, $RC", IIC_IntMulHD, [(set i64:$RT, (add_without_simm16 (mul_without_simm16 i64:$RA, i64:$RB), i64:$RC))]>, isPPC64; def SETB8 : XForm_44<31, 128, (outs g8rc:$RT), (ins crrc:$BFA), "setb $RT, $BFA", IIC_IntGeneral>, isPPC64; } def DARN : XForm_45<31, 755, (outs g8rc:$RT), (ins i32imm:$L), "darn $RT, $L", IIC_LdStLD>, isPPC64; def ADDPCIS : DXForm<19, 2, (outs g8rc:$RT), (ins i32imm:$D), "addpcis $RT, $D", IIC_BrB, []>, isPPC64; def MODSD : XForm_8<31, 777, (outs g8rc:$rT), (ins g8rc:$rA, g8rc:$rB), "modsd $rT, $rA, $rB", IIC_IntDivW, [(set i64:$rT, (srem i64:$rA, i64:$rB))]>; def MODUD : XForm_8<31, 265, (outs g8rc:$rT), (ins g8rc:$rA, g8rc:$rB), "modud $rT, $rA, $rB", IIC_IntDivW, [(set i64:$rT, (urem i64:$rA, i64:$rB))]>; } defm DIVDEU : XOForm_1rcr<31, 393, 0, (outs g8rc:$rT), (ins g8rc:$rA, g8rc:$rB), "divdeu", "$rT, $rA, $rB", IIC_IntDivD, [(set i64:$rT, (int_ppc_divdeu g8rc:$rA, g8rc:$rB))]>, isPPC64, Requires<[HasExtDiv]>; let isCommutable = 1 in defm MULLD : XOForm_1rx<31, 233, 0, (outs g8rc:$rT), (ins g8rc:$rA, g8rc:$rB), "mulld", "$rT, $rA, $rB", IIC_IntMulHD, [(set i64:$rT, (mul i64:$rA, i64:$rB))]>, isPPC64; let Interpretation64Bit = 1, isCodeGenOnly = 1 in def MULLI8 : DForm_2<7, (outs g8rc:$rD), (ins g8rc:$rA, s16imm64:$imm), "mulli $rD, $rA, $imm", IIC_IntMulLI, [(set i64:$rD, (mul i64:$rA, imm64SExt16:$imm))]>; } let hasSideEffects = 0 in { defm RLDIMI : MDForm_1r<30, 3, (outs g8rc:$rA), (ins g8rc:$rSi, g8rc:$rS, u6imm:$SH, u6imm:$MBE), "rldimi", "$rA, $rS, $SH, $MBE", IIC_IntRotateDI, []>, isPPC64, RegConstraint<"$rSi = $rA">, NoEncode<"$rSi">; // Rotate instructions. defm RLDCL : MDSForm_1r<30, 8, (outs g8rc:$rA), (ins g8rc:$rS, gprc:$rB, u6imm:$MBE), "rldcl", "$rA, $rS, $rB, $MBE", IIC_IntRotateD, []>, isPPC64; defm RLDCR : MDSForm_1r<30, 9, (outs g8rc:$rA), (ins g8rc:$rS, gprc:$rB, u6imm:$MBE), "rldcr", "$rA, $rS, $rB, $MBE", IIC_IntRotateD, []>, isPPC64; defm RLDICL : MDForm_1r<30, 0, (outs g8rc:$rA), (ins g8rc:$rS, u6imm:$SH, u6imm:$MBE), "rldicl", "$rA, $rS, $SH, $MBE", IIC_IntRotateDI, []>, isPPC64; // For fast-isel: let isCodeGenOnly = 1 in def RLDICL_32_64 : MDForm_1<30, 0, (outs g8rc:$rA), (ins gprc:$rS, u6imm:$SH, u6imm:$MBE), "rldicl $rA, $rS, $SH, $MBE", IIC_IntRotateDI, []>, isPPC64; // End fast-isel. let Interpretation64Bit = 1, isCodeGenOnly = 1 in defm RLDICL_32 : MDForm_1r<30, 0, (outs gprc:$rA), (ins gprc:$rS, u6imm:$SH, u6imm:$MBE), "rldicl", "$rA, $rS, $SH, $MBE", IIC_IntRotateDI, []>, isPPC64; defm RLDICR : MDForm_1r<30, 1, (outs g8rc:$rA), (ins g8rc:$rS, u6imm:$SH, u6imm:$MBE), "rldicr", "$rA, $rS, $SH, $MBE", IIC_IntRotateDI, []>, isPPC64; let isCodeGenOnly = 1 in def RLDICR_32 : MDForm_1<30, 1, (outs gprc:$rA), (ins gprc:$rS, u6imm:$SH, u6imm:$MBE), "rldicr $rA, $rS, $SH, $MBE", IIC_IntRotateDI, []>, isPPC64; defm RLDIC : MDForm_1r<30, 2, (outs g8rc:$rA), (ins g8rc:$rS, u6imm:$SH, u6imm:$MBE), "rldic", "$rA, $rS, $SH, $MBE", IIC_IntRotateDI, []>, isPPC64; let Interpretation64Bit = 1, isCodeGenOnly = 1 in { defm RLWINM8 : MForm_2r<21, (outs g8rc:$rA), (ins g8rc:$rS, u5imm:$SH, u5imm:$MB, u5imm:$ME), "rlwinm", "$rA, $rS, $SH, $MB, $ME", IIC_IntGeneral, []>; defm RLWNM8 : MForm_2r<23, (outs g8rc:$rA), (ins g8rc:$rS, g8rc:$rB, u5imm:$MB, u5imm:$ME), "rlwnm", "$rA, $rS, $rB, $MB, $ME", IIC_IntGeneral, []>; // RLWIMI can be commuted if the rotate amount is zero. let Interpretation64Bit = 1, isCodeGenOnly = 1 in defm RLWIMI8 : MForm_2r<20, (outs g8rc:$rA), (ins g8rc:$rSi, g8rc:$rS, u5imm:$SH, u5imm:$MB, u5imm:$ME), "rlwimi", "$rA, $rS, $SH, $MB, $ME", IIC_IntRotate, []>, PPC970_DGroup_Cracked, RegConstraint<"$rSi = $rA">, NoEncode<"$rSi">; let isSelect = 1 in def ISEL8 : AForm_4<31, 15, (outs g8rc:$rT), (ins g8rc_nox0:$rA, g8rc:$rB, crbitrc:$cond), "isel $rT, $rA, $rB, $cond", IIC_IntISEL, []>; } // Interpretation64Bit } // hasSideEffects = 0 } // End FXU Operations. def : InstAlias<"mr $rA, $rB", (OR8 g8rc:$rA, g8rc:$rB, g8rc:$rB)>; def : InstAlias<"mr. $rA, $rB", (OR8_rec g8rc:$rA, g8rc:$rB, g8rc:$rB)>; def : InstAlias<"not $rA, $rB", (NOR8 g8rc:$rA, g8rc:$rB, g8rc:$rB)>; def : InstAlias<"not. $rA, $rB", (NOR8_rec g8rc:$rA, g8rc:$rB, g8rc:$rB)>; def : InstAlias<"mtcr $rA", (MTCRF8 255, g8rc:$rA)>; def : InstAlias<"sub $rA, $rB, $rC", (SUBF8 g8rc:$rA, g8rc:$rC, g8rc:$rB)>; def : InstAlias<"sub. $rA, $rB, $rC", (SUBF8_rec g8rc:$rA, g8rc:$rC, g8rc:$rB)>; def : InstAlias<"subc $rA, $rB, $rC", (SUBFC8 g8rc:$rA, g8rc:$rC, g8rc:$rB)>; def : InstAlias<"subc. $rA, $rB, $rC", (SUBFC8_rec g8rc:$rA, g8rc:$rC, g8rc:$rB)>; def : InstAlias<"rotlwi $rA, $rS, $n", (RLWINM8 g8rc:$rA, g8rc:$rS, u5imm:$n, 0, 31)>; def : InstAlias<"rotlwi. $rA, $rS, $n", (RLWINM8_rec g8rc:$rA, g8rc:$rS, u5imm:$n, 0, 31)>; def : InstAlias<"rotlw $rA, $rS, $rB", (RLWNM8 g8rc:$rA, g8rc:$rS, g8rc:$rB, 0, 31)>; def : InstAlias<"rotlw. $rA, $rS, $rB", (RLWNM8_rec g8rc:$rA, g8rc:$rS, g8rc:$rB, 0, 31)>; //===----------------------------------------------------------------------===// // Load/Store instructions. // // Sign extending loads. let PPC970_Unit = 2 in { let Interpretation64Bit = 1, isCodeGenOnly = 1 in def LHA8: DForm_1<42, (outs g8rc:$rD), (ins memri:$src), "lha $rD, $src", IIC_LdStLHA, [(set i64:$rD, (sextloadi16 iaddr:$src))]>, PPC970_DGroup_Cracked; def LWA : DSForm_1<58, 2, (outs g8rc:$rD), (ins memrix:$src), "lwa $rD, $src", IIC_LdStLWA, [(set i64:$rD, (aligned4sextloadi32 iaddrX4:$src))]>, isPPC64, PPC970_DGroup_Cracked; let Interpretation64Bit = 1, isCodeGenOnly = 1 in def LHAX8: XForm_1_memOp<31, 343, (outs g8rc:$rD), (ins memrr:$src), "lhax $rD, $src", IIC_LdStLHA, [(set i64:$rD, (sextloadi16 xaddr:$src))]>, PPC970_DGroup_Cracked; def LWAX : XForm_1_memOp<31, 341, (outs g8rc:$rD), (ins memrr:$src), "lwax $rD, $src", IIC_LdStLHA, [(set i64:$rD, (sextloadi32 xaddrX4:$src))]>, isPPC64, PPC970_DGroup_Cracked; // For fast-isel: let isCodeGenOnly = 1, mayLoad = 1, hasSideEffects = 0 in { def LWA_32 : DSForm_1<58, 2, (outs gprc:$rD), (ins memrix:$src), "lwa $rD, $src", IIC_LdStLWA, []>, isPPC64, PPC970_DGroup_Cracked; def LWAX_32 : XForm_1_memOp<31, 341, (outs gprc:$rD), (ins memrr:$src), "lwax $rD, $src", IIC_LdStLHA, []>, isPPC64, PPC970_DGroup_Cracked; } // end fast-isel isCodeGenOnly // Update forms. let mayLoad = 1, hasSideEffects = 0 in { let Interpretation64Bit = 1, isCodeGenOnly = 1 in def LHAU8 : DForm_1<43, (outs g8rc:$rD, ptr_rc_nor0:$ea_result), (ins memri:$addr), "lhau $rD, $addr", IIC_LdStLHAU, []>, RegConstraint<"$addr.reg = $ea_result">, NoEncode<"$ea_result">; // NO LWAU! let Interpretation64Bit = 1, isCodeGenOnly = 1 in def LHAUX8 : XForm_1_memOp<31, 375, (outs g8rc:$rD, ptr_rc_nor0:$ea_result), (ins memrr:$addr), "lhaux $rD, $addr", IIC_LdStLHAUX, []>, RegConstraint<"$addr.ptrreg = $ea_result">, NoEncode<"$ea_result">; def LWAUX : XForm_1_memOp<31, 373, (outs g8rc:$rD, ptr_rc_nor0:$ea_result), (ins memrr:$addr), "lwaux $rD, $addr", IIC_LdStLHAUX, []>, RegConstraint<"$addr.ptrreg = $ea_result">, NoEncode<"$ea_result">, isPPC64; } } let Interpretation64Bit = 1, isCodeGenOnly = 1 in { // Zero extending loads. let PPC970_Unit = 2 in { def LBZ8 : DForm_1<34, (outs g8rc:$rD), (ins memri:$src), "lbz $rD, $src", IIC_LdStLoad, [(set i64:$rD, (zextloadi8 iaddr:$src))]>; def LHZ8 : DForm_1<40, (outs g8rc:$rD), (ins memri:$src), "lhz $rD, $src", IIC_LdStLoad, [(set i64:$rD, (zextloadi16 iaddr:$src))]>; def LWZ8 : DForm_1<32, (outs g8rc:$rD), (ins memri:$src), "lwz $rD, $src", IIC_LdStLoad, [(set i64:$rD, (zextloadi32 iaddr:$src))]>, isPPC64; def LBZX8 : XForm_1_memOp<31, 87, (outs g8rc:$rD), (ins memrr:$src), "lbzx $rD, $src", IIC_LdStLoad, [(set i64:$rD, (zextloadi8 xaddr:$src))]>; def LHZX8 : XForm_1_memOp<31, 279, (outs g8rc:$rD), (ins memrr:$src), "lhzx $rD, $src", IIC_LdStLoad, [(set i64:$rD, (zextloadi16 xaddr:$src))]>; def LWZX8 : XForm_1_memOp<31, 23, (outs g8rc:$rD), (ins memrr:$src), "lwzx $rD, $src", IIC_LdStLoad, [(set i64:$rD, (zextloadi32 xaddr:$src))]>; // Update forms. let mayLoad = 1, hasSideEffects = 0 in { def LBZU8 : DForm_1<35, (outs g8rc:$rD, ptr_rc_nor0:$ea_result), (ins memri:$addr), "lbzu $rD, $addr", IIC_LdStLoadUpd, []>, RegConstraint<"$addr.reg = $ea_result">, NoEncode<"$ea_result">; def LHZU8 : DForm_1<41, (outs g8rc:$rD, ptr_rc_nor0:$ea_result), (ins memri:$addr), "lhzu $rD, $addr", IIC_LdStLoadUpd, []>, RegConstraint<"$addr.reg = $ea_result">, NoEncode<"$ea_result">; def LWZU8 : DForm_1<33, (outs g8rc:$rD, ptr_rc_nor0:$ea_result), (ins memri:$addr), "lwzu $rD, $addr", IIC_LdStLoadUpd, []>, RegConstraint<"$addr.reg = $ea_result">, NoEncode<"$ea_result">; def LBZUX8 : XForm_1_memOp<31, 119, (outs g8rc:$rD, ptr_rc_nor0:$ea_result), (ins memrr:$addr), "lbzux $rD, $addr", IIC_LdStLoadUpdX, []>, RegConstraint<"$addr.ptrreg = $ea_result">, NoEncode<"$ea_result">; def LHZUX8 : XForm_1_memOp<31, 311, (outs g8rc:$rD, ptr_rc_nor0:$ea_result), (ins memrr:$addr), "lhzux $rD, $addr", IIC_LdStLoadUpdX, []>, RegConstraint<"$addr.ptrreg = $ea_result">, NoEncode<"$ea_result">; def LWZUX8 : XForm_1_memOp<31, 55, (outs g8rc:$rD, ptr_rc_nor0:$ea_result), (ins memrr:$addr), "lwzux $rD, $addr", IIC_LdStLoadUpdX, []>, RegConstraint<"$addr.ptrreg = $ea_result">, NoEncode<"$ea_result">; } } } // Interpretation64Bit // Full 8-byte loads. let PPC970_Unit = 2 in { def LD : DSForm_1<58, 0, (outs g8rc:$rD), (ins memrix:$src), "ld $rD, $src", IIC_LdStLD, [(set i64:$rD, (aligned4load iaddrX4:$src))]>, isPPC64; // The following four definitions are selected for small code model only. // Otherwise, we need to create two instructions to form a 32-bit offset, // so we have a custom matcher for TOC_ENTRY in PPCDAGToDAGIsel::Select(). def LDtoc: PPCEmitTimePseudo<(outs g8rc:$rD), (ins tocentry:$disp, g8rc:$reg), "#LDtoc", [(set i64:$rD, (PPCtoc_entry tglobaladdr:$disp, i64:$reg))]>, isPPC64; def LDtocJTI: PPCEmitTimePseudo<(outs g8rc:$rD), (ins tocentry:$disp, g8rc:$reg), "#LDtocJTI", [(set i64:$rD, (PPCtoc_entry tjumptable:$disp, i64:$reg))]>, isPPC64; def LDtocCPT: PPCEmitTimePseudo<(outs g8rc:$rD), (ins tocentry:$disp, g8rc:$reg), "#LDtocCPT", [(set i64:$rD, (PPCtoc_entry tconstpool:$disp, i64:$reg))]>, isPPC64; def LDtocBA: PPCEmitTimePseudo<(outs g8rc:$rD), (ins tocentry:$disp, g8rc:$reg), "#LDtocCPT", [(set i64:$rD, (PPCtoc_entry tblockaddress:$disp, i64:$reg))]>, isPPC64; def LDX : XForm_1_memOp<31, 21, (outs g8rc:$rD), (ins memrr:$src), "ldx $rD, $src", IIC_LdStLD, [(set i64:$rD, (load xaddrX4:$src))]>, isPPC64; def LDBRX : XForm_1_memOp<31, 532, (outs g8rc:$rD), (ins memrr:$src), "ldbrx $rD, $src", IIC_LdStLoad, [(set i64:$rD, (PPClbrx xoaddr:$src, i64))]>, isPPC64; let mayLoad = 1, hasSideEffects = 0, isCodeGenOnly = 1 in { def LHBRX8 : XForm_1_memOp<31, 790, (outs g8rc:$rD), (ins memrr:$src), "lhbrx $rD, $src", IIC_LdStLoad, []>; def LWBRX8 : XForm_1_memOp<31, 534, (outs g8rc:$rD), (ins memrr:$src), "lwbrx $rD, $src", IIC_LdStLoad, []>; } let mayLoad = 1, hasSideEffects = 0 in { def LDU : DSForm_1<58, 1, (outs g8rc:$rD, ptr_rc_nor0:$ea_result), (ins memrix:$addr), "ldu $rD, $addr", IIC_LdStLDU, []>, RegConstraint<"$addr.reg = $ea_result">, isPPC64, NoEncode<"$ea_result">; def LDUX : XForm_1_memOp<31, 53, (outs g8rc:$rD, ptr_rc_nor0:$ea_result), (ins memrr:$addr), "ldux $rD, $addr", IIC_LdStLDUX, []>, RegConstraint<"$addr.ptrreg = $ea_result">, NoEncode<"$ea_result">, isPPC64; def LDMX : XForm_1<31, 309, (outs g8rc:$rD), (ins memrr:$src), "ldmx $rD, $src", IIC_LdStLD, []>, isPPC64, Requires<[IsISA3_0]>; } } // Support for medium and large code model. let hasSideEffects = 0 in { let isReMaterializable = 1 in { def ADDIStocHA8: PPCEmitTimePseudo<(outs g8rc:$rD), (ins g8rc_nox0:$reg, tocentry:$disp), "#ADDIStocHA8", []>, isPPC64; def ADDItocL: PPCEmitTimePseudo<(outs g8rc:$rD), (ins g8rc_nox0:$reg, tocentry:$disp), "#ADDItocL", []>, isPPC64; } let mayLoad = 1 in def LDtocL: PPCEmitTimePseudo<(outs g8rc:$rD), (ins tocentry:$disp, g8rc_nox0:$reg), "#LDtocL", []>, isPPC64; } // Support for thread-local storage. def ADDISgotTprelHA: PPCEmitTimePseudo<(outs g8rc:$rD), (ins g8rc_nox0:$reg, s16imm64:$disp), "#ADDISgotTprelHA", [(set i64:$rD, (PPCaddisGotTprelHA i64:$reg, tglobaltlsaddr:$disp))]>, isPPC64; def LDgotTprelL: PPCEmitTimePseudo<(outs g8rc_nox0:$rD), (ins s16imm64:$disp, g8rc_nox0:$reg), "#LDgotTprelL", [(set i64:$rD, (PPCldGotTprelL tglobaltlsaddr:$disp, i64:$reg))]>, isPPC64; let Defs = [CR7], Itinerary = IIC_LdStSync in def CFENCE8 : PPCPostRAExpPseudo<(outs), (ins g8rc:$cr), "#CFENCE8", []>; def : Pat<(PPCaddTls i64:$in, tglobaltlsaddr:$g), (ADD8TLS $in, tglobaltlsaddr:$g)>; def ADDIStlsgdHA: PPCEmitTimePseudo<(outs g8rc:$rD), (ins g8rc_nox0:$reg, s16imm64:$disp), "#ADDIStlsgdHA", [(set i64:$rD, (PPCaddisTlsgdHA i64:$reg, tglobaltlsaddr:$disp))]>, isPPC64; def ADDItlsgdL : PPCEmitTimePseudo<(outs g8rc:$rD), (ins g8rc_nox0:$reg, s16imm64:$disp), "#ADDItlsgdL", [(set i64:$rD, (PPCaddiTlsgdL i64:$reg, tglobaltlsaddr:$disp))]>, isPPC64; // LR8 is a true define, while the rest of the Defs are clobbers. X3 is // explicitly defined when this op is created, so not mentioned here. // This is lowered to BL8_NOP_TLS by the assembly printer, so the size must be // correct because the branch select pass is relying on it. let hasExtraSrcRegAllocReq = 1, hasExtraDefRegAllocReq = 1, Size = 8, Defs = [X0,X4,X5,X6,X7,X8,X9,X10,X11,X12,LR8,CTR8,CR0,CR1,CR5,CR6,CR7] in def GETtlsADDR : PPCEmitTimePseudo<(outs g8rc:$rD), (ins g8rc:$reg, tlsgd:$sym), "#GETtlsADDR", [(set i64:$rD, (PPCgetTlsAddr i64:$reg, tglobaltlsaddr:$sym))]>, isPPC64; // Combined op for ADDItlsgdL and GETtlsADDR, late expanded. X3 and LR8 // are true defines while the rest of the Defs are clobbers. let hasExtraSrcRegAllocReq = 1, hasExtraDefRegAllocReq = 1, Defs = [X0,X3,X4,X5,X6,X7,X8,X9,X10,X11,X12,LR8,CTR8,CR0,CR1,CR5,CR6,CR7] in def ADDItlsgdLADDR : PPCEmitTimePseudo<(outs g8rc:$rD), (ins g8rc_nox0:$reg, s16imm64:$disp, tlsgd:$sym), "#ADDItlsgdLADDR", [(set i64:$rD, (PPCaddiTlsgdLAddr i64:$reg, tglobaltlsaddr:$disp, tglobaltlsaddr:$sym))]>, isPPC64; def ADDIStlsldHA: PPCEmitTimePseudo<(outs g8rc:$rD), (ins g8rc_nox0:$reg, s16imm64:$disp), "#ADDIStlsldHA", [(set i64:$rD, (PPCaddisTlsldHA i64:$reg, tglobaltlsaddr:$disp))]>, isPPC64; def ADDItlsldL : PPCEmitTimePseudo<(outs g8rc:$rD), (ins g8rc_nox0:$reg, s16imm64:$disp), "#ADDItlsldL", [(set i64:$rD, (PPCaddiTlsldL i64:$reg, tglobaltlsaddr:$disp))]>, isPPC64; // LR8 is a true define, while the rest of the Defs are clobbers. X3 is // explicitly defined when this op is created, so not mentioned here. let hasExtraSrcRegAllocReq = 1, hasExtraDefRegAllocReq = 1, Defs = [X0,X4,X5,X6,X7,X8,X9,X10,X11,X12,LR8,CTR8,CR0,CR1,CR5,CR6,CR7] in def GETtlsldADDR : PPCEmitTimePseudo<(outs g8rc:$rD), (ins g8rc:$reg, tlsgd:$sym), "#GETtlsldADDR", [(set i64:$rD, (PPCgetTlsldAddr i64:$reg, tglobaltlsaddr:$sym))]>, isPPC64; // Combined op for ADDItlsldL and GETtlsADDR, late expanded. X3 and LR8 // are true defines, while the rest of the Defs are clobbers. let hasExtraSrcRegAllocReq = 1, hasExtraDefRegAllocReq = 1, Defs = [X0,X3,X4,X5,X6,X7,X8,X9,X10,X11,X12,LR8,CTR8,CR0,CR1,CR5,CR6,CR7] in def ADDItlsldLADDR : PPCEmitTimePseudo<(outs g8rc:$rD), (ins g8rc_nox0:$reg, s16imm64:$disp, tlsgd:$sym), "#ADDItlsldLADDR", [(set i64:$rD, (PPCaddiTlsldLAddr i64:$reg, tglobaltlsaddr:$disp, tglobaltlsaddr:$sym))]>, isPPC64; def ADDISdtprelHA: PPCEmitTimePseudo<(outs g8rc:$rD), (ins g8rc_nox0:$reg, s16imm64:$disp), "#ADDISdtprelHA", [(set i64:$rD, (PPCaddisDtprelHA i64:$reg, tglobaltlsaddr:$disp))]>, isPPC64; def ADDIdtprelL : PPCEmitTimePseudo<(outs g8rc:$rD), (ins g8rc_nox0:$reg, s16imm64:$disp), "#ADDIdtprelL", [(set i64:$rD, (PPCaddiDtprelL i64:$reg, tglobaltlsaddr:$disp))]>, isPPC64; let PPC970_Unit = 2 in { let Interpretation64Bit = 1, isCodeGenOnly = 1 in { // Truncating stores. def STB8 : DForm_1<38, (outs), (ins g8rc:$rS, memri:$src), "stb $rS, $src", IIC_LdStStore, [(truncstorei8 i64:$rS, iaddr:$src)]>; def STH8 : DForm_1<44, (outs), (ins g8rc:$rS, memri:$src), "sth $rS, $src", IIC_LdStStore, [(truncstorei16 i64:$rS, iaddr:$src)]>; def STW8 : DForm_1<36, (outs), (ins g8rc:$rS, memri:$src), "stw $rS, $src", IIC_LdStStore, [(truncstorei32 i64:$rS, iaddr:$src)]>; def STBX8 : XForm_8_memOp<31, 215, (outs), (ins g8rc:$rS, memrr:$dst), "stbx $rS, $dst", IIC_LdStStore, [(truncstorei8 i64:$rS, xaddr:$dst)]>, PPC970_DGroup_Cracked; def STHX8 : XForm_8_memOp<31, 407, (outs), (ins g8rc:$rS, memrr:$dst), "sthx $rS, $dst", IIC_LdStStore, [(truncstorei16 i64:$rS, xaddr:$dst)]>, PPC970_DGroup_Cracked; def STWX8 : XForm_8_memOp<31, 151, (outs), (ins g8rc:$rS, memrr:$dst), "stwx $rS, $dst", IIC_LdStStore, [(truncstorei32 i64:$rS, xaddr:$dst)]>, PPC970_DGroup_Cracked; } // Interpretation64Bit // Normal 8-byte stores. def STD : DSForm_1<62, 0, (outs), (ins g8rc:$rS, memrix:$dst), "std $rS, $dst", IIC_LdStSTD, [(aligned4store i64:$rS, iaddrX4:$dst)]>, isPPC64; def STDX : XForm_8_memOp<31, 149, (outs), (ins g8rc:$rS, memrr:$dst), "stdx $rS, $dst", IIC_LdStSTD, [(store i64:$rS, xaddrX4:$dst)]>, isPPC64, PPC970_DGroup_Cracked; def STDBRX: XForm_8_memOp<31, 660, (outs), (ins g8rc:$rS, memrr:$dst), "stdbrx $rS, $dst", IIC_LdStStore, [(PPCstbrx i64:$rS, xoaddr:$dst, i64)]>, isPPC64, PPC970_DGroup_Cracked; } // Stores with Update (pre-inc). let PPC970_Unit = 2, mayStore = 1, mayLoad = 0 in { let Interpretation64Bit = 1, isCodeGenOnly = 1 in { def STBU8 : DForm_1<39, (outs ptr_rc_nor0:$ea_res), (ins g8rc:$rS, memri:$dst), "stbu $rS, $dst", IIC_LdStSTU, []>, RegConstraint<"$dst.reg = $ea_res">, NoEncode<"$ea_res">; def STHU8 : DForm_1<45, (outs ptr_rc_nor0:$ea_res), (ins g8rc:$rS, memri:$dst), "sthu $rS, $dst", IIC_LdStSTU, []>, RegConstraint<"$dst.reg = $ea_res">, NoEncode<"$ea_res">; def STWU8 : DForm_1<37, (outs ptr_rc_nor0:$ea_res), (ins g8rc:$rS, memri:$dst), "stwu $rS, $dst", IIC_LdStSTU, []>, RegConstraint<"$dst.reg = $ea_res">, NoEncode<"$ea_res">; def STBUX8: XForm_8_memOp<31, 247, (outs ptr_rc_nor0:$ea_res), (ins g8rc:$rS, memrr:$dst), "stbux $rS, $dst", IIC_LdStSTUX, []>, RegConstraint<"$dst.ptrreg = $ea_res">, NoEncode<"$ea_res">, PPC970_DGroup_Cracked; def STHUX8: XForm_8_memOp<31, 439, (outs ptr_rc_nor0:$ea_res), (ins g8rc:$rS, memrr:$dst), "sthux $rS, $dst", IIC_LdStSTUX, []>, RegConstraint<"$dst.ptrreg = $ea_res">, NoEncode<"$ea_res">, PPC970_DGroup_Cracked; def STWUX8: XForm_8_memOp<31, 183, (outs ptr_rc_nor0:$ea_res), (ins g8rc:$rS, memrr:$dst), "stwux $rS, $dst", IIC_LdStSTUX, []>, RegConstraint<"$dst.ptrreg = $ea_res">, NoEncode<"$ea_res">, PPC970_DGroup_Cracked; } // Interpretation64Bit def STDU : DSForm_1<62, 1, (outs ptr_rc_nor0:$ea_res), (ins g8rc:$rS, memrix:$dst), "stdu $rS, $dst", IIC_LdStSTU, []>, RegConstraint<"$dst.reg = $ea_res">, NoEncode<"$ea_res">, isPPC64; def STDUX : XForm_8_memOp<31, 181, (outs ptr_rc_nor0:$ea_res), (ins g8rc:$rS, memrr:$dst), "stdux $rS, $dst", IIC_LdStSTUX, []>, RegConstraint<"$dst.ptrreg = $ea_res">, NoEncode<"$ea_res">, PPC970_DGroup_Cracked, isPPC64; } // Patterns to match the pre-inc stores. We can't put the patterns on // the instruction definitions directly as ISel wants the address base // and offset to be separate operands, not a single complex operand. def : Pat<(pre_truncsti8 i64:$rS, iPTR:$ptrreg, iaddroff:$ptroff), (STBU8 $rS, iaddroff:$ptroff, $ptrreg)>; def : Pat<(pre_truncsti16 i64:$rS, iPTR:$ptrreg, iaddroff:$ptroff), (STHU8 $rS, iaddroff:$ptroff, $ptrreg)>; def : Pat<(pre_truncsti32 i64:$rS, iPTR:$ptrreg, iaddroff:$ptroff), (STWU8 $rS, iaddroff:$ptroff, $ptrreg)>; def : Pat<(aligned4pre_store i64:$rS, iPTR:$ptrreg, iaddroff:$ptroff), (STDU $rS, iaddroff:$ptroff, $ptrreg)>; def : Pat<(pre_truncsti8 i64:$rS, iPTR:$ptrreg, iPTR:$ptroff), (STBUX8 $rS, $ptrreg, $ptroff)>; def : Pat<(pre_truncsti16 i64:$rS, iPTR:$ptrreg, iPTR:$ptroff), (STHUX8 $rS, $ptrreg, $ptroff)>; def : Pat<(pre_truncsti32 i64:$rS, iPTR:$ptrreg, iPTR:$ptroff), (STWUX8 $rS, $ptrreg, $ptroff)>; def : Pat<(pre_store i64:$rS, iPTR:$ptrreg, iPTR:$ptroff), (STDUX $rS, $ptrreg, $ptroff)>; //===----------------------------------------------------------------------===// // Floating point instructions. // let PPC970_Unit = 3, hasSideEffects = 0, Uses = [RM] in { // FPU Operations. defm FCFID : XForm_26r<63, 846, (outs f8rc:$frD), (ins f8rc:$frB), "fcfid", "$frD, $frB", IIC_FPGeneral, [(set f64:$frD, (PPCfcfid f64:$frB))]>, isPPC64; defm FCTID : XForm_26r<63, 814, (outs f8rc:$frD), (ins f8rc:$frB), "fctid", "$frD, $frB", IIC_FPGeneral, []>, isPPC64; defm FCTIDU : XForm_26r<63, 942, (outs f8rc:$frD), (ins f8rc:$frB), "fctidu", "$frD, $frB", IIC_FPGeneral, []>, isPPC64; defm FCTIDZ : XForm_26r<63, 815, (outs f8rc:$frD), (ins f8rc:$frB), "fctidz", "$frD, $frB", IIC_FPGeneral, [(set f64:$frD, (PPCfctidz f64:$frB))]>, isPPC64; defm FCFIDU : XForm_26r<63, 974, (outs f8rc:$frD), (ins f8rc:$frB), "fcfidu", "$frD, $frB", IIC_FPGeneral, [(set f64:$frD, (PPCfcfidu f64:$frB))]>, isPPC64; defm FCFIDS : XForm_26r<59, 846, (outs f4rc:$frD), (ins f8rc:$frB), "fcfids", "$frD, $frB", IIC_FPGeneral, [(set f32:$frD, (PPCfcfids f64:$frB))]>, isPPC64; defm FCFIDUS : XForm_26r<59, 974, (outs f4rc:$frD), (ins f8rc:$frB), "fcfidus", "$frD, $frB", IIC_FPGeneral, [(set f32:$frD, (PPCfcfidus f64:$frB))]>, isPPC64; defm FCTIDUZ : XForm_26r<63, 943, (outs f8rc:$frD), (ins f8rc:$frB), "fctiduz", "$frD, $frB", IIC_FPGeneral, [(set f64:$frD, (PPCfctiduz f64:$frB))]>, isPPC64; defm FCTIWUZ : XForm_26r<63, 143, (outs f8rc:$frD), (ins f8rc:$frB), "fctiwuz", "$frD, $frB", IIC_FPGeneral, [(set f64:$frD, (PPCfctiwuz f64:$frB))]>, isPPC64; } //===----------------------------------------------------------------------===// // Instruction Patterns // // Extensions and truncates to/from 32-bit regs. def : Pat<(i64 (zext i32:$in)), (RLDICL (INSERT_SUBREG (i64 (IMPLICIT_DEF)), $in, sub_32), 0, 32)>; def : Pat<(i64 (anyext i32:$in)), (INSERT_SUBREG (i64 (IMPLICIT_DEF)), $in, sub_32)>; def : Pat<(i32 (trunc i64:$in)), (EXTRACT_SUBREG $in, sub_32)>; // Implement the 'not' operation with the NOR instruction. // (we could use the default xori pattern, but nor has lower latency on some // cores (such as the A2)). def i64not : OutPatFrag<(ops node:$in), (NOR8 $in, $in)>; def : Pat<(not i64:$in), (i64not $in)>; // Extending loads with i64 targets. def : Pat<(zextloadi1 iaddr:$src), (LBZ8 iaddr:$src)>; def : Pat<(zextloadi1 xaddr:$src), (LBZX8 xaddr:$src)>; def : Pat<(extloadi1 iaddr:$src), (LBZ8 iaddr:$src)>; def : Pat<(extloadi1 xaddr:$src), (LBZX8 xaddr:$src)>; def : Pat<(extloadi8 iaddr:$src), (LBZ8 iaddr:$src)>; def : Pat<(extloadi8 xaddr:$src), (LBZX8 xaddr:$src)>; def : Pat<(extloadi16 iaddr:$src), (LHZ8 iaddr:$src)>; def : Pat<(extloadi16 xaddr:$src), (LHZX8 xaddr:$src)>; def : Pat<(extloadi32 iaddr:$src), (LWZ8 iaddr:$src)>; def : Pat<(extloadi32 xaddr:$src), (LWZX8 xaddr:$src)>; // Standard shifts. These are represented separately from the real shifts above // so that we can distinguish between shifts that allow 6-bit and 7-bit shift // amounts. def : Pat<(sra i64:$rS, i32:$rB), (SRAD $rS, $rB)>; def : Pat<(srl i64:$rS, i32:$rB), (SRD $rS, $rB)>; def : Pat<(shl i64:$rS, i32:$rB), (SLD $rS, $rB)>; // SUBFIC def : Pat<(sub imm64SExt16:$imm, i64:$in), (SUBFIC8 $in, imm:$imm)>; // SHL/SRL def : Pat<(shl i64:$in, (i32 imm:$imm)), (RLDICR $in, imm:$imm, (SHL64 imm:$imm))>; def : Pat<(srl i64:$in, (i32 imm:$imm)), (RLDICL $in, (SRL64 imm:$imm), imm:$imm)>; // ROTL def : Pat<(rotl i64:$in, i32:$sh), (RLDCL $in, $sh, 0)>; def : Pat<(rotl i64:$in, (i32 imm:$imm)), (RLDICL $in, imm:$imm, 0)>; // Hi and Lo for Darwin Global Addresses. def : Pat<(PPChi tglobaladdr:$in, 0), (LIS8 tglobaladdr:$in)>; def : Pat<(PPClo tglobaladdr:$in, 0), (LI8 tglobaladdr:$in)>; def : Pat<(PPChi tconstpool:$in , 0), (LIS8 tconstpool:$in)>; def : Pat<(PPClo tconstpool:$in , 0), (LI8 tconstpool:$in)>; def : Pat<(PPChi tjumptable:$in , 0), (LIS8 tjumptable:$in)>; def : Pat<(PPClo tjumptable:$in , 0), (LI8 tjumptable:$in)>; def : Pat<(PPChi tblockaddress:$in, 0), (LIS8 tblockaddress:$in)>; def : Pat<(PPClo tblockaddress:$in, 0), (LI8 tblockaddress:$in)>; def : Pat<(PPChi tglobaltlsaddr:$g, i64:$in), (ADDIS8 $in, tglobaltlsaddr:$g)>; def : Pat<(PPClo tglobaltlsaddr:$g, i64:$in), (ADDI8 $in, tglobaltlsaddr:$g)>; def : Pat<(add i64:$in, (PPChi tglobaladdr:$g, 0)), (ADDIS8 $in, tglobaladdr:$g)>; def : Pat<(add i64:$in, (PPChi tconstpool:$g, 0)), (ADDIS8 $in, tconstpool:$g)>; def : Pat<(add i64:$in, (PPChi tjumptable:$g, 0)), (ADDIS8 $in, tjumptable:$g)>; def : Pat<(add i64:$in, (PPChi tblockaddress:$g, 0)), (ADDIS8 $in, tblockaddress:$g)>; // Patterns to match r+r indexed loads and stores for // addresses without at least 4-byte alignment. def : Pat<(i64 (unaligned4sextloadi32 xoaddr:$src)), (LWAX xoaddr:$src)>; def : Pat<(i64 (unaligned4load xoaddr:$src)), (LDX xoaddr:$src)>; def : Pat<(unaligned4store i64:$rS, xoaddr:$dst), (STDX $rS, xoaddr:$dst)>; // 64-bits atomic loads and stores def : Pat<(atomic_load_64 iaddrX4:$src), (LD memrix:$src)>; def : Pat<(atomic_load_64 xaddrX4:$src), (LDX memrr:$src)>; def : Pat<(atomic_store_64 iaddrX4:$ptr, i64:$val), (STD g8rc:$val, memrix:$ptr)>; def : Pat<(atomic_store_64 xaddrX4:$ptr, i64:$val), (STDX g8rc:$val, memrr:$ptr)>; let Predicates = [IsISA3_0] in { class X_L1_RA5_RB5 opcode, bits<10> xo, string opc, RegisterOperand ty, InstrItinClass itin, list pattern> : X_L1_RS5_RS5; let Interpretation64Bit = 1, isCodeGenOnly = 1 in { def CP_COPY8 : X_L1_RA5_RB5<31, 774, "copy" , g8rc, IIC_LdStCOPY, []>; def CP_PASTE8 : X_L1_RA5_RB5<31, 902, "paste" , g8rc, IIC_LdStPASTE, []>; def CP_PASTE8_rec : X_L1_RA5_RB5<31, 902, "paste.", g8rc, IIC_LdStPASTE, []>,isRecordForm; } // SLB Invalidate Entry Global def SLBIEG : XForm_26<31, 466, (outs), (ins gprc:$RS, gprc:$RB), "slbieg $RS, $RB", IIC_SprSLBIEG, []>; // SLB Synchronize def SLBSYNC : XForm_0<31, 338, (outs), (ins), "slbsync", IIC_SprSLBSYNC, []>; } // IsISA3_0 diff --git a/llvm/lib/Target/PowerPC/PPCInstrInfo.td b/llvm/lib/Target/PowerPC/PPCInstrInfo.td index 189dd77ab7ed..7a7128d99016 100644 --- a/llvm/lib/Target/PowerPC/PPCInstrInfo.td +++ b/llvm/lib/Target/PowerPC/PPCInstrInfo.td @@ -1,5143 +1,5147 @@ //===-- PPCInstrInfo.td - The PowerPC Instruction Set ------*- tablegen -*-===// // // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. // See https://llvm.org/LICENSE.txt for license information. // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // // This file describes the subset of the 32-bit PowerPC instruction set, as used // by the PowerPC instruction selector. // //===----------------------------------------------------------------------===// include "PPCInstrFormats.td" //===----------------------------------------------------------------------===// // PowerPC specific type constraints. // def SDT_PPCstfiwx : SDTypeProfile<0, 2, [ // stfiwx SDTCisVT<0, f64>, SDTCisPtrTy<1> ]>; def SDT_PPClfiwx : SDTypeProfile<1, 1, [ // lfiw[az]x SDTCisVT<0, f64>, SDTCisPtrTy<1> ]>; def SDT_PPCLxsizx : SDTypeProfile<1, 2, [ SDTCisVT<0, f64>, SDTCisPtrTy<1>, SDTCisPtrTy<2> ]>; def SDT_PPCstxsix : SDTypeProfile<0, 3, [ SDTCisVT<0, f64>, SDTCisPtrTy<1>, SDTCisPtrTy<2> ]>; def SDT_PPCcv_fp_to_int : SDTypeProfile<1, 1, [ SDTCisFP<0>, SDTCisFP<1> ]>; def SDT_PPCstore_scal_int_from_vsr : SDTypeProfile<0, 3, [ SDTCisVT<0, f64>, SDTCisPtrTy<1>, SDTCisPtrTy<2> ]>; def SDT_PPCVexts : SDTypeProfile<1, 2, [ SDTCisVT<0, f64>, SDTCisVT<1, f64>, SDTCisPtrTy<2> ]>; def SDT_PPCCallSeqStart : SDCallSeqStart<[ SDTCisVT<0, i32>, SDTCisVT<1, i32> ]>; def SDT_PPCCallSeqEnd : SDCallSeqEnd<[ SDTCisVT<0, i32>, SDTCisVT<1, i32> ]>; def SDT_PPCvperm : SDTypeProfile<1, 3, [ SDTCisVT<3, v16i8>, SDTCisSameAs<0, 1>, SDTCisSameAs<0, 2> ]>; def SDT_PPCVecSplat : SDTypeProfile<1, 2, [ SDTCisVec<0>, SDTCisVec<1>, SDTCisInt<2> ]>; def SDT_PPCVecShift : SDTypeProfile<1, 3, [ SDTCisVec<0>, SDTCisVec<1>, SDTCisVec<2>, SDTCisPtrTy<3> ]>; def SDT_PPCVecInsert : SDTypeProfile<1, 3, [ SDTCisVec<0>, SDTCisVec<1>, SDTCisVec<2>, SDTCisInt<3> ]>; def SDT_PPCxxpermdi: SDTypeProfile<1, 3, [ SDTCisVec<0>, SDTCisVec<1>, SDTCisVec<2>, SDTCisInt<3> ]>; def SDT_PPCvcmp : SDTypeProfile<1, 3, [ SDTCisSameAs<0, 1>, SDTCisSameAs<1, 2>, SDTCisVT<3, i32> ]>; def SDT_PPCcondbr : SDTypeProfile<0, 3, [ SDTCisVT<0, i32>, SDTCisVT<2, OtherVT> ]>; def SDT_PPClbrx : SDTypeProfile<1, 2, [ SDTCisInt<0>, SDTCisPtrTy<1>, SDTCisVT<2, OtherVT> ]>; def SDT_PPCstbrx : SDTypeProfile<0, 3, [ SDTCisInt<0>, SDTCisPtrTy<1>, SDTCisVT<2, OtherVT> ]>; def SDT_PPCTC_ret : SDTypeProfile<0, 2, [ SDTCisPtrTy<0>, SDTCisVT<1, i32> ]>; def tocentry32 : Operand { let MIOperandInfo = (ops i32imm:$imm); } def SDT_PPCqvfperm : SDTypeProfile<1, 3, [ SDTCisVec<0>, SDTCisSameAs<0, 1>, SDTCisSameAs<0, 2>, SDTCisVec<3> ]>; def SDT_PPCqvgpci : SDTypeProfile<1, 1, [ SDTCisVec<0>, SDTCisInt<1> ]>; def SDT_