|
13 | 13 |
|
14 | 14 | #include "InstCombineInternal.h"
|
15 | 15 | #include "llvm/ADT/APSInt.h"
|
| 16 | +#include "llvm/ADT/SetVector.h" |
16 | 17 | #include "llvm/ADT/Statistic.h"
|
17 | 18 | #include "llvm/Analysis/ConstantFolding.h"
|
18 | 19 | #include "llvm/Analysis/InstructionSimplify.h"
|
@@ -595,6 +596,318 @@ static Value *EvaluateGEPOffsetExpression(User *GEP, InstCombiner &IC,
|
595 | 596 | return IC.Builder->CreateAdd(VariableIdx, OffsetVal, "offset");
|
596 | 597 | }
|
597 | 598 |
|
| 599 | +/// Returns true if we can rewrite Start as a GEP with pointer Base |
| 600 | +/// and some integer offset. The nodes that need to be re-written |
| 601 | +/// for this transformation will be added to Explored. |
| 602 | +static bool canRewriteGEPAsOffset(Value *Start, Value *Base, |
| 603 | + const DataLayout &DL, |
| 604 | + SetVector<Value *> &Explored) { |
| 605 | + SmallVector<Value *, 16> WorkList(1, Start); |
| 606 | + Explored.insert(Base); |
| 607 | + |
| 608 | + // The following traversal gives us an order which can be used |
| 609 | + // when doing the final transformation. Since in the final |
| 610 | + // transformation we create the PHI replacement instructions first, |
| 611 | + // we don't have to get them in any particular order. |
| 612 | + // |
| 613 | + // However, for other instructions we will have to traverse the |
| 614 | + // operands of an instruction first, which means that we have to |
| 615 | + // do a post-order traversal. |
| 616 | + while (!WorkList.empty()) { |
| 617 | + SetVector<PHINode *> PHIs; |
| 618 | + |
| 619 | + while (!WorkList.empty()) { |
| 620 | + if (Explored.size() >= 100) |
| 621 | + return false; |
| 622 | + |
| 623 | + Value *V = WorkList.back(); |
| 624 | + |
| 625 | + if (Explored.count(V) != 0) { |
| 626 | + WorkList.pop_back(); |
| 627 | + continue; |
| 628 | + } |
| 629 | + |
| 630 | + if (!isa<IntToPtrInst>(V) && !isa<PtrToIntInst>(V) && |
| 631 | + !isa<GEPOperator>(V) && !isa<PHINode>(V)) |
| 632 | + // We've found some value that we can't explore which is different from |
| 633 | + // the base. Therefore we can't do this transformation. |
| 634 | + return false; |
| 635 | + |
| 636 | + if (isa<IntToPtrInst>(V) || isa<PtrToIntInst>(V)) { |
| 637 | + auto *CI = dyn_cast<CastInst>(V); |
| 638 | + if (!CI->isNoopCast(DL)) |
| 639 | + return false; |
| 640 | + |
| 641 | + if (Explored.count(CI->getOperand(0)) == 0) |
| 642 | + WorkList.push_back(CI->getOperand(0)); |
| 643 | + } |
| 644 | + |
| 645 | + if (auto *GEP = dyn_cast<GEPOperator>(V)) { |
| 646 | + // We're limiting the GEP to having one index. This will preserve |
| 647 | + // the original pointer type. We could handle more cases in the |
| 648 | + // future. |
| 649 | + if (GEP->getNumIndices() != 1 || !GEP->isInBounds() || |
| 650 | + GEP->getType() != Start->getType()) |
| 651 | + return false; |
| 652 | + |
| 653 | + if (Explored.count(GEP->getOperand(0)) == 0) |
| 654 | + WorkList.push_back(GEP->getOperand(0)); |
| 655 | + } |
| 656 | + |
| 657 | + if (WorkList.back() == V) { |
| 658 | + WorkList.pop_back(); |
| 659 | + // We've finished visiting this node, mark it as such. |
| 660 | + Explored.insert(V); |
| 661 | + } |
| 662 | + |
| 663 | + if (auto *PN = dyn_cast<PHINode>(V)) { |
| 664 | + Explored.insert(PN); |
| 665 | + PHIs.insert(PN); |
| 666 | + } |
| 667 | + } |
| 668 | + |
| 669 | + // Explore the PHI nodes further. |
| 670 | + for (auto *PN : PHIs) |
| 671 | + for (Value *Op : PN->incoming_values()) |
| 672 | + if (Explored.count(Op) == 0) |
| 673 | + WorkList.push_back(Op); |
| 674 | + } |
| 675 | + |
| 676 | + // Make sure that we can do this. Since we can't insert GEPs in a basic |
| 677 | + // block before a PHI node, we can't easily do this transformation if |
| 678 | + // we have PHI node users of transformed instructions. |
| 679 | + for (Value *Val : Explored) { |
| 680 | + for (Value *Use : Val->uses()) { |
| 681 | + |
| 682 | + auto *PHI = dyn_cast<PHINode>(Use); |
| 683 | + auto *Inst = dyn_cast<Instruction>(Val); |
| 684 | + |
| 685 | + if (Inst == Base || Inst == PHI || !Inst || !PHI || |
| 686 | + Explored.count(PHI) == 0) |
| 687 | + continue; |
| 688 | + |
| 689 | + if (PHI->getParent() == Inst->getParent()) |
| 690 | + return false; |
| 691 | + } |
| 692 | + } |
| 693 | + return true; |
| 694 | +} |
| 695 | + |
| 696 | +// Sets the appropriate insert point on Builder where we can add |
| 697 | +// a replacement Instruction for V (if that is possible). |
| 698 | +static void setInsertionPoint(IRBuilder<> &Builder, Value *V, |
| 699 | + bool Before = true) { |
| 700 | + if (auto *PHI = dyn_cast<PHINode>(V)) { |
| 701 | + Builder.SetInsertPoint(&*PHI->getParent()->getFirstInsertionPt()); |
| 702 | + return; |
| 703 | + } |
| 704 | + if (auto *I = dyn_cast<Instruction>(V)) { |
| 705 | + if (!Before) |
| 706 | + I = &*std::next(I->getIterator()); |
| 707 | + Builder.SetInsertPoint(I); |
| 708 | + return; |
| 709 | + } |
| 710 | + if (auto *A = dyn_cast<Argument>(V)) { |
| 711 | + // Set the insertion point in the entry block. |
| 712 | + BasicBlock &Entry = A->getParent()->getEntryBlock(); |
| 713 | + Builder.SetInsertPoint(&*Entry.getFirstInsertionPt()); |
| 714 | + return; |
| 715 | + } |
| 716 | + // Otherwise, this is a constant and we don't need to set a new |
| 717 | + // insertion point. |
| 718 | + assert(isa<Constant>(V) && "Setting insertion point for unknown value!"); |
| 719 | +} |
| 720 | + |
| 721 | +/// Returns a re-written value of Start as an indexed GEP using Base as a |
| 722 | +/// pointer. |
| 723 | +static Value *rewriteGEPAsOffset(Value *Start, Value *Base, |
| 724 | + const DataLayout &DL, |
| 725 | + SetVector<Value *> &Explored) { |
| 726 | + // Perform all the substitutions. This is a bit tricky because we can |
| 727 | + // have cycles in our use-def chains. |
| 728 | + // 1. Create the PHI nodes without any incoming values. |
| 729 | + // 2. Create all the other values. |
| 730 | + // 3. Add the edges for the PHI nodes. |
| 731 | + // 4. Emit GEPs to get the original pointers. |
| 732 | + // 5. Remove the original instructions. |
| 733 | + Type *IndexType = IntegerType::get( |
| 734 | + Base->getContext(), DL.getPointerTypeSizeInBits(Start->getType())); |
| 735 | + |
| 736 | + DenseMap<Value *, Value *> NewInsts; |
| 737 | + NewInsts[Base] = ConstantInt::getNullValue(IndexType); |
| 738 | + |
| 739 | + // Create the new PHI nodes, without adding any incoming values. |
| 740 | + for (Value *Val : Explored) { |
| 741 | + if (Val == Base) |
| 742 | + continue; |
| 743 | + // Create empty phi nodes. This avoids cyclic dependencies when creating |
| 744 | + // the remaining instructions. |
| 745 | + if (auto *PHI = dyn_cast<PHINode>(Val)) |
| 746 | + NewInsts[PHI] = PHINode::Create(IndexType, PHI->getNumIncomingValues(), |
| 747 | + PHI->getName() + ".idx", PHI); |
| 748 | + } |
| 749 | + IRBuilder<> Builder(Base->getContext()); |
| 750 | + |
| 751 | + // Create all the other instructions. |
| 752 | + for (Value *Val : Explored) { |
| 753 | + |
| 754 | + if (NewInsts.find(Val) != NewInsts.end()) |
| 755 | + continue; |
| 756 | + |
| 757 | + if (auto *CI = dyn_cast<CastInst>(Val)) { |
| 758 | + NewInsts[CI] = NewInsts[CI->getOperand(0)]; |
| 759 | + continue; |
| 760 | + } |
| 761 | + if (auto *GEP = dyn_cast<GEPOperator>(Val)) { |
| 762 | + Value *Index = NewInsts[GEP->getOperand(1)] ? NewInsts[GEP->getOperand(1)] |
| 763 | + : GEP->getOperand(1); |
| 764 | + setInsertionPoint(Builder, GEP); |
| 765 | + // Indices might need to be sign extended. GEPs will magically do |
| 766 | + // this, but we need to do it ourselves here. |
| 767 | + if (Index->getType()->getScalarSizeInBits() != |
| 768 | + NewInsts[GEP->getOperand(0)]->getType()->getScalarSizeInBits()) { |
| 769 | + Index = Builder.CreateSExtOrTrunc( |
| 770 | + Index, NewInsts[GEP->getOperand(0)]->getType(), |
| 771 | + GEP->getOperand(0)->getName() + ".sext"); |
| 772 | + } |
| 773 | + |
| 774 | + auto *Op = NewInsts[GEP->getOperand(0)]; |
| 775 | + if (isa<ConstantInt>(Op) && dyn_cast<ConstantInt>(Op)->isZero()) |
| 776 | + NewInsts[GEP] = Index; |
| 777 | + else |
| 778 | + NewInsts[GEP] = Builder.CreateNSWAdd( |
| 779 | + Op, Index, GEP->getOperand(0)->getName() + ".add"); |
| 780 | + continue; |
| 781 | + } |
| 782 | + if (isa<PHINode>(Val)) |
| 783 | + continue; |
| 784 | + |
| 785 | + llvm_unreachable("Unexpected instruction type"); |
| 786 | + } |
| 787 | + |
| 788 | + // Add the incoming values to the PHI nodes. |
| 789 | + for (Value *Val : Explored) { |
| 790 | + if (Val == Base) |
| 791 | + continue; |
| 792 | + // All the instructions have been created, we can now add edges to the |
| 793 | + // phi nodes. |
| 794 | + if (auto *PHI = dyn_cast<PHINode>(Val)) { |
| 795 | + PHINode *NewPhi = static_cast<PHINode *>(NewInsts[PHI]); |
| 796 | + for (unsigned I = 0, E = PHI->getNumIncomingValues(); I < E; ++I) { |
| 797 | + Value *NewIncoming = PHI->getIncomingValue(I); |
| 798 | + |
| 799 | + if (NewInsts.find(NewIncoming) != NewInsts.end()) |
| 800 | + NewIncoming = NewInsts[NewIncoming]; |
| 801 | + |
| 802 | + NewPhi->addIncoming(NewIncoming, PHI->getIncomingBlock(I)); |
| 803 | + } |
| 804 | + } |
| 805 | + } |
| 806 | + |
| 807 | + for (Value *Val : Explored) { |
| 808 | + if (Val == Base) |
| 809 | + continue; |
| 810 | + |
| 811 | + // Depending on the type, for external users we have to emit |
| 812 | + // a GEP or a GEP + ptrtoint. |
| 813 | + setInsertionPoint(Builder, Val, false); |
| 814 | + |
| 815 | + // If required, create an inttoptr instruction for Base. |
| 816 | + Value *NewBase = Base; |
| 817 | + if (!Base->getType()->isPointerTy()) |
| 818 | + NewBase = Builder.CreateBitOrPointerCast(Base, Start->getType(), |
| 819 | + Start->getName() + "to.ptr"); |
| 820 | + |
| 821 | + Value *GEP = Builder.CreateInBoundsGEP( |
| 822 | + Start->getType()->getPointerElementType(), NewBase, |
| 823 | + makeArrayRef(NewInsts[Val]), Val->getName() + ".ptr"); |
| 824 | + |
| 825 | + if (!Val->getType()->isPointerTy()) { |
| 826 | + Value *Cast = Builder.CreatePointerCast(GEP, Val->getType(), |
| 827 | + Val->getName() + ".conv"); |
| 828 | + GEP = Cast; |
| 829 | + } |
| 830 | + Val->replaceAllUsesWith(GEP); |
| 831 | + } |
| 832 | + |
| 833 | + return NewInsts[Start]; |
| 834 | +} |
| 835 | + |
| 836 | +/// Looks through GEPs, IntToPtrInsts and PtrToIntInsts in order to express |
| 837 | +/// the input Value as a constant indexed GEP. Returns a pair containing |
| 838 | +/// the GEPs Pointer and Index. |
| 839 | +static std::pair<Value *, Value *> |
| 840 | +getAsConstantIndexedAddress(Value *V, const DataLayout &DL) { |
| 841 | + Type *IndexType = IntegerType::get(V->getContext(), |
| 842 | + DL.getPointerTypeSizeInBits(V->getType())); |
| 843 | + |
| 844 | + Constant *Index = ConstantInt::getNullValue(IndexType); |
| 845 | + while (true) { |
| 846 | + if (GEPOperator *GEP = dyn_cast<GEPOperator>(V)) { |
| 847 | + // We accept only inbouds GEPs here to exclude the possibility of |
| 848 | + // overflow. |
| 849 | + if (!GEP->isInBounds()) |
| 850 | + break; |
| 851 | + if (GEP->hasAllConstantIndices() && GEP->getNumIndices() == 1 && |
| 852 | + GEP->getType() == V->getType()) { |
| 853 | + V = GEP->getOperand(0); |
| 854 | + Constant *GEPIndex = static_cast<Constant *>(GEP->getOperand(1)); |
| 855 | + Index = ConstantExpr::getAdd( |
| 856 | + Index, ConstantExpr::getSExtOrBitCast(GEPIndex, IndexType)); |
| 857 | + continue; |
| 858 | + } |
| 859 | + break; |
| 860 | + } |
| 861 | + if (auto *CI = dyn_cast<IntToPtrInst>(V)) { |
| 862 | + if (!CI->isNoopCast(DL)) |
| 863 | + break; |
| 864 | + V = CI->getOperand(0); |
| 865 | + continue; |
| 866 | + } |
| 867 | + if (auto *CI = dyn_cast<PtrToIntInst>(V)) { |
| 868 | + if (!CI->isNoopCast(DL)) |
| 869 | + break; |
| 870 | + V = CI->getOperand(0); |
| 871 | + continue; |
| 872 | + } |
| 873 | + break; |
| 874 | + } |
| 875 | + return {V, Index}; |
| 876 | +} |
| 877 | + |
| 878 | +// Converts (CMP GEPLHS, RHS) if this change would make RHS a constant. |
| 879 | +// We can look through PHIs, GEPs and casts in order to determine a |
| 880 | +// common base between GEPLHS and RHS. |
| 881 | +static Instruction *transformToIndexedCompare(GEPOperator *GEPLHS, Value *RHS, |
| 882 | + ICmpInst::Predicate Cond, |
| 883 | + const DataLayout &DL) { |
| 884 | + if (!GEPLHS->hasAllConstantIndices()) |
| 885 | + return nullptr; |
| 886 | + |
| 887 | + Value *PtrBase, *Index; |
| 888 | + std::tie(PtrBase, Index) = getAsConstantIndexedAddress(GEPLHS, DL); |
| 889 | + |
| 890 | + // The set of nodes that will take part in this transformation. |
| 891 | + SetVector<Value *> Nodes; |
| 892 | + |
| 893 | + if (!canRewriteGEPAsOffset(RHS, PtrBase, DL, Nodes)) |
| 894 | + return nullptr; |
| 895 | + |
| 896 | + // We know we can re-write this as |
| 897 | + // ((gep Ptr, OFFSET1) cmp (gep Ptr, OFFSET2) |
| 898 | + // Since we've only looked through inbouds GEPs we know that we |
| 899 | + // can't have overflow on either side. We can therefore re-write |
| 900 | + // this as: |
| 901 | + // OFFSET1 cmp OFFSET2 |
| 902 | + Value *NewRHS = rewriteGEPAsOffset(RHS, PtrBase, DL, Nodes); |
| 903 | + |
| 904 | + // RewriteGEPAsOffset has replaced RHS and all of its uses with a re-written |
| 905 | + // GEP having PtrBase as the pointer base, and has returned in NewRHS the |
| 906 | + // offset. Since Index is the offset of LHS to the base pointer, we will now |
| 907 | + // compare the offsets instead of comparing the pointers. |
| 908 | + return new ICmpInst(ICmpInst::getSignedPredicate(Cond), Index, NewRHS); |
| 909 | +} |
| 910 | + |
598 | 911 | /// FoldGEPICmp - Fold comparisons between a GEP instruction and something
|
599 | 912 | /// else. At this point we know that the GEP is on the LHS of the comparison.
|
600 | 913 | Instruction *InstCombiner::FoldGEPICmp(GEPOperator *GEPLHS, Value *RHS,
|
@@ -674,8 +987,9 @@ Instruction *InstCombiner::FoldGEPICmp(GEPOperator *GEPLHS, Value *RHS,
|
674 | 987 | }
|
675 | 988 |
|
676 | 989 | // Otherwise, the base pointers are different and the indices are
|
677 |
| - // different, bail out. |
678 |
| - return nullptr; |
| 990 | + // different. Try convert this to an indexed compare by looking through |
| 991 | + // PHIs/casts. |
| 992 | + return transformToIndexedCompare(GEPLHS, RHS, Cond, DL); |
679 | 993 | }
|
680 | 994 |
|
681 | 995 | // If one of the GEPs has all zero indices, recurse.
|
@@ -727,7 +1041,10 @@ Instruction *InstCombiner::FoldGEPICmp(GEPOperator *GEPLHS, Value *RHS,
|
727 | 1041 | return new ICmpInst(ICmpInst::getSignedPredicate(Cond), L, R);
|
728 | 1042 | }
|
729 | 1043 | }
|
730 |
| - return nullptr; |
| 1044 | + |
| 1045 | + // Try convert this to an indexed compare by looking through PHIs/casts as a |
| 1046 | + // last resort. |
| 1047 | + return transformToIndexedCompare(GEPLHS, RHS, Cond, DL); |
731 | 1048 | }
|
732 | 1049 |
|
733 | 1050 | Instruction *InstCombiner::FoldAllocaCmp(ICmpInst &ICI, AllocaInst *Alloca,
|
|
0 commit comments