[ExpandMemCmp] Improve non-equality comparisons with zero.

[courbet]

Summary

This is one step towards fixing PR45147.

[courbet]

clang-format

[courbet]

Benchmark results from test-suite. TBH I'm not exactly sure why only the "greater" case gets better, but overall the results are promising.

BM_MemCmp<1, EqZero, None>                        2.51µs ± 2%             2.47µs ± 2%   -1.48%         (p=0.017 n=9+10)
BM_MemCmp<1, EqZero, First>                       2.50µs ± 3%             2.49µs ± 2%     ~            (p=0.356 n=10+9)
BM_MemCmp<1, EqZero, Mid>                         2.49µs ± 3%             2.48µs ± 2%     ~            (p=0.447 n=10+9)
BM_MemCmp<1, EqZero, Last>                        2.51µs ± 2%             2.48µs ± 2%     ~             (p=0.113 n=9+9)
BM_MemCmp<1, LessThanZero, None>                  2.48µs ± 3%             2.48µs ± 2%     ~             (p=0.605 n=9+9)
BM_MemCmp<1, LessThanZero, First>                 2.49µs ± 3%             2.49µs ± 2%     ~             (p=1.000 n=9+9)
BM_MemCmp<1, LessThanZero, Mid>                   2.49µs ± 3%             2.49µs ± 2%     ~             (p=0.931 n=9+9)
BM_MemCmp<1, LessThanZero, Last>                  2.49µs ± 3%             2.48µs ± 2%     ~            (p=0.905 n=9+10)
BM_MemCmp<1, GreaterThanZero, None>               2.89µs ± 3%             2.89µs ± 2%     ~            (p=0.720 n=9+10)
BM_MemCmp<1, GreaterThanZero, First>              2.89µs ± 3%             2.90µs ± 2%     ~            (p=0.515 n=8+10)
BM_MemCmp<1, GreaterThanZero, Mid>                2.91µs ± 5%             2.91µs ± 1%     ~             (p=0.546 n=9+9)
BM_MemCmp<1, GreaterThanZero, Last>               2.89µs ± 2%             2.90µs ± 2%     ~             (p=0.606 n=8+9)
BM_MemCmp<2, EqZero, None>                        1.24µs ± 2%             1.24µs ± 2%     ~            (p=0.696 n=8+10)
BM_MemCmp<2, EqZero, First>                       1.24µs ± 2%             1.24µs ± 2%     ~            (p=0.460 n=8+10)
BM_MemCmp<2, EqZero, Mid>                         1.24µs ± 2%             1.24µs ± 2%     ~             (p=1.000 n=8+9)
BM_MemCmp<2, EqZero, Last>                        1.23µs ± 2%             1.24µs ± 2%     ~             (p=0.743 n=8+9)
BM_MemCmp<2, LessThanZero, None>                  1.70µs ± 4%             1.71µs ± 2%     ~             (p=0.481 n=9+8)
BM_MemCmp<2, LessThanZero, First>                 1.71µs ± 3%             1.83µs ±15%     ~           (p=0.247 n=10+10)
BM_MemCmp<2, LessThanZero, Mid>                   1.71µs ± 3%             1.83µs ±16%     ~            (p=0.497 n=9+10)
BM_MemCmp<2, LessThanZero, Last>                  1.71µs ± 3%             1.83µs ±16%     ~            (p=0.604 n=9+10)
BM_MemCmp<2, GreaterThanZero, None>               2.16µs ±15%             1.65µs ±16%  -23.59%        (p=0.000 n=10+10)
BM_MemCmp<2, GreaterThanZero, First>              2.16µs ±15%             1.55µs ± 2%  -28.48%         (p=0.000 n=10+8)
BM_MemCmp<2, GreaterThanZero, Mid>                2.17µs ±15%             1.55µs ± 4%  -28.40%         (p=0.000 n=10+8)
BM_MemCmp<2, GreaterThanZero, Last>               2.16µs ±15%             1.56µs ± 6%  -27.65%         (p=0.000 n=10+8)
BM_MemCmp<3, EqZero, None>                        1.14µs ± 3%             1.22µs ±15%     ~            (p=0.068 n=8+10)
BM_MemCmp<3, EqZero, First>                       1.15µs ± 3%             1.23µs ±15%     ~            (p=0.315 n=8+10)
BM_MemCmp<3, EqZero, Mid>                         1.15µs ± 5%             1.22µs ±15%     ~            (p=0.356 n=9+10)
BM_MemCmp<3, EqZero, Last>                        1.14µs ± 3%             1.22µs ±16%     ~            (p=0.315 n=9+10)
BM_MemCmp<3, LessThanZero, None>                  1.24µs ± 3%             1.25µs ± 3%     ~             (p=0.370 n=9+8)
BM_MemCmp<3, LessThanZero, First>                 1.53µs ± 3%             1.52µs ± 3%     ~             (p=0.796 n=9+9)
BM_MemCmp<3, LessThanZero, Mid>                   1.47µs ± 3%             1.47µs ± 2%     ~            (p=0.897 n=8+10)
BM_MemCmp<3, LessThanZero, Last>                  1.24µs ± 3%             1.24µs ± 2%     ~             (p=0.963 n=8+9)
BM_MemCmp<3, GreaterThanZero, None>               1.44µs ± 3%             1.38µs ± 2%   -4.56%          (p=0.000 n=9+9)
BM_MemCmp<3, GreaterThanZero, First>              1.66µs ± 3%             1.38µs ± 2%  -16.60%          (p=0.000 n=9+9)
BM_MemCmp<3, GreaterThanZero, Mid>                1.66µs ± 3%             0.93µs ± 2%  -43.92%          (p=0.000 n=9+9)
BM_MemCmp<3, GreaterThanZero, Last>               1.45µs ± 3%             1.37µs ± 2%   -5.07%          (p=0.000 n=9+9)
BM_MemCmp<4, EqZero, None>                         624ns ± 3%              621ns ± 2%     ~             (p=0.796 n=9+9)
BM_MemCmp<4, EqZero, First>                        625ns ± 3%              621ns ± 2%     ~            (p=0.549 n=9+10)
BM_MemCmp<4, EqZero, Mid>                          625ns ± 3%              620ns ± 2%     ~             (p=0.340 n=9+9)
BM_MemCmp<4, EqZero, Last>                         624ns ± 3%              620ns ± 2%     ~             (p=0.546 n=9+9)
BM_MemCmp<4, LessThanZero, None>                  1.01µs ± 3%             1.01µs ± 2%     ~             (p=0.605 n=9+9)
BM_MemCmp<4, LessThanZero, First>                 1.02µs ± 4%             1.01µs ± 2%     ~             (p=0.190 n=9+9)
BM_MemCmp<4, LessThanZero, Mid>                   1.01µs ± 3%             1.01µs ± 2%     ~             (p=0.963 n=9+8)
BM_MemCmp<4, LessThanZero, Last>                  1.01µs ± 3%             1.01µs ± 2%     ~             (p=0.815 n=9+8)
BM_MemCmp<4, GreaterThanZero, None>               1.17µs ± 3%             0.77µs ± 2%  -33.58%          (p=0.000 n=9+9)
BM_MemCmp<4, GreaterThanZero, First>              1.17µs ± 3%             0.77µs ± 2%  -33.64%          (p=0.000 n=9+9)
BM_MemCmp<4, GreaterThanZero, Mid>                1.16µs ± 4%             0.78µs ± 3%  -33.20%          (p=0.000 n=9+8)
BM_MemCmp<4, GreaterThanZero, Last>               1.16µs ± 3%             0.78µs ± 2%  -33.33%          (p=0.000 n=9+9)
BM_MemCmp<5, EqZero, None>                         687ns ± 4%              688ns ± 2%     ~            (p=0.549 n=10+9)
BM_MemCmp<5, EqZero, First>                        688ns ± 3%              688ns ± 2%     ~            (p=0.661 n=10+9)
BM_MemCmp<5, EqZero, Mid>                          748ns ± 3%              691ns ± 1%   -7.61%         (p=0.000 n=10+9)
BM_MemCmp<5, EqZero, Last>                         747ns ± 5%              691ns ± 1%   -7.61%        (p=0.000 n=10+10)
BM_MemCmp<5, LessThanZero, None>                   745ns ± 4%              744ns ± 2%     ~            (p=0.842 n=9+10)
BM_MemCmp<5, LessThanZero, First>                  875ns ± 4%              887ns ± 3%   +1.36%         (p=0.043 n=9+10)
BM_MemCmp<5, LessThanZero, Mid>                    884ns ± 4%              879ns ± 2%     ~            (p=0.720 n=9+10)
BM_MemCmp<5, LessThanZero, Last>                   751ns ± 6%              744ns ± 2%     ~            (p=0.842 n=9+10)
BM_MemCmp<5, GreaterThanZero, None>                921ns ±17%              826ns ± 2%  -10.37%        (p=0.000 n=10+10)
BM_MemCmp<5, GreaterThanZero, First>               995ns ± 3%              822ns ± 2%  -17.41%          (p=0.000 n=8+9)
BM_MemCmp<5, GreaterThanZero, Mid>                 990ns ± 2%              558ns ± 2%  -43.67%          (p=0.000 n=8+8)
BM_MemCmp<5, GreaterThanZero, Last>                864ns ± 2%              824ns ± 2%   -4.63%          (p=0.000 n=8+8)
BM_MemCmp<6, EqZero, None>                         569ns ± 2%              611ns ±16%     ~            (p=0.274 n=8+10)
BM_MemCmp<6, EqZero, First>                        576ns ± 6%              610ns ±16%     ~            (p=0.780 n=9+10)
BM_MemCmp<6, EqZero, Mid>                          576ns ± 3%              654ns ±18%  +13.51%         (p=0.000 n=8+10)
BM_MemCmp<6, EqZero, Last>                         577ns ± 3%              664ns ±16%  +15.10%         (p=0.000 n=8+10)
BM_MemCmp<6, LessThanZero, None>                  1.00µs ±15%             1.00µs ±15%     ~           (p=0.631 n=10+10)
BM_MemCmp<6, LessThanZero, First>                  785ns ±15%              780ns ±16%     ~           (p=0.853 n=10+10)
BM_MemCmp<6, LessThanZero, Mid>                    745ns ±11%              781ns ±16%     ~            (p=0.497 n=9+10)
BM_MemCmp<6, LessThanZero, Last>                  1.14µs ± 3%             1.21µs ±17%     ~            (p=0.633 n=8+10)
BM_MemCmp<6, GreaterThanZero, None>               1.08µs ± 4%             0.77µs ± 1%  -28.73%          (p=0.000 n=8+8)
BM_MemCmp<6, GreaterThanZero, First>               832ns ± 3%              772ns ± 2%   -7.24%         (p=0.000 n=9+10)
BM_MemCmp<6, GreaterThanZero, Mid>                 831ns ± 3%              464ns ± 1%  -44.12%          (p=0.000 n=8+9)
BM_MemCmp<6, GreaterThanZero, Last>               1.24µs ± 3%             0.77µs ± 1%  -37.93%          (p=0.000 n=8+9)
BM_MemCmp<7, EqZero, None>                         452ns ± 3%              446ns ± 1%   -1.50%          (p=0.027 n=8+9)
BM_MemCmp<7, EqZero, First>                        451ns ± 3%              447ns ± 2%     ~             (p=0.222 n=9+9)
BM_MemCmp<7, EqZero, Mid>                          452ns ± 3%              446ns ± 2%     ~             (p=0.094 n=9+9)
BM_MemCmp<7, EqZero, Last>                         451ns ± 3%              444ns ± 1%   -1.38%         (p=0.022 n=10+9)
BM_MemCmp<7, LessThanZero, None>                  1.90µs ± 3%             1.89µs ± 3%     ~             (p=0.546 n=9+9)
BM_MemCmp<7, LessThanZero, First>                 1.59µs ± 3%             1.58µs ± 2%     ~             (p=0.489 n=9+9)
BM_MemCmp<7, LessThanZero, Mid>                   1.59µs ± 3%             1.40µs ± 2%  -11.88%          (p=0.000 n=9+9)
BM_MemCmp<7, LessThanZero, Last>                  1.90µs ± 3%             1.89µs ± 2%     ~             (p=0.863 n=9+9)
BM_MemCmp<7, GreaterThanZero, None>               1.99µs ± 3%             1.98µs ± 4%     ~             (p=0.931 n=9+9)
BM_MemCmp<7, GreaterThanZero, First>              1.59µs ± 6%             1.58µs ± 1%     ~             (p=0.606 n=9+8)
BM_MemCmp<7, GreaterThanZero, Mid>                1.59µs ± 3%             1.40µs ± 2%  -11.46%          (p=0.000 n=9+8)
BM_MemCmp<7, GreaterThanZero, Last>               1.98µs ± 4%             1.98µs ± 3%     ~             (p=0.863 n=9+9)
BM_MemCmp<8, EqZero, None>                         313ns ± 4%              311ns ± 2%     ~             (p=0.863 n=9+9)
BM_MemCmp<8, EqZero, First>                        313ns ± 4%              312ns ± 4%     ~             (p=1.000 n=9+9)
BM_MemCmp<8, EqZero, Mid>                          313ns ± 4%              312ns ± 2%     ~             (p=0.888 n=9+8)
BM_MemCmp<8, EqZero, Last>                         313ns ± 4%              312ns ± 1%     ~             (p=0.796 n=9+9)
BM_MemCmp<8, LessThanZero, None>                   586ns ± 3%              584ns ± 2%     ~            (p=0.842 n=10+9)
BM_MemCmp<8, LessThanZero, First>                  585ns ± 2%              584ns ± 2%     ~             (p=0.931 n=9+9)
BM_MemCmp<8, LessThanZero, Mid>                    584ns ± 2%              583ns ± 1%     ~             (p=0.931 n=9+9)
BM_MemCmp<8, LessThanZero, Last>                   584ns ± 3%              582ns ± 1%     ~             (p=0.673 n=9+8)
BM_MemCmp<8, GreaterThanZero, None>                655ns ± 1%              464ns ± 1%  -29.07%         (p=0.000 n=8+10)
BM_MemCmp<8, GreaterThanZero, First>               657ns ± 2%              464ns ± 1%  -29.39%          (p=0.000 n=9+9)
BM_MemCmp<8, GreaterThanZero, Mid>                 661ns ± 3%              465ns ± 2%  -29.73%         (p=0.000 n=9+10)
BM_MemCmp<8, GreaterThanZero, Last>                655ns ± 1%              465ns ± 2%  -29.08%         (p=0.000 n=8+10)
BM_MemCmp<15, EqZero, None>                        213ns ± 1%              213ns ± 1%     ~            (p=0.897 n=8+10)
BM_MemCmp<15, EqZero, First>                       214ns ± 3%              214ns ± 2%     ~            (p=0.356 n=9+10)
BM_MemCmp<15, EqZero, Mid>                         214ns ± 1%              214ns ± 2%     ~            (p=0.897 n=8+10)
BM_MemCmp<15, EqZero, Last>                        227ns ±17%              214ns ± 1%     ~           (p=0.218 n=10+10)
BM_MemCmp<15, LessThanZero, None>                  856ns ± 2%              859ns ± 3%     ~             (p=0.606 n=8+9)
BM_MemCmp<15, LessThanZero, First>                 578ns ± 1%              576ns ± 2%     ~             (p=0.574 n=8+8)
BM_MemCmp<15, LessThanZero, Mid>                   656ns ± 2%              617ns ±17%     ~            (p=0.173 n=8+10)
BM_MemCmp<15, LessThanZero, Last>                  854ns ± 2%              920ns ±16%     ~            (p=0.203 n=8+10)
BM_MemCmp<15, GreaterThanZero, None>               962ns ±17%              969ns ±16%     ~           (p=0.796 n=10+10)
BM_MemCmp<15, GreaterThanZero, First>              656ns ± 3%              720ns ±13%     ~            (p=0.101 n=8+10)
BM_MemCmp<15, GreaterThanZero, Mid>                704ns ±16%              724ns ±14%     ~           (p=0.218 n=10+10)
BM_MemCmp<15, GreaterThanZero, Last>               973ns ±16%              980ns ±15%     ~           (p=0.436 n=10+10)
BM_MemCmp<16, EqZero, None>                        212ns ±16%              212ns ±16%     ~           (p=0.739 n=10+10)
BM_MemCmp<16, EqZero, First>                       200ns ± 8%              212ns ±16%     ~            (p=1.000 n=9+10)
BM_MemCmp<16, EqZero, Mid>                         197ns ± 3%              206ns ±19%     ~            (p=0.515 n=8+10)
BM_MemCmp<16, EqZero, Last>                        198ns ± 3%              196ns ± 2%     ~             (p=0.277 n=8+9)
BM_MemCmp<16, LessThanZero, None>                  390ns ± 3%              388ns ± 2%     ~            (p=0.447 n=9+10)
BM_MemCmp<16, LessThanZero, First>                 315ns ± 3%              313ns ± 1%     ~             (p=0.321 n=8+9)
BM_MemCmp<16, LessThanZero, Mid>                   467ns ± 3%              465ns ± 1%     ~             (p=0.481 n=8+9)
BM_MemCmp<16, LessThanZero, Last>                  471ns ± 4%              465ns ± 2%     ~             (p=0.258 n=9+9)
BM_MemCmp<16, GreaterThanZero, None>               427ns ± 3%              369ns ± 1%  -13.68%          (p=0.000 n=9+8)
BM_MemCmp<16, GreaterThanZero, First>              355ns ± 3%              369ns ± 1%   +3.86%          (p=0.000 n=9+8)
BM_MemCmp<16, GreaterThanZero, Mid>                503ns ± 2%              371ns ± 3%  -26.12%          (p=0.000 n=9+9)
BM_MemCmp<16, GreaterThanZero, Last>               505ns ± 3%              371ns ± 2%  -26.56%         (p=0.000 n=9+10)
BM_MemCmp<31, EqZero, None>                        146ns ± 3%              145ns ± 2%     ~             (p=0.796 n=9+9)
BM_MemCmp<31, EqZero, First>                       145ns ± 4%              145ns ± 2%     ~             (p=0.546 n=9+9)
BM_MemCmp<31, EqZero, Mid>                         146ns ± 4%              145ns ± 1%     ~             (p=0.666 n=9+9)
BM_MemCmp<31, EqZero, Last>                        146ns ± 3%              146ns ± 1%     ~             (p=0.666 n=9+9)
BM_MemCmp<31, LessThanZero, None>                  465ns ± 8%              463ns ± 4%     ~            (p=0.762 n=10+8)
BM_MemCmp<31, LessThanZero, First>                 434ns ± 3%              435ns ± 2%     ~             (p=0.888 n=9+8)
BM_MemCmp<31, LessThanZero, Mid>                   480ns ± 3%              480ns ± 2%     ~             (p=0.666 n=9+9)
BM_MemCmp<31, LessThanZero, Last>                  549ns ± 1%              552ns ± 2%     ~             (p=0.525 n=8+9)
BM_MemCmp<31, GreaterThanZero, None>               446ns ± 4%              447ns ± 2%     ~             (p=0.743 n=9+8)
BM_MemCmp<31, GreaterThanZero, First>              433ns ± 3%              435ns ± 2%     ~             (p=0.222 n=9+9)
BM_MemCmp<31, GreaterThanZero, Mid>                474ns ± 3%              475ns ± 2%     ~            (p=0.447 n=10+9)
BM_MemCmp<31, GreaterThanZero, Last>               552ns ± 2%              551ns ± 1%     ~            (p=0.965 n=10+8)
BM_MemCmp<32, EqZero, None>                        140ns ± 2%              141ns ± 3%     ~            (p=0.447 n=10+9)
BM_MemCmp<32, EqZero, First>                       139ns ± 1%              140ns ± 1%     ~             (p=0.297 n=9+9)
BM_MemCmp<32, EqZero, Mid>                         139ns ± 2%              140ns ± 1%     ~            (p=0.356 n=9+10)
BM_MemCmp<32, EqZero, Last>                        139ns ± 2%              139ns ± 2%     ~            (p=0.447 n=9+10)
BM_MemCmp<32, LessThanZero, None>                  519ns ± 4%              513ns ± 3%     ~            (p=0.133 n=10+9)
BM_MemCmp<32, LessThanZero, First>                 418ns ± 1%              418ns ± 3%     ~            (p=0.573 n=8+10)
BM_MemCmp<32, LessThanZero, Mid>                   494ns ± 1%              493ns ± 1%     ~             (p=0.815 n=8+9)
BM_MemCmp<32, LessThanZero, Last>                  535ns ± 3%              534ns ± 2%     ~            (p=0.720 n=9+10)
BM_MemCmp<32, GreaterThanZero, None>               544ns ± 3%              545ns ± 2%     ~            (p=0.661 n=9+10)
BM_MemCmp<32, GreaterThanZero, First>              419ns ± 3%              419ns ± 1%     ~            (p=0.842 n=9+10)
BM_MemCmp<32, GreaterThanZero, Mid>                508ns ± 2%              510ns ± 2%     ~            (p=0.356 n=9+10)
BM_MemCmp<32, GreaterThanZero, Last>               549ns ± 2%              552ns ± 1%     ~             (p=0.321 n=8+9)
BM_MemCmp<63, EqZero, None>                        364ns ± 1%              363ns ± 2%     ~             (p=0.234 n=8+8)
BM_MemCmp<63, EqZero, First>                       161ns ± 2%              161ns ± 2%     ~             (p=0.878 n=8+8)
BM_MemCmp<63, EqZero, Mid>                         315ns ± 1%              316ns ± 2%     ~             (p=0.574 n=8+8)
BM_MemCmp<63, EqZero, Last>                        161ns ± 2%              173ns ±16%     ~            (p=0.101 n=8+10)
BM_MemCmp<63, LessThanZero, None>                  350ns ± 2%              375ns ±17%     ~            (p=0.515 n=8+10)
BM_MemCmp<63, LessThanZero, First>                 214ns ± 2%              215ns ± 3%     ~             (p=0.798 n=8+8)
BM_MemCmp<63, LessThanZero, Mid>                   288ns ±18%              293ns ±16%     ~           (p=0.853 n=10+10)
BM_MemCmp<63, LessThanZero, Last>                  376ns ±16%              384ns ±14%     ~           (p=0.912 n=10+10)
BM_MemCmp<63, GreaterThanZero, None>               326ns ±16%              328ns ±16%     ~           (p=0.631 n=10+10)
BM_MemCmp<63, GreaterThanZero, First>              228ns ±18%              229ns ±17%     ~           (p=0.393 n=10+10)
BM_MemCmp<63, GreaterThanZero, Mid>                279ns ± 3%              299ns ±17%     ~            (p=0.573 n=8+10)
BM_MemCmp<63, GreaterThanZero, Last>               350ns ± 4%              352ns ± 3%     ~             (p=0.423 n=8+9)
BM_MemCmp<64, EqZero, None>                        407ns ± 2%              406ns ± 2%     ~            (p=0.720 n=9+10)
BM_MemCmp<64, EqZero, First>                       160ns ± 2%              158ns ± 1%     ~            (p=0.068 n=8+10)
BM_MemCmp<64, EqZero, Mid>                         321ns ± 2%              316ns ± 2%   -1.78%          (p=0.006 n=8+9)
BM_MemCmp<64, EqZero, Last>                        159ns ± 2%              158ns ± 1%     ~             (p=0.606 n=8+9)
BM_MemCmp<64, LessThanZero, None>                  401ns ± 2%              396ns ± 2%     ~             (p=0.063 n=9+9)
BM_MemCmp<64, LessThanZero, First>                 210ns ± 1%              209ns ± 1%     ~             (p=0.142 n=7+9)
BM_MemCmp<64, LessThanZero, Mid>                   285ns ± 1%              285ns ± 1%     ~             (p=0.606 n=8+9)
BM_MemCmp<64, LessThanZero, Last>                  402ns ± 3%              407ns ± 1%     ~            (p=0.053 n=10+9)
BM_MemCmp<64, GreaterThanZero, None>               375ns ± 4%              377ns ± 4%     ~            (p=0.497 n=9+10)
BM_MemCmp<64, GreaterThanZero, First>              210ns ± 2%              210ns ± 1%     ~             (p=0.340 n=9+9)
BM_MemCmp<64, GreaterThanZero, Mid>                306ns ± 2%              306ns ± 1%     ~             (p=0.863 n=9+9)
BM_MemCmp<64, GreaterThanZero, Last>               406ns ± 3%              407ns ± 1%     ~             (p=0.546 n=9+9)

[spatel]

llvm/lib/CodeGen/ExpandMemCmp.cpp

682–686

Is there a need to check for multiple users that are identical? Ie, doesn't CSE make this unlikely?

[courbet]

llvm/lib/CodeGen/ExpandMemCmp.cpp

682–686

It might be unlikely, but it's not guaranteed. I'm not sure how I would handle diverging users, did yo have anything specific in mind ?

[spatel]

llvm/lib/CodeGen/ExpandMemCmp.cpp

371–375

This is not correct. memcmp() compares as unsigned bytes:
http://www.cplusplus.com/reference/cstring/memcmp/

This program will show a failure compiled with -O1 vs -O0 (do we have some kind of correctness test like this in the test-suite?):

#include <string.h>
#include <stdio.h>

void wrapper(char *c, char *d) {
  if (memcmp(c, d, 4) > 0)
    printf(">0\n");
  else
    printf("<=0\n");
}

int main() {
  int A = 0x00000000;
  int B = 0x00000080;
  wrapper(&A, &B);
  return 0;
}

llvm/test/CodeGen/X86/memcmp-more-load-pairs.ll

128

I think that's a miscompile...should be "setb"?

[courbet]

Use the right unsigned equivalent for comparing loads instead of the signed predicate from the zero relational compare.

[courbet]

llvm/lib/CodeGen/ExpandMemCmp.cpp

371–375

Thank for the catch. I've added more tests in 4edd050c7e97 to make the issue more obvious to the eye than the setl/setb issue.

I though the test-suite benchmark was validating the result, but it's only benchmarking and throwing the result away. I'll fix the benchmarks.

[spatel]

llvm/lib/CodeGen/ExpandMemCmp.cpp

371–375

Great - thanks!
If we want another option that's easier to spot diffs in than x86 asm - add to the IR tests in "llvm/test/Transforms/ExpandMemCmp/".

682–686

We could just give up if the memcmp() call !hasOneUse(). It simplifies the code, but it might miss some strange case...but if there's no test coverage/evidence that the multi-user case exists, it should be fine?

[spatel]

Side note: I was wondering what these patterns look like in DAG/IR, and why we're missing them, so:
D75961

We could argue that it's worth solving these more generally in DAGCombiner, but if we can produce the optimal code here without too much effort, that's good too.

llvm/lib/CodeGen/ExpandMemCmp.cpp

397

That looks like the right fix, but it could use a comment so we'll remember why. Something like this:

// The result of memcmp is a signed value, so the relational 
// compare kind predicate is always signed. But the byte
// comparison used within memcmp is always unsigned.

[spatel]

llvm/test/Transforms/ExpandMemCmp/X86/memcmp.ll

455

Wait - this is another miscompile. Can we split the equality compare part of this into its own patch?

If the 1st load pair compares equal, we must check the 2nd load pair, not go to the result block.

[courbet]

Actually I'm actually now convinced that the whole approach is incorrect when there are several blocks. I'm going to take some time to rethink this.

llvm/lib/CodeGen/ExpandMemCmp.cpp

class MemCmpExpansion {
   ResultBlock ResBlock;
   const uint64_t Size;
   unsigned MaxLoadSize;
   uint64_t NumLoadsNonOneByte;
   const uint64_t NumLoadsPerBlockForZeroCmp;
   std::vector<BasicBlock *> LoadCmpBlocks;
   BasicBlock *EndBlock;
   PHINode *PhiRes;
-  const bool IsUsedForZeroCmp;
+  const CmpInst::Predicate ZeroCmpKind;
   const DataLayout &DL;
   IRBuilder<> Builder;
   // Represents the decomposition in blocks of the expansion. For example,
   // comparing 33 bytes on X86+sse can be done with 2x16-byte loads and
   // 1x1-byte load, which would be represented as [{16, 0}, {16, 16}, {1, 32}.
   struct LoadEntry {
     LoadEntry(unsigned LoadSize, uint64_t Offset)
         : LoadSize(LoadSize), Offset(Offset) {
     }
 
     // The size of the load for this block, in bytes.
     unsigned LoadSize;
     // The offset of this load from the base pointer, in bytes.
     uint64_t Offset;
   };
   using LoadEntryVector = SmallVector<LoadEntry, 8>;
   LoadEntryVector LoadSequence;
 
   void createLoadCmpBlocks();
   void createResultBlock();
   void setupResultBlockPHINodes();
   void setupEndBlockPHINodes();
   Value *getCompareLoadPairs(unsigned BlockIndex, unsigned &LoadIndex);
+  Value *getCompareLoadPairsForZeroEquality(unsigned BlockIndex,
+                                            unsigned &LoadIndex);
+  Value *getCompareLoadPairsForZeroRelational(unsigned BlockIndex,
+                                              unsigned &LoadIndex);
   void emitLoadCompareBlock(unsigned BlockIndex);
   void emitLoadCompareBlockMultipleLoads(unsigned BlockIndex,
                                          unsigned &LoadIndex);
   void emitLoadCompareByteBlock(unsigned BlockIndex, unsigned OffsetBytes);
   void emitMemCmpResultBlock();
-  Value *getMemCmpExpansionZeroCase();
-  Value *getMemCmpEqZeroOneBlock();
+  Value *getMemCmpWithZeroSeveralBlocks();
+  Value *getMemCmpWithZeroOneBlock();
   Value *getMemCmpOneBlock();
   struct LoadPair {
     Value *Lhs = nullptr;
     Value *Rhs = nullptr;
   };
   LoadPair getLoadPair(Type *LoadSizeType, bool NeedsBSwap, Type *CmpSizeType,
                        unsigned OffsetBytes);
 
   static LoadEntryVector
   computeGreedyLoadSequence(uint64_t Size, llvm::ArrayRef<unsigned> LoadSizes,
                             unsigned MaxNumLoads, unsigned &NumLoadsNonOneByte);
   static LoadEntryVector
   computeOverlappingLoadSequence(uint64_t Size, unsigned MaxLoadSize,
                                  unsigned MaxNumLoads,
                                  unsigned &NumLoadsNonOneByte);
 
 public:
   MemCmpExpansion(CallInst *CI, uint64_t Size,
                   const TargetTransformInfo::MemCmpExpansionOptions &Options,
-                  const bool IsUsedForZeroCmp, const DataLayout &TheDataLayout);
+                  CmpInst::Predicate ZeroCmpKind,
+                  const DataLayout &TheDataLayout);
 
   unsigned getNumBlocks();
   uint64_t getNumLoads() const { return LoadSequence.size(); }
 
   Value *getMemCmpExpansion();
 };
 
 MemCmpExpansion::LoadEntryVector MemCmpExpansion::computeGreedyLoadSequence(

 // 1. A list of load compare blocks - LoadCmpBlocks.
 // 2. An EndBlock, split from original instruction point, which is the block to
 // return from.
 // 3. ResultBlock, block to branch to for early exit when a
 // LoadCmpBlock finds a difference.
 MemCmpExpansion::MemCmpExpansion(
     CallInst *const CI, uint64_t Size,
     const TargetTransformInfo::MemCmpExpansionOptions &Options,
-    const bool IsUsedForZeroCmp, const DataLayout &TheDataLayout)
+    const CmpInst::Predicate ZeroCmpKind, const DataLayout &TheDataLayout)
     : CI(CI), Size(Size), MaxLoadSize(0), NumLoadsNonOneByte(0),
       NumLoadsPerBlockForZeroCmp(Options.NumLoadsPerBlock),
-      IsUsedForZeroCmp(IsUsedForZeroCmp), DL(TheDataLayout), Builder(CI) {
+      ZeroCmpKind(ZeroCmpKind), DL(TheDataLayout), Builder(CI) {
   assert(Size > 0 && "zero blocks");
   // Scale the max size down if the target can load more bytes than we need.
   llvm::ArrayRef<unsigned> LoadSizes(Options.LoadSizes);
   while (!LoadSizes.empty() && LoadSizes.front() > Size) {
     LoadSizes = LoadSizes.drop_front();
   }
   assert(!LoadSizes.empty() && "cannot load Size bytes");
   MaxLoadSize = LoadSizes.front();
   // Compute the decomposition.
   unsigned GreedyNumLoadsNonOneByte = 0;
   LoadSequence = computeGreedyLoadSequence(Size, LoadSizes, Options.MaxNumLoads,
                                            GreedyNumLoadsNonOneByte);
   NumLoadsNonOneByte = GreedyNumLoadsNonOneByte;
   assert(LoadSequence.size() <= Options.MaxNumLoads && "broken invariant");
   // If we allow overlapping loads and the load sequence is not already optimal,
   // use overlapping loads.
-  if (Options.AllowOverlappingLoads &&
+  // FIXME: Allow overlapping for any zero cmp, not just zero equality cmp.
+  if (ICmpInst::isEquality(ZeroCmpKind) && Options.AllowOverlappingLoads &&
       (LoadSequence.empty() || LoadSequence.size() > 2)) {
     unsigned OverlappingNumLoadsNonOneByte = 0;
     auto OverlappingLoads = computeOverlappingLoadSequence(
         Size, MaxLoadSize, Options.MaxNumLoads, OverlappingNumLoadsNonOneByte);
     if (!OverlappingLoads.empty() &&
         (LoadSequence.empty() ||
          OverlappingLoads.size() < LoadSequence.size())) {
       LoadSequence = OverlappingLoads;
       NumLoadsNonOneByte = OverlappingNumLoadsNonOneByte;
     }
   }
   assert(LoadSequence.size() <= Options.MaxNumLoads && "broken invariant");
 }
 
 unsigned MemCmpExpansion::getNumBlocks() {
-  if (IsUsedForZeroCmp)
+  // FIXME: Allow multiple loads per block for any zero cmp, not just zero
+  // equality cmp.
+  if (ICmpInst::isEquality(ZeroCmpKind))
     return getNumLoads() / NumLoadsPerBlockForZeroCmp +
            (getNumLoads() % NumLoadsPerBlockForZeroCmp != 0 ? 1 : 0);
   return getNumLoads();
 }
 
 void MemCmpExpansion::createLoadCmpBlocks() {
   for (unsigned i = 0; i < getNumBlocks(); i++) {
     BasicBlock *BB = BasicBlock::Create(CI->getContext(), "loadbb",
if (BlockIndex < (LoadCmpBlocks.size() - 1)) {
     Builder.Insert(CmpBr);
   } else {
     // The last block has an unconditional branch to EndBlock.
     BranchInst *CmpBr = BranchInst::Create(EndBlock);
     Builder.Insert(CmpBr);
   }
 }
 
-/// Generate an equality comparison for one or more pairs of loaded values.
-/// This is used in the case where the memcmp() call is compared equal or not
-/// equal to zero.
+/// Generate an equality/inequality comparison for one or more pairs of loaded
+/// values. This is used in the case where the memcmp() call is compared to
+/// zero.
 Value *MemCmpExpansion::getCompareLoadPairs(unsigned BlockIndex,
                                             unsigned &LoadIndex) {
   assert(LoadIndex < getNumLoads() &&
          "getCompareLoadPairs() called with no remaining loads");
-  std::vector<Value *> XorList, OrList;
-  Value *Diff = nullptr;
-
-  const unsigned NumLoads =
-      std::min(getNumLoads() - LoadIndex, NumLoadsPerBlockForZeroCmp);
-
   // For a single-block expansion, start inserting before the memcmp call.
   if (LoadCmpBlocks.empty())
     Builder.SetInsertPoint(CI);
   else
     Builder.SetInsertPoint(LoadCmpBlocks[BlockIndex]);
 
+  switch (ZeroCmpKind) {
+  case CmpInst::ICMP_EQ:
+  case CmpInst::ICMP_NE:
+    return getCompareLoadPairsForZeroEquality(BlockIndex, LoadIndex);
+  case CmpInst::ICMP_SLT:
+  case CmpInst::ICMP_SGT:
+  case CmpInst::ICMP_SLE:
+  case CmpInst::ICMP_SGE:
+    return getCompareLoadPairsForZeroRelational(BlockIndex, LoadIndex);
+  default:
+    llvm_unreachable("getCompareLoadPairs only works for zero comparison");
+  }
+  return nullptr;
+}
+
+Value *
+MemCmpExpansion::getCompareLoadPairsForZeroRelational(unsigned BlockIndex,
+                                                      unsigned &LoadIndex) {
+  const LoadEntry &CurLoadEntry = LoadSequence[LoadIndex];
+  Type *LoadType =
+      IntegerType::get(CI->getContext(), CurLoadEntry.LoadSize * 8);
+
+  const LoadPair Loads =
+      getLoadPair(LoadType, /*NeedsBSwap=*/DL.isLittleEndian() &&
+                                CurLoadEntry.LoadSize != 1,
+                  LoadType, CurLoadEntry.Offset);
+
+  Value *Cmp = nullptr;
+  // If there's only one load per block, we just compare the loaded values.
+  Cmp = Builder.CreateICmp(
+      ICmpInst::getUnsignedPredicate(ZeroCmpKind), Loads.Lhs, Loads.Rhs);
+
+  ++LoadIndex;
+  return Cmp;
+}
+
+/// Generate an equality comparison for one or more pairs of loaded values.
+/// This is used in the case where the memcmp() call is compared equal or not
+/// equal to zero.
+Value *
+MemCmpExpansion::getCompareLoadPairsForZeroEquality(unsigned BlockIndex,
+                                                    unsigned &LoadIndex) {
+  std::vector<Value *> XorList, OrList;
+  Value *Diff = nullptr;
+
+  const unsigned NumLoads =
+      std::min(getNumLoads() - LoadIndex, NumLoadsPerBlockForZeroCmp);
+
   Value *Cmp = nullptr;
   // If we have multiple loads per block, we need to generate a composite
   // comparison using xor+or. The type for the combinations is the largest load
   // type.
   IntegerType *const MaxLoadType =
       NumLoads == 1 ? nullptr
                     : IntegerType::get(CI->getContext(), MaxLoadSize * 8);
   for (unsigned i = 0; i < NumLoads; ++i, ++LoadIndex) {
     const LoadEntry &CurLoadEntry = LoadSequence[LoadIndex];
     const LoadPair Loads = getLoadPair(
         IntegerType::get(CI->getContext(), CurLoadEntry.LoadSize * 8),
         /*NeedsBSwap=*/false, MaxLoadType, CurLoadEntry.Offset);
 
     if (NumLoads != 1) {
       // If we have multiple loads per block, we need to generate a composite
       // comparison using xor+or.
       Diff = Builder.CreateXor(Loads.Lhs, Loads.Rhs);
       Diff = Builder.CreateZExt(Diff, MaxLoadType);
       XorList.push_back(Diff);
     } else {
       // If there's only one load per block, we just compare the loaded values.
-      Cmp = Builder.CreateICmpNE(Loads.Lhs, Loads.Rhs);
+      Cmp = Builder.CreateICmp(ZeroCmpKind, Loads.Lhs, Loads.Rhs);
     }
   }
 
   auto pairWiseOr = [&](std::vector<Value *> &InList) -> std::vector<Value *> {
     std::vector<Value *> OutList;
     for (unsigned i = 0; i < InList.size() - 1; i = i + 2) {
       Value *Or = Builder.CreateOr(InList[i], InList[i + 1]);
       OutList.push_back(Or);
     }
     if (InList.size() % 2 != 0)
       OutList.push_back(InList.back());
     return OutList;
   };
 
   if (!Cmp) {
     // Pairwise OR the XOR results.
     OrList = pairWiseOr(XorList);
 
     // Pairwise OR the OR results until one result left.
     while (OrList.size() != 1) {
       OrList = pairWiseOr(OrList);
     }
 
     assert(Diff && "Failed to find comparison diff");
-    Cmp = Builder.CreateICmpNE(OrList[0], ConstantInt::get(Diff->getType(), 0));
+    Cmp = Builder.CreateICmp(ZeroCmpKind, OrList[0],
+                             ConstantInt::get(Diff->getType(), 0));
   }
 
   return Cmp;
 }
 
 void MemCmpExpansion::emitLoadCompareBlockMultipleLoads(unsigned BlockIndex,
                                                         unsigned &LoadIndex) {
   Value *Cmp = getCompareLoadPairs(BlockIndex, LoadIndex);
void MemCmpExpansion::emitLoadCompareBlock(unsigned BlockIndex) {
 
   Builder.SetInsertPoint(LoadCmpBlocks[BlockIndex]);
 
   const LoadPair Loads =
       getLoadPair(LoadSizeType, /*NeedsBSwap=*/DL.isLittleEndian(), MaxLoadType,
                   CurLoadEntry.Offset);
 
   // Add the loaded values to the phi nodes for calculating memcmp result only
-  // if result is not used in a zero equality.
-  if (!IsUsedForZeroCmp) {
+  // if result is not used in a zero comparison.
+  if (ZeroCmpKind == CmpInst::BAD_ICMP_PREDICATE) {
     ResBlock.PhiSrc1->addIncoming(Loads.Lhs, LoadCmpBlocks[BlockIndex]);
     ResBlock.PhiSrc2->addIncoming(Loads.Rhs, LoadCmpBlocks[BlockIndex]);
   }
 
   Value *Cmp = Builder.CreateICmp(ICmpInst::ICMP_EQ, Loads.Lhs, Loads.Rhs);
   BasicBlock *NextBB = (BlockIndex == (LoadCmpBlocks.size() - 1))
                            ? EndBlock
                            : LoadCmpBlocks[BlockIndex + 1];
if (BlockIndex == LoadCmpBlocks.size() - 1) {
     PhiRes->addIncoming(Zero, LoadCmpBlocks[BlockIndex]);
   }
 }
 
 // This function populates the ResultBlock with a sequence to calculate the
 // memcmp result. It compares the two loaded source values and returns -1 if
 // src1 < src2 and 1 if src1 > src2.
 void MemCmpExpansion::emitMemCmpResultBlock() {
-  // Special case: if memcmp result is used in a zero equality, result does not
-  // need to be calculated and can simply return 1.
-  if (IsUsedForZeroCmp) {
+  // Special case: if memcmp result is used in a zero comparison, result does
+  // not need to be calculated and can simply return 1.
+  if (ZeroCmpKind != CmpInst::BAD_ICMP_PREDICATE) {
     BasicBlock::iterator InsertPt = ResBlock.BB->getFirstInsertionPt();
     Builder.SetInsertPoint(ResBlock.BB, InsertPt);
     Value *Res = ConstantInt::get(Type::getInt32Ty(CI->getContext()), 1);
     PhiRes->addIncoming(Res, ResBlock.BB);
     BranchInst *NewBr = BranchInst::Create(EndBlock);
     Builder.Insert(NewBr);
     return;
   }
ResBlock.PhiSrc2 =
       Builder.CreatePHI(MaxLoadType, NumLoadsNonOneByte, "phi.src2");
 }
 
 void MemCmpExpansion::setupEndBlockPHINodes() {
   Builder.SetInsertPoint(&EndBlock->front());
   PhiRes = Builder.CreatePHI(Type::getInt32Ty(CI->getContext()), 2, "phi.res");
 }
 
-Value *MemCmpExpansion::getMemCmpExpansionZeroCase() {
+Value *MemCmpExpansion::getMemCmpWithZeroSeveralBlocks() {
   unsigned LoadIndex = 0;
   // This loop populates each of the LoadCmpBlocks with the IR sequence to
   // handle multiple loads per block.
   for (unsigned I = 0; I < getNumBlocks(); ++I) {
     emitLoadCompareBlockMultipleLoads(I, LoadIndex);
   }
 
   emitMemCmpResultBlock();
   return PhiRes;
 }
 
 /// A memcmp expansion that compares equality with 0 and only has one block of
 /// load and compare can bypass the compare, branch, and phi IR that is required
 /// in the general case.
-Value *MemCmpExpansion::getMemCmpEqZeroOneBlock() {
+Value *MemCmpExpansion::getMemCmpWithZeroOneBlock() {
   unsigned LoadIndex = 0;
   Value *Cmp = getCompareLoadPairs(0, LoadIndex);
   assert(LoadIndex == getNumLoads() && "some entries were not consumed");
   return Builder.CreateZExt(Cmp, Type::getInt32Ty(CI->getContext()));
 }
 
 /// A memcmp expansion that only has one block of load and compare can bypass
 /// the compare, branch, and phi IR that is required in the general case.
if (getNumBlocks() != 1) {
     EndBlock = StartBlock->splitBasicBlock(CI, "endblock");
     setupEndBlockPHINodes();
     createResultBlock();
 
     // If return value of memcmp is not used in a zero equality, we need to
     // calculate which source was larger. The calculation requires the
     // two loaded source values of each load compare block.
     // These will be saved in the phi nodes created by setupResultBlockPHINodes.
-    if (!IsUsedForZeroCmp) setupResultBlockPHINodes();
+    if (ZeroCmpKind == CmpInst::BAD_ICMP_PREDICATE)
+      setupResultBlockPHINodes();
 
     // Create the number of required load compare basic blocks.
     createLoadCmpBlocks();
 
     // Update the terminator added by splitBasicBlock to branch to the first
     // LoadCmpBlock.
     StartBlock->getTerminator()->setSuccessor(0, LoadCmpBlocks[0]);
   }
 
   Builder.SetCurrentDebugLocation(CI->getDebugLoc());
 
-  if (IsUsedForZeroCmp)
-    return getNumBlocks() == 1 ? getMemCmpEqZeroOneBlock()
-                               : getMemCmpExpansionZeroCase();
+  if (ZeroCmpKind != CmpInst::BAD_ICMP_PREDICATE)
+    return getNumBlocks() == 1 ? getMemCmpWithZeroOneBlock()
+                               : getMemCmpWithZeroSeveralBlocks();
 
   if (getNumBlocks() == 1)
     return getMemCmpOneBlock();
 
   for (unsigned I = 0; I < getNumBlocks(); ++I) {
     emitLoadCompareBlock(I);
   }
 
   emitMemCmpResultBlock();
   return PhiRes;
 }
 
+// If the memcmp is only used for a given comparison against zero,
+// return that predicate.
+static CmpInst::Predicate isOnlyUsedInZeroComparison(const Instruction *CxtI) {
+  CmpInst::Predicate CommonPred = CmpInst::BAD_ICMP_PREDICATE;
+  for (const User *U : CxtI->users()) {
+    // The user must be an icmp.
+    const ICmpInst *IC = dyn_cast<ICmpInst>(U);
+    if (!IC)
+      return CmpInst::BAD_ICMP_PREDICATE;
+    // The rhs must be zero.
+    Constant *C = dyn_cast<Constant>(IC->getOperand(1));
+    if (C == nullptr || !C->isNullValue())
+      return CmpInst::BAD_ICMP_PREDICATE;
+    // The predicate must match that of other users.
+    const CmpInst::Predicate Pred = IC->getPredicate();
+    if (CommonPred == CmpInst::BAD_ICMP_PREDICATE)
+      CommonPred = Pred;
+    else if (Pred != CommonPred)
+      return CmpInst::BAD_ICMP_PREDICATE;
+  }
+  return CommonPred;
+}
+
+// Change all user icmp predicates to `!= 0`.
+// Requires `isOnlyUsedInZeroComparison(CxtI)`.
+static void updateZeroComparisonPredicates(Instruction *CxtI) {
+  for (User *U : CxtI->users()) {
+    ICmpInst *IC = dyn_cast<ICmpInst>(U);
+    assert(IC && "isOnlyUsedInZeroComparison must be true");
+    IC->setPredicate(ICmpInst::ICMP_NE);
+  }
+}
+
 // This function checks to see if an expansion of memcmp can be generated.
 // It checks for constant compare size that is less than the max inline size.
 // If an expansion cannot occur, returns false to leave as a library call.
 // Otherwise, the library call is replaced with a new IR instruction sequence.
 /// We want to transform:
 /// %call = call signext i32 @memcmp(i8* %0, i8* %1, i64 15)
 /// To:
 /// loadbb:
static bool expandMemCmp(CallInst *CI, const TargetTransformInfo *TTI,
   }
   const uint64_t SizeVal = SizeCast->getZExtValue();
 
   if (SizeVal == 0) {
     return false;
   }
   // TTI call to check if target would like to expand memcmp. Also, get the
   // available load sizes.
-  const bool IsUsedForZeroCmp = isOnlyUsedInZeroEqualityComparison(CI);
+  const CmpInst::Predicate ZeroCmpKind = isOnlyUsedInZeroComparison(CI);
   bool OptForSize = CI->getFunction()->hasOptSize() ||
                     llvm::shouldOptimizeForSize(CI->getParent(), PSI, BFI);
-  auto Options = TTI->enableMemCmpExpansion(OptForSize,
-                                            IsUsedForZeroCmp);
+  auto Options =
+      TTI->enableMemCmpExpansion(OptForSize, ICmpInst::isEquality(ZeroCmpKind));
   if (!Options) return false;
 
   if (MemCmpEqZeroNumLoadsPerBlock.getNumOccurrences())
     Options.NumLoadsPerBlock = MemCmpEqZeroNumLoadsPerBlock;
 
   if (OptForSize &&
       MaxLoadsPerMemcmpOptSize.getNumOccurrences())
     Options.MaxNumLoads = MaxLoadsPerMemcmpOptSize;
 
   if (!OptForSize && MaxLoadsPerMemcmp.getNumOccurrences())
     Options.MaxNumLoads = MaxLoadsPerMemcmp;
 
-  MemCmpExpansion Expansion(CI, SizeVal, Options, IsUsedForZeroCmp, *DL);
+  MemCmpExpansion Expansion(CI, SizeVal, Options, ZeroCmpKind, *DL);
 
   // Don't expand if this will require more loads than desired by the target.
   if (Expansion.getNumLoads() == 0) {
     NumMemCmpGreaterThanMax++;
     return false;
   }
 
   NumMemCmpInlined++;
 
   Value *Res = Expansion.getMemCmpExpansion();
 
   // Replace call with result of expansion and erase call.
+  if (ZeroCmpKind != CmpInst::BAD_ICMP_PREDICATE) {
+    updateZeroComparisonPredicates(CI);
+  }
   CI->replaceAllUsesWith(Res);
   CI->eraseFromParent();
 
   return true;
 }
 
 
 

llvm/test/CodeGen/PowerPC/memCmpUsedInZeroEqualityComparison.ll

 
 define signext i32 @equalityFoldOneConstant(i8* %X) {
 ; CHECK-LABEL: equalityFoldOneConstant:
 ; CHECK:       # %bb.0:
 ; CHECK-NEXT:    ld 4, 0(3)
 ; CHECK-NEXT:    li 5, 1
 ; CHECK-NEXT:    sldi 5, 5, 32
 ; CHECK-NEXT:    cmpld 4, 5
-; CHECK-NEXT:    bne 0, .LBB6_2
+; CHECK-NEXT:    beq 0, .LBB6_2
 ; CHECK-NEXT:  # %bb.1: # %loadbb1
 ; CHECK-NEXT:    li 4, 3
 ; CHECK-NEXT:    ld 3, 8(3)
 ; CHECK-NEXT:    sldi 4, 4, 32
 ; CHECK-NEXT:    ori 4, 4, 2
 ; CHECK-NEXT:    cmpld 3, 4
 ; CHECK-NEXT:    li 3, 0
-; CHECK-NEXT:    beq 0, .LBB6_3
+; CHECK-NEXT:    bne 0, .LBB6_3
 ; CHECK-NEXT:  .LBB6_2: # %res_block
 ; CHECK-NEXT:    li 3, 1
 ; CHECK-NEXT:  .LBB6_3: # %endblock
-; CHECK-NEXT:    cntlzw 3, 3
-; CHECK-NEXT:    srwi 3, 3, 5
+; CHECK-NEXT:    clrldi 3, 3, 32
 ; CHECK-NEXT:    blr
   %call = tail call signext i32 @memcmp(i8* bitcast ([15 x i32]* @zeroEqualityTest04.buffer1 to i8*), i8* %X, i64 16)
   %not.tobool = icmp eq i32 %call, 0
   %cond = zext i1 %not.tobool to i32
   ret i32 %cond
 }
 
 define i1 @length2_eq_nobuiltin_attr(i8* %X, i8* %Y) nounwind {

llvm/test/CodeGen/X86/memcmp-more-load-pairs.ll

 }
 
 define i1 @length2_lt(i8* %X, i8* %Y) nounwind {
 ; X86-LABEL: length2_lt:
 ; X86:       # %bb.0:
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; X86-NEXT:    movzwl (%ecx), %ecx
-; X86-NEXT:    movzwl (%eax), %edx
+; X86-NEXT:    movzwl (%eax), %eax
 ; X86-NEXT:    rolw $8, %cx
-; X86-NEXT:    rolw $8, %dx
-; X86-NEXT:    movzwl %cx, %eax
-; X86-NEXT:    movzwl %dx, %ecx
-; X86-NEXT:    subl %ecx, %eax
-; X86-NEXT:    shrl $31, %eax
-; X86-NEXT:    # kill: def $al killed $al killed $eax
+; X86-NEXT:    rolw $8, %ax
+; X86-NEXT:    cmpw %ax, %cx
+; X86-NEXT:    setb %al
 ; X86-NEXT:    retl
 ;
 ; X64-LABEL: length2_lt:
 ; X64:       # %bb.0:
 ; X64-NEXT:    movzwl (%rdi), %eax
 ; X64-NEXT:    movzwl (%rsi), %ecx
 ; X64-NEXT:    rolw $8, %ax
 ; X64-NEXT:    rolw $8, %cx
-; X64-NEXT:    movzwl %ax, %eax
-; X64-NEXT:    movzwl %cx, %ecx
-; X64-NEXT:    subl %ecx, %eax
-; X64-NEXT:    shrl $31, %eax
-; X64-NEXT:    # kill: def $al killed $al killed $eax
+; X64-NEXT:    cmpw %cx, %ax
+; X64-NEXT:    setb %al
 ; X64-NEXT:    retq
   %m = tail call i32 @memcmp(i8* %X, i8* %Y, i64 2) nounwind
   %c = icmp slt i32 %m, 0
   ret i1 %c
 }
 
 define i1 @length2_gt(i8* %X, i8* %Y) nounwind {
 ; X86-LABEL: length2_gt:
 ; X86:       # %bb.0:
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; X86-NEXT:    movzwl (%ecx), %ecx
 ; X86-NEXT:    movzwl (%eax), %eax
 ; X86-NEXT:    rolw $8, %cx
 ; X86-NEXT:    rolw $8, %ax
-; X86-NEXT:    movzwl %cx, %ecx
-; X86-NEXT:    movzwl %ax, %eax
-; X86-NEXT:    subl %eax, %ecx
-; X86-NEXT:    testl %ecx, %ecx
-; X86-NEXT:    setg %al
+; X86-NEXT:    cmpw %ax, %cx
+; X86-NEXT:    seta %al
 ; X86-NEXT:    retl
 ;
 ; X64-LABEL: length2_gt:
 ; X64:       # %bb.0:
 ; X64-NEXT:    movzwl (%rdi), %eax
 ; X64-NEXT:    movzwl (%rsi), %ecx
 ; X64-NEXT:    rolw $8, %ax
 ; X64-NEXT:    rolw $8, %cx
-; X64-NEXT:    movzwl %ax, %eax
-; X64-NEXT:    movzwl %cx, %ecx
-; X64-NEXT:    subl %ecx, %eax
-; X64-NEXT:    testl %eax, %eax
-; X64-NEXT:    setg %al
+; X64-NEXT:    cmpw %cx, %ax
+; X64-NEXT:    seta %al
 ; X64-NEXT:    retq
   %m = tail call i32 @memcmp(i8* %X, i8* %Y, i64 2) nounwind
   %c = icmp sgt i32 %m, 0
   ret i1 %c
 }
 
 define i1 @length2_eq_const(i8* %X) nounwind {
 ; X86-LABEL: length2_eq_const:

 }
 
 define i1 @length4_lt(i8* %X, i8* %Y) nounwind {
 ; X86-LABEL: length4_lt:
 ; X86:       # %bb.0:
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; X86-NEXT:    movl (%ecx), %ecx
-; X86-NEXT:    movl (%eax), %edx
+; X86-NEXT:    movl (%eax), %eax
 ; X86-NEXT:    bswapl %ecx
-; X86-NEXT:    bswapl %edx
-; X86-NEXT:    xorl %eax, %eax
-; X86-NEXT:    cmpl %edx, %ecx
-; X86-NEXT:    seta %al
-; X86-NEXT:    sbbl $0, %eax
-; X86-NEXT:    shrl $31, %eax
-; X86-NEXT:    # kill: def $al killed $al killed $eax
+; X86-NEXT:    bswapl %eax
+; X86-NEXT:    cmpl %eax, %ecx
+; X86-NEXT:    setb %al
 ; X86-NEXT:    retl
 ;
 ; X64-LABEL: length4_lt:
 ; X64:       # %bb.0:
-; X64-NEXT:    movl (%rdi), %ecx
-; X64-NEXT:    movl (%rsi), %edx
+; X64-NEXT:    movl (%rdi), %eax
+; X64-NEXT:    movl (%rsi), %ecx
+; X64-NEXT:    bswapl %eax
 ; X64-NEXT:    bswapl %ecx
-; X64-NEXT:    bswapl %edx
-; X64-NEXT:    xorl %eax, %eax
-; X64-NEXT:    cmpl %edx, %ecx
-; X64-NEXT:    seta %al
-; X64-NEXT:    sbbl $0, %eax
-; X64-NEXT:    shrl $31, %eax
-; X64-NEXT:    # kill: def $al killed $al killed $eax
+; X64-NEXT:    cmpl %ecx, %eax
+; X64-NEXT:    setb %al
 ; X64-NEXT:    retq
   %m = tail call i32 @memcmp(i8* %X, i8* %Y, i64 4) nounwind
   %c = icmp slt i32 %m, 0
   ret i1 %c
 }
 
 define i1 @length4_gt(i8* %X, i8* %Y) nounwind {
 ; X86-LABEL: length4_gt:
 ; X86:       # %bb.0:
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; X86-NEXT:    movl (%ecx), %ecx
 ; X86-NEXT:    movl (%eax), %eax
 ; X86-NEXT:    bswapl %ecx
 ; X86-NEXT:    bswapl %eax
-; X86-NEXT:    xorl %edx, %edx
 ; X86-NEXT:    cmpl %eax, %ecx
-; X86-NEXT:    seta %dl
-; X86-NEXT:    sbbl $0, %edx
-; X86-NEXT:    testl %edx, %edx
-; X86-NEXT:    setg %al
+; X86-NEXT:    seta %al
 ; X86-NEXT:    retl
 ;
 ; X64-LABEL: length4_gt:
 ; X64:       # %bb.0:
 ; X64-NEXT:    movl (%rdi), %eax
 ; X64-NEXT:    movl (%rsi), %ecx
 ; X64-NEXT:    bswapl %eax
 ; X64-NEXT:    bswapl %ecx
-; X64-NEXT:    xorl %edx, %edx
 ; X64-NEXT:    cmpl %ecx, %eax
-; X64-NEXT:    seta %dl
-; X64-NEXT:    sbbl $0, %edx
-; X64-NEXT:    testl %edx, %edx
-; X64-NEXT:    setg %al
+; X64-NEXT:    seta %al
 ; X64-NEXT:    retq
   %m = tail call i32 @memcmp(i8* %X, i8* %Y, i64 4) nounwind
   %c = icmp sgt i32 %m, 0
   ret i1 %c
 }
 
 define i1 @length4_eq_const(i8* %X) nounwind {
 ; X86-LABEL: length4_eq_const:
; X64-NEXT:    retq
   %c = icmp ne i32 %m, 0
   ret i1 %c
 }
 
 define i1 @length5_lt(i8* %X, i8* %Y) nounwind {
 ; X86-LABEL: length5_lt:
 ; X86:       # %bb.0:
 ; X86-NEXT:    pushl %esi
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT:    movl (%eax), %edx
-; X86-NEXT:    movl (%ecx), %esi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl (%ecx), %edx
+; X86-NEXT:    movl (%eax), %esi
 ; X86-NEXT:    bswapl %edx
 ; X86-NEXT:    bswapl %esi
 ; X86-NEXT:    cmpl %esi, %edx
-; X86-NEXT:    jne .LBB18_3
+; X86-NEXT:    jb .LBB18_2
 ; X86-NEXT:  # %bb.1: # %loadbb1
-; X86-NEXT:    movzbl 4(%eax), %eax
-; X86-NEXT:    movzbl 4(%ecx), %ecx
-; X86-NEXT:    subl %ecx, %eax
-; X86-NEXT:    jmp .LBB18_2
-; X86-NEXT:  .LBB18_3: # %res_block
-; X86-NEXT:    setae %al
-; X86-NEXT:    movzbl %al, %eax
-; X86-NEXT:    leal -1(%eax,%eax), %eax
-; X86-NEXT:  .LBB18_2: # %endblock
-; X86-NEXT:    shrl $31, %eax
-; X86-NEXT:    # kill: def $al killed $al killed $eax
+; X86-NEXT:    movb 4(%ecx), %dl
+; X86-NEXT:    xorl %ecx, %ecx
+; X86-NEXT:    cmpb 4(%eax), %dl
+; X86-NEXT:    jae .LBB18_3
+; X86-NEXT:  .LBB18_2: # %res_block
+; X86-NEXT:    movl $1, %ecx
+; X86-NEXT:  .LBB18_3: # %endblock
+; X86-NEXT:    testl %ecx, %ecx
+; X86-NEXT:    setne %al
 ; X86-NEXT:    popl %esi
 ; X86-NEXT:    retl
 ;
 ; X64-LABEL: length5_lt:
 ; X64:       # %bb.0:
 ; X64-NEXT:    movl (%rdi), %eax
 ; X64-NEXT:    movl (%rsi), %ecx
 ; X64-NEXT:    bswapl %eax
 ; X64-NEXT:    bswapl %ecx
 ; X64-NEXT:    cmpl %ecx, %eax
-; X64-NEXT:    jne .LBB18_3
+; X64-NEXT:    jb .LBB18_2
 ; X64-NEXT:  # %bb.1: # %loadbb1
-; X64-NEXT:    movzbl 4(%rdi), %eax
-; X64-NEXT:    movzbl 4(%rsi), %ecx
-; X64-NEXT:    subl %ecx, %eax
-; X64-NEXT:    shrl $31, %eax
-; X64-NEXT:    # kill: def $al killed $al killed $eax
-; X64-NEXT:    retq
-; X64-NEXT:  .LBB18_3: # %res_block
-; X64-NEXT:    setae %al
-; X64-NEXT:    movzbl %al, %eax
-; X64-NEXT:    leal -1(%rax,%rax), %eax
-; X64-NEXT:    shrl $31, %eax
-; X64-NEXT:    # kill: def $al killed $al killed $eax
+; X64-NEXT:    movb 4(%rdi), %cl
+; X64-NEXT:    xorl %eax, %eax
+; X64-NEXT:    cmpb 4(%rsi), %cl
+; X64-NEXT:    jae .LBB18_3
+; X64-NEXT:  .LBB18_2: # %res_block
+; X64-NEXT:    movl $1, %eax
+; X64-NEXT:  .LBB18_3: # %endblock
+; X64-NEXT:    testl %eax, %eax
+; X64-NEXT:    setne %al
 ; X64-NEXT:    retq
   %m = tail call i32 @memcmp(i8* %X, i8* %Y, i64 5) nounwind
   %c = icmp slt i32 %m, 0
   ret i1 %c
 }
 
 define i1 @length7_eq(i8* %X, i8* %Y) nounwind {
 ; X86-LABEL: length7_eq:
; X64-MIC-AVX-NEXT:    retq
   ret i1 %cmp
 }
 
 define i1 @length16_lt(i8* %x, i8* %y) nounwind {
 ; X86-LABEL: length16_lt:
 ; X86:       # %bb.0:
 ; X86-NEXT:    pushl %esi
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
-; X86-NEXT:    movl (%esi), %ecx
-; X86-NEXT:    movl (%eax), %edx
-; X86-NEXT:    bswapl %ecx
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl (%ecx), %edx
+; X86-NEXT:    movl (%eax), %esi
 ; X86-NEXT:    bswapl %edx
-; X86-NEXT:    cmpl %edx, %ecx
-; X86-NEXT:    jne .LBB33_4
+; X86-NEXT:    bswapl %esi
+; X86-NEXT:    cmpl %esi, %edx
+; X86-NEXT:    jb .LBB33_4
 ; X86-NEXT:  # %bb.1: # %loadbb1
-; X86-NEXT:    movl 4(%esi), %ecx
-; X86-NEXT:    movl 4(%eax), %edx
-; X86-NEXT:    bswapl %ecx
+; X86-NEXT:    movl 4(%ecx), %edx
+; X86-NEXT:    movl 4(%eax), %esi
 ; X86-NEXT:    bswapl %edx
-; X86-NEXT:    cmpl %edx, %ecx
-; X86-NEXT:    jne .LBB33_4
+; X86-NEXT:    bswapl %esi
+; X86-NEXT:    cmpl %esi, %edx
+; X86-NEXT:    jb .LBB33_4
 ; X86-NEXT:  # %bb.2: # %loadbb2
-; X86-NEXT:    movl 8(%esi), %ecx
-; X86-NEXT:    movl 8(%eax), %edx
-; X86-NEXT:    bswapl %ecx
+; X86-NEXT:    movl 8(%ecx), %edx
+; X86-NEXT:    movl 8(%eax), %esi
 ; X86-NEXT:    bswapl %edx
-; X86-NEXT:    cmpl %edx, %ecx
-; X86-NEXT:    jne .LBB33_4
+; X86-NEXT:    bswapl %esi
+; X86-NEXT:    cmpl %esi, %edx
+; X86-NEXT:    jb .LBB33_4
 ; X86-NEXT:  # %bb.3: # %loadbb3
-; X86-NEXT:    movl 12(%esi), %ecx
+; X86-NEXT:    movl 12(%ecx), %ecx
 ; X86-NEXT:    movl 12(%eax), %edx
 ; X86-NEXT:    bswapl %ecx
 ; X86-NEXT:    bswapl %edx
 ; X86-NEXT:    xorl %eax, %eax
 ; X86-NEXT:    cmpl %edx, %ecx
-; X86-NEXT:    je .LBB33_5
+; X86-NEXT:    jae .LBB33_5
 ; X86-NEXT:  .LBB33_4: # %res_block
-; X86-NEXT:    xorl %eax, %eax
-; X86-NEXT:    cmpl %edx, %ecx
-; X86-NEXT:    setae %al
-; X86-NEXT:    leal -1(%eax,%eax), %eax
+; X86-NEXT:    movl $1, %eax
 ; X86-NEXT:  .LBB33_5: # %endblock
-; X86-NEXT:    shrl $31, %eax
-; X86-NEXT:    # kill: def $al killed $al killed $eax
+; X86-NEXT:    testl %eax, %eax
+; X86-NEXT:    setne %al
 ; X86-NEXT:    popl %esi
 ; X86-NEXT:    retl
 ;
 ; X64-LABEL: length16_lt:
 ; X64:       # %bb.0:
-; X64-NEXT:    movq (%rdi), %rcx
-; X64-NEXT:    movq (%rsi), %rdx
+; X64-NEXT:    movq (%rdi), %rax
+; X64-NEXT:    movq (%rsi), %rcx
+; X64-NEXT:    bswapq %rax
 ; X64-NEXT:    bswapq %rcx
-; X64-NEXT:    bswapq %rdx
-; X64-NEXT:    cmpq %rdx, %rcx
-; X64-NEXT:    jne .LBB33_2
+; X64-NEXT:    cmpq %rcx, %rax
+; X64-NEXT:    jb .LBB33_2
 ; X64-NEXT:  # %bb.1: # %loadbb1
 ; X64-NEXT:    movq 8(%rdi), %rcx
 ; X64-NEXT:    movq 8(%rsi), %rdx
 ; X64-NEXT:    bswapq %rcx
 ; X64-NEXT:    bswapq %rdx
 ; X64-NEXT:    xorl %eax, %eax
 ; X64-NEXT:    cmpq %rdx, %rcx
-; X64-NEXT:    je .LBB33_3
+; X64-NEXT:    jae .LBB33_3
 ; X64-NEXT:  .LBB33_2: # %res_block
-; X64-NEXT:    xorl %eax, %eax
-; X64-NEXT:    cmpq %rdx, %rcx
-; X64-NEXT:    setae %al
-; X64-NEXT:    leal -1(%rax,%rax), %eax
+; X64-NEXT:    movl $1, %eax
 ; X64-NEXT:  .LBB33_3: # %endblock
-; X64-NEXT:    shrl $31, %eax
-; X64-NEXT:    # kill: def $al killed $al killed $eax
+; X64-NEXT:    testl %eax, %eax
+; X64-NEXT:    setne %al
 ; X64-NEXT:    retq
   %call = tail call i32 @memcmp(i8* %x, i8* %y, i64 16) nounwind
   %cmp = icmp slt i32 %call, 0
   ret i1 %cmp
 }
 
 define i1 @length16_gt(i8* %x, i8* %y) nounwind {
 ; X86-LABEL: length16_gt:
 ; X86:       # %bb.0:
 ; X86-NEXT:    pushl %esi
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %edx
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %esi
-; X86-NEXT:    movl (%esi), %eax
-; X86-NEXT:    movl (%edx), %ecx
-; X86-NEXT:    bswapl %eax
-; X86-NEXT:    bswapl %ecx
-; X86-NEXT:    cmpl %ecx, %eax
-; X86-NEXT:    jne .LBB34_4
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl (%ecx), %edx
+; X86-NEXT:    movl (%eax), %esi
+; X86-NEXT:    bswapl %edx
+; X86-NEXT:    bswapl %esi
+; X86-NEXT:    cmpl %esi, %edx
+; X86-NEXT:    ja .LBB34_4
 ; X86-NEXT:  # %bb.1: # %loadbb1
-; X86-NEXT:    movl 4(%esi), %eax
-; X86-NEXT:    movl 4(%edx), %ecx
-; X86-NEXT:    bswapl %eax
-; X86-NEXT:    bswapl %ecx
-; X86-NEXT:    cmpl %ecx, %eax
-; X86-NEXT:    jne .LBB34_4
+; X86-NEXT:    movl 4(%ecx), %edx
+; X86-NEXT:    movl 4(%eax), %esi
+; X86-NEXT:    bswapl %edx
+; X86-NEXT:    bswapl %esi
+; X86-NEXT:    cmpl %esi, %edx
+; X86-NEXT:    ja .LBB34_4
 ; X86-NEXT:  # %bb.2: # %loadbb2
-; X86-NEXT:    movl 8(%esi), %eax
-; X86-NEXT:    movl 8(%edx), %ecx
-; X86-NEXT:    bswapl %eax
-; X86-NEXT:    bswapl %ecx
-; X86-NEXT:    cmpl %ecx, %eax
-; X86-NEXT:    jne .LBB34_4
+; X86-NEXT:    movl 8(%ecx), %edx
+; X86-NEXT:    movl 8(%eax), %esi
+; X86-NEXT:    bswapl %edx
+; X86-NEXT:    bswapl %esi
+; X86-NEXT:    cmpl %esi, %edx
+; X86-NEXT:    ja .LBB34_4
 ; X86-NEXT:  # %bb.3: # %loadbb3
-; X86-NEXT:    movl 12(%esi), %eax
-; X86-NEXT:    movl 12(%edx), %ecx
-; X86-NEXT:    bswapl %eax
+; X86-NEXT:    movl 12(%ecx), %ecx
+; X86-NEXT:    movl 12(%eax), %edx
 ; X86-NEXT:    bswapl %ecx
-; X86-NEXT:    xorl %edx, %edx
-; X86-NEXT:    cmpl %ecx, %eax
-; X86-NEXT:    je .LBB34_5
+; X86-NEXT:    bswapl %edx
+; X86-NEXT:    xorl %eax, %eax
+; X86-NEXT:    cmpl %edx, %ecx
+; X86-NEXT:    jbe .LBB34_5
 ; X86-NEXT:  .LBB34_4: # %res_block
-; X86-NEXT:    xorl %edx, %edx
-; X86-NEXT:    cmpl %ecx, %eax
-; X86-NEXT:    setae %dl
-; X86-NEXT:    leal -1(%edx,%edx), %edx
+; X86-NEXT:    movl $1, %eax
 ; X86-NEXT:  .LBB34_5: # %endblock
-; X86-NEXT:    testl %edx, %edx
-; X86-NEXT:    setg %al
+; X86-NEXT:    testl %eax, %eax
+; X86-NEXT:    setne %al
 ; X86-NEXT:    popl %esi
 ; X86-NEXT:    retl
 ;
 ; X64-LABEL: length16_gt:
 ; X64:       # %bb.0:
 ; X64-NEXT:    movq (%rdi), %rax
 ; X64-NEXT:    movq (%rsi), %rcx
 ; X64-NEXT:    bswapq %rax
 ; X64-NEXT:    bswapq %rcx
 ; X64-NEXT:    cmpq %rcx, %rax
-; X64-NEXT:    jne .LBB34_2
+; X64-NEXT:    ja .LBB34_2
 ; X64-NEXT:  # %bb.1: # %loadbb1
-; X64-NEXT:    movq 8(%rdi), %rax
-; X64-NEXT:    movq 8(%rsi), %rcx
-; X64-NEXT:    bswapq %rax
+; X64-NEXT:    movq 8(%rdi), %rcx
+; X64-NEXT:    movq 8(%rsi), %rdx
 ; X64-NEXT:    bswapq %rcx
-; X64-NEXT:    xorl %edx, %edx
-; X64-NEXT:    cmpq %rcx, %rax
-; X64-NEXT:    je .LBB34_3
+; X64-NEXT:    bswapq %rdx
+; X64-NEXT:    xorl %eax, %eax
+; X64-NEXT:    cmpq %rdx, %rcx
+; X64-NEXT:    jbe .LBB34_3
 ; X64-NEXT:  .LBB34_2: # %res_block
-; X64-NEXT:    xorl %edx, %edx
-; X64-NEXT:    cmpq %rcx, %rax
-; X64-NEXT:    setae %dl
-; X64-NEXT:    leal -1(%rdx,%rdx), %edx
+; X64-NEXT:    movl $1, %eax
 ; X64-NEXT:  .LBB34_3: # %endblock
-; X64-NEXT:    testl %edx, %edx
-; X64-NEXT:    setg %al
+; X64-NEXT:    testl %eax, %eax
+; X64-NEXT:    setne %al
 ; X64-NEXT:    retq
   %call = tail call i32 @memcmp(i8* %x, i8* %y, i64 16) nounwind
   %cmp = icmp sgt i32 %call, 0
   ret i1 %cmp
 }
 
 define i1 @length16_eq_const(i8* %X) nounwind {
 ; X86-NOSSE-LABEL: length16_eq_const:

 ; X86-NEXT:    calll memcmp
 ; X86-NEXT:    addl $16, %esp
 ; X86-NEXT:    shrl $31, %eax
 ; X86-NEXT:    # kill: def $al killed $al killed $eax
 ; X86-NEXT:    retl
 ;
 ; X64-LABEL: length24_lt:
 ; X64:       # %bb.0:
-; X64-NEXT:    movq (%rdi), %rcx
-; X64-NEXT:    movq (%rsi), %rdx
+; X64-NEXT:    movq (%rdi), %rax
+; X64-NEXT:    movq (%rsi), %rcx
+; X64-NEXT:    bswapq %rax
 ; X64-NEXT:    bswapq %rcx
-; X64-NEXT:    bswapq %rdx
-; X64-NEXT:    cmpq %rdx, %rcx
-; X64-NEXT:    jne .LBB38_3
+; X64-NEXT:    cmpq %rcx, %rax
+; X64-NEXT:    jb .LBB38_3
 ; X64-NEXT:  # %bb.1: # %loadbb1
-; X64-NEXT:    movq 8(%rdi), %rcx
-; X64-NEXT:    movq 8(%rsi), %rdx
+; X64-NEXT:    movq 8(%rdi), %rax
+; X64-NEXT:    movq 8(%rsi), %rcx
+; X64-NEXT:    bswapq %rax
 ; X64-NEXT:    bswapq %rcx
-; X64-NEXT:    bswapq %rdx
-; X64-NEXT:    cmpq %rdx, %rcx
-; X64-NEXT:    jne .LBB38_3
+; X64-NEXT:    cmpq %rcx, %rax
+; X64-NEXT:    jb .LBB38_3
 ; X64-NEXT:  # %bb.2: # %loadbb2
 ; X64-NEXT:    movq 16(%rdi), %rcx
 ; X64-NEXT:    movq 16(%rsi), %rdx
 ; X64-NEXT:    bswapq %rcx
 ; X64-NEXT:    bswapq %rdx
 ; X64-NEXT:    xorl %eax, %eax
 ; X64-NEXT:    cmpq %rdx, %rcx
-; X64-NEXT:    je .LBB38_4
+; X64-NEXT:    jae .LBB38_4
 ; X64-NEXT:  .LBB38_3: # %res_block
-; X64-NEXT:    xorl %eax, %eax
-; X64-NEXT:    cmpq %rdx, %rcx
-; X64-NEXT:    setae %al
-; X64-NEXT:    leal -1(%rax,%rax), %eax
+; X64-NEXT:    movl $1, %eax
 ; X64-NEXT:  .LBB38_4: # %endblock
-; X64-NEXT:    shrl $31, %eax
-; X64-NEXT:    # kill: def $al killed $al killed $eax
+; X64-NEXT:    testl %eax, %eax
+; X64-NEXT:    setne %al
 ; X64-NEXT:    retq
   %call = tail call i32 @memcmp(i8* %x, i8* %y, i64 24) nounwind
   %cmp = icmp slt i32 %call, 0
   ret i1 %cmp
 }
 
 define i1 @length24_gt(i8* %x, i8* %y) nounwind {
 ; X86-LABEL: length24_gt:

 ;
 ; X64-LABEL: length24_gt:
 ; X64:       # %bb.0:
 ; X64-NEXT:    movq (%rdi), %rax
 ; X64-NEXT:    movq (%rsi), %rcx
 ; X64-NEXT:    bswapq %rax
 ; X64-NEXT:    bswapq %rcx
 ; X64-NEXT:    cmpq %rcx, %rax
-; X64-NEXT:    jne .LBB39_3
+; X64-NEXT:    ja .LBB39_3
 ; X64-NEXT:  # %bb.1: # %loadbb1
 ; X64-NEXT:    movq 8(%rdi), %rax
 ; X64-NEXT:    movq 8(%rsi), %rcx
 ; X64-NEXT:    bswapq %rax
 ; X64-NEXT:    bswapq %rcx
 ; X64-NEXT:    cmpq %rcx, %rax
-; X64-NEXT:    jne .LBB39_3
+; X64-NEXT:    ja .LBB39_3
 ; X64-NEXT:  # %bb.2: # %loadbb2
-; X64-NEXT:    movq 16(%rdi), %rax
-; X64-NEXT:    movq 16(%rsi), %rcx
-; X64-NEXT:    bswapq %rax
+; X64-NEXT:    movq 16(%rdi), %rcx
+; X64-NEXT:    movq 16(%rsi), %rdx
 ; X64-NEXT:    bswapq %rcx
-; X64-NEXT:    xorl %edx, %edx
-; X64-NEXT:    cmpq %rcx, %rax
-; X64-NEXT:    je .LBB39_4
+; X64-NEXT:    bswapq %rdx
+; X64-NEXT:    xorl %eax, %eax
+; X64-NEXT:    cmpq %rdx, %rcx
+; X64-NEXT:    jbe .LBB39_4
 ; X64-NEXT:  .LBB39_3: # %res_block
-; X64-NEXT:    xorl %edx, %edx
-; X64-NEXT:    cmpq %rcx, %rax
-; X64-NEXT:    setae %dl
-; X64-NEXT:    leal -1(%rdx,%rdx), %edx
+; X64-NEXT:    movl $1, %eax
 ; X64-NEXT:  .LBB39_4: # %endblock
-; X64-NEXT:    testl %edx, %edx
-; X64-NEXT:    setg %al
+; X64-NEXT:    testl %eax, %eax
+; X64-NEXT:    setne %al
 ; X64-NEXT:    retq
   %call = tail call i32 @memcmp(i8* %x, i8* %y, i64 24) nounwind
   %cmp = icmp sgt i32 %call, 0
   ret i1 %cmp
 }
 
 define i1 @length24_eq_const(i8* %X) nounwind {
 ; X86-NOSSE-LABEL: length24_eq_const:

 ; X86-NEXT:    calll memcmp
 ; X86-NEXT:    addl $16, %esp
 ; X86-NEXT:    shrl $31, %eax
 ; X86-NEXT:    # kill: def $al killed $al killed $eax
 ; X86-NEXT:    retl
 ;
 ; X64-LABEL: length32_lt:
 ; X64:       # %bb.0:
-; X64-NEXT:    movq (%rdi), %rcx
-; X64-NEXT:    movq (%rsi), %rdx
+; X64-NEXT:    movq (%rdi), %rax
+; X64-NEXT:    movq (%rsi), %rcx
+; X64-NEXT:    bswapq %rax
 ; X64-NEXT:    bswapq %rcx
-; X64-NEXT:    bswapq %rdx
-; X64-NEXT:    cmpq %rdx, %rcx
-; X64-NEXT:    jne .LBB49_4
+; X64-NEXT:    cmpq %rcx, %rax
+; X64-NEXT:    jb .LBB49_4
 ; X64-NEXT:  # %bb.1: # %loadbb1
-; X64-NEXT:    movq 8(%rdi), %rcx
-; X64-NEXT:    movq 8(%rsi), %rdx
+; X64-NEXT:    movq 8(%rdi), %rax
+; X64-NEXT:    movq 8(%rsi), %rcx
+; X64-NEXT:    bswapq %rax
 ; X64-NEXT:    bswapq %rcx
-; X64-NEXT:    bswapq %rdx
-; X64-NEXT:    cmpq %rdx, %rcx
-; X64-NEXT:    jne .LBB49_4
+; X64-NEXT:    cmpq %rcx, %rax
+; X64-NEXT:    jb .LBB49_4
 ; X64-NEXT:  # %bb.2: # %loadbb2
-; X64-NEXT:    movq 16(%rdi), %rcx
-; X64-NEXT:    movq 16(%rsi), %rdx
+; X64-NEXT:    movq 16(%rdi), %rax
+; X64-NEXT:    movq 16(%rsi), %rcx
+; X64-NEXT:    bswapq %rax
 ; X64-NEXT:    bswapq %rcx
-; X64-NEXT:    bswapq %rdx
-; X64-NEXT:    cmpq %rdx, %rcx
-; X64-NEXT:    jne .LBB49_4
+; X64-NEXT:    cmpq %rcx, %rax
+; X64-NEXT:    jb .LBB49_4
 ; X64-NEXT:  # %bb.3: # %loadbb3
 ; X64-NEXT:    movq 24(%rdi), %rcx
 ; X64-NEXT:    movq 24(%rsi), %rdx
 ; X64-NEXT:    bswapq %rcx
 ; X64-NEXT:    bswapq %rdx
 ; X64-NEXT:    xorl %eax, %eax
 ; X64-NEXT:    cmpq %rdx, %rcx
-; X64-NEXT:    je .LBB49_5
+; X64-NEXT:    jae .LBB49_5
 ; X64-NEXT:  .LBB49_4: # %res_block
-; X64-NEXT:    xorl %eax, %eax
-; X64-NEXT:    cmpq %rdx, %rcx
-; X64-NEXT:    setae %al
-; X64-NEXT:    leal -1(%rax,%rax), %eax
+; X64-NEXT:    movl $1, %eax
 ; X64-NEXT:  .LBB49_5: # %endblock
-; X64-NEXT:    shrl $31, %eax
-; X64-NEXT:    # kill: def $al killed $al killed $eax
+; X64-NEXT:    testl %eax, %eax
+; X64-NEXT:    setne %al
 ; X64-NEXT:    retq
   %call = tail call i32 @memcmp(i8* %x, i8* %y, i64 32) nounwind
   %cmp = icmp slt i32 %call, 0
   ret i1 %cmp
 }
 
 define i1 @length32_gt(i8* %x, i8* %y) nounwind {
 ; X86-LABEL: length32_gt:

 ;
 ; X64-LABEL: length32_gt:
 ; X64:       # %bb.0:
 ; X64-NEXT:    movq (%rdi), %rax
 ; X64-NEXT:    movq (%rsi), %rcx
 ; X64-NEXT:    bswapq %rax
 ; X64-NEXT:    bswapq %rcx
 ; X64-NEXT:    cmpq %rcx, %rax
-; X64-NEXT:    jne .LBB50_4
+; X64-NEXT:    ja .LBB50_4
 ; X64-NEXT:  # %bb.1: # %loadbb1
 ; X64-NEXT:    movq 8(%rdi), %rax
 ; X64-NEXT:    movq 8(%rsi), %rcx
 ; X64-NEXT:    bswapq %rax
 ; X64-NEXT:    bswapq %rcx
 ; X64-NEXT:    cmpq %rcx, %rax
-; X64-NEXT:    jne .LBB50_4
+; X64-NEXT:    ja .LBB50_4
 ; X64-NEXT:  # %bb.2: # %loadbb2
 ; X64-NEXT:    movq 16(%rdi), %rax
 ; X64-NEXT:    movq 16(%rsi), %rcx
 ; X64-NEXT:    bswapq %rax
 ; X64-NEXT:    bswapq %rcx
 ; X64-NEXT:    cmpq %rcx, %rax
-; X64-NEXT:    jne .LBB50_4
+; X64-NEXT:    ja .LBB50_4
 ; X64-NEXT:  # %bb.3: # %loadbb3
-; X64-NEXT:    movq 24(%rdi), %rax
-; X64-NEXT:    movq 24(%rsi), %rcx
-; X64-NEXT:    bswapq %rax
+; X64-NEXT:    movq 24(%rdi), %rcx
+; X64-NEXT:    movq 24(%rsi), %rdx
 ; X64-NEXT:    bswapq %rcx
-; X64-NEXT:    xorl %edx, %edx
-; X64-NEXT:    cmpq %rcx, %rax
-; X64-NEXT:    je .LBB50_5
+; X64-NEXT:    bswapq %rdx
+; X64-NEXT:    xorl %eax, %eax
+; X64-NEXT:    cmpq %rdx, %rcx
+; X64-NEXT:    jbe .LBB50_5
 ; X64-NEXT:  .LBB50_4: # %res_block
-; X64-NEXT:    xorl %edx, %edx
-; X64-NEXT:    cmpq %rcx, %rax
-; X64-NEXT:    setae %dl
-; X64-NEXT:    leal -1(%rdx,%rdx), %edx
+; X64-NEXT:    movl $1, %eax
 ; X64-NEXT:  .LBB50_5: # %endblock
-; X64-NEXT:    testl %edx, %edx
-; X64-NEXT:    setg %al
+; X64-NEXT:    testl %eax, %eax
+; X64-NEXT:    setne %al
 ; X64-NEXT:    retq
   %call = tail call i32 @memcmp(i8* %x, i8* %y, i64 32) nounwind
   %cmp = icmp sgt i32 %call, 0
   ret i1 %cmp
 }
 
 define i1 @length32_eq_prefer128(i8* %x, i8* %y) nounwind "prefer-vector-width"="128" {
 ; X86-NOSSE-LABEL: length32_eq_prefer128:

llvm/test/CodeGen/X86/memcmp.ll

 
 define i1 @length2_gt_const(i8* %X, i8* %Y) nounwind {
 ; X86-LABEL: length2_gt_const:
 ; X86:       # %bb.0:
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X86-NEXT:    movzwl (%eax), %eax
 ; X86-NEXT:    rolw $8, %ax
 ; X86-NEXT:    movzwl %ax, %eax
-; X86-NEXT:    addl $-12594, %eax # imm = 0xCECE
-; X86-NEXT:    testl %eax, %eax
-; X86-NEXT:    setg %al
+; X86-NEXT:    cmpl $12594, %eax # imm = 0x3132
+; X86-NEXT:    seta %al
 ; X86-NEXT:    retl
 ;
 ; X64-LABEL: length2_gt_const:
 ; X64:       # %bb.0:
 ; X64-NEXT:    movzwl (%rdi), %eax
 ; X64-NEXT:    rolw $8, %ax
 ; X64-NEXT:    movzwl %ax, %eax
-; X64-NEXT:    addl $-12594, %eax # imm = 0xCECE
-; X64-NEXT:    testl %eax, %eax
-; X64-NEXT:    setg %al
+; X64-NEXT:    cmpl $12594, %eax # imm = 0x3132
+; X64-NEXT:    seta %al
 ; X64-NEXT:    retq
   %m = tail call i32 @memcmp(i8* %X, i8* getelementptr inbounds ([513 x i8], [513 x i8]* @.str, i32 0, i32 1), i64 2) nounwind
   %c = icmp sgt i32 %m, 0
   ret i1 %c
 }
 
 define i1 @length2_eq(i8* %X, i8* %Y) nounwind {
 ; X86-LABEL: length2_eq:

 }
 
 define i1 @length2_lt(i8* %X, i8* %Y) nounwind {
 ; X86-LABEL: length2_lt:
 ; X86:       # %bb.0:
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; X86-NEXT:    movzwl (%ecx), %ecx
-; X86-NEXT:    movzwl (%eax), %edx
+; X86-NEXT:    movzwl (%eax), %eax
 ; X86-NEXT:    rolw $8, %cx
-; X86-NEXT:    rolw $8, %dx
-; X86-NEXT:    movzwl %cx, %eax
-; X86-NEXT:    movzwl %dx, %ecx
-; X86-NEXT:    subl %ecx, %eax
-; X86-NEXT:    shrl $31, %eax
-; X86-NEXT:    # kill: def $al killed $al killed $eax
+; X86-NEXT:    rolw $8, %ax
+; X86-NEXT:    cmpw %ax, %cx
+; X86-NEXT:    setb %al
 ; X86-NEXT:    retl
 ;
 ; X64-LABEL: length2_lt:
 ; X64:       # %bb.0:
 ; X64-NEXT:    movzwl (%rdi), %eax
 ; X64-NEXT:    movzwl (%rsi), %ecx
 ; X64-NEXT:    rolw $8, %ax
 ; X64-NEXT:    rolw $8, %cx
-; X64-NEXT:    movzwl %ax, %eax
-; X64-NEXT:    movzwl %cx, %ecx
-; X64-NEXT:    subl %ecx, %eax
-; X64-NEXT:    shrl $31, %eax
-; X64-NEXT:    # kill: def $al killed $al killed $eax
+; X64-NEXT:    cmpw %cx, %ax
+; X64-NEXT:    setb %al
 ; X64-NEXT:    retq
   %m = tail call i32 @memcmp(i8* %X, i8* %Y, i64 2) nounwind
   %c = icmp slt i32 %m, 0
   ret i1 %c
 }
 
 define i1 @length2_gt(i8* %X, i8* %Y) nounwind {
 ; X86-LABEL: length2_gt:
 ; X86:       # %bb.0:
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; X86-NEXT:    movzwl (%ecx), %ecx
 ; X86-NEXT:    movzwl (%eax), %eax
 ; X86-NEXT:    rolw $8, %cx
 ; X86-NEXT:    rolw $8, %ax
-; X86-NEXT:    movzwl %cx, %ecx
-; X86-NEXT:    movzwl %ax, %eax
-; X86-NEXT:    subl %eax, %ecx
-; X86-NEXT:    testl %ecx, %ecx
-; X86-NEXT:    setg %al
+; X86-NEXT:    cmpw %ax, %cx
+; X86-NEXT:    seta %al
 ; X86-NEXT:    retl
 ;
 ; X64-LABEL: length2_gt:
 ; X64:       # %bb.0:
 ; X64-NEXT:    movzwl (%rdi), %eax
 ; X64-NEXT:    movzwl (%rsi), %ecx
 ; X64-NEXT:    rolw $8, %ax
 ; X64-NEXT:    rolw $8, %cx
-; X64-NEXT:    movzwl %ax, %eax
-; X64-NEXT:    movzwl %cx, %ecx
-; X64-NEXT:    subl %ecx, %eax
-; X64-NEXT:    testl %eax, %eax
-; X64-NEXT:    setg %al
+; X64-NEXT:    cmpw %cx, %ax
+; X64-NEXT:    seta %al
 ; X64-NEXT:    retq
   %m = tail call i32 @memcmp(i8* %X, i8* %Y, i64 2) nounwind
   %c = icmp sgt i32 %m, 0
   ret i1 %c
 }
 
 define i1 @length2_eq_const(i8* %X) nounwind {
 ; X86-LABEL: length2_eq_const:

 }
 
 define i1 @length4_lt(i8* %X, i8* %Y) nounwind {
 ; X86-LABEL: length4_lt:
 ; X86:       # %bb.0:
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; X86-NEXT:    movl (%ecx), %ecx
-; X86-NEXT:    movl (%eax), %edx
+; X86-NEXT:    movl (%eax), %eax
 ; X86-NEXT:    bswapl %ecx
-; X86-NEXT:    bswapl %edx
-; X86-NEXT:    xorl %eax, %eax
-; X86-NEXT:    cmpl %edx, %ecx
-; X86-NEXT:    seta %al
-; X86-NEXT:    sbbl $0, %eax
-; X86-NEXT:    shrl $31, %eax
-; X86-NEXT:    # kill: def $al killed $al killed $eax
+; X86-NEXT:    bswapl %eax
+; X86-NEXT:    cmpl %eax, %ecx
+; X86-NEXT:    setb %al
 ; X86-NEXT:    retl
 ;
 ; X64-LABEL: length4_lt:
 ; X64:       # %bb.0:
-; X64-NEXT:    movl (%rdi), %ecx
-; X64-NEXT:    movl (%rsi), %edx
+; X64-NEXT:    movl (%rdi), %eax
+; X64-NEXT:    movl (%rsi), %ecx
+; X64-NEXT:    bswapl %eax
 ; X64-NEXT:    bswapl %ecx
-; X64-NEXT:    bswapl %edx
-; X64-NEXT:    xorl %eax, %eax
-; X64-NEXT:    cmpl %edx, %ecx
-; X64-NEXT:    seta %al
-; X64-NEXT:    sbbl $0, %eax
-; X64-NEXT:    shrl $31, %eax
-; X64-NEXT:    # kill: def $al killed $al killed $eax
+; X64-NEXT:    cmpl %ecx, %eax
+; X64-NEXT:    setb %al
 ; X64-NEXT:    retq
   %m = tail call i32 @memcmp(i8* %X, i8* %Y, i64 4) nounwind
   %c = icmp slt i32 %m, 0
   ret i1 %c
 }
 
 define i1 @length4_gt(i8* %X, i8* %Y) nounwind {
 ; X86-LABEL: length4_gt:
 ; X86:       # %bb.0:
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; X86-NEXT:    movl (%ecx), %ecx
 ; X86-NEXT:    movl (%eax), %eax
 ; X86-NEXT:    bswapl %ecx
 ; X86-NEXT:    bswapl %eax
-; X86-NEXT:    xorl %edx, %edx
 ; X86-NEXT:    cmpl %eax, %ecx
-; X86-NEXT:    seta %dl
-; X86-NEXT:    sbbl $0, %edx
-; X86-NEXT:    testl %edx, %edx
-; X86-NEXT:    setg %al
+; X86-NEXT:    seta %al
 ; X86-NEXT:    retl
 ;
 ; X64-LABEL: length4_gt:
 ; X64:       # %bb.0:
 ; X64-NEXT:    movl (%rdi), %eax
 ; X64-NEXT:    movl (%rsi), %ecx
 ; X64-NEXT:    bswapl %eax
 ; X64-NEXT:    bswapl %ecx
-; X64-NEXT:    xorl %edx, %edx
 ; X64-NEXT:    cmpl %ecx, %eax
-; X64-NEXT:    seta %dl
-; X64-NEXT:    sbbl $0, %edx
-; X64-NEXT:    testl %edx, %edx
-; X64-NEXT:    setg %al
+; X64-NEXT:    seta %al
 ; X64-NEXT:    retq
   %m = tail call i32 @memcmp(i8* %X, i8* %Y, i64 4) nounwind
   %c = icmp sgt i32 %m, 0
   ret i1 %c
 }
 
 define i1 @length4_eq_const(i8* %X) nounwind {
 ; X86-LABEL: length4_eq_const:
; X64-NEXT:    retq
   %c = icmp ne i32 %m, 0
   ret i1 %c
 }
 
 define i1 @length5_lt(i8* %X, i8* %Y) nounwind {
 ; X86-LABEL: length5_lt:
 ; X86:       # %bb.0:
 ; X86-NEXT:    pushl %esi
-; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; X86-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT:    movl (%eax), %edx
-; X86-NEXT:    movl (%ecx), %esi
+; X86-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT:    movl (%ecx), %edx
+; X86-NEXT:    movl (%eax), %esi
 ; X86-NEXT:    bswapl %edx
 ; X86-NEXT:    bswapl %esi
 ; X86-NEXT:    cmpl %esi, %edx
-; X86-NEXT:    jne .LBB20_3
+; X86-NEXT:    jb .LBB20_2
 ; X86-NEXT:  # %bb.1: # %loadbb1
-; X86-NEXT:    movzbl 4(%eax), %eax
-; X86-NEXT:    movzbl 4(%ecx), %ecx
-; X86-NEXT:    subl %ecx, %eax
-; X86-NEXT:    jmp .LBB20_2
-; X86-NEXT:  .LBB20_3: # %res_block
-; X86-NEXT:    setae %al
-; X86-NEXT:    movzbl %al, %eax
-; X86-NEXT:    leal -1(%eax,%eax), %eax
-; X86-NEXT:  .LBB20_2: # %endblock
-; X86-NEXT:    shrl $31, %eax
-; X86-NEXT:    # kill: def $al killed $al killed $eax
+; X86-NEXT:    movb 4(%ecx), %dl
+; X86-NEXT:    xorl %ecx, %ecx
+; X86-NEXT:    cmpb 4(%eax), %dl
+; X86-NEXT:    jae .LBB20_3
+; X86-NEXT:  .LBB20_2: # %res_block
+; X86-NEXT:    movl $1, %ecx
+; X86-NEXT:  .LBB20_3: # %endblock
+; X86-NEXT:    testl %ecx, %ecx
+; X86-NEXT:    setne %al
 ; X86-NEXT:    popl %esi
 ; X86-NEXT:    retl
 ;
 ; X64-LABEL: length5_lt:
 ; X64:       # %bb.0:
 ; X64-NEXT:    movl (%rdi), %eax
 ; X64-NEXT:    movl (%rsi), %ecx
 ; X64-NEXT:    bswapl %eax
 ; X64-NEXT:    bswapl %ecx
 ; X64-NEXT:    cmpl %ecx, %eax
-; X64-NEXT:    jne .LBB20_3
+; X64-NEXT:    jb .LBB20_2
 ; X64-NEXT:  # %bb.1: # %loadbb1
-; X64-NEXT:    movzbl 4(%rdi), %eax
-; X64-NEXT:    movzbl 4(%rsi), %ecx
-; X64-NEXT:    subl %ecx, %eax
-; X64-NEXT:    shrl $31, %eax
-; X64-NEXT:    # kill: def $al killed $al killed $eax
-; X64-NEXT:    retq
-; X64-NEXT:  .LBB20_3: # %res_block
-; X64-NEXT:    setae %al
-; X64-NEXT:    movzbl %al, %eax
-; X64-NEXT:    leal -1(%rax,%rax), %eax
-; X64-NEXT:    shrl $31, %eax
-; X64-NEXT:    # kill: def $al killed $al killed $eax
+; X64-NEXT:    movb 4(%rdi), %cl
+; X64-NEXT:    xorl %eax, %eax
+; X64-NEXT:    cmpb 4(%rsi), %cl
+; X64-NEXT:    jae .LBB20_3
+; X64-NEXT:  .LBB20_2: # %res_block
+; X64-NEXT:    movl $1, %eax
+; X64-NEXT:  .LBB20_3: # %endblock
+; X64-NEXT:    testl %eax, %eax
+; X64-NEXT:    setne %al
 ; X64-NEXT:    retq
   %m = tail call i32 @memcmp(i8* %X, i8* %Y, i64 5) nounwind
   %c = icmp slt i32 %m, 0
   ret i1 %c
 }
 
 define i32 @length7(i8* %X, i8* %Y) nounwind {
 ; X86-LABEL: length7:

 ; X86-NEXT:    calll memcmp
 ; X86-NEXT:    addl $16, %esp
 ; X86-NEXT:    shrl $31, %eax
 ; X86-NEXT:    # kill: def $al killed $al killed $eax
 ; X86-NEXT:    retl
 ;
 ; X64-LABEL: length16_lt:
 ; X64:       # %bb.0:
-; X64-NEXT:    movq (%rdi), %rcx
-; X64-NEXT:    movq (%rsi), %rdx
+; X64-NEXT:    movq (%rdi), %rax
+; X64-NEXT:    movq (%rsi), %rcx
+; X64-NEXT:    bswapq %rax
 ; X64-NEXT:    bswapq %rcx
-; X64-NEXT:    bswapq %rdx
-; X64-NEXT:    cmpq %rdx, %rcx
-; X64-NEXT:    jne .LBB41_2
+; X64-NEXT:    cmpq %rcx, %rax
+; X64-NEXT:    jb .LBB41_2
 ; X64-NEXT:  # %bb.1: # %loadbb1
 ; X64-NEXT:    movq 8(%rdi), %rcx
 ; X64-NEXT:    movq 8(%rsi), %rdx
 ; X64-NEXT:    bswapq %rcx
 ; X64-NEXT:    bswapq %rdx
 ; X64-NEXT:    xorl %eax, %eax
 ; X64-NEXT:    cmpq %rdx, %rcx
-; X64-NEXT:    je .LBB41_3
+; X64-NEXT:    jae .LBB41_3
 ; X64-NEXT:  .LBB41_2: # %res_block
-; X64-NEXT:    xorl %eax, %eax
-; X64-NEXT:    cmpq %rdx, %rcx
-; X64-NEXT:    setae %al
-; X64-NEXT:    leal -1(%rax,%rax), %eax
+; X64-NEXT:    movl $1, %eax
 ; X64-NEXT:  .LBB41_3: # %endblock
-; X64-NEXT:    shrl $31, %eax
-; X64-NEXT:    # kill: def $al killed $al killed $eax
+; X64-NEXT:    testl %eax, %eax
+; X64-NEXT:    setne %al
 ; X64-NEXT:    retq
   %call = tail call i32 @memcmp(i8* %x, i8* %y, i64 16) nounwind
   %cmp = icmp slt i32 %call, 0
   ret i1 %cmp
 }
 
 define i1 @length16_gt(i8* %x, i8* %y) nounwind {
 ; X86-LABEL: length16_gt:

 ;
 ; X64-LABEL: length16_gt:
 ; X64:       # %bb.0:
 ; X64-NEXT:    movq (%rdi), %rax
 ; X64-NEXT:    movq (%rsi), %rcx
 ; X64-NEXT:    bswapq %rax
 ; X64-NEXT:    bswapq %rcx
 ; X64-NEXT:    cmpq %rcx, %rax
-; X64-NEXT:    jne .LBB42_2
+; X64-NEXT:    ja .LBB42_2
 ; X64-NEXT:  # %bb.1: # %loadbb1
-; X64-NEXT:    movq 8(%rdi), %rax
-; X64-NEXT:    movq 8(%rsi), %rcx
-; X64-NEXT:    bswapq %rax
+; X64-NEXT:    movq 8(%rdi), %rcx
+; X64-NEXT:    movq 8(%rsi), %rdx
 ; X64-NEXT:    bswapq %rcx
-; X64-NEXT:    xorl %edx, %edx
-; X64-NEXT:    cmpq %rcx, %rax
-; X64-NEXT:    je .LBB42_3
+; X64-NEXT:    bswapq %rdx
+; X64-NEXT:    xorl %eax, %eax
+; X64-NEXT:    cmpq %rdx, %rcx
+; X64-NEXT:    jbe .LBB42_3
 ; X64-NEXT:  .LBB42_2: # %res_block
-; X64-NEXT:    xorl %edx, %edx
-; X64-NEXT:    cmpq %rcx, %rax
-; X64-NEXT:    setae %dl
-; X64-NEXT:    leal -1(%rdx,%rdx), %edx
+; X64-NEXT:    movl $1, %eax
 ; X64-NEXT:  .LBB42_3: # %endblock
-; X64-NEXT:    testl %edx, %edx
-; X64-NEXT:    setg %al
+; X64-NEXT:    testl %eax, %eax
+; X64-NEXT:    setne %al
 ; X64-NEXT:    retq
   %call = tail call i32 @memcmp(i8* %x, i8* %y, i64 16) nounwind
   %cmp = icmp sgt i32 %call, 0
   ret i1 %cmp
 }
 
 define i1 @length16_eq_const(i8* %X) nounwind {
 ; X86-NOSSE-LABEL: length16_eq_const:

llvm/test/Transforms/ExpandMemCmp/X86/memcmp.ll

 }
 
 define i32 @cmp_eq2(i8* nocapture readonly %x, i8* nocapture readonly %y)  {
 ; ALL-LABEL: @cmp_eq2(
 ; ALL-NEXT:    [[TMP1:%.*]] = bitcast i8* [[X:%.*]] to i16*
 ; ALL-NEXT:    [[TMP2:%.*]] = bitcast i8* [[Y:%.*]] to i16*
 ; ALL-NEXT:    [[TMP3:%.*]] = load i16, i16* [[TMP1]]
 ; ALL-NEXT:    [[TMP4:%.*]] = load i16, i16* [[TMP2]]
-; ALL-NEXT:    [[TMP5:%.*]] = icmp ne i16 [[TMP3]], [[TMP4]]
+; ALL-NEXT:    [[TMP5:%.*]] = icmp eq i16 [[TMP3]], [[TMP4]]
 ; ALL-NEXT:    [[TMP6:%.*]] = zext i1 [[TMP5]] to i32
-; ALL-NEXT:    [[CMP:%.*]] = icmp eq i32 [[TMP6]], 0
-; ALL-NEXT:    [[CONV:%.*]] = zext i1 [[CMP]] to i32
+; ALL-NEXT:    [[CONV:%.*]] = zext i1 [[TMP5]] to i32
 ; ALL-NEXT:    ret i32 [[CONV]]
 ;
   %call = tail call i32 @memcmp(i8* %x, i8* %y, i64 2)
   %cmp = icmp eq i32 %call, 0
   %conv = zext i1 %cmp to i32
   ret i32 %conv
 }
 
 define i32 @cmp_eq3(i8* nocapture readonly %x, i8* nocapture readonly %y)  {
 ; X32-LABEL: @cmp_eq3(
 ; X32-NEXT:    [[TMP1:%.*]] = bitcast i8* [[X:%.*]] to i16*
 ; X32-NEXT:    [[TMP2:%.*]] = bitcast i8* [[Y:%.*]] to i16*
 ; X32-NEXT:    [[TMP3:%.*]] = load i16, i16* [[TMP1]]
 ; X32-NEXT:    [[TMP4:%.*]] = load i16, i16* [[TMP2]]
 ; X32-NEXT:    [[TMP5:%.*]] = xor i16 [[TMP3]], [[TMP4]]
 ; X32-NEXT:    [[TMP6:%.*]] = getelementptr i8, i8* [[X]], i64 2
 ; X32-NEXT:    [[TMP7:%.*]] = getelementptr i8, i8* [[Y]], i64 2
 ; X32-NEXT:    [[TMP8:%.*]] = load i8, i8* [[TMP6]]
 ; X32-NEXT:    [[TMP9:%.*]] = load i8, i8* [[TMP7]]
 ; X32-NEXT:    [[TMP10:%.*]] = zext i8 [[TMP8]] to i16
 ; X32-NEXT:    [[TMP11:%.*]] = zext i8 [[TMP9]] to i16
 ; X32-NEXT:    [[TMP12:%.*]] = xor i16 [[TMP10]], [[TMP11]]
 ; X32-NEXT:    [[TMP13:%.*]] = or i16 [[TMP5]], [[TMP12]]
-; X32-NEXT:    [[TMP14:%.*]] = icmp ne i16 [[TMP13]], 0
+; X32-NEXT:    [[TMP14:%.*]] = icmp eq i16 [[TMP13]], 0
 ; X32-NEXT:    [[TMP15:%.*]] = zext i1 [[TMP14]] to i32
-; X32-NEXT:    [[CMP:%.*]] = icmp eq i32 [[TMP15]], 0
-; X32-NEXT:    [[CONV:%.*]] = zext i1 [[CMP]] to i32
+; X32-NEXT:    [[CONV:%.*]] = zext i1 [[TMP14]] to i32
 ; X32-NEXT:    ret i32 [[CONV]]
 ;
 ; X64_1LD-LABEL: @cmp_eq3(
 ; X64_1LD-NEXT:    br label [[LOADBB:%.*]]
 ; X64_1LD:       res_block:
 ; X64_1LD-NEXT:    br label [[ENDBLOCK:%.*]]
 ; X64_1LD:       loadbb:
 ; X64_1LD-NEXT:    [[TMP1:%.*]] = bitcast i8* [[X:%.*]] to i16*
 ; X64_1LD-NEXT:    [[TMP2:%.*]] = bitcast i8* [[Y:%.*]] to i16*
 ; X64_1LD-NEXT:    [[TMP3:%.*]] = load i16, i16* [[TMP1]]
 ; X64_1LD-NEXT:    [[TMP4:%.*]] = load i16, i16* [[TMP2]]
-; X64_1LD-NEXT:    [[TMP5:%.*]] = icmp ne i16 [[TMP3]], [[TMP4]]
+; X64_1LD-NEXT:    [[TMP5:%.*]] = icmp eq i16 [[TMP3]], [[TMP4]]
 ; X64_1LD-NEXT:    br i1 [[TMP5]], label [[RES_BLOCK:%.*]], label [[LOADBB1:%.*]]
 ; X64_1LD:       loadbb1:
 ; X64_1LD-NEXT:    [[TMP6:%.*]] = getelementptr i8, i8* [[X]], i64 2
 ; X64_1LD-NEXT:    [[TMP7:%.*]] = getelementptr i8, i8* [[Y]], i64 2
 ; X64_1LD-NEXT:    [[TMP8:%.*]] = load i8, i8* [[TMP6]]
 ; X64_1LD-NEXT:    [[TMP9:%.*]] = load i8, i8* [[TMP7]]
-; X64_1LD-NEXT:    [[TMP10:%.*]] = icmp ne i8 [[TMP8]], [[TMP9]]
+; X64_1LD-NEXT:    [[TMP10:%.*]] = icmp eq i8 [[TMP8]], [[TMP9]]
 ; X64_1LD-NEXT:    br i1 [[TMP10]], label [[RES_BLOCK]], label [[ENDBLOCK]]
 ; X64_1LD:       endblock:
 ; X64_1LD-NEXT:    [[PHI_RES:%.*]] = phi i32 [ 0, [[LOADBB1]] ], [ 1, [[RES_BLOCK]] ]
-; X64_1LD-NEXT:    [[CMP:%.*]] = icmp eq i32 [[PHI_RES]], 0
+; X64_1LD-NEXT:    [[CMP:%.*]] = icmp ne i32 [[PHI_RES]], 0
 ; X64_1LD-NEXT:    [[CONV:%.*]] = zext i1 [[CMP]] to i32
 ; X64_1LD-NEXT:    ret i32 [[CONV]]
 ;
 ; X64_2LD-LABEL: @cmp_eq3(
 ; X64_2LD-NEXT:    [[TMP1:%.*]] = bitcast i8* [[X:%.*]] to i16*
 ; X64_2LD-NEXT:    [[TMP2:%.*]] = bitcast i8* [[Y:%.*]] to i16*
 ; X64_2LD-NEXT:    [[TMP3:%.*]] = load i16, i16* [[TMP1]]
 ; X64_2LD-NEXT:    [[TMP4:%.*]] = load i16, i16* [[TMP2]]
 ; X64_2LD-NEXT:    [[TMP5:%.*]] = xor i16 [[TMP3]], [[TMP4]]
 ; X64_2LD-NEXT:    [[TMP6:%.*]] = getelementptr i8, i8* [[X]], i64 2
 ; X64_2LD-NEXT:    [[TMP7:%.*]] = getelementptr i8, i8* [[Y]], i64 2
 ; X64_2LD-NEXT:    [[TMP8:%.*]] = load i8, i8* [[TMP6]]
 ; X64_2LD-NEXT:    [[TMP9:%.*]] = load i8, i8* [[TMP7]]
 ; X64_2LD-NEXT:    [[TMP10:%.*]] = zext i8 [[TMP8]] to i16
 ; X64_2LD-NEXT:    [[TMP11:%.*]] = zext i8 [[TMP9]] to i16
 ; X64_2LD-NEXT:    [[TMP12:%.*]] = xor i16 [[TMP10]], [[TMP11]]
 ; X64_2LD-NEXT:    [[TMP13:%.*]] = or i16 [[TMP5]], [[TMP12]]
-; X64_2LD-NEXT:    [[TMP14:%.*]] = icmp ne i16 [[TMP13]], 0
+; X64_2LD-NEXT:    [[TMP14:%.*]] = icmp eq i16 [[TMP13]], 0
 ; X64_2LD-NEXT:    [[TMP15:%.*]] = zext i1 [[TMP14]] to i32
-; X64_2LD-NEXT:    [[CMP:%.*]] = icmp eq i32 [[TMP15]], 0
-; X64_2LD-NEXT:    [[CONV:%.*]] = zext i1 [[CMP]] to i32
+; X64_2LD-NEXT:    [[CONV:%.*]] = zext i1 [[TMP14]] to i32
 ; X64_2LD-NEXT:    ret i32 [[CONV]]
 ;
   %call = tail call i32 @memcmp(i8* %x, i8* %y, i64 3)
   %cmp = icmp eq i32 %call, 0
   %conv = zext i1 %cmp to i32
   ret i32 %conv
 }
 
 define i32 @cmp_eq4(i8* nocapture readonly %x, i8* nocapture readonly %y)  {
 ; ALL-LABEL: @cmp_eq4(
 ; ALL-NEXT:    [[TMP1:%.*]] = bitcast i8* [[X:%.*]] to i32*
 ; ALL-NEXT:    [[TMP2:%.*]] = bitcast i8* [[Y:%.*]] to i32*
 ; ALL-NEXT:    [[TMP3:%.*]] = load i32, i32* [[TMP1]]
 ; ALL-NEXT:    [[TMP4:%.*]] = load i32, i32* [[TMP2]]
-; ALL-NEXT:    [[TMP5:%.*]] = icmp ne i32 [[TMP3]], [[TMP4]]
+; ALL-NEXT:    [[TMP5:%.*]] = icmp eq i32 [[TMP3]], [[TMP4]]
 ; ALL-NEXT:    [[TMP6:%.*]] = zext i1 [[TMP5]] to i32
-; ALL-NEXT:    [[CMP:%.*]] = icmp eq i32 [[TMP6]], 0
-; ALL-NEXT:    [[CONV:%.*]] = zext i1 [[CMP]] to i32
+; ALL-NEXT:    [[CONV:%.*]] = zext i1 [[TMP5]] to i32
 ; ALL-NEXT:    ret i32 [[CONV]]
 ;
   %call = tail call i32 @memcmp(i8* %x, i8* %y, i64 4)
   %cmp = icmp eq i32 %call, 0
   %conv = zext i1 %cmp to i32
   ret i32 %conv
 }
 
 define i32 @cmp_eq5(i8* nocapture readonly %x, i8* nocapture readonly %y)  {
 ; X32-LABEL: @cmp_eq5(
 ; X32-NEXT:    [[TMP1:%.*]] = bitcast i8* [[X:%.*]] to i32*
 ; X32-NEXT:    [[TMP2:%.*]] = bitcast i8* [[Y:%.*]] to i32*
 ; X32-NEXT:    [[TMP3:%.*]] = load i32, i32* [[TMP1]]
 ; X32-NEXT:    [[TMP4:%.*]] = load i32, i32* [[TMP2]]
 ; X32-NEXT:    [[TMP5:%.*]] = xor i32 [[TMP3]], [[TMP4]]
 ; X32-NEXT:    [[TMP6:%.*]] = getelementptr i8, i8* [[X]], i64 4
 ; X32-NEXT:    [[TMP7:%.*]] = getelementptr i8, i8* [[Y]], i64 4
 ; X32-NEXT:    [[TMP8:%.*]] = load i8, i8* [[TMP6]]
 ; X32-NEXT:    [[TMP9:%.*]] = load i8, i8* [[TMP7]]
 ; X32-NEXT:    [[TMP10:%.*]] = zext i8 [[TMP8]] to i32
 ; X32-NEXT:    [[TMP11:%.*]] = zext i8 [[TMP9]] to i32
 ; X32-NEXT:    [[TMP12:%.*]] = xor i32 [[TMP10]], [[TMP11]]
 ; X32-NEXT:    [[TMP13:%.*]] = or i32 [[TMP5]], [[TMP12]]
-; X32-NEXT:    [[TMP14:%.*]] = icmp ne i32 [[TMP13]], 0
+; X32-NEXT:    [[TMP14:%.*]] = icmp eq i32 [[TMP13]], 0
 ; X32-NEXT:    [[TMP15:%.*]] = zext i1 [[TMP14]] to i32
-; X32-NEXT:    [[CMP:%.*]] = icmp eq i32 [[TMP15]], 0
-; X32-NEXT:    [[CONV:%.*]] = zext i1 [[CMP]] to i32
+; X32-NEXT:    [[CONV:%.*]] = zext i1 [[TMP14]] to i32
 ; X32-NEXT:    ret i32 [[CONV]]
 ;
 ; X64_1LD-LABEL: @cmp_eq5(
 ; X64_1LD-NEXT:    br label [[LOADBB:%.*]]
 ; X64_1LD:       res_block:
 ; X64_1LD-NEXT:    br label [[ENDBLOCK:%.*]]
 ; X64_1LD:       loadbb:
 ; X64_1LD-NEXT:    [[TMP1:%.*]] = bitcast i8* [[X:%.*]] to i32*
 ; X64_1LD-NEXT:    [[TMP2:%.*]] = bitcast i8* [[Y:%.*]] to i32*
 ; X64_1LD-NEXT:    [[TMP3:%.*]] = load i32, i32* [[TMP1]]
 ; X64_1LD-NEXT:    [[TMP4:%.*]] = load i32, i32* [[TMP2]]
-; X64_1LD-NEXT:    [[TMP5:%.*]] = icmp ne i32 [[TMP3]], [[TMP4]]
+; X64_1LD-NEXT:    [[TMP5:%.*]] = icmp eq i32 [[TMP3]], [[TMP4]]
 ; X64_1LD-NEXT:    br i1 [[TMP5]], label [[RES_BLOCK:%.*]], label [[LOADBB1:%.*]]
 ; X64_1LD:       loadbb1:
 ; X64_1LD-NEXT:    [[TMP6:%.*]] = getelementptr i8, i8* [[X]], i64 4
 ; X64_1LD-NEXT:    [[TMP7:%.*]] = getelementptr i8, i8* [[Y]], i64 4
 ; X64_1LD-NEXT:    [[TMP8:%.*]] = load i8, i8* [[TMP6]]
 ; X64_1LD-NEXT:    [[TMP9:%.*]] = load i8, i8* [[TMP7]]
-; X64_1LD-NEXT:    [[TMP10:%.*]] = icmp ne i8 [[TMP8]], [[TMP9]]
+; X64_1LD-NEXT:    [[TMP10:%.*]] = icmp eq i8 [[TMP8]], [[TMP9]]
 ; X64_1LD-NEXT:    br i1 [[TMP10]], label [[RES_BLOCK]], label [[ENDBLOCK]]
 ; X64_1LD:       endblock:
 ; X64_1LD-NEXT:    [[PHI_RES:%.*]] = phi i32 [ 0, [[LOADBB1]] ], [ 1, [[RES_BLOCK]] ]
-; X64_1LD-NEXT:    [[CMP:%.*]] = icmp eq i32 [[PHI_RES]], 0
+; X64_1LD-NEXT:    [[CMP:%.*]] = icmp ne i32 [[PHI_RES]], 0
 ; X64_1LD-NEXT:    [[CONV:%.*]] = zext i1 [[CMP]] to i32
 ; X64_1LD-NEXT:    ret i32 [[CONV]]
 ;
 ; X64_2LD-LABEL: @cmp_eq5(
 ; X64_2LD-NEXT:    [[TMP1:%.*]] = bitcast i8* [[X:%.*]] to i32*
 ; X64_2LD-NEXT:    [[TMP2:%.*]] = bitcast i8* [[Y:%.*]] to i32*
 ; X64_2LD-NEXT:    [[TMP3:%.*]] = load i32, i32* [[TMP1]]
 ; X64_2LD-NEXT:    [[TMP4:%.*]] = load i32, i32* [[TMP2]]
 ; X64_2LD-NEXT:    [[TMP5:%.*]] = xor i32 [[TMP3]], [[TMP4]]
 ; X64_2LD-NEXT:    [[TMP6:%.*]] = getelementptr i8, i8* [[X]], i64 4
 ; X64_2LD-NEXT:    [[TMP7:%.*]] = getelementptr i8, i8* [[Y]], i64 4
 ; X64_2LD-NEXT:    [[TMP8:%.*]] = load i8, i8* [[TMP6]]
 ; X64_2LD-NEXT:    [[TMP9:%.*]] = load i8, i8* [[TMP7]]
 ; X64_2LD-NEXT:    [[TMP10:%.*]] = zext i8 [[TMP8]] to i32
 ; X64_2LD-NEXT:    [[TMP11:%.*]] = zext i8 [[TMP9]] to i32
 ; X64_2LD-NEXT:    [[TMP12:%.*]] = xor i32 [[TMP10]], [[TMP11]]
 ; X64_2LD-NEXT:    [[TMP13:%.*]] = or i32 [[TMP5]], [[TMP12]]
-; X64_2LD-NEXT:    [[TMP14:%.*]] = icmp ne i32 [[TMP13]], 0
+; X64_2LD-NEXT:    [[TMP14:%.*]] = icmp eq i32 [[TMP13]], 0
 ; X64_2LD-NEXT:    [[TMP15:%.*]] = zext i1 [[TMP14]] to i32
-; X64_2LD-NEXT:    [[CMP:%.*]] = icmp eq i32 [[TMP15]], 0
-; X64_2LD-NEXT:    [[CONV:%.*]] = zext i1 [[CMP]] to i32
+; X64_2LD-NEXT:    [[CONV:%.*]] = zext i1 [[TMP14]] to i32
 ; X64_2LD-NEXT:    ret i32 [[CONV]]
 ;
   %call = tail call i32 @memcmp(i8* %x, i8* %y, i64 5)
   %cmp = icmp eq i32 %call, 0
   %conv = zext i1 %cmp to i32
   ret i32 %conv
 }
 

 ; X32-NEXT:    [[TMP8:%.*]] = bitcast i8* [[TMP6]] to i16*
 ; X32-NEXT:    [[TMP9:%.*]] = bitcast i8* [[TMP7]] to i16*
 ; X32-NEXT:    [[TMP10:%.*]] = load i16, i16* [[TMP8]]
 ; X32-NEXT:    [[TMP11:%.*]] = load i16, i16* [[TMP9]]
 ; X32-NEXT:    [[TMP12:%.*]] = zext i16 [[TMP10]] to i32
 ; X32-NEXT:    [[TMP13:%.*]] = zext i16 [[TMP11]] to i32
 ; X32-NEXT:    [[TMP14:%.*]] = xor i32 [[TMP12]], [[TMP13]]
 ; X32-NEXT:    [[TMP15:%.*]] = or i32 [[TMP5]], [[TMP14]]
-; X32-NEXT:    [[TMP16:%.*]] = icmp ne i32 [[TMP15]], 0
+; X32-NEXT:    [[TMP16:%.*]] = icmp eq i32 [[TMP15]], 0
 ; X32-NEXT:    [[TMP17:%.*]] = zext i1 [[TMP16]] to i32
-; X32-NEXT:    [[CMP:%.*]] = icmp eq i32 [[TMP17]], 0
-; X32-NEXT:    [[CONV:%.*]] = zext i1 [[CMP]] to i32
+; X32-NEXT:    [[CONV:%.*]] = zext i1 [[TMP16]] to i32
 ; X32-NEXT:    ret i32 [[CONV]]
 ;
 ; X64_1LD-LABEL: @cmp_eq6(
 ; X64_1LD-NEXT:    br label [[LOADBB:%.*]]
 ; X64_1LD:       res_block:
 ; X64_1LD-NEXT:    br label [[ENDBLOCK:%.*]]
 ; X64_1LD:       loadbb:
 ; X64_1LD-NEXT:    [[TMP1:%.*]] = bitcast i8* [[X:%.*]] to i32*
 ; X64_1LD-NEXT:    [[TMP2:%.*]] = bitcast i8* [[Y:%.*]] to i32*
 ; X64_1LD-NEXT:    [[TMP3:%.*]] = load i32, i32* [[TMP1]]
 ; X64_1LD-NEXT:    [[TMP4:%.*]] = load i32, i32* [[TMP2]]
-; X64_1LD-NEXT:    [[TMP5:%.*]] = icmp ne i32 [[TMP3]], [[TMP4]]
+; X64_1LD-NEXT:    [[TMP5:%.*]] = icmp eq i32 [[TMP3]], [[TMP4]]
 ; X64_1LD-NEXT:    br i1 [[TMP5]], label [[RES_BLOCK:%.*]], label [[LOADBB1:%.*]]
 ; X64_1LD:       loadbb1:
 ; X64_1LD-NEXT:    [[TMP6:%.*]] = getelementptr i8, i8* [[X]], i64 4
 ; X64_1LD-NEXT:    [[TMP7:%.*]] = getelementptr i8, i8* [[Y]], i64 4
 ; X64_1LD-NEXT:    [[TMP8:%.*]] = bitcast i8* [[TMP6]] to i16*
 ; X64_1LD-NEXT:    [[TMP9:%.*]] = bitcast i8* [[TMP7]] to i16*
 ; X64_1LD-NEXT:    [[TMP10:%.*]] = load i16, i16* [[TMP8]]
 ; X64_1LD-NEXT:    [[TMP11:%.*]] = load i16, i16* [[TMP9]]
-; X64_1LD-NEXT:    [[TMP12:%.*]] = icmp ne i16 [[TMP10]], [[TMP11]]
+; X64_1LD-NEXT:    [[TMP12:%.*]] = icmp eq i16 [[TMP10]], [[TMP11]]
 ; X64_1LD-NEXT:    br i1 [[TMP12]], label [[RES_BLOCK]], label [[ENDBLOCK]]
 ; X64_1LD:       endblock:
 ; X64_1LD-NEXT:    [[PHI_RES:%.*]] = phi i32 [ 0, [[LOADBB1]] ], [ 1, [[RES_BLOCK]] ]
-; X64_1LD-NEXT:    [[CMP:%.*]] = icmp eq i32 [[PHI_RES]], 0
+; X64_1LD-NEXT:    [[CMP:%.*]] = icmp ne i32 [[PHI_RES]], 0
 ; X64_1LD-NEXT:    [[CONV:%.*]] = zext i1 [[CMP]] to i32
 ; X64_1LD-NEXT:    ret i32 [[CONV]]
 ;
 ; X64_2LD-LABEL: @cmp_eq6(
 ; X64_2LD-NEXT:    [[TMP1:%.*]] = bitcast i8* [[X:%.*]] to i32*
 ; X64_2LD-NEXT:    [[TMP2:%.*]] = bitcast i8* [[Y:%.*]] to i32*
 ; X64_2LD-NEXT:    [[TMP3:%.*]] = load i32, i32* [[TMP1]]
 ; X64_2LD-NEXT:    [[TMP4:%.*]] = load i32, i32* [[TMP2]]
 ; X64_2LD-NEXT:    [[TMP5:%.*]] = xor i32 [[TMP3]], [[TMP4]]
 ; X64_2LD-NEXT:    [[TMP6:%.*]] = getelementptr i8, i8* [[X]], i64 4
 ; X64_2LD-NEXT:    [[TMP7:%.*]] = getelementptr i8, i8* [[Y]], i64 4
 ; X64_2LD-NEXT:    [[TMP8:%.*]] = bitcast i8* [[TMP6]] to i16*
 ; X64_2LD-NEXT:    [[TMP9:%.*]] = bitcast i8* [[TMP7]] to i16*
 ; X64_2LD-NEXT:    [[TMP10:%.*]] = load i16, i16* [[TMP8]]
 ; X64_2LD-NEXT:    [[TMP11:%.*]] = load i16, i16* [[TMP9]]
 ; X64_2LD-NEXT:    [[TMP12:%.*]] = zext i16 [[TMP10]] to i32
 ; X64_2LD-NEXT:    [[TMP13:%.*]] = zext i16 [[TMP11]] to i32
 ; X64_2LD-NEXT:    [[TMP14:%.*]] = xor i32 [[TMP12]], [[TMP13]]
 ; X64_2LD-NEXT:    [[TMP15:%.*]] = or i32 [[TMP5]], [[TMP14]]
-; X64_2LD-NEXT:    [[TMP16:%.*]] = icmp ne i32 [[TMP15]], 0
+; X64_2LD-NEXT:    [[TMP16:%.*]] = icmp eq i32 [[TMP15]], 0
 ; X64_2LD-NEXT:    [[TMP17:%.*]] = zext i1 [[TMP16]] to i32
-; X64_2LD-NEXT:    [[CMP:%.*]] = icmp eq i32 [[TMP17]], 0
-; X64_2LD-NEXT:    [[CONV:%.*]] = zext i1 [[CMP]] to i32
+; X64_2LD-NEXT:    [[CONV:%.*]] = zext i1 [[TMP16]] to i32
 ; X64_2LD-NEXT:    ret i32 [[CONV]]
 ;
   %call = tail call i32 @memcmp(i8* %x, i8* %y, i64 6)
   %cmp = icmp eq i32 %call, 0
   %conv = zext i1 %cmp to i32
   ret i32 %conv
 }
 
 define i32 @cmp_eq7(i8* nocapture readonly %x, i8* nocapture readonly %y)  {
 ; X32-LABEL: @cmp_eq7(
 ; X32-NEXT:    [[TMP1:%.*]] = bitcast i8* [[X:%.*]] to i32*
 ; X32-NEXT:    [[TMP2:%.*]] = bitcast i8* [[Y:%.*]] to i32*
 ; X32-NEXT:    [[TMP3:%.*]] = load i32, i32* [[TMP1]]
 ; X32-NEXT:    [[TMP4:%.*]] = load i32, i32* [[TMP2]]
 ; X32-NEXT:    [[TMP5:%.*]] = xor i32 [[TMP3]], [[TMP4]]
 ; X32-NEXT:    [[TMP6:%.*]] = getelementptr i8, i8* [[X]], i64 3
 ; X32-NEXT:    [[TMP7:%.*]] = getelementptr i8, i8* [[Y]], i64 3
 ; X32-NEXT:    [[TMP8:%.*]] = bitcast i8* [[TMP6]] to i32*
 ; X32-NEXT:    [[TMP9:%.*]] = bitcast i8* [[TMP7]] to i32*
 ; X32-NEXT:    [[TMP10:%.*]] = load i32, i32* [[TMP8]]
 ; X32-NEXT:    [[TMP11:%.*]] = load i32, i32* [[TMP9]]
 ; X32-NEXT:    [[TMP12:%.*]] = xor i32 [[TMP10]], [[TMP11]]
 ; X32-NEXT:    [[TMP13:%.*]] = or i32 [[TMP5]], [[TMP12]]
-; X32-NEXT:    [[TMP14:%.*]] = icmp ne i32 [[TMP13]], 0
+; X32-NEXT:    [[TMP14:%.*]] = icmp eq i32 [[TMP13]], 0
 ; X32-NEXT:    [[TMP15:%.*]] = zext i1 [[TMP14]] to i32
-; X32-NEXT:    [[CMP:%.*]] = icmp eq i32 [[TMP15]], 0
-; X32-NEXT:    [[CONV:%.*]] = zext i1 [[CMP]] to i32
+; X32-NEXT:    [[CONV:%.*]] = zext i1 [[TMP14]] to i32
 ; X32-NEXT:    ret i32 [[CONV]]
 ;
 ; X64_1LD-LABEL: @cmp_eq7(
 ; X64_1LD-NEXT:    br label [[LOADBB:%.*]]
 ; X64_1LD:       res_block:
 ; X64_1LD-NEXT:    br label [[ENDBLOCK:%.*]]
 ; X64_1LD:       loadbb:
 ; X64_1LD-NEXT:    [[TMP1:%.*]] = bitcast i8* [[X:%.*]] to i32*
 ; X64_1LD-NEXT:    [[TMP2:%.*]] = bitcast i8* [[Y:%.*]] to i32*
 ; X64_1LD-NEXT:    [[TMP3:%.*]] = load i32, i32* [[TMP1]]
 ; X64_1LD-NEXT:    [[TMP4:%.*]] = load i32, i32* [[TMP2]]
-; X64_1LD-NEXT:    [[TMP5:%.*]] = icmp ne i32 [[TMP3]], [[TMP4]]
+; X64_1LD-NEXT:    [[TMP5:%.*]] = icmp eq i32 [[TMP3]], [[TMP4]]
 ; X64_1LD-NEXT:    br i1 [[TMP5]], label [[RES_BLOCK:%.*]], label [[LOADBB1:%.*]]
 ; X64_1LD:       loadbb1:
 ; X64_1LD-NEXT:    [[TMP6:%.*]] = getelementptr i8, i8* [[X]], i64 3
 ; X64_1LD-NEXT:    [[TMP7:%.*]] = getelementptr i8, i8* [[Y]], i64 3
 ; X64_1LD-NEXT:    [[TMP8:%.*]] = bitcast i8* [[TMP6]] to i32*
 ; X64_1LD-NEXT:    [[TMP9:%.*]] = bitcast i8* [[TMP7]] to i32*
 ; X64_1LD-NEXT:    [[TMP10:%.*]] = load i32, i32* [[TMP8]]
 ; X64_1LD-NEXT:    [[TMP11:%.*]] = load i32, i32* [[TMP9]]
-; X64_1LD-NEXT:    [[TMP12:%.*]] = icmp ne i32 [[TMP10]], [[TMP11]]
+; X64_1LD-NEXT:    [[TMP12:%.*]] = icmp eq i32 [[TMP10]], [[TMP11]]
 ; X64_1LD-NEXT:    br i1 [[TMP12]], label [[RES_BLOCK]], label [[ENDBLOCK]]
 ; X64_1LD:       endblock:
 ; X64_1LD-NEXT:    [[PHI_RES:%.*]] = phi i32 [ 0, [[LOADBB1]] ], [ 1, [[RES_BLOCK]] ]
-; X64_1LD-NEXT:    [[CMP:%.*]] = icmp eq i32 [[PHI_RES]], 0
+; X64_1LD-NEXT:    [[CMP:%.*]] = icmp ne i32 [[PHI_RES]], 0
 ; X64_1LD-NEXT:    [[CONV:%.*]] = zext i1 [[CMP]] to i32
 ; X64_1LD-NEXT:    ret i32 [[CONV]]
 ;
 ; X64_2LD-LABEL: @cmp_eq7(
 ; X64_2LD-NEXT:    [[TMP1:%.*]] = bitcast i8* [[X:%.*]] to i32*
 ; X64_2LD-NEXT:    [[TMP2:%.*]] = bitcast i8* [[Y:%.*]] to i32*
 ; X64_2LD-NEXT:    [[TMP3:%.*]] = load i32, i32* [[TMP1]]
 ; X64_2LD-NEXT:    [[TMP4:%.*]] = load i32, i32* [[TMP2]]
 ; X64_2LD-NEXT:    [[TMP5:%.*]] = xor i32 [[TMP3]], [[TMP4]]
 ; X64_2LD-NEXT:    [[TMP6:%.*]] = getelementptr i8, i8* [[X]], i64 3
 ; X64_2LD-NEXT:    [[TMP7:%.*]] = getelementptr i8, i8* [[Y]], i64 3
 ; X64_2LD-NEXT:    [[TMP8:%.*]] = bitcast i8* [[TMP6]] to i32*
 ; X64_2LD-NEXT:    [[TMP9:%.*]] = bitcast i8* [[TMP7]] to i32*
 ; X64_2LD-NEXT:    [[TMP10:%.*]] = load i32, i32* [[TMP8]]
 ; X64_2LD-NEXT:    [[TMP11:%.*]] = load i32, i32* [[TMP9]]
 ; X64_2LD-NEXT:    [[TMP12:%.*]] = xor i32 [[TMP10]], [[TMP11]]
 ; X64_2LD-NEXT:    [[TMP13:%.*]] = or i32 [[TMP5]], [[TMP12]]
-; X64_2LD-NEXT:    [[TMP14:%.*]] = icmp ne i32 [[TMP13]], 0
+; X64_2LD-NEXT:    [[TMP14:%.*]] = icmp eq i32 [[TMP13]], 0
 ; X64_2LD-NEXT:    [[TMP15:%.*]] = zext i1 [[TMP14]] to i32
-; X64_2LD-NEXT:    [[CMP:%.*]] = icmp eq i32 [[TMP15]], 0
-; X64_2LD-NEXT:    [[CONV:%.*]] = zext i1 [[CMP]] to i32
+; X64_2LD-NEXT:    [[CONV:%.*]] = zext i1 [[TMP14]] to i32
 ; X64_2LD-NEXT:    ret i32 [[CONV]]
 ;
   %call = tail call i32 @memcmp(i8* %x, i8* %y, i64 7)
   %cmp = icmp eq i32 %call, 0
   %conv = zext i1 %cmp to i32
   ret i32 %conv
 }
 
 define i32 @cmp_eq8(i8* nocapture readonly %x, i8* nocapture readonly %y)  {
 ; X32-LABEL: @cmp_eq8(
 ; X32-NEXT:    [[TMP1:%.*]] = bitcast i8* [[X:%.*]] to i32*
 ; X32-NEXT:    [[TMP2:%.*]] = bitcast i8* [[Y:%.*]] to i32*
 ; X32-NEXT:    [[TMP3:%.*]] = load i32, i32* [[TMP1]]
 ; X32-NEXT:    [[TMP4:%.*]] = load i32, i32* [[TMP2]]
 ; X32-NEXT:    [[TMP5:%.*]] = xor i32 [[TMP3]], [[TMP4]]
 ; X32-NEXT:    [[TMP6:%.*]] = getelementptr i8, i8* [[X]], i64 4
 ; X32-NEXT:    [[TMP7:%.*]] = getelementptr i8, i8* [[Y]], i64 4
 ; X32-NEXT:    [[TMP8:%.*]] = bitcast i8* [[TMP6]] to i32*
 ; X32-NEXT:    [[TMP9:%.*]] = bitcast i8* [[TMP7]] to i32*
 ; X32-NEXT:    [[TMP10:%.*]] = load i32, i32* [[TMP8]]
 ; X32-NEXT:    [[TMP11:%.*]] = load i32, i32* [[TMP9]]
 ; X32-NEXT:    [[TMP12:%.*]] = xor i32 [[TMP10]], [[TMP11]]
 ; X32-NEXT:    [[TMP13:%.*]] = or i32 [[TMP5]], [[TMP12]]
-; X32-NEXT:    [[TMP14:%.*]] = icmp ne i32 [[TMP13]], 0
+; X32-NEXT:    [[TMP14:%.*]] = icmp eq i32 [[TMP13]], 0
 ; X32-NEXT:    [[TMP15:%.*]] = zext i1 [[TMP14]] to i32
-; X32-NEXT:    [[CMP:%.*]] = icmp eq i32 [[TMP15]], 0
-; X32-NEXT:    [[CONV:%.*]] = zext i1 [[CMP]] to i32
+; X32-NEXT:    [[CONV:%.*]] = zext i1 [[TMP14]] to i32
 ; X32-NEXT:    ret i32 [[CONV]]
 ;
 ; X64-LABEL: @cmp_eq8(
 ; X64-NEXT:    [[TMP1:%.*]] = bitcast i8* [[X:%.*]] to i64*
 ; X64-NEXT:    [[TMP2:%.*]] = bitcast i8* [[Y:%.*]] to i64*
 ; X64-NEXT:    [[TMP3:%.*]] = load i64, i64* [[TMP1]]
 ; X64-NEXT:    [[TMP4:%.*]] = load i64, i64* [[TMP2]]
-; X64-NEXT:    [[TMP5:%.*]] = icmp ne i64 [[TMP3]], [[TMP4]]
+; X64-NEXT:    [[TMP5:%.*]] = icmp eq i64 [[TMP3]], [[TMP4]]
 ; X64-NEXT:    [[TMP6:%.*]] = zext i1 [[TMP5]] to i32
-; X64-NEXT:    [[CMP:%.*]] = icmp eq i32 [[TMP6]], 0
-; X64-NEXT:    [[CONV:%.*]] = zext i1 [[CMP]] to i32
+; X64-NEXT:    [[CONV:%.*]] = zext i1 [[TMP5]] to i32
 ; X64-NEXT:    ret i32 [[CONV]]
 ;
   %call = tail call i32 @memcmp(i8* %x, i8* %y, i64 8)
   %cmp = icmp eq i32 %call, 0
   %conv = zext i1 %cmp to i32
   ret i32 %conv
 }
 
 define i32 @cmp_eq9(i8* nocapture readonly %x, i8* nocapture readonly %y)  {
 ; X32-LABEL: @cmp_eq9(
 ; X32-NEXT:    [[CALL:%.*]] = tail call i32 @memcmp(i8* [[X:%.*]], i8* [[Y:%.*]], i64 9)
 ; X32-NEXT:    [[CMP:%.*]] = icmp eq i32 [[CALL]], 0
 ; X32-NEXT:    [[CONV:%.*]] = zext i1 [[CMP]] to i32
 ; X32-NEXT:    ret i32 [[CONV]]
 ;
 ; X64_1LD-LABEL: @cmp_eq9(
 ; X64_1LD-NEXT:    br label [[LOADBB:%.*]]
 ; X64_1LD:       res_block:
 ; X64_1LD-NEXT:    br label [[ENDBLOCK:%.*]]
 ; X64_1LD:       loadbb:
 ; X64_1LD-NEXT:    [[TMP1:%.*]] = bitcast i8* [[X:%.*]] to i64*
 ; X64_1LD-NEXT:    [[TMP2:%.*]] = bitcast i8* [[Y:%.*]] to i64*
 ; X64_1LD-NEXT:    [[TMP3:%.*]] = load i64, i64* [[TMP1]]
 ; X64_1LD-NEXT:    [[TMP4:%.*]] = load i64, i64* [[TMP2]]
-; X64_1LD-NEXT:    [[TMP5:%.*]] = icmp ne i64 [[TMP3]], [[TMP4]]
+; X64_1LD-NEXT:    [[TMP5:%.*]] = icmp eq i64 [[TMP3]], [[TMP4]]
 ; X64_1LD-NEXT:    br i1 [[TMP5]], label [[RES_BLOCK:%.*]], label [[LOADBB1:%.*]]
 ; X64_1LD:       loadbb1:
 ; X64_1LD-NEXT:    [[TMP6:%.*]] = getelementptr i8, i8* [[X]], i64 8
 ; X64_1LD-NEXT:    [[TMP7:%.*]] = getelementptr i8, i8* [[Y]], i64 8
 ; X64_1LD-NEXT:    [[TMP8:%.*]] = load i8, i8* [[TMP6]]
 ; X64_1LD-NEXT:    [[TMP9:%.*]] = load i8, i8* [[TMP7]]
-; X64_1LD-NEXT:    [[TMP10:%.*]] = icmp ne i8 [[TMP8]], [[TMP9]]
+; X64_1LD-NEXT:    [[TMP10:%.*]] = icmp eq i8 [[TMP8]], [[TMP9]]
 ; X64_1LD-NEXT:    br i1 [[TMP10]], label [[RES_BLOCK]], label [[ENDBLOCK]]
 ; X64_1LD:       endblock:
 ; X64_1LD-NEXT:    [[PHI_RES:%.*]] = phi i32 [ 0, [[LOADBB1]] ], [ 1, [[RES_BLOCK]] ]
-; X64_1LD-NEXT:    [[CMP:%.*]] = icmp eq i32 [[PHI_RES]], 0
+; X64_1LD-NEXT:    [[CMP:%.*]] = icmp ne i32 [[PHI_RES]], 0
 ; X64_1LD-NEXT:    [[CONV:%.*]] = zext i1 [[CMP]] to i32
 ; X64_1LD-NEXT:    ret i32 [[CONV]]
 ;
 ; X64_2LD-LABEL: @cmp_eq9(
 ; X64_2LD-NEXT:    [[TMP1:%.*]] = bitcast i8* [[X:%.*]] to i64*
 ; X64_2LD-NEXT:    [[TMP2:%.*]] = bitcast i8* [[Y:%.*]] to i64*
 ; X64_2LD-NEXT:    [[TMP3:%.*]] = load i64, i64* [[TMP1]]
 ; X64_2LD-NEXT:    [[TMP4:%.*]] = load i64, i64* [[TMP2]]
 ; X64_2LD-NEXT:    [[TMP5:%.*]] = xor i64 [[TMP3]], [[TMP4]]
 ; X64_2LD-NEXT:    [[TMP6:%.*]] = getelementptr i8, i8* [[X]], i64 8
 ; X64_2LD-NEXT:    [[TMP7:%.*]] = getelementptr i8, i8* [[Y]], i64 8
 ; X64_2LD-NEXT:    [[TMP8:%.*]] = load i8, i8* [[TMP6]]
 ; X64_2LD-NEXT:    [[TMP9:%.*]] = load i8, i8* [[TMP7]]
 ; X64_2LD-NEXT:    [[TMP10:%.*]] = zext i8 [[TMP8]] to i64
 ; X64_2LD-NEXT:    [[TMP11:%.*]] = zext i8 [[TMP9]] to i64
 ; X64_2LD-NEXT:    [[TMP12:%.*]] = xor i64 [[TMP10]], [[TMP11]]
 ; X64_2LD-NEXT:    [[TMP13:%.*]] = or i64 [[TMP5]], [[TMP12]]
-; X64_2LD-NEXT:    [[TMP14:%.*]] = icmp ne i64 [[TMP13]], 0
+; X64_2LD-NEXT:    [[TMP14:%.*]] = icmp eq i64 [[TMP13]], 0
 ; X64_2LD-NEXT:    [[TMP15:%.*]] = zext i1 [[TMP14]] to i32
-; X64_2LD-NEXT:    [[CMP:%.*]] = icmp eq i32 [[TMP15]], 0
-; X64_2LD-NEXT:    [[CONV:%.*]] = zext i1 [[CMP]] to i32
+; X64_2LD-NEXT:    [[CONV:%.*]] = zext i1 [[TMP14]] to i32
 ; X64_2LD-NEXT:    ret i32 [[CONV]]
 ;
   %call = tail call i32 @memcmp(i8* %x, i8* %y, i64 9)
   %cmp = icmp eq i32 %call, 0
   %conv = zext i1 %cmp to i32
   ret i32 %conv
 }
 
 define i32 @cmp_eq10(i8* nocapture readonly %x, i8* nocapture readonly %y)  {
 ; X32-LABEL: @cmp_eq10(
 ; X32-NEXT:    [[CALL:%.*]] = tail call i32 @memcmp(i8* [[X:%.*]], i8* [[Y:%.*]], i64 10)
 ; X32-NEXT:    [[CMP:%.*]] = icmp eq i32 [[CALL]], 0
 ; X32-NEXT:    [[CONV:%.*]] = zext i1 [[CMP]] to i32
 ; X32-NEXT:    ret i32 [[CONV]]
 ;
 ; X64_1LD-LABEL: @cmp_eq10(
 ; X64_1LD-NEXT:    br label [[LOADBB:%.*]]
 ; X64_1LD:       res_block:
 ; X64_1LD-NEXT:    br label [[ENDBLOCK:%.*]]
 ; X64_1LD:       loadbb:
 ; X64_1LD-NEXT:    [[TMP1:%.*]] = bitcast i8* [[X:%.*]] to i64*
 ; X64_1LD-NEXT:    [[TMP2:%.*]] = bitcast i8* [[Y:%.*]] to i64*
 ; X64_1LD-NEXT:    [[TMP3:%.*]] = load i64, i64* [[TMP1]]
 ; X64_1LD-NEXT:    [[TMP4:%.*]] = load i64, i64* [[TMP2]]
-; X64_1LD-NEXT:    [[TMP5:%.*]] = icmp ne i64 [[TMP3]], [[TMP4]]
+; X64_1LD-NEXT:    [[TMP5:%.*]] = icmp eq i64 [[TMP3]], [[TMP4]]
 ; X64_1LD-NEXT:    br i1 [[TMP5]], label [[RES_BLOCK:%.*]], label [[LOADBB1:%.*]]
 ; X64_1LD:       loadbb1:
 ; X64_1LD-NEXT:    [[TMP6:%.*]] = getelementptr i8, i8* [[X]], i64 8
 ; X64_1LD-NEXT:    [[TMP7:%.*]] = getelementptr i8, i8* [[Y]], i64 8
 ; X64_1LD-NEXT:    [[TMP8:%.*]] = bitcast i8* [[TMP6]] to i16*
 ; X64_1LD-NEXT:    [[TMP9:%.*]] = bitcast i8* [[TMP7]] to i16*
 ; X64_1LD-NEXT:    [[TMP10:%.*]] = load i16, i16* [[TMP8]]
 ; X64_1LD-NEXT:    [[TMP11:%.*]] = load i16, i16* [[TMP9]]
-; X64_1LD-NEXT:    [[TMP12:%.*]] = icmp ne i16 [[TMP10]], [[TMP11]]
+; X64_1LD-NEXT:    [[TMP12:%.*]] = icmp eq i16 [[TMP10]], [[TMP11]]
 ; X64_1LD-NEXT:    br i1 [[TMP12]], label [[RES_BLOCK]], label [[ENDBLOCK]]
 ; X64_1LD:       endblock:
 ; X64_1LD-NEXT:    [[PHI_RES:%.*]] = phi i32 [ 0, [[LOADBB1]] ], [ 1, [[RES_BLOCK]] ]
-; X64_1LD-NEXT:    [[CMP:%.*]] = icmp eq i32 [[PHI_RES]], 0
+; X64_1LD-NEXT:    [[CMP:%.*]] = icmp ne i32 [[PHI_RES]], 0
 ; X64_1LD-NEXT:    [[CONV:%.*]] = zext i1 [[CMP]] to i32
 ; X64_1LD-NEXT:    ret i32 [[CONV]]
 ;
 ; X64_2LD-LABEL: @cmp_eq10(
 ; X64_2LD-NEXT:    [[TMP1:%.*]] = bitcast i8* [[X:%.*]] to i64*
 ; X64_2LD-NEXT:    [[TMP2:%.*]] = bitcast i8* [[Y:%.*]] to i64*
 ; X64_2LD-NEXT:    [[TMP3:%.*]] = load i64, i64* [[TMP1]]
 ; X64_2LD-NEXT:    [[TMP4:%.*]] = load i64, i64* [[TMP2]]
 ; X64_2LD-NEXT:    [[TMP5:%.*]] = xor i64 [[TMP3]], [[TMP4]]
 ; X64_2LD-NEXT:    [[TMP6:%.*]] = getelementptr i8, i8* [[X]], i64 8
 ; X64_2LD-NEXT:    [[TMP7:%.*]] = getelementptr i8, i8* [[Y]], i64 8
 ; X64_2LD-NEXT:    [[TMP8:%.*]] = bitcast i8* [[TMP6]] to i16*
 ; X64_2LD-NEXT:    [[TMP9:%.*]] = bitcast i8* [[TMP7]] to i16*
 ; X64_2LD-NEXT:    [[TMP10:%.*]] = load i16, i16* [[TMP8]]
 ; X64_2LD-NEXT:    [[TMP11:%.*]] = load i16, i16* [[TMP9]]
 ; X64_2LD-NEXT:    [[TMP12:%.*]] = zext i16 [[TMP10]] to i64
 ; X64_2LD-NEXT:    [[TMP13:%.*]] = zext i16 [[TMP11]] to i64
 ; X64_2LD-NEXT:    [[TMP14:%.*]] = xor i64 [[TMP12]], [[TMP13]]
 ; X64_2LD-NEXT:    [[TMP15:%.*]] = or i64 [[TMP5]], [[TMP14]]
-; X64_2LD-NEXT:    [[TMP16:%.*]] = icmp ne i64 [[TMP15]], 0
+; X64_2LD-NEXT:    [[TMP16:%.*]] = icmp eq i64 [[TMP15]], 0
 ; X64_2LD-NEXT:    [[TMP17:%.*]] = zext i1 [[TMP16]] to i32
-; X64_2LD-NEXT:    [[CMP:%.*]] = icmp eq i32 [[TMP17]], 0
-; X64_2LD-NEXT:    [[CONV:%.*]] = zext i1 [[CMP]] to i32
+; X64_2LD-NEXT:    [[CONV:%.*]] = zext i1 [[TMP16]] to i32
 ; X64_2LD-NEXT:    ret i32 [[CONV]]
 ;
   %call = tail call i32 @memcmp(i8* %x, i8* %y, i64 10)
   %cmp = icmp eq i32 %call, 0
   %conv = zext i1 %cmp to i32
   ret i32 %conv
 }
 
 define i32 @cmp_eq11(i8* nocapture readonly %x, i8* nocapture readonly %y)  {
 ; X32-LABEL: @cmp_eq11(
 ; X32-NEXT:    [[CALL:%.*]] = tail call i32 @memcmp(i8* [[X:%.*]], i8* [[Y:%.*]], i64 11)
 ; X32-NEXT:    [[CMP:%.*]] = icmp eq i32 [[CALL]], 0
 ; X32-NEXT:    [[CONV:%.*]] = zext i1 [[CMP]] to i32
 ; X32-NEXT:    ret i32 [[CONV]]
 ;
 ; X64_1LD-LABEL: @cmp_eq11(
 ; X64_1LD-NEXT:    br label [[LOADBB:%.*]]
 ; X64_1LD:       res_block:
 ; X64_1LD-NEXT:    br label [[ENDBLOCK:%.*]]
 ; X64_1LD:       loadbb:
 ; X64_1LD-NEXT:    [[TMP1:%.*]] = bitcast i8* [[X:%.*]] to i64*
 ; X64_1LD-NEXT:    [[TMP2:%.*]] = bitcast i8* [[Y:%.*]] to i64*
 ; X64_1LD-NEXT:    [[TMP3:%.*]] = load i64, i64* [[TMP1]]
 ; X64_1LD-NEXT:    [[TMP4:%.*]] = load i64, i64* [[TMP2]]
-; X64_1LD-NEXT:    [[TMP5:%.*]] = icmp ne i64 [[TMP3]], [[TMP4]]
+; X64_1LD-NEXT:    [[TMP5:%.*]] = icmp eq i64 [[TMP3]], [[TMP4]]
 ; X64_1LD-NEXT:    br i1 [[TMP5]], label [[RES_BLOCK:%.*]], label [[LOADBB1:%.*]]
 ; X64_1LD:       loadbb1:
 ; X64_1LD-NEXT:    [[TMP6:%.*]] = getelementptr i8, i8* [[X]], i64 3
 ; X64_1LD-NEXT:    [[TMP7:%.*]] = getelementptr i8, i8* [[Y]], i64 3
 ; X64_1LD-NEXT:    [[TMP8:%.*]] = bitcast i8* [[TMP6]] to i64*
 ; X64_1LD-NEXT:    [[TMP9:%.*]] = bitcast i8* [[TMP7]] to i64*
 ; X64_1LD-NEXT:    [[TMP10:%.*]] = load i64, i64* [[TMP8]]
 ; X64_1LD-NEXT:    [[TMP11:%.*]] = load i64, i64* [[TMP9]]
-; X64_1LD-NEXT:    [[TMP12:%.*]] = icmp ne i64 [[TMP10]], [[TMP11]]
+; X64_1LD-NEXT:    [[TMP12:%.*]] = icmp eq i64 [[TMP10]], [[TMP11]]
 ; X64_1LD-NEXT:    br i1 [[TMP12]], label [[RES_BLOCK]], label [[ENDBLOCK]]
 ; X64_1LD:       endblock:
 ; X64_1LD-NEXT:    [[PHI_RES:%.*]] = phi i32 [ 0, [[LOADBB1]] ], [ 1, [[RES_BLOCK]] ]
-; X64_1LD-NEXT:    [[CMP:%.*]] = icmp eq i32 [[PHI_RES]], 0
+; X64_1LD-NEXT:    [[CMP:%.*]] = icmp ne i32 [[PHI_RES]], 0
 ; X64_1LD-NEXT:    [[CONV:%.*]] = zext i1 [[CMP]] to i32
 ; X64_1LD-NEXT:    ret i32 [[CONV]]
 ;
 ; X64_2LD-LABEL: @cmp_eq11(
 ; X64_2LD-NEXT:    [[TMP1:%.*]] = bitcast i8* [[X:%.*]] to i64*
 ; X64_2LD-NEXT:    [[TMP2:%.*]] = bitcast i8* [[Y:%.*]] to i64*
 ; X64_2LD-NEXT:    [[TMP3:%.*]] = load i64, i64* [[TMP1]]
 ; X64_2LD-NEXT:    [[TMP4:%.*]] = load i64, i64* [[TMP2]]
 ; X64_2LD-NEXT:    [[TMP5:%.*]] = xor i64 [[TMP3]], [[TMP4]]
 ; X64_2LD-NEXT:    [[TMP6:%.*]] = getelementptr i8, i8* [[X]], i64 3
 ; X64_2LD-NEXT:    [[TMP7:%.*]] = getelementptr i8, i8* [[Y]], i64 3
 ; X64_2LD-NEXT:    [[TMP8:%.*]] = bitcast i8* [[TMP6]] to i64*
 ; X64_2LD-NEXT:    [[TMP9:%.*]] = bitcast i8* [[TMP7]] to i64*
 ; X64_2LD-NEXT:    [[TMP10:%.*]] = load i64, i64* [[TMP8]]
 ; X64_2LD-NEXT:    [[TMP11:%.*]] = load i64, i64* [[TMP9]]
 ; X64_2LD-NEXT:    [[TMP12:%.*]] = xor i64 [[TMP10]], [[TMP11]]
 ; X64_2LD-NEXT:    [[TMP13:%.*]] = or i64 [[TMP5]], [[TMP12]]
-; X64_2LD-NEXT:    [[TMP14:%.*]] = icmp ne i64 [[TMP13]], 0
+; X64_2LD-NEXT:    [[TMP14:%.*]] = icmp eq i64 [[TMP13]], 0
 ; X64_2LD-NEXT:    [[TMP15:%.*]] = zext i1 [[TMP14]] to i32
-; X64_2LD-NEXT:    [[CMP:%.*]] = icmp eq i32 [[TMP15]], 0
-; X64_2LD-NEXT:    [[CONV:%.*]] = zext i1 [[CMP]] to i32
+; X64_2LD-NEXT:    [[CONV:%.*]] = zext i1 [[TMP14]] to i32
 ; X64_2LD-NEXT:    ret i32 [[CONV]]
 ;
   %call = tail call i32 @memcmp(i8* %x, i8* %y, i64 11)
   %cmp = icmp eq i32 %call, 0
   %conv = zext i1 %cmp to i32
   ret i32 %conv
 }
 
 define i32 @cmp_eq12(i8* nocapture readonly %x, i8* nocapture readonly %y)  {
 ; X32-LABEL: @cmp_eq12(
 ; X32-NEXT:    [[CALL:%.*]] = tail call i32 @memcmp(i8* [[X:%.*]], i8* [[Y:%.*]], i64 12)
 ; X32-NEXT:    [[CMP:%.*]] = icmp eq i32 [[CALL]], 0
 ; X32-NEXT:    [[CONV:%.*]] = zext i1 [[CMP]] to i32
 ; X32-NEXT:    ret i32 [[CONV]]
 ;
 ; X64_1LD-LABEL: @cmp_eq12(
 ; X64_1LD-NEXT:    br label [[LOADBB:%.*]]
 ; X64_1LD:       res_block:
 ; X64_1LD-NEXT:    br label [[ENDBLOCK:%.*]]
 ; X64_1LD:       loadbb:
 ; X64_1LD-NEXT:    [[TMP1:%.*]] = bitcast i8* [[X:%.*]] to i64*
 ; X64_1LD-NEXT:    [[TMP2:%.*]] = bitcast i8* [[Y:%.*]] to i64*
 ; X64_1LD-NEXT:    [[TMP3:%.*]] = load i64, i64* [[TMP1]]
 ; X64_1LD-NEXT:    [[TMP4:%.*]] = load i64, i64* [[TMP2]]
-; X64_1LD-NEXT:    [[TMP5:%.*]] = icmp ne i64 [[TMP3]], [[TMP4]]
+; X64_1LD-NEXT:    [[TMP5:%.*]] = icmp eq i64 [[TMP3]], [[TMP4]]
 ; X64_1LD-NEXT:    br i1 [[TMP5]], label [[RES_BLOCK:%.*]], label [[LOADBB1:%.*]]
 ; X64_1LD:       loadbb1:
 ; X64_1LD-NEXT:    [[TMP6:%.*]] = getelementptr i8, i8* [[X]], i64 8
 ; X64_1LD-NEXT:    [[TMP7:%.*]] = getelementptr i8, i8* [[Y]], i64 8
 ; X64_1LD-NEXT:    [[TMP8:%.*]] = bitcast i8* [[TMP6]] to i32*
 ; X64_1LD-NEXT:    [[TMP9:%.*]] = bitcast i8* [[TMP7]] to i32*
 ; X64_1LD-NEXT:    [[TMP10:%.*]] = load i32, i32* [[TMP8]]
 ; X64_1LD-NEXT:    [[TMP11:%.*]] = load i32, i32* [[TMP9]]
-; X64_1LD-NEXT:    [[TMP12:%.*]] = icmp ne i32 [[TMP10]], [[TMP11]]
+; X64_1LD-NEXT:    [[TMP12:%.*]] = icmp eq i32 [[TMP10]], [[TMP11]]
 ; X64_1LD-NEXT:    br i1 [[TMP12]], label [[RES_BLOCK]], label [[ENDBLOCK]]
 ; X64_1LD:       endblock:
 ; X64_1LD-NEXT:    [[PHI_RES:%.*]] = phi i32 [ 0, [[LOADBB1]] ], [ 1, [[RES_BLOCK]] ]
-; X64_1LD-NEXT:    [[CMP:%.*]] = icmp eq i32 [[PHI_RES]], 0
+; X64_1LD-NEXT:    [[CMP:%.*]] = icmp ne i32 [[PHI_RES]], 0
 ; X64_1LD-NEXT:    [[CONV:%.*]] = zext i1 [[CMP]] to i32
 ; X64_1LD-NEXT:    ret i32 [[CONV]]
 ;
 ; X64_2LD-LABEL: @cmp_eq12(
 ; X64_2LD-NEXT:    [[TMP1:%.*]] = bitcast i8* [[X:%.*]] to i64*
 ; X64_2LD-NEXT:    [[TMP2:%.*]] = bitcast i8* [[Y:%.*]] to i64*
 ; X64_2LD-NEXT:    [[TMP3:%.*]] = load i64, i64* [[TMP1]]
 ; X64_2LD-NEXT:    [[TMP4:%.*]] = load i64, i64* [[TMP2]]
 ; X64_2LD-NEXT:    [[TMP5:%.*]] = xor i64 [[TMP3]], [[TMP4]]
 ; X64_2LD-NEXT:    [[TMP6:%.*]] = getelementptr i8, i8* [[X]], i64 8
 ; X64_2LD-NEXT:    [[TMP7:%.*]] = getelementptr i8, i8* [[Y]], i64 8
 ; X64_2LD-NEXT:    [[TMP8:%.*]] = bitcast i8* [[TMP6]] to i32*
 ; X64_2LD-NEXT:    [[TMP9:%.*]] = bitcast i8* [[TMP7]] to i32*
 ; X64_2LD-NEXT:    [[TMP10:%.*]] = load i32, i32* [[TMP8]]
 ; X64_2LD-NEXT:    [[TMP11:%.*]] = load i32, i32* [[TMP9]]
 ; X64_2LD-NEXT:    [[TMP12:%.*]] = zext i32 [[TMP10]] to i64
 ; X64_2LD-NEXT:    [[TMP13:%.*]] = zext i32 [[TMP11]] to i64
 ; X64_2LD-NEXT:    [[TMP14:%.*]] = xor i64 [[TMP12]], [[TMP13]]
 ; X64_2LD-NEXT:    [[TMP15:%.*]] = or i64 [[TMP5]], [[TMP14]]
-; X64_2LD-NEXT:    [[TMP16:%.*]] = icmp ne i64 [[TMP15]], 0
+; X64_2LD-NEXT:    [[TMP16:%.*]] = icmp eq i64 [[TMP15]], 0
 ; X64_2LD-NEXT:    [[TMP17:%.*]] = zext i1 [[TMP16]] to i32
-; X64_2LD-NEXT:    [[CMP:%.*]] = icmp eq i32 [[TMP17]], 0
-; X64_2LD-NEXT:    [[CONV:%.*]] = zext i1 [[CMP]] to i32
+; X64_2LD-NEXT:    [[CONV:%.*]] = zext i1 [[TMP16]] to i32
 ; X64_2LD-NEXT:    ret i32 [[CONV]]
 ;
   %call = tail call i32 @memcmp(i8* %x, i8* %y, i64 12)
   %cmp = icmp eq i32 %call, 0
   %conv = zext i1 %cmp to i32
   ret i32 %conv
 }
 
 define i32 @cmp_eq13(i8* nocapture readonly %x, i8* nocapture readonly %y)  {
 ; X32-LABEL: @cmp_eq13(
 ; X32-NEXT:    [[CALL:%.*]] = tail call i32 @memcmp(i8* [[X:%.*]], i8* [[Y:%.*]], i64 13)
 ; X32-NEXT:    [[CMP:%.*]] = icmp eq i32 [[CALL]], 0
 ; X32-NEXT:    [[CONV:%.*]] = zext i1 [[CMP]] to i32
 ; X32-NEXT:    ret i32 [[CONV]]
 ;
 ; X64_1LD-LABEL: @cmp_eq13(
 ; X64_1LD-NEXT:    br label [[LOADBB:%.*]]
 ; X64_1LD:       res_block:
 ; X64_1LD-NEXT:    br label [[ENDBLOCK:%.*]]
 ; X64_1LD:       loadbb:
 ; X64_1LD-NEXT:    [[TMP1:%.*]] = bitcast i8* [[X:%.*]] to i64*
 ; X64_1LD-NEXT:    [[TMP2:%.*]] = bitcast i8* [[Y:%.*]] to i64*
 ; X64_1LD-NEXT:    [[TMP3:%.*]] = load i64, i64* [[TMP1]]
 ; X64_1LD-NEXT:    [[TMP4:%.*]] = load i64, i64* [[TMP2]]
-; X64_1LD-NEXT:    [[TMP5:%.*]] = icmp ne i64 [[TMP3]], [[TMP4]]
+; X64_1LD-NEXT:    [[TMP5:%.*]] = icmp eq i64 [[TMP3]], [[TMP4]]
 ; X64_1LD-NEXT:    br i1 [[TMP5]], label [[RES_BLOCK:%.*]], label [[LOADBB1:%.*]]
 ; X64_1LD:       loadbb1:
 ; X64_1LD-NEXT:    [[TMP6:%.*]] = getelementptr i8, i8* [[X]], i64 5
 ; X64_1LD-NEXT:    [[TMP7:%.*]] = getelementptr i8, i8* [[Y]], i64 5
 ; X64_1LD-NEXT:    [[TMP8:%.*]] = bitcast i8* [[TMP6]] to i64*
 ; X64_1LD-NEXT:    [[TMP9:%.*]] = bitcast i8* [[TMP7]] to i64*
 ; X64_1LD-NEXT:    [[TMP10:%.*]] = load i64, i64* [[TMP8]]
 ; X64_1LD-NEXT:    [[TMP11:%.*]] = load i64, i64* [[TMP9]]
-; X64_1LD-NEXT:    [[TMP12:%.*]] = icmp ne i64 [[TMP10]], [[TMP11]]
+; X64_1LD-NEXT:    [[TMP12:%.*]] = icmp eq i64 [[TMP10]], [[TMP11]]
 ; X64_1LD-NEXT:    br i1 [[TMP12]], label [[RES_BLOCK]], label [[ENDBLOCK]]
 ; X64_1LD:       endblock:
 ; X64_1LD-NEXT:    [[PHI_RES:%.*]] = phi i32 [ 0, [[LOADBB1]] ], [ 1, [[RES_BLOCK]] ]
-; X64_1LD-NEXT:    [[CMP:%.*]] = icmp eq i32 [[PHI_RES]], 0
+; X64_1LD-NEXT:    [[CMP:%.*]] = icmp ne i32 [[PHI_RES]], 0
 ; X64_1LD-NEXT:    [[CONV:%.*]] = zext i1 [[CMP]] to i32
 ; X64_1LD-NEXT:    ret i32 [[CONV]]
 ;
 ; X64_2LD-LABEL: @cmp_eq13(
 ; X64_2LD-NEXT:    [[TMP1:%.*]] = bitcast i8* [[X:%.*]] to i64*
 ; X64_2LD-NEXT:    [[TMP2:%.*]] = bitcast i8* [[Y:%.*]] to i64*
 ; X64_2LD-NEXT:    [[TMP3:%.*]] = load i64, i64* [[TMP1]]
 ; X64_2LD-NEXT:    [[TMP4:%.*]] = load i64, i64* [[TMP2]]
 ; X64_2LD-NEXT:    [[TMP5:%.*]] = xor i64 [[TMP3]], [[TMP4]]
 ; X64_2LD-NEXT:    [[TMP6:%.*]] = getelementptr i8, i8* [[X]], i64 5
 ; X64_2LD-NEXT:    [[TMP7:%.*]] = getelementptr i8, i8* [[Y]], i64 5
 ; X64_2LD-NEXT:    [[TMP8:%.*]] = bitcast i8* [[TMP6]] to i64*
 ; X64_2LD-NEXT:    [[TMP9:%.*]] = bitcast i8* [[TMP7]] to i64*
 ; X64_2LD-NEXT:    [[TMP10:%.*]] = load i64, i64* [[TMP8]]
 ; X64_2LD-NEXT:    [[TMP11:%.*]] = load i64, i64* [[TMP9]]
 ; X64_2LD-NEXT:    [[TMP12:%.*]] = xor i64 [[TMP10]], [[TMP11]]
 ; X64_2LD-NEXT:    [[TMP13:%.*]] = or i64 [[TMP5]], [[TMP12]]
-; X64_2LD-NEXT:    [[TMP14:%.*]] = icmp ne i64 [[TMP13]], 0
+; X64_2LD-NEXT:    [[TMP14:%.*]] = icmp eq i64 [[TMP13]], 0
 ; X64_2LD-NEXT:    [[TMP15:%.*]] = zext i1 [[TMP14]] to i32
-; X64_2LD-NEXT:    [[CMP:%.*]] = icmp eq i32 [[TMP15]], 0
-; X64_2LD-NEXT:    [[CONV:%.*]] = zext i1 [[CMP]] to i32
+; X64_2LD-NEXT:    [[CONV:%.*]] = zext i1 [[TMP14]] to i32
 ; X64_2LD-NEXT:    ret i32 [[CONV]]
 ;
   %call = tail call i32 @memcmp(i8* %x, i8* %y, i64 13)
   %cmp = icmp eq i32 %call, 0
   %conv = zext i1 %cmp to i32
   ret i32 %conv
 }
 
 define i32 @cmp_eq14(i8* nocapture readonly %x, i8* nocapture readonly %y)  {
 ; X32-LABEL: @cmp_eq14(
 ; X32-NEXT:    [[CALL:%.*]] = tail call i32 @memcmp(i8* [[X:%.*]], i8* [[Y:%.*]], i64 14)
 ; X32-NEXT:    [[CMP:%.*]] = icmp eq i32 [[CALL]], 0
 ; X32-NEXT:    [[CONV:%.*]] = zext i1 [[CMP]] to i32
 ; X32-NEXT:    ret i32 [[CONV]]
 ;
 ; X64_1LD-LABEL: @cmp_eq14(
 ; X64_1LD-NEXT:    br label [[LOADBB:%.*]]
 ; X64_1LD:       res_block:
 ; X64_1LD-NEXT:    br label [[ENDBLOCK:%.*]]
 ; X64_1LD:       loadbb:
 ; X64_1LD-NEXT:    [[TMP1:%.*]] = bitcast i8* [[X:%.*]] to i64*
 ; X64_1LD-NEXT:    [[TMP2:%.*]] = bitcast i8* [[Y:%.*]] to i64*
 ; X64_1LD-NEXT:    [[TMP3:%.*]] = load i64, i64* [[TMP1]]
 ; X64_1LD-NEXT:    [[TMP4:%.*]] = load i64, i64* [[TMP2]]
-; X64_1LD-NEXT:    [[TMP5:%.*]] = icmp ne i64 [[TMP3]], [[TMP4]]
+; X64_1LD-NEXT:    [[TMP5:%.*]] = icmp eq i64 [[TMP3]], [[TMP4]]
 ; X64_1LD-NEXT:    br i1 [[TMP5]], label [[RES_BLOCK:%.*]], label [[LOADBB1:%.*]]
 ; X64_1LD:       loadbb1:
 ; X64_1LD-NEXT:    [[TMP6:%.*]] = getelementptr i8, i8* [[X]], i64 6
 ; X64_1LD-NEXT:    [[TMP7:%.*]] = getelementptr i8, i8* [[Y]], i64 6
 ; X64_1LD-NEXT:    [[TMP8:%.*]] = bitcast i8* [[TMP6]] to i64*
 ; X64_1LD-NEXT:    [[TMP9:%.*]] = bitcast i8* [[TMP7]] to i64*
 ; X64_1LD-NEXT:    [[TMP10:%.*]] = load i64, i64* [[TMP8]]
 ; X64_1LD-NEXT:    [[TMP11:%.*]] = load i64, i64* [[TMP9]]
-; X64_1LD-NEXT:    [[TMP12:%.*]] = icmp ne i64 [[TMP10]], [[TMP11]]
+; X64_1LD-NEXT:    [[TMP12:%.*]] = icmp eq i64 [[TMP10]], [[TMP11]]
 ; X64_1LD-NEXT:    br i1 [[TMP12]], label [[RES_BLOCK]], label [[ENDBLOCK]]
 ; X64_1LD:       endblock:
 ; X64_1LD-NEXT:    [[PHI_RES:%.*]] = phi i32 [ 0, [[LOADBB1]] ], [ 1, [[RES_BLOCK]] ]
-; X64_1LD-NEXT:    [[CMP:%.*]] = icmp eq i32 [[PHI_RES]], 0
+; X64_1LD-NEXT:    [[CMP:%.*]] = icmp ne i32 [[PHI_RES]], 0
 ; X64_1LD-NEXT:    [[CONV:%.*]] = zext i1 [[CMP]] to i32
 ; X64_1LD-NEXT:    ret i32 [[CONV]]
 ;
 ; X64_2LD-LABEL: @cmp_eq14(
 ; X64_2LD-NEXT:    [[TMP1:%.*]] = bitcast i8* [[X:%.*]] to i64*
 ; X64_2LD-NEXT:    [[TMP2:%.*]] = bitcast i8* [[Y:%.*]] to i64*
 ; X64_2LD-NEXT:    [[TMP3:%.*]] = load i64, i64* [[TMP1]]
 ; X64_2LD-NEXT:    [[TMP4:%.*]] = load i64, i64* [[TMP2]]
 ; X64_2LD-NEXT:    [[TMP5:%.*]] = xor i64 [[TMP3]], [[TMP4]]
 ; X64_2LD-NEXT:    [[TMP6:%.*]] = getelementptr i8, i8* [[X]], i64 6
 ; X64_2LD-NEXT:    [[TMP7:%.*]] = getelementptr i8, i8* [[Y]], i64 6
 ; X64_2LD-NEXT:    [[TMP8:%.*]] = bitcast i8* [[TMP6]] to i64*
 ; X64_2LD-NEXT:    [[TMP9:%.*]] = bitcast i8* [[TMP7]] to i64*
 ; X64_2LD-NEXT:    [[TMP10:%.*]] = load i64, i64* [[TMP8]]
 ; X64_2LD-NEXT:    [[TMP11:%.*]] = load i64, i64* [[TMP9]]
 ; X64_2LD-NEXT:    [[TMP12:%.*]] = xor i64 [[TMP10]], [[TMP11]]
 ; X64_2LD-NEXT:    [[TMP13:%.*]] = or i64 [[TMP5]], [[TMP12]]
-; X64_2LD-NEXT:    [[TMP14:%.*]] = icmp ne i64 [[TMP13]], 0
+; X64_2LD-NEXT:    [[TMP14:%.*]] = icmp eq i64 [[TMP13]], 0
 ; X64_2LD-NEXT:    [[TMP15:%.*]] = zext i1 [[TMP14]] to i32
-; X64_2LD-NEXT:    [[CMP:%.*]] = icmp eq i32 [[TMP15]], 0
-; X64_2LD-NEXT:    [[CONV:%.*]] = zext i1 [[CMP]] to i32
+; X64_2LD-NEXT:    [[CONV:%.*]] = zext i1 [[TMP14]] to i32
 ; X64_2LD-NEXT:    ret i32 [[CONV]]
 ;
   %call = tail call i32 @memcmp(i8* %x, i8* %y, i64 14)
   %cmp = icmp eq i32 %call, 0
   %conv = zext i1 %cmp to i32
   ret i32 %conv
 }
 
 define i32 @cmp_eq15(i8* nocapture readonly %x, i8* nocapture readonly %y)  {
 ; X32-LABEL: @cmp_eq15(
 ; X32-NEXT:    [[CALL:%.*]] = tail call i32 @memcmp(i8* [[X:%.*]], i8* [[Y:%.*]], i64 15)
 ; X32-NEXT:    [[CMP:%.*]] = icmp eq i32 [[CALL]], 0
 ; X32-NEXT:    [[CONV:%.*]] = zext i1 [[CMP]] to i32
 ; X32-NEXT:    ret i32 [[CONV]]
 ;
 ; X64_1LD-LABEL: @cmp_eq15(
 ; X64_1LD-NEXT:    br label [[LOADBB:%.*]]
 ; X64_1LD:       res_block:
 ; X64_1LD-NEXT:    br label [[ENDBLOCK:%.*]]
 ; X64_1LD:       loadbb:
 ; X64_1LD-NEXT:    [[TMP1:%.*]] = bitcast i8* [[X:%.*]] to i64*
 ; X64_1LD-NEXT:    [[TMP2:%.*]] = bitcast i8* [[Y:%.*]] to i64*
 ; X64_1LD-NEXT:    [[TMP3:%.*]] = load i64, i64* [[TMP1]]
 ; X64_1LD-NEXT:    [[TMP4:%.*]] = load i64, i64* [[TMP2]]
-; X64_1LD-NEXT:    [[TMP5:%.*]] = icmp ne i64 [[TMP3]], [[TMP4]]
+; X64_1LD-NEXT:    [[TMP5:%.*]] = icmp eq i64 [[TMP3]], [[TMP4]]
 ; X64_1LD-NEXT:    br i1 [[TMP5]], label [[RES_BLOCK:%.*]], label [[LOADBB1:%.*]]
 ; X64_1LD:       loadbb1:
 ; X64_1LD-NEXT:    [[TMP6:%.*]] = getelementptr i8, i8* [[X]], i64 7
 ; X64_1LD-NEXT:    [[TMP7:%.*]] = getelementptr i8, i8* [[Y]], i64 7
 ; X64_1LD-NEXT:    [[TMP8:%.*]] = bitcast i8* [[TMP6]] to i64*
 ; X64_1LD-NEXT:    [[TMP9:%.*]] = bitcast i8* [[TMP7]] to i64*
 ; X64_1LD-NEXT:    [[TMP10:%.*]] = load i64, i64* [[TMP8]]
 ; X64_1LD-NEXT:    [[TMP11:%.*]] = load i64, i64* [[TMP9]]
-; X64_1LD-NEXT:    [[TMP12:%.*]] = icmp ne i64 [[TMP10]], [[TMP11]]
+; X64_1LD-NEXT:    [[TMP12:%.*]] = icmp eq i64 [[TMP10]], [[TMP11]]
 ; X64_1LD-NEXT:    br i1 [[TMP12]], label [[RES_BLOCK]], label [[ENDBLOCK]]
 ; X64_1LD:       endblock:
 ; X64_1LD-NEXT:    [[PHI_RES:%.*]] = phi i32 [ 0, [[LOADBB1]] ], [ 1, [[RES_BLOCK]] ]
-; X64_1LD-NEXT:    [[CMP:%.*]] = icmp eq i32 [[PHI_RES]], 0
+; X64_1LD-NEXT:    [[CMP:%.*]] = icmp ne i32 [[PHI_RES]], 0
 ; X64_1LD-NEXT:    [[CONV:%.*]] = zext i1 [[CMP]] to i32
 ; X64_1LD-NEXT:    ret i32 [[CONV]]
 ;
 ; X64_2LD-LABEL: @cmp_eq15(
 ; X64_2LD-NEXT:    [[TMP1:%.*]] = bitcast i8* [[X:%.*]] to i64*
 ; X64_2LD-NEXT:    [[TMP2:%.*]] = bitcast i8* [[Y:%.*]] to i64*
 ; X64_2LD-NEXT:    [[TMP3:%.*]] = load i64, i64* [[TMP1]]
 ; X64_2LD-NEXT:    [[TMP4:%.*]] = load i64, i64* [[TMP2]]
 ; X64_2LD-NEXT:    [[TMP5:%.*]] = xor i64 [[TMP3]], [[TMP4]]
 ; X64_2LD-NEXT:    [[TMP6:%.*]] = getelementptr i8, i8* [[X]], i64 7
 ; X64_2LD-NEXT:    [[TMP7:%.*]] = getelementptr i8, i8* [[Y]], i64 7
 ; X64_2LD-NEXT:    [[TMP8:%.*]] = bitcast i8* [[TMP6]] to i64*
 ; X64_2LD-NEXT:    [[TMP9:%.*]] = bitcast i8* [[TMP7]] to i64*
 ; X64_2LD-NEXT:    [[TMP10:%.*]] = load i64, i64* [[TMP8]]
 ; X64_2LD-NEXT:    [[TMP11:%.*]] = load i64, i64* [[TMP9]]
 ; X64_2LD-NEXT:    [[TMP12:%.*]] = xor i64 [[TMP10]], [[TMP11]]
 ; X64_2LD-NEXT:    [[TMP13:%.*]] = or i64 [[TMP5]], [[TMP12]]
-; X64_2LD-NEXT:    [[TMP14:%.*]] = icmp ne i64 [[TMP13]], 0
+; X64_2LD-NEXT:    [[TMP14:%.*]] = icmp eq i64 [[TMP13]], 0
 ; X64_2LD-NEXT:    [[TMP15:%.*]] = zext i1 [[TMP14]] to i32
-; X64_2LD-NEXT:    [[CMP:%.*]] = icmp eq i32 [[TMP15]], 0
-; X64_2LD-NEXT:    [[CONV:%.*]] = zext i1 [[CMP]] to i32
+; X64_2LD-NEXT:    [[CONV:%.*]] = zext i1 [[TMP14]] to i32
 ; X64_2LD-NEXT:    ret i32 [[CONV]]
 ;
   %call = tail call i32 @memcmp(i8* %x, i8* %y, i64 15)
   %cmp = icmp eq i32 %call, 0
   %conv = zext i1 %cmp to i32
   ret i32 %conv
 }
 
 define i32 @cmp_eq16(i8* nocapture readonly %x, i8* nocapture readonly %y)  {
 ; X32-LABEL: @cmp_eq16(
 ; X32-NEXT:    [[CALL:%.*]] = tail call i32 @memcmp(i8* [[X:%.*]], i8* [[Y:%.*]], i64 16)
 ; X32-NEXT:    [[CMP:%.*]] = icmp eq i32 [[CALL]], 0
 ; X32-NEXT:    [[CONV:%.*]] = zext i1 [[CMP]] to i32
 ; X32-NEXT:    ret i32 [[CONV]]
 ;
 ; X64-LABEL: @cmp_eq16(
 ; X64-NEXT:    [[TMP1:%.*]] = bitcast i8* [[X:%.*]] to i128*
 ; X64-NEXT:    [[TMP2:%.*]] = bitcast i8* [[Y:%.*]] to i128*
 ; X64-NEXT:    [[TMP3:%.*]] = load i128, i128* [[TMP1]]
 ; X64-NEXT:    [[TMP4:%.*]] = load i128, i128* [[TMP2]]
-; X64-NEXT:    [[TMP5:%.*]] = icmp ne i128 [[TMP3]], [[TMP4]]
+; X64-NEXT:    [[TMP5:%.*]] = icmp eq i128 [[TMP3]], [[TMP4]]
 ; X64-NEXT:    [[TMP6:%.*]] = zext i1 [[TMP5]] to i32
-; X64-NEXT:    [[CMP:%.*]] = icmp eq i32 [[TMP6]], 0
-; X64-NEXT:    [[CONV:%.*]] = zext i1 [[CMP]] to i32
+; X64-NEXT:    [[CONV:%.*]] = zext i1 [[TMP5]] to i32
 ; X64-NEXT:    ret i32 [[CONV]]
 ;
   %call = tail call i32 @memcmp(i8* %x, i8* %y, i64 16)
   %cmp = icmp eq i32 %call, 0
   %conv = zext i1 %cmp to i32
   ret i32 %conv
 }