[ARM-BE] Generate correct NEON intrinsics for big endian systems.

The NEON intrinsics in arm_neon.h are designed to work on vectors
"as-if" loaded by (V)LDR. We load vectors "as-if" (V)LD1, so the
intrinsics are currently incorrect.

This patch adds big-endian versions of the intrinsics that does the
"obvious but dumb" thing of reversing all vector inputs and all
vector outputs. This will produce extra REVs, but we trust the
optimizer to remove them.

git-svn-id: https://llvm.org/svn/llvm-project/cfe/trunk@211893 91177308-0d34-0410-b5e6-96231b3b80d8

3 files changed, 192 insertions, 64 deletions

diff --git a/include/clang/Basic/arm_neon.td b/include/clang/Basic/arm_neon.td
index 4dba0f1058..f68ccea655 100644
--- a/include/clang/Basic/arm_neon.td
+++ b/include/clang/Basic/arm_neon.td
@@ -261,6 +261,7 @@ class Inst <string n, string p, string t, Operation o> {
 
   Operation Operation = o;
   bit CartesianProductOfTypes = 0;
+  bit BigEndianSafe = 0;
   bit isShift = 0;
   bit isScalarShift = 0;
   bit isScalarNarrowShift = 0;
@@ -654,7 +655,9 @@ def VSET_LANE : IInst<"vset_lane", "dsdi",
 
 ////////////////////////////////////////////////////////////////////////////////
 // E.3.18 Initialize a vector from bit pattern
-def VCREATE : NoTestOpInst<"vcreate", "dl", "csihfUcUsUiUlPcPsl", OP_CAST>;
+def VCREATE : NoTestOpInst<"vcreate", "dl", "csihfUcUsUiUlPcPsl", OP_CAST> {
+  let BigEndianSafe = 1;
+}
 
 ////////////////////////////////////////////////////////////////////////////////
 // E.3.19 Set all lanes to same value
@@ -791,6 +794,7 @@ def VREINTERPRET
          "csilUcUsUiUlhfPcPsQcQsQiQlQUcQUsQUiQUlQhQfQPcQPs", OP_REINT> {
   let CartesianProductOfTypes = 1;
   let ArchGuard = "!defined(__aarch64__)";
+  let BigEndianSafe = 1;
 }
 
 ////////////////////////////////////////////////////////////////////////////////
@@ -1092,7 +1096,9 @@ def COMBINE : NoTestOpInst<"vcombine", "kdd", "dPl", OP_CONC>;
 
 ////////////////////////////////////////////////////////////////////////////////
 //Initialize a vector from bit pattern
-def CREATE : NoTestOpInst<"vcreate", "dl", "dPl", OP_CAST>;
+def CREATE : NoTestOpInst<"vcreate", "dl", "dPl", OP_CAST> {
+  let BigEndianSafe = 1;
+}
 
 ////////////////////////////////////////////////////////////////////////////////
 
@@ -1256,6 +1262,7 @@ def VVREINTERPRET
   : NoTestOpInst<"vreinterpret", "dd",
        "csilUcUsUiUlhfdPcPsPlQcQsQiQlQUcQUsQUiQUlQhQfQdQPcQPsQPlQPk", OP_REINT> {
   let CartesianProductOfTypes = 1;
+  let BigEndianSafe = 1;
   let ArchGuard = "__ARM_ARCH >= 8 && defined(__aarch64__)";
 }
 
diff --git a/test/CodeGen/arm64-lanes.c b/test/CodeGen/arm64-lanes.c
index b0d4694677..8ab2bd4c66 100644
--- a/test/CodeGen/arm64-lanes.c
+++ b/test/CodeGen/arm64-lanes.c
@@ -1,4 +1,5 @@
 // RUN: %clang_cc1 -O3 -triple arm64-apple-ios7 -target-feature +neon -ffreestanding -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -O3 -triple arm64_be-linux-gnu -target-feature +neon -ffreestanding -emit-llvm -o - %s | FileCheck %s --check-prefix CHECK-BE
 
 #include <arm_neon.h>
 
@@ -6,58 +7,68 @@
 int8_t test_vdupb_lane_s8(int8x8_t src) {
   return vdupb_lane_s8(src, 2);
   // CHECK: extractelement <8 x i8> %src, i32 2
+  // CHECK-BE: extractelement <8 x i8> %src, i32 5
 }
 
 // CHECK-LABEL: @test_vdupb_lane_u8
 uint8_t test_vdupb_lane_u8(uint8x8_t src) {
   return vdupb_lane_u8(src, 2);
   // CHECK: extractelement <8 x i8> %src, i32 2
+  // CHECK-BE: extractelement <8 x i8> %src, i32 5
 }
 
 // CHECK-LABEL: @test_vduph_lane_s16
 int16_t test_vduph_lane_s16(int16x4_t src) {
   return vduph_lane_s16(src, 2);
   // CHECK: extractelement <4 x i16> %src, i32 2
+  // CHECK-BE: extractelement <4 x i16> %src, i32 1
 }
 
 // CHECK-LABEL: @test_vduph_lane_u16
 uint16_t test_vduph_lane_u16(uint16x4_t src) {
   return vduph_lane_u16(src, 2);
   // CHECK: extractelement <4 x i16> %src, i32 2
+  // CHECK-BE: extractelement <4 x i16> %src, i32 1
 }
 
 // CHECK-LABEL: @test_vdups_lane_s32
 int32_t test_vdups_lane_s32(int32x2_t src) {
   return vdups_lane_s32(src, 0);
   // CHECK: extractelement <2 x i32> %src, i32 0
+  // CHECK-BE: extractelement <2 x i32> %src, i32 1
 }
 
 // CHECK-LABEL: @test_vdups_lane_u32
 uint32_t test_vdups_lane_u32(uint32x2_t src) {
   return vdups_lane_u32(src, 0);
   // CHECK: extractelement <2 x i32> %src, i32 0
+  // CHECK-BE: extractelement <2 x i32> %src, i32 1
 }
 
 // CHECK-LABEL: @test_vdups_lane_f32
 float32_t test_vdups_lane_f32(float32x2_t src) {
   return vdups_lane_f32(src, 0);
   // CHECK: extractelement <2 x float> %src, i32 0
+  // CHECK-BE: extractelement <2 x float> %src, i32 1
 }
 
 // CHECK-LABEL: @test_vdupd_lane_s64
 int64_t test_vdupd_lane_s64(int64x1_t src) {
   return vdupd_lane_s64(src, 0);
   // CHECK: extractelement <1 x i64> %src, i32 0
+  // CHECK-BE: extractelement <1 x i64> %src, i32 0
 }
 
 // CHECK-LABEL: @test_vdupd_lane_u64
 uint64_t test_vdupd_lane_u64(uint64x1_t src) {
   return vdupd_lane_u64(src, 0);
   // CHECK: extractelement <1 x i64> %src, i32 0
+  // CHECK-BE: extractelement <1 x i64> %src, i32 0
 }
 
 // CHECK-LABEL: @test_vdupd_lane_f64
 float64_t test_vdupd_lane_f64(float64x1_t src) {
   return vdupd_lane_f64(src, 0);
   // CHECK: extractelement <1 x double> %src, i32 0
+  // CHECK-BE: extractelement <1 x double> %src, i32 0
 }
diff --git a/utils/TableGen/NeonEmitter.cpp b/utils/TableGen/NeonEmitter.cpp
index 9d6ae3497b..d7e418a810 100644
--- a/utils/TableGen/NeonEmitter.cpp
+++ b/utils/TableGen/NeonEmitter.cpp
@@ -259,6 +259,8 @@ public:
 /// The main grunt class. This represents an instantiation of an intrinsic with
 /// a particular typespec and prototype.
 class Intrinsic {
+  friend class DagEmitter;
+
   /// The Record this intrinsic was created from.
   Record *R;
   /// The unmangled name and prototype.
@@ -279,6 +281,9 @@ class Intrinsic {
   /// Set if the Unvailable bit is 1. This means we don't generate a body,
   /// just an "unavailable" attribute on a declaration.
   bool IsUnavailable;
+  /// Is this intrinsic safe for big-endian? or does it need its arguments
+  /// reversing?
+  bool BigEndianSafe;
 
   /// The types of return value [0] and parameters [1..].
   std::vector<Type> Types;
@@ -305,11 +310,11 @@ class Intrinsic {
 public:
   Intrinsic(Record *R, StringRef Name, StringRef Proto, TypeSpec OutTS,
             TypeSpec InTS, ClassKind CK, ListInit *Body, NeonEmitter &Emitter,
-            StringRef Guard, bool IsUnavailable)
+            StringRef Guard, bool IsUnavailable, bool BigEndianSafe)
       : R(R), Name(Name.str()), Proto(Proto.str()), OutTS(OutTS), InTS(InTS),
         CK(CK), Body(Body), Guard(Guard.str()), IsUnavailable(IsUnavailable),
-        NeededEarly(false), UseMacro(false), BaseType(OutTS, 'd'),
-        InBaseType(InTS, 'd'), Emitter(Emitter) {
+        BigEndianSafe(BigEndianSafe), NeededEarly(false), UseMacro(false),
+        BaseType(OutTS, 'd'), InBaseType(InTS, 'd'), Emitter(Emitter) {
     // If this builtin takes an immediate argument, we need to #define it rather
     // than use a standard declaration, so that SemaChecking can range check
     // the immediate passed by the user.
@@ -435,25 +440,41 @@ private:
   std::string replaceParamsIn(std::string S);
 
   void emitBodyAsBuiltinCall();
-  std::pair<Type, std::string> emitDagArg(Init *Arg, std::string ArgName);
-  std::pair<Type, std::string> emitDagSaveTemp(DagInit *DI);
-  std::pair<Type, std::string> emitDagSplat(DagInit *DI);
-  std::pair<Type, std::string> emitDagDup(DagInit *DI);
-  std::pair<Type, std::string> emitDagShuffle(DagInit *DI);
-  std::pair<Type, std::string> emitDagCast(DagInit *DI, bool IsBitCast);
-  std::pair<Type, std::string> emitDagCall(DagInit *DI);
-  std::pair<Type, std::string> emitDagNameReplace(DagInit *DI);
-  std::pair<Type, std::string> emitDagLiteral(DagInit *DI);
-  std::pair<Type, std::string> emitDagOp(DagInit *DI);
-  std::pair<Type, std::string> emitDag(DagInit *DI);
 
+  void generateImpl(bool ReverseArguments,
+                    StringRef NamePrefix, StringRef CallPrefix);
   void emitReturn();
-  void emitBody();
+  void emitBody(StringRef CallPrefix);
   void emitShadowedArgs();
+  void emitArgumentReversal();
+  void emitReturnReversal();
+  void emitReverseVariable(Variable &Dest, Variable &Src);
   void emitNewLine();
   void emitClosingBrace();
   void emitOpeningBrace();
-  void emitPrototype();
+  void emitPrototype(StringRef NamePrefix);
+
+  class DagEmitter {
+    Intrinsic &Intr;
+    StringRef CallPrefix;
+
+  public:
+    DagEmitter(Intrinsic &Intr, StringRef CallPrefix) :
+      Intr(Intr), CallPrefix(CallPrefix) {
+    }
+    std::pair<Type, std::string> emitDagArg(Init *Arg, std::string ArgName);
+    std::pair<Type, std::string> emitDagSaveTemp(DagInit *DI);
+    std::pair<Type, std::string> emitDagSplat(DagInit *DI);
+    std::pair<Type, std::string> emitDagDup(DagInit *DI);
+    std::pair<Type, std::string> emitDagShuffle(DagInit *DI);
+    std::pair<Type, std::string> emitDagCast(DagInit *DI, bool IsBitCast);
+    std::pair<Type, std::string> emitDagCall(DagInit *DI);
+    std::pair<Type, std::string> emitDagNameReplace(DagInit *DI);
+    std::pair<Type, std::string> emitDagLiteral(DagInit *DI);
+    std::pair<Type, std::string> emitDagOp(DagInit *DI);
+    std::pair<Type, std::string> emitDag(DagInit *DI);
+  };
+
 };
 
 //===----------------------------------------------------------------------===//
@@ -1103,13 +1124,13 @@ void Intrinsic::initVariables() {
   RetVar = Variable(Types[0], "ret" + VariablePostfix);
 }
 
-void Intrinsic::emitPrototype() {
+void Intrinsic::emitPrototype(StringRef NamePrefix) {
   if (UseMacro)
     OS << "#define ";
   else
     OS << "__ai " << Types[0].str() << " ";
 
-  OS << mangleName(Name, ClassS) << "(";
+  OS << NamePrefix.str() << mangleName(Name, ClassS) << "(";
 
   for (unsigned I = 0; I < getNumParams(); ++I) {
     if (I != 0)
@@ -1151,6 +1172,61 @@ void Intrinsic::emitNewLine() {
     OS << "\n";
 }
 
+void Intrinsic::emitReverseVariable(Variable &Dest, Variable &Src) {
+  if (Dest.getType().getNumVectors() > 1) {
+    emitNewLine();
+
+    for (unsigned K = 0; K < Dest.getType().getNumVectors(); ++K) {
+      OS << "  " << Dest.getName() << ".val[" << utostr(K) << "] = "
+         << "__builtin_shufflevector("
+         << Src.getName() << ".val[" << utostr(K) << "], "
+         << Src.getName() << ".val[" << utostr(K) << "]";
+      for (int J = Dest.getType().getNumElements() - 1; J >= 0; --J)
+        OS << ", " << utostr(J);
+      OS << ");";
+      emitNewLine();
+    }
+  } else {
+    OS << "  " << Dest.getName()
+       << " = __builtin_shufflevector(" << Src.getName() << ", " << Src.getName();
+    for (int J = Dest.getType().getNumElements() - 1; J >= 0; --J)
+      OS << ", " << utostr(J);
+    OS << ");";
+    emitNewLine();
+  }
+}
+
+void Intrinsic::emitArgumentReversal() {
+  if (BigEndianSafe)
+    return;
+
+  // Reverse all vector arguments.
+  for (unsigned I = 0; I < getNumParams(); ++I) {
+    std::string Name = "p" + utostr(I);
+    std::string NewName = "rev" + utostr(I);
+
+    Variable &V = Variables[Name];
+    Variable NewV(V.getType(), NewName + VariablePostfix);
+
+    if (!NewV.getType().isVector() || NewV.getType().getNumElements() == 1)
+      continue;
+
+    OS << "  " << NewV.getType().str() << " " << NewV.getName() << ";";
+    emitReverseVariable(NewV, V);
+    V = NewV;
+  }
+}
+
+void Intrinsic::emitReturnReversal() {
+  if (BigEndianSafe)
+    return;
+  if (!getReturnType().isVector() || getReturnType().isVoid() ||
+      getReturnType().getNumElements() == 1)
+    return;
+  emitReverseVariable(RetVar, RetVar);
+}
+
+
 void Intrinsic::emitShadowedArgs() {
   // Macro arguments are not type-checked like inline function arguments,
   // so assign them to local temporaries to get the right type checking.
@@ -1167,9 +1243,7 @@ void Intrinsic::emitShadowedArgs() {
     if (getParamType(I).isPointer())
       continue;
 
-    char NameC = '0' + I;
-    std::string Name = "p";
-    Name.push_back(NameC);
+    std::string Name = "p" + utostr(I);
 
     assert(Variables.find(Name) != Variables.end());
     Variable &V = Variables[Name];
@@ -1293,7 +1367,7 @@ void Intrinsic::emitBodyAsBuiltinCall() {
   emitNewLine();
 }
 
-void Intrinsic::emitBody() {
+void Intrinsic::emitBody(StringRef CallPrefix) {
   std::vector<std::string> Lines;
 
   assert(RetVar.getType() == Types[0]);
@@ -1314,7 +1388,8 @@ void Intrinsic::emitBody() {
     if (StringInit *SI = dyn_cast<StringInit>(I)) {
       Lines.push_back(replaceParamsIn(SI->getAsString()));
     } else if (DagInit *DI = dyn_cast<DagInit>(I)) {
-      Lines.push_back(emitDag(DI).second + ";");
+      DagEmitter DE(*this, CallPrefix);
+      Lines.push_back(DE.emitDag(DI).second + ";");
     }
   }
 
@@ -1338,7 +1413,7 @@ void Intrinsic::emitReturn() {
   emitNewLine();
 }
 
-std::pair<Type, std::string> Intrinsic::emitDag(DagInit *DI) {
+std::pair<Type, std::string> Intrinsic::DagEmitter::emitDag(DagInit *DI) {
   // At this point we should only be seeing a def.
   DefInit *DefI = cast<DefInit>(DI->getOperator());
   std::string Op = DefI->getAsString();
@@ -1365,7 +1440,7 @@ std::pair<Type, std::string> Intrinsic::emitDag(DagInit *DI) {
   return std::make_pair(Type::getVoid(), "");
 }
 
-std::pair<Type, std::string> Intrinsic::emitDagOp(DagInit *DI) {
+std::pair<Type, std::string> Intrinsic::DagEmitter::emitDagOp(DagInit *DI) {
   std::string Op = cast<StringInit>(DI->getArg(0))->getAsUnquotedString();
   if (DI->getNumArgs() == 2) {
     // Unary op.
@@ -1383,7 +1458,7 @@ std::pair<Type, std::string> Intrinsic::emitDagOp(DagInit *DI) {
   }
 }
 
-std::pair<Type, std::string> Intrinsic::emitDagCall(DagInit *DI) {
+std::pair<Type, std::string> Intrinsic::DagEmitter::emitDagCall(DagInit *DI) {
   std::vector<Type> Types;
   std::vector<std::string> Values;
   for (unsigned I = 0; I < DI->getNumArgs() - 1; ++I) {
@@ -1399,15 +1474,15 @@ std::pair<Type, std::string> Intrinsic::emitDagCall(DagInit *DI) {
     N = SI->getAsUnquotedString();
   else
     N = emitDagArg(DI->getArg(0), "").second;
-  Intrinsic *Callee = Emitter.getIntrinsic(N, Types);
+  Intrinsic *Callee = Intr.Emitter.getIntrinsic(N, Types);
   assert(Callee && "getIntrinsic should not return us nullptr!");
 
   // Make sure the callee is known as an early def.
   Callee->setNeededEarly();
-  Dependencies.insert(Callee);
+  Intr.Dependencies.insert(Callee);
 
   // Now create the call itself.
-  std::string S = Callee->getMangledName(true) + "(";
+  std::string S = CallPrefix.str() + Callee->getMangledName(true) + "(";
   for (unsigned I = 0; I < DI->getNumArgs() - 1; ++I) {
     if (I != 0)
       S += ", ";
@@ -1418,8 +1493,8 @@ std::pair<Type, std::string> Intrinsic::emitDagCall(DagInit *DI) {
   return std::make_pair(Callee->getReturnType(), S);
 }
 
-std::pair<Type, std::string> Intrinsic::emitDagCast(DagInit *DI,
-                                                    bool IsBitCast) {
+std::pair<Type, std::string> Intrinsic::DagEmitter::emitDagCast(DagInit *DI,
+                                                                bool IsBitCast){
   // (cast MOD* VAL) -> cast VAL to type given by MOD.
   std::pair<Type, std::string> R = emitDagArg(
       DI->getArg(DI->getNumArgs() - 1), DI->getArgName(DI->getNumArgs() - 1));
@@ -1434,15 +1509,16 @@ std::pair<Type, std::string> Intrinsic::emitDagCast(DagInit *DI,
     //   5. The value "H" or "D" to half or double the bitwidth.
     //   6. The value "8" to convert to 8-bit (signed) integer lanes.
     if (DI->getArgName(ArgIdx).size()) {
-      assert_with_loc(Variables.find(DI->getArgName(ArgIdx)) != Variables.end(),
+      assert_with_loc(Intr.Variables.find(DI->getArgName(ArgIdx)) !=
+                      Intr.Variables.end(),
                       "Variable not found");
-      castToType = Variables[DI->getArgName(ArgIdx)].getType();
+      castToType = Intr.Variables[DI->getArgName(ArgIdx)].getType();
     } else {
       StringInit *SI = dyn_cast<StringInit>(DI->getArg(ArgIdx));
       assert_with_loc(SI, "Expected string type or $Name for cast type");
 
       if (SI->getAsUnquotedString() == "R") {
-        castToType = getReturnType();
+        castToType = Intr.getReturnType();
       } else if (SI->getAsUnquotedString() == "U") {
         castToType.makeUnsigned();
       } else if (SI->getAsUnquotedString() == "S") {
@@ -1466,15 +1542,15 @@ std::pair<Type, std::string> Intrinsic::emitDagCast(DagInit *DI,
     // a temporary.
     std::string N = "reint";
     unsigned I = 0;
-    while (Variables.find(N) != Variables.end())
+    while (Intr.Variables.find(N) != Intr.Variables.end())
       N = "reint" + utostr(++I);
-    Variables[N] = Variable(R.first, N + VariablePostfix);
+    Intr.Variables[N] = Variable(R.first, N + Intr.VariablePostfix);
 
-    OS << R.first.str() << " " << Variables[N].getName() << " = " << R.second
-       << ";";
-    emitNewLine();
+    Intr.OS << R.first.str() << " " << Intr.Variables[N].getName() << " = "
+            << R.second << ";";
+    Intr.emitNewLine();
 
-    S = "*(" + castToType.str() + " *) &" + Variables[N].getName() + "";
+    S = "*(" + castToType.str() + " *) &" + Intr.Variables[N].getName() + "";
   } else {
     // Emit a normal (static) cast.
     S = "(" + castToType.str() + ")(" + R.second + ")";
@@ -1483,7 +1559,7 @@ std::pair<Type, std::string> Intrinsic::emitDagCast(DagInit *DI,
   return std::make_pair(castToType, S);
 }
 
-std::pair<Type, std::string> Intrinsic::emitDagShuffle(DagInit *DI) {
+std::pair<Type, std::string> Intrinsic::DagEmitter::emitDagShuffle(DagInit *DI){
   // See the documentation in arm_neon.td for a description of these operators.
   class LowHalf : public SetTheory::Operator {
   public:
@@ -1598,12 +1674,12 @@ std::pair<Type, std::string> Intrinsic::emitDagShuffle(DagInit *DI) {
   return std::make_pair(T, S);
 }
 
-std::pair<Type, std::string> Intrinsic::emitDagDup(DagInit *DI) {
+std::pair<Type, std::string> Intrinsic::DagEmitter::emitDagDup(DagInit *DI) {
   assert_with_loc(DI->getNumArgs() == 1, "dup() expects one argument");
   std::pair<Type, std::string> A = emitDagArg(DI->getArg(0), DI->getArgName(0));
   assert_with_loc(A.first.isScalar(), "dup() expects a scalar argument");
 
-  Type T = getBaseType();
+  Type T = Intr.getBaseType();
   assert_with_loc(T.isVector(), "dup() used but default type is scalar!");
   std::string S = "(" + T.str() + ") {";
   for (unsigned I = 0; I < T.getNumElements(); ++I) {
@@ -1616,7 +1692,7 @@ std::pair<Type, std::string> Intrinsic::emitDagDup(DagInit *DI) {
   return std::make_pair(T, S);
 }
 
-std::pair<Type, std::string> Intrinsic::emitDagSplat(DagInit *DI) {
+std::pair<Type, std::string> Intrinsic::DagEmitter::emitDagSplat(DagInit *DI) {
   assert_with_loc(DI->getNumArgs() == 2, "splat() expects two arguments");
   std::pair<Type, std::string> A = emitDagArg(DI->getArg(0), DI->getArgName(0));
   std::pair<Type, std::string> B = emitDagArg(DI->getArg(1), DI->getArgName(1));
@@ -1625,15 +1701,15 @@ std::pair<Type, std::string> Intrinsic::emitDagSplat(DagInit *DI) {
                   "splat() requires a scalar int as the second argument");
 
   std::string S = "__builtin_shufflevector(" + A.second + ", " + A.second;
-  for (unsigned I = 0; I < BaseType.getNumElements(); ++I) {
+  for (unsigned I = 0; I < Intr.getBaseType().getNumElements(); ++I) {
     S += ", " + B.second;
   }
   S += ")";
 
-  return std::make_pair(BaseType, S);
+  return std::make_pair(Intr.getBaseType(), S);
 }
 
-std::pair<Type, std::string> Intrinsic::emitDagSaveTemp(DagInit *DI) {
+std::pair<Type, std::string> Intrinsic::DagEmitter::emitDagSaveTemp(DagInit *DI) {
   assert_with_loc(DI->getNumArgs() == 2, "save_temp() expects two arguments");
   std::pair<Type, std::string> A = emitDagArg(DI->getArg(1), DI->getArgName(1));
 
@@ -1643,18 +1719,19 @@ std::pair<Type, std::string> Intrinsic::emitDagSaveTemp(DagInit *DI) {
   std::string N = DI->getArgName(0);
   assert_with_loc(N.size(), "save_temp() expects a name as the first argument");
 
-  assert_with_loc(Variables.find(N) == Variables.end(),
+  assert_with_loc(Intr.Variables.find(N) == Intr.Variables.end(),
                   "Variable already defined!");
-  Variables[N] = Variable(A.first, N + VariablePostfix);
+  Intr.Variables[N] = Variable(A.first, N + Intr.VariablePostfix);
 
   std::string S =
-      A.first.str() + " " + Variables[N].getName() + " = " + A.second;
+      A.first.str() + " " + Intr.Variables[N].getName() + " = " + A.second;
 
   return std::make_pair(Type::getVoid(), S);
 }
 
-std::pair<Type, std::string> Intrinsic::emitDagNameReplace(DagInit *DI) {
-  std::string S = Name;
+std::pair<Type, std::string>
+Intrinsic::DagEmitter::emitDagNameReplace(DagInit *DI) {
+  std::string S = Intr.Name;
 
   assert_with_loc(DI->getNumArgs() == 2, "name_replace requires 2 arguments!");
   std::string ToReplace = cast<StringInit>(DI->getArg(0))->getAsUnquotedString();
@@ -1668,20 +1745,20 @@ std::pair<Type, std::string> Intrinsic::emitDagNameReplace(DagInit *DI) {
   return std::make_pair(Type::getVoid(), S);
 }
 
-std::pair<Type, std::string> Intrinsic::emitDagLiteral(DagInit *DI) {
+std::pair<Type, std::string> Intrinsic::DagEmitter::emitDagLiteral(DagInit *DI){
   std::string Ty = cast<StringInit>(DI->getArg(0))->getAsUnquotedString();
   std::string Value = cast<StringInit>(DI->getArg(1))->getAsUnquotedString();
   return std::make_pair(Type::fromTypedefName(Ty), Value);
 }
 
-std::pair<Type, std::string> Intrinsic::emitDagArg(Init *Arg,
-                                                   std::string ArgName) {
+std::pair<Type, std::string>
+Intrinsic::DagEmitter::emitDagArg(Init *Arg, std::string ArgName) {
   if (ArgName.size()) {
     assert_with_loc(!Arg->isComplete(),
                     "Arguments must either be DAGs or names, not both!");
-    assert_with_loc(Variables.find(ArgName) != Variables.end(),
+    assert_with_loc(Intr.Variables.find(ArgName) != Intr.Variables.end(),
                     "Variable not defined!");
-    Variable &V = Variables[ArgName];
+    Variable &V = Intr.Variables[ArgName];
     return std::make_pair(V.getType(), V.getName());
   }
 
@@ -1693,6 +1770,35 @@ std::pair<Type, std::string> Intrinsic::emitDagArg(Init *Arg,
 }
 
 std::string Intrinsic::generate() {
+  // Little endian intrinsics are simple and don't require any argument
+  // swapping.
+  OS << "#ifdef __LITTLE_ENDIAN__\n";
+
+  generateImpl(false, "", "");
+
+  OS << "#else\n";
+
+  // Big endian intrinsics are more complex. The user intended these
+  // intrinsics to operate on a vector "as-if" loaded by (V)LDR,
+  // but we load as-if (V)LD1. So we should swap all arguments and
+  // swap the return value too.
+  //
+  // If we call sub-intrinsics, we should call a version that does
+  // not re-swap the arguments!
+  generateImpl(true, "", "__noswap_");
+
+  // If we're needed early, create a non-swapping variant for
+  // big-endian.
+  if (NeededEarly) {
+    generateImpl(false, "__noswap_", "__noswap_");
+  }
+  OS << "#endif\n\n";
+
+  return OS.str();
+}
+
+void Intrinsic::generateImpl(bool ReverseArguments,
+                             StringRef NamePrefix, StringRef CallPrefix) {
   CurrentRecord = R;
 
   // If we call a macro, our local variables may be corrupted due to
@@ -1708,28 +1814,31 @@ std::string Intrinsic::generate() {
 
   initVariables();
 
-  emitPrototype();
+  emitPrototype(NamePrefix);
 
   if (IsUnavailable) {
     OS << " __attribute__((unavailable));";
   } else {
     emitOpeningBrace();
     emitShadowedArgs();
-    emitBody();
+    if (ReverseArguments)
+      emitArgumentReversal();
+    emitBody(CallPrefix);
+    if (ReverseArguments)
+      emitReturnReversal();
     emitReturn();
     emitClosingBrace();
   }
   OS << "\n";
 
   CurrentRecord = nullptr;
-  return OS.str();
 }
 
 void Intrinsic::indexBody() {
   CurrentRecord = R;
 
   initVariables();
-  emitBody();
+  emitBody("");
   OS.str("");
 
   CurrentRecord = nullptr;
@@ -1796,6 +1905,7 @@ void NeonEmitter::createIntrinsic(Record *R,
   std::string Types = R->getValueAsString("Types");
   Record *OperationRec = R->getValueAsDef("Operation");
   bool CartesianProductOfTypes = R->getValueAsBit("CartesianProductOfTypes");
+  bool BigEndianSafe  = R->getValueAsBit("BigEndianSafe");
   std::string Guard = R->getValueAsString("ArchGuard");
   bool IsUnavailable = OperationRec->getValueAsBit("Unavailable");
 
@@ -1832,7 +1942,7 @@ void NeonEmitter::createIntrinsic(Record *R,
 
   for (auto &I : NewTypeSpecs) {
     Intrinsic *IT = new Intrinsic(R, Name, Proto, I.first, I.second, CK, Body,
-                                  *this, Guard, IsUnavailable);
+                                  *this, Guard, IsUnavailable, BigEndianSafe);
 
     IntrinsicMap[Name].push_back(IT);
     Out.push_back(IT);