[stubs]: Support 1->2 byte copies in CopyStringCharacters

src/code-stub-assembler.cc

Index: src/code-stub-assembler.cc
diff --git a/src/code-stub-assembler.cc b/src/code-stub-assembler.cc
index be7286e7c7365e8c064a80b6f5d98ddd3ed7ba00..170ad235e316f3f21141e296282c5252e47e402a 100644
--- a/src/code-stub-assembler.cc
+++ b/src/code-stub-assembler.cc
@@ -2073,47 +2073,66 @@ void CodeStubAssembler::CopyFixedArrayElements(
   Comment("] CopyFixedArrayElements");
 }
 
-void CodeStubAssembler::CopyStringCharacters(compiler::Node* from_string,
-                                             compiler::Node* to_string,
-                                             compiler::Node* from_index,
-                                             compiler::Node* to_index,
-                                             compiler::Node* character_count,
-                                             String::Encoding encoding,
-                                             ParameterMode mode) {
-  bool one_byte = encoding == String::ONE_BYTE_ENCODING;
-  Comment(one_byte ? "CopyStringCharacters ONE_BYTE_ENCODING"
-                   : "CopyStringCharacters TWO_BYTE_ENCODING");
-
-  ElementsKind kind = one_byte ? UINT8_ELEMENTS : UINT16_ELEMENTS;
-  int header_size = (one_byte ? SeqOneByteString::kHeaderSize
-                              : SeqTwoByteString::kHeaderSize) -
-                    kHeapObjectTag;
-  Node* from_offset = ElementOffsetFromIndex(from_index, kind, mode);
-  Node* to_offset = ElementOffsetFromIndex(to_index, kind, mode);
-  Node* byte_count = ElementOffsetFromIndex(character_count, kind, mode);
+void CodeStubAssembler::CopyStringCharacters(
+    compiler::Node* from_string, compiler::Node* to_string,
+    compiler::Node* from_index, compiler::Node* to_index,
+    compiler::Node* character_count, String::Encoding from_encoding,
+    String::Encoding to_encoding, ParameterMode mode) {
+  bool from_one_byte = from_encoding == String::ONE_BYTE_ENCODING;
+  bool to_one_byte = to_encoding == String::ONE_BYTE_ENCODING;
+  DCHECK_IMPLIES(to_one_byte, from_one_byte);
+  Comment("CopyStringCharacters %s -> %s",
+          from_one_byte ? "ONE_BYTE_ENCODING" : "TWO_BYTE_ENCODING",
+          to_one_byte ? "ONE_BYTE_ENCODING" : "TWO_BYTE_ENCODING");
+
+  ElementsKind from_kind = from_one_byte ? UINT8_ELEMENTS : UINT16_ELEMENTS;
+  ElementsKind to_kind = to_one_byte ? UINT8_ELEMENTS : UINT16_ELEMENTS;
+  STATIC_ASSERT(SeqOneByteString::kHeaderSize == SeqTwoByteString::kHeaderSize);
+  int header_size = SeqOneByteString::kHeaderSize - kHeapObjectTag;
+  Node* from_offset =
+      ElementOffsetFromIndex(from_index, from_kind, mode, header_size);
+  Node* to_offset =
+      ElementOffsetFromIndex(to_index, to_kind, mode, header_size);
+  Node* byte_count = ElementOffsetFromIndex(character_count, from_kind, mode);
   Node* limit_offset = IntPtrAddFoldConstants(from_offset, byte_count);
 
   // Prepare the fast loop
-  MachineType type = one_byte ? MachineType::Uint8() : MachineType::Uint16();
-  MachineRepresentation rep =
-      one_byte ? MachineRepresentation::kWord8 : MachineRepresentation::kWord16;
-  int increment = -(1 << ElementsKindToShiftSize(kind));
-
-  Node* to_string_adjusted = IntPtrAddFoldConstants(
-      to_string, IntPtrSubFoldConstants(to_offset, from_offset));
-  limit_offset =
-      IntPtrAddFoldConstants(limit_offset, IntPtrConstant(header_size));
-  from_offset =
-      IntPtrAddFoldConstants(from_offset, IntPtrConstant(header_size));
-
-  BuildFastLoop(MachineType::PointerRepresentation(), limit_offset, from_offset,
-                [from_string, to_string_adjusted, type, rep](
-                    CodeStubAssembler* assembler, Node* offset) {
+  MachineType type =
+      from_one_byte ? MachineType::Uint8() : MachineType::Uint16();
+  MachineRepresentation rep = to_one_byte ? MachineRepresentation::kWord8
+                                          : MachineRepresentation::kWord16;
+  int from_increment = 1 << ElementsKindToShiftSize(from_kind);
+  int to_increment = 1 << ElementsKindToShiftSize(to_kind);
+
+  Variable current_to_offset(this, MachineType::PointerRepresentation());
+  VariableList vars({&current_to_offset}, zone());
+  current_to_offset.Bind(to_offset);
+  int to_index_constant = 0, from_index_constant = 0;
+  Smi* to_index_smi = nullptr;
+  Smi* from_index_smi = nullptr;
+  bool index_same = (from_encoding == to_encoding) &&
+                    (from_index == to_index ||
+                     (ToInt32Constant(from_index, from_index_constant) &&
+                      ToInt32Constant(to_index, to_index_constant) &&
+                      from_index_constant == to_index_constant) ||
+                     (ToSmiConstant(from_index, from_index_smi) &&
+                      ToSmiConstant(to_index, to_index_smi) &&
+                      to_index_smi == from_index_smi));
+  BuildFastLoop(vars, MachineType::PointerRepresentation(), from_offset,
+                limit_offset,
+                [from_string, to_string, &current_to_offset, to_increment, type,
+                 rep, index_same](CodeStubAssembler* assembler, Node* offset) {
                   Node* value = assembler->Load(type, from_string, offset);
-                  assembler->StoreNoWriteBarrier(rep, to_string_adjusted,
-                                                 offset, value);
+                  assembler->StoreNoWriteBarrier(
+                      rep, to_string,
+                      index_same ? offset : current_to_offset.value(), value);
+                  if (!index_same) {
+                    current_to_offset.Bind(assembler->IntPtrAdd(
+                        current_to_offset.value(),
+                        assembler->IntPtrConstant(to_increment)));
+                  }
                 },
-                increment);
+                from_increment, IndexAdvanceMode::kPost);
 }
 
 Node* CodeStubAssembler::LoadElementAndPrepareForStore(Node* array,
@@ -2983,6 +3002,7 @@ Node* AllocAndCopyStringCharacters(CodeStubAssembler* a, Node* context,
         a->AllocateSeqOneByteString(context, a->SmiToWord(character_count));
     a->CopyStringCharacters(from, result, from_index, smi_zero, character_count,
                             String::ONE_BYTE_ENCODING,
+                            String::ONE_BYTE_ENCODING,
                             CodeStubAssembler::SMI_PARAMETERS);
     var_result.Bind(result);
 
@@ -2996,6 +3016,7 @@ Node* AllocAndCopyStringCharacters(CodeStubAssembler* a, Node* context,
         a->AllocateSeqTwoByteString(context, a->SmiToWord(character_count));
     a->CopyStringCharacters(from, result, from_index, smi_zero, character_count,
                             String::TWO_BYTE_ENCODING,
+                            String::TWO_BYTE_ENCODING,
                             CodeStubAssembler::SMI_PARAMETERS);
     var_result.Bind(result);
 
@@ -3297,9 +3318,11 @@ Node* CodeStubAssembler::StringAdd(Node* context, Node* left, Node* right,
       AllocateSeqOneByteString(context, new_length, SMI_PARAMETERS);
   CopyStringCharacters(left, new_string, SmiConstant(Smi::kZero),
                        SmiConstant(Smi::kZero), left_length,
-                       String::ONE_BYTE_ENCODING, SMI_PARAMETERS);
+                       String::ONE_BYTE_ENCODING, String::ONE_BYTE_ENCODING,
+                       SMI_PARAMETERS);
   CopyStringCharacters(right, new_string, SmiConstant(Smi::kZero), left_length,
-                       right_length, String::ONE_BYTE_ENCODING, SMI_PARAMETERS);
+                       right_length, String::ONE_BYTE_ENCODING,
+                       String::ONE_BYTE_ENCODING, SMI_PARAMETERS);
   result.Bind(new_string);
   Goto(&done_native);
 
@@ -3309,10 +3332,11 @@ Node* CodeStubAssembler::StringAdd(Node* context, Node* left, Node* right,
     new_string = AllocateSeqTwoByteString(context, new_length, SMI_PARAMETERS);
     CopyStringCharacters(left, new_string, SmiConstant(Smi::kZero),
                          SmiConstant(Smi::kZero), left_length,
-                         String::TWO_BYTE_ENCODING, SMI_PARAMETERS);
+                         String::TWO_BYTE_ENCODING, String::TWO_BYTE_ENCODING,
+                         SMI_PARAMETERS);
     CopyStringCharacters(right, new_string, SmiConstant(Smi::kZero),
                          left_length, right_length, String::TWO_BYTE_ENCODING,
-                         SMI_PARAMETERS);
+                         String::TWO_BYTE_ENCODING, SMI_PARAMETERS);
     result.Bind(new_string);
     Goto(&done_native);
   }
@@ -6964,12 +6988,15 @@ Node* CodeStubAssembler::CreateWeakCellInFeedbackVector(Node* feedback_vector,
 }
 
 void CodeStubAssembler::BuildFastLoop(
+    const CodeStubAssembler::VariableList& vars,
     MachineRepresentation index_rep, Node* start_index, Node* end_index,
     std::function<void(CodeStubAssembler* assembler, Node* index)> body,
     int increment, IndexAdvanceMode mode) {
   Variable var(this, index_rep);
+  VariableList vars_copy(vars, zone());
+  vars_copy.Add(&var, zone());
   var.Bind(start_index);
-  Label loop(this, &var);
+  Label loop(this, vars_copy);
   Label after_loop(this);
   // Introduce an explicit second check of the termination condition before the
   // loop that helps turbofan generate better code. If there's only a single