OLD | NEW |
(Empty) | |
| 1 // Copyright 2017 the V8 project authors. All rights reserved. |
| 2 // Use of this source code is governed by a BSD-style license that can be |
| 3 // found in the LICENSE file. |
| 4 |
| 5 #include "src/wasm/wasm-code-specialization.h" |
| 6 |
| 7 #include "src/assembler-inl.h" |
| 8 #include "src/objects-inl.h" |
| 9 #include "src/source-position-table.h" |
| 10 #include "src/wasm/decoder.h" |
| 11 #include "src/wasm/wasm-module.h" |
| 12 #include "src/wasm/wasm-opcodes.h" |
| 13 |
| 14 using namespace v8::internal; |
| 15 using namespace v8::internal::wasm; |
| 16 |
| 17 namespace { |
| 18 |
| 19 int ExtractDirectCallIndex(wasm::Decoder& decoder, const byte* pc) { |
| 20 DCHECK_EQ(static_cast<int>(kExprCallFunction), static_cast<int>(*pc)); |
| 21 decoder.Reset(pc + 1, pc + 6); |
| 22 uint32_t call_idx = decoder.consume_u32v("call index"); |
| 23 DCHECK(decoder.ok()); |
| 24 DCHECK_GE(kMaxInt, call_idx); |
| 25 return static_cast<int>(call_idx); |
| 26 } |
| 27 |
| 28 int AdvanceSourcePositionTableIterator(SourcePositionTableIterator& iterator, |
| 29 size_t offset_l) { |
| 30 DCHECK_GE(kMaxInt, offset_l); |
| 31 int offset = static_cast<int>(offset_l); |
| 32 DCHECK(!iterator.done()); |
| 33 int byte_pos; |
| 34 do { |
| 35 byte_pos = iterator.source_position().ScriptOffset(); |
| 36 iterator.Advance(); |
| 37 } while (!iterator.done() && iterator.code_offset() <= offset); |
| 38 return byte_pos; |
| 39 } |
| 40 |
| 41 class PatchDirectCallsHelper { |
| 42 public: |
| 43 PatchDirectCallsHelper(WasmInstanceObject* instance, Code* code) |
| 44 : source_pos_it(code->source_position_table()), |
| 45 decoder(nullptr, nullptr) { |
| 46 FixedArray* deopt_data = code->deoptimization_data(); |
| 47 DCHECK_EQ(2, deopt_data->length()); |
| 48 WasmCompiledModule* comp_mod = instance->compiled_module(); |
| 49 int func_index = Smi::cast(deopt_data->get(1))->value(); |
| 50 func_bytes = comp_mod->module_bytes()->GetChars() + |
| 51 comp_mod->module()->functions[func_index].code_start_offset; |
| 52 } |
| 53 |
| 54 SourcePositionTableIterator source_pos_it; |
| 55 Decoder decoder; |
| 56 const byte* func_bytes; |
| 57 }; |
| 58 |
| 59 } // namespace |
| 60 |
| 61 CodeSpecialization::CodeSpecialization(Isolate* isolate, Zone* zone) |
| 62 : objects_to_relocate(isolate->heap(), zone) {} |
| 63 |
| 64 CodeSpecialization::~CodeSpecialization() {} |
| 65 |
| 66 void CodeSpecialization::RelocateMemoryReferences(Address old_start, |
| 67 uint32_t old_size, |
| 68 Address new_start, |
| 69 uint32_t new_size) { |
| 70 DCHECK(old_mem_start == 0 && new_mem_start == 0); |
| 71 DCHECK(old_start != 0 || new_start != 0); |
| 72 old_mem_start = old_start; |
| 73 old_mem_size = old_size; |
| 74 new_mem_start = new_start; |
| 75 new_mem_size = new_size; |
| 76 } |
| 77 |
| 78 void CodeSpecialization::RelocateGlobals(Address old_start, Address new_start) { |
| 79 DCHECK(old_globals_start == 0 && new_globals_start == 0); |
| 80 DCHECK(old_start != 0 || new_start != 0); |
| 81 old_globals_start = old_start; |
| 82 new_globals_start = new_start; |
| 83 } |
| 84 |
| 85 void CodeSpecialization::PatchTableSize(uint32_t old_size, uint32_t new_size) { |
| 86 DCHECK(old_function_table_size == 0 && new_function_table_size == 0); |
| 87 DCHECK(old_size != 0 || new_size != 0); |
| 88 old_function_table_size = old_size; |
| 89 new_function_table_size = new_size; |
| 90 } |
| 91 |
| 92 void CodeSpecialization::RelocateDirectCalls( |
| 93 Handle<WasmInstanceObject> instance) { |
| 94 DCHECK(relocate_direct_calls_instance.is_null()); |
| 95 DCHECK(!instance.is_null()); |
| 96 relocate_direct_calls_instance = instance; |
| 97 } |
| 98 |
| 99 void CodeSpecialization::RelocateObject(Handle<Object> old_obj, |
| 100 Handle<Object> new_obj) { |
| 101 DCHECK(!old_obj.is_null() && !new_obj.is_null()); |
| 102 has_objects_to_relocate = true; |
| 103 objects_to_relocate.Set(*old_obj, new_obj); |
| 104 } |
| 105 |
| 106 bool CodeSpecialization::ApplyToWholeInstance( |
| 107 WasmInstanceObject* instance, ICacheFlushMode icache_flush_mode) { |
| 108 DisallowHeapAllocation no_gc; |
| 109 WasmCompiledModule* compiled_module = instance->compiled_module(); |
| 110 FixedArray* code_table = compiled_module->ptr_to_code_table(); |
| 111 WasmModule* module = compiled_module->module(); |
| 112 std::vector<WasmFunction>* wasm_functions = |
| 113 &compiled_module->module()->functions; |
| 114 DCHECK_EQ(wasm_functions->size() + |
| 115 compiled_module->module()->num_exported_functions, |
| 116 code_table->length()); |
| 117 |
| 118 bool changed = false; |
| 119 int func_index = module->num_imported_functions; |
| 120 |
| 121 // Patch all wasm functions. |
| 122 for (int num_wasm_functions = static_cast<int>(wasm_functions->size()); |
| 123 func_index < num_wasm_functions; ++func_index) { |
| 124 Code* wasm_function = Code::cast(code_table->get(func_index)); |
| 125 changed |= ApplyToWasmCode(wasm_function, icache_flush_mode); |
| 126 } |
| 127 |
| 128 // Patch all exported functions. |
| 129 for (auto exp : module->export_table) { |
| 130 if (exp.kind != kExternalFunction) continue; |
| 131 Code* export_wrapper = Code::cast(code_table->get(func_index)); |
| 132 DCHECK_EQ(Code::JS_TO_WASM_FUNCTION, export_wrapper->kind()); |
| 133 // There must be exactly one call to WASM_FUNCTION or WASM_TO_JS_FUNCTION. |
| 134 int num_wasm_calls = 0; |
| 135 for (RelocIterator it(export_wrapper, |
| 136 RelocInfo::ModeMask(RelocInfo::CODE_TARGET)); |
| 137 !it.done(); it.next()) { |
| 138 DCHECK(RelocInfo::IsCodeTarget(it.rinfo()->rmode())); |
| 139 Code* code = Code::GetCodeFromTargetAddress(it.rinfo()->target_address()); |
| 140 // Ignore calls to other builtins like ToNumber. |
| 141 if (code->kind() != Code::WASM_FUNCTION && |
| 142 code->kind() != Code::WASM_TO_JS_FUNCTION && |
| 143 code->builtin_index() != Builtins::kIllegal) |
| 144 continue; |
| 145 ++num_wasm_calls; |
| 146 Code* new_code = Code::cast(code_table->get(exp.index)); |
| 147 DCHECK(new_code->kind() == Code::WASM_FUNCTION || |
| 148 new_code->kind() == Code::WASM_TO_JS_FUNCTION); |
| 149 it.rinfo()->set_target_address(new_code->instruction_start(), |
| 150 UPDATE_WRITE_BARRIER, SKIP_ICACHE_FLUSH); |
| 151 changed = true; |
| 152 } |
| 153 DCHECK_EQ(1, num_wasm_calls); |
| 154 func_index++; |
| 155 } |
| 156 DCHECK_EQ(code_table->length(), func_index); |
| 157 return changed; |
| 158 } |
| 159 |
| 160 bool CodeSpecialization::ApplyToWasmCode(Code* code, |
| 161 ICacheFlushMode icache_flush_mode) { |
| 162 DisallowHeapAllocation no_gc; |
| 163 DCHECK_EQ(Code::WASM_FUNCTION, code->kind()); |
| 164 |
| 165 bool reloc_mem = old_mem_start || new_mem_start; |
| 166 bool reloc_globals = old_globals_start || new_globals_start; |
| 167 bool patch_table_size = old_function_table_size || new_function_table_size; |
| 168 bool reloc_direct_calls = !relocate_direct_calls_instance.is_null(); |
| 169 bool reloc_objects = has_objects_to_relocate; |
| 170 |
| 171 int reloc_mode = 0; |
| 172 auto add_mode = [&reloc_mode](bool cond, RelocInfo::Mode mode) { |
| 173 if (cond) reloc_mode |= RelocInfo::ModeMask(mode); |
| 174 }; |
| 175 add_mode(reloc_mem, RelocInfo::WASM_MEMORY_REFERENCE); |
| 176 add_mode(reloc_mem, RelocInfo::WASM_MEMORY_SIZE_REFERENCE); |
| 177 add_mode(reloc_globals, RelocInfo::WASM_GLOBAL_REFERENCE); |
| 178 add_mode(patch_table_size, RelocInfo::WASM_FUNCTION_TABLE_SIZE_REFERENCE); |
| 179 add_mode(reloc_direct_calls, RelocInfo::CODE_TARGET); |
| 180 add_mode(reloc_objects, RelocInfo::EMBEDDED_OBJECT); |
| 181 |
| 182 std::unique_ptr<PatchDirectCallsHelper> patch_direct_calls_helper; |
| 183 bool changed = false; |
| 184 |
| 185 for (RelocIterator it(code, reloc_mode); !it.done(); it.next()) { |
| 186 RelocInfo::Mode mode = it.rinfo()->rmode(); |
| 187 switch (mode) { |
| 188 case RelocInfo::WASM_MEMORY_REFERENCE: |
| 189 case RelocInfo::WASM_MEMORY_SIZE_REFERENCE: |
| 190 DCHECK(reloc_mem); |
| 191 it.rinfo()->update_wasm_memory_reference(old_mem_start, new_mem_start, |
| 192 old_mem_size, new_mem_size, |
| 193 icache_flush_mode); |
| 194 changed = true; |
| 195 break; |
| 196 case RelocInfo::WASM_GLOBAL_REFERENCE: |
| 197 DCHECK(reloc_globals); |
| 198 it.rinfo()->update_wasm_global_reference( |
| 199 old_globals_start, new_globals_start, icache_flush_mode); |
| 200 changed = true; |
| 201 break; |
| 202 case RelocInfo::CODE_TARGET: { |
| 203 DCHECK(reloc_direct_calls); |
| 204 Code* old_code = |
| 205 Code::GetCodeFromTargetAddress(it.rinfo()->target_address()); |
| 206 // Skip everything which is not a wasm call (stack checks, traps, ...). |
| 207 if (old_code->kind() != Code::WASM_FUNCTION && |
| 208 old_code->kind() != Code::WASM_TO_JS_FUNCTION && |
| 209 old_code->builtin_index() != Builtins::kIllegal) |
| 210 continue; |
| 211 // Iterate simultaneously over the relocation information and the source |
| 212 // position table. For each call in the reloc info, move the source |
| 213 // position iterator forward to that position to find the byte offset of |
| 214 // the respective call. Then extract the call index from the module wire |
| 215 // bytes to find the new compiled function. |
| 216 size_t offset = it.rinfo()->pc() - code->instruction_start(); |
| 217 if (!patch_direct_calls_helper) { |
| 218 patch_direct_calls_helper.reset(new PatchDirectCallsHelper( |
| 219 *relocate_direct_calls_instance, code)); |
| 220 } |
| 221 int byte_pos = AdvanceSourcePositionTableIterator( |
| 222 patch_direct_calls_helper->source_pos_it, offset); |
| 223 int called_func_index = ExtractDirectCallIndex( |
| 224 patch_direct_calls_helper->decoder, |
| 225 patch_direct_calls_helper->func_bytes + byte_pos); |
| 226 FixedArray* code_table = |
| 227 relocate_direct_calls_instance->compiled_module() |
| 228 ->ptr_to_code_table(); |
| 229 Code* new_code = Code::cast(code_table->get(called_func_index)); |
| 230 it.rinfo()->set_target_address(new_code->instruction_start(), |
| 231 UPDATE_WRITE_BARRIER, icache_flush_mode); |
| 232 changed = true; |
| 233 } break; |
| 234 case RelocInfo::EMBEDDED_OBJECT: { |
| 235 DCHECK(reloc_objects); |
| 236 Object* old = it.rinfo()->target_object(); |
| 237 Handle<Object>* new_obj = objects_to_relocate.Find(old); |
| 238 if (new_obj) { |
| 239 it.rinfo()->set_target_object(**new_obj, UPDATE_WRITE_BARRIER, |
| 240 icache_flush_mode); |
| 241 changed = true; |
| 242 } |
| 243 } break; |
| 244 case RelocInfo::WASM_FUNCTION_TABLE_SIZE_REFERENCE: |
| 245 DCHECK(patch_table_size); |
| 246 it.rinfo()->update_wasm_function_table_size_reference( |
| 247 old_function_table_size, new_function_table_size, |
| 248 icache_flush_mode); |
| 249 changed = true; |
| 250 break; |
| 251 default: |
| 252 UNREACHABLE(); |
| 253 } |
| 254 } |
| 255 |
| 256 return changed; |
| 257 } |
OLD | NEW |