diff options
Diffstat (limited to 'deps/v8/src/x64/ic-x64.cc')
| -rw-r--r-- | deps/v8/src/x64/ic-x64.cc | 379 |
1 files changed, 248 insertions, 131 deletions
diff --git a/deps/v8/src/x64/ic-x64.cc b/deps/v8/src/x64/ic-x64.cc index 9d55594dcb..0632ce439f 100644 --- a/deps/v8/src/x64/ic-x64.cc +++ b/deps/v8/src/x64/ic-x64.cc @@ -1,4 +1,4 @@ -// Copyright 2011 the V8 project authors. All rights reserved. +// Copyright 2012 the V8 project authors. All rights reserved. // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions are // met: @@ -221,7 +221,7 @@ static void GenerateDictionaryStore(MacroAssembler* masm, // Update write barrier. Make sure not to clobber the value. __ movq(scratch0, value); - __ RecordWrite(elements, scratch1, scratch0); + __ RecordWrite(elements, scratch1, scratch0, kDontSaveFPRegs); } @@ -462,30 +462,58 @@ void KeyedLoadIC::GenerateGeneric(MacroAssembler* masm) { __ movl(rdi, FieldOperand(rax, String::kHashFieldOffset)); __ shr(rdi, Immediate(String::kHashShift)); __ xor_(rcx, rdi); - __ and_(rcx, Immediate(KeyedLookupCache::kCapacityMask)); + int mask = (KeyedLookupCache::kCapacityMask & KeyedLookupCache::kHashMask); + __ and_(rcx, Immediate(mask)); // Load the key (consisting of map and symbol) from the cache and // check for match. + Label load_in_object_property; + static const int kEntriesPerBucket = KeyedLookupCache::kEntriesPerBucket; + Label hit_on_nth_entry[kEntriesPerBucket]; ExternalReference cache_keys = ExternalReference::keyed_lookup_cache_keys(masm->isolate()); - __ movq(rdi, rcx); - __ shl(rdi, Immediate(kPointerSizeLog2 + 1)); - __ LoadAddress(kScratchRegister, cache_keys); - __ cmpq(rbx, Operand(kScratchRegister, rdi, times_1, 0)); + + for (int i = 0; i < kEntriesPerBucket - 1; i++) { + Label try_next_entry; + __ movq(rdi, rcx); + __ shl(rdi, Immediate(kPointerSizeLog2 + 1)); + __ LoadAddress(kScratchRegister, cache_keys); + int off = kPointerSize * i * 2; + __ cmpq(rbx, Operand(kScratchRegister, rdi, times_1, off)); + __ j(not_equal, &try_next_entry); + __ cmpq(rax, Operand(kScratchRegister, rdi, times_1, off + kPointerSize)); + __ j(equal, &hit_on_nth_entry[i]); + __ bind(&try_next_entry); + } + + int off = kPointerSize * (kEntriesPerBucket - 1) * 2; + __ cmpq(rbx, Operand(kScratchRegister, rdi, times_1, off)); __ j(not_equal, &slow); - __ cmpq(rax, Operand(kScratchRegister, rdi, times_1, kPointerSize)); + __ cmpq(rax, Operand(kScratchRegister, rdi, times_1, off + kPointerSize)); __ j(not_equal, &slow); // Get field offset, which is a 32-bit integer. ExternalReference cache_field_offsets = ExternalReference::keyed_lookup_cache_field_offsets(masm->isolate()); - __ LoadAddress(kScratchRegister, cache_field_offsets); - __ movl(rdi, Operand(kScratchRegister, rcx, times_4, 0)); - __ movzxbq(rcx, FieldOperand(rbx, Map::kInObjectPropertiesOffset)); - __ subq(rdi, rcx); - __ j(above_equal, &property_array_property); + + // Hit on nth entry. + for (int i = kEntriesPerBucket - 1; i >= 0; i--) { + __ bind(&hit_on_nth_entry[i]); + if (i != 0) { + __ addl(rcx, Immediate(i)); + } + __ LoadAddress(kScratchRegister, cache_field_offsets); + __ movl(rdi, Operand(kScratchRegister, rcx, times_4, 0)); + __ movzxbq(rcx, FieldOperand(rbx, Map::kInObjectPropertiesOffset)); + __ subq(rdi, rcx); + __ j(above_equal, &property_array_property); + if (i != 0) { + __ jmp(&load_in_object_property); + } + } // Load in-object property. + __ bind(&load_in_object_property); __ movzxbq(rcx, FieldOperand(rbx, Map::kInstanceSizeOffset)); __ addq(rcx, rdi); __ movq(rax, FieldOperand(rdx, rcx, times_pointer_size, 0)); @@ -531,14 +559,12 @@ void KeyedLoadIC::GenerateString(MacroAssembler* masm) { Register receiver = rdx; Register index = rax; - Register scratch1 = rbx; - Register scratch2 = rcx; + Register scratch = rcx; Register result = rax; StringCharAtGenerator char_at_generator(receiver, index, - scratch1, - scratch2, + scratch, result, &miss, // When not a string. &miss, // When not a number. @@ -606,45 +632,42 @@ void KeyedStoreIC::GenerateGeneric(MacroAssembler* masm, // -- rdx : receiver // -- rsp[0] : return address // ----------------------------------- - Label slow, slow_with_tagged_index, fast, array, extra; + Label slow, slow_with_tagged_index, fast, array, extra, check_extra_double; + Label fast_object_with_map_check, fast_object_without_map_check; + Label fast_double_with_map_check, fast_double_without_map_check; + Label transition_smi_elements, finish_object_store, non_double_value; + Label transition_double_elements; // Check that the object isn't a smi. __ JumpIfSmi(rdx, &slow_with_tagged_index); // Get the map from the receiver. - __ movq(rbx, FieldOperand(rdx, HeapObject::kMapOffset)); + __ movq(r9, FieldOperand(rdx, HeapObject::kMapOffset)); // Check that the receiver does not require access checks. We need // to do this because this generic stub does not perform map checks. - __ testb(FieldOperand(rbx, Map::kBitFieldOffset), + __ testb(FieldOperand(r9, Map::kBitFieldOffset), Immediate(1 << Map::kIsAccessCheckNeeded)); __ j(not_zero, &slow_with_tagged_index); // Check that the key is a smi. __ JumpIfNotSmi(rcx, &slow_with_tagged_index); __ SmiToInteger32(rcx, rcx); - __ CmpInstanceType(rbx, JS_ARRAY_TYPE); + __ CmpInstanceType(r9, JS_ARRAY_TYPE); __ j(equal, &array); // Check that the object is some kind of JSObject. - __ CmpInstanceType(rbx, FIRST_JS_RECEIVER_TYPE); + __ CmpInstanceType(r9, FIRST_JS_OBJECT_TYPE); __ j(below, &slow); - __ CmpInstanceType(rbx, JS_PROXY_TYPE); - __ j(equal, &slow); - __ CmpInstanceType(rbx, JS_FUNCTION_PROXY_TYPE); - __ j(equal, &slow); // Object case: Check key against length in the elements array. // rax: value // rdx: JSObject // rcx: index __ movq(rbx, FieldOperand(rdx, JSObject::kElementsOffset)); - // Check that the object is in fast mode and writable. - __ CompareRoot(FieldOperand(rbx, HeapObject::kMapOffset), - Heap::kFixedArrayMapRootIndex); - __ j(not_equal, &slow); + // Check array bounds. __ SmiCompareInteger32(FieldOperand(rbx, FixedArray::kLengthOffset), rcx); // rax: value // rbx: FixedArray // rcx: index - __ j(above, &fast); + __ j(above, &fast_object_with_map_check); // Slow case: call runtime. __ bind(&slow); @@ -666,9 +689,20 @@ void KeyedStoreIC::GenerateGeneric(MacroAssembler* masm, __ SmiCompareInteger32(FieldOperand(rbx, FixedArray::kLengthOffset), rcx); __ j(below_equal, &slow); // Increment index to get new length. + __ movq(rdi, FieldOperand(rbx, HeapObject::kMapOffset)); + __ CompareRoot(rdi, Heap::kFixedArrayMapRootIndex); + __ j(not_equal, &check_extra_double); + __ leal(rdi, Operand(rcx, 1)); + __ Integer32ToSmiField(FieldOperand(rdx, JSArray::kLengthOffset), rdi); + __ jmp(&fast_object_without_map_check); + + __ bind(&check_extra_double); + // rdi: elements array's map + __ CompareRoot(rdi, Heap::kFixedDoubleArrayMapRootIndex); + __ j(not_equal, &slow); __ leal(rdi, Operand(rcx, 1)); __ Integer32ToSmiField(FieldOperand(rdx, JSArray::kLengthOffset), rdi); - __ jmp(&fast); + __ jmp(&fast_double_without_map_check); // Array case: Get the length and the elements array from the JS // array. Check that the array is in fast mode (and writable); if it @@ -678,9 +712,6 @@ void KeyedStoreIC::GenerateGeneric(MacroAssembler* masm, // rdx: receiver (a JSArray) // rcx: index __ movq(rbx, FieldOperand(rdx, JSObject::kElementsOffset)); - __ CompareRoot(FieldOperand(rbx, HeapObject::kMapOffset), - Heap::kFixedArrayMapRootIndex); - __ j(not_equal, &slow); // Check the key against the length in the array, compute the // address to store into and fall through to fast case. @@ -688,30 +719,100 @@ void KeyedStoreIC::GenerateGeneric(MacroAssembler* masm, __ j(below_equal, &extra); // Fast case: Do the store. - __ bind(&fast); + __ bind(&fast_object_with_map_check); // rax: value // rbx: receiver's elements array (a FixedArray) // rcx: index + // rdx: receiver (a JSArray) + __ movq(rdi, FieldOperand(rbx, HeapObject::kMapOffset)); + __ CompareRoot(rdi, Heap::kFixedArrayMapRootIndex); + __ j(not_equal, &fast_double_with_map_check); + __ bind(&fast_object_without_map_check); + // Smi stores don't require further checks. Label non_smi_value; + __ JumpIfNotSmi(rax, &non_smi_value); + // It's irrelevant whether array is smi-only or not when writing a smi. __ movq(FieldOperand(rbx, rcx, times_pointer_size, FixedArray::kHeaderSize), rax); - __ JumpIfNotSmi(rax, &non_smi_value, Label::kNear); __ ret(0); + __ bind(&non_smi_value); - // Slow case that needs to retain rcx for use by RecordWrite. - // Update write barrier for the elements array address. - __ movq(rdx, rax); - __ RecordWriteNonSmi(rbx, 0, rdx, rcx); + // Writing a non-smi, check whether array allows non-smi elements. + // r9: receiver's map + __ CheckFastObjectElements(r9, &transition_smi_elements); + __ bind(&finish_object_store); + __ movq(FieldOperand(rbx, rcx, times_pointer_size, FixedArray::kHeaderSize), + rax); + __ movq(rdx, rax); // Preserve the value which is returned. + __ RecordWriteArray( + rbx, rdx, rcx, kDontSaveFPRegs, EMIT_REMEMBERED_SET, OMIT_SMI_CHECK); __ ret(0); + + __ bind(&fast_double_with_map_check); + // Check for fast double array case. If this fails, call through to the + // runtime. + // rdi: elements array's map + __ CompareRoot(rdi, Heap::kFixedDoubleArrayMapRootIndex); + __ j(not_equal, &slow); + __ bind(&fast_double_without_map_check); + // If the value is a number, store it as a double in the FastDoubleElements + // array. + __ StoreNumberToDoubleElements(rax, rbx, rcx, xmm0, + &transition_double_elements); + __ ret(0); + + __ bind(&transition_smi_elements); + __ movq(rbx, FieldOperand(rdx, HeapObject::kMapOffset)); + + // Transition the array appropriately depending on the value type. + __ movq(r9, FieldOperand(rax, HeapObject::kMapOffset)); + __ CompareRoot(r9, Heap::kHeapNumberMapRootIndex); + __ j(not_equal, &non_double_value); + + // Value is a double. Transition FAST_SMI_ONLY_ELEMENTS -> + // FAST_DOUBLE_ELEMENTS and complete the store. + __ LoadTransitionedArrayMapConditional(FAST_SMI_ONLY_ELEMENTS, + FAST_DOUBLE_ELEMENTS, + rbx, + rdi, + &slow); + ElementsTransitionGenerator::GenerateSmiOnlyToDouble(masm, &slow); + __ movq(rbx, FieldOperand(rdx, JSObject::kElementsOffset)); + __ jmp(&fast_double_without_map_check); + + __ bind(&non_double_value); + // Value is not a double, FAST_SMI_ONLY_ELEMENTS -> FAST_ELEMENTS + __ LoadTransitionedArrayMapConditional(FAST_SMI_ONLY_ELEMENTS, + FAST_ELEMENTS, + rbx, + rdi, + &slow); + ElementsTransitionGenerator::GenerateSmiOnlyToObject(masm); + __ movq(rbx, FieldOperand(rdx, JSObject::kElementsOffset)); + __ jmp(&finish_object_store); + + __ bind(&transition_double_elements); + // Elements are FAST_DOUBLE_ELEMENTS, but value is an Object that's not a + // HeapNumber. Make sure that the receiver is a Array with FAST_ELEMENTS and + // transition array from FAST_DOUBLE_ELEMENTS to FAST_ELEMENTS + __ movq(rbx, FieldOperand(rdx, HeapObject::kMapOffset)); + __ LoadTransitionedArrayMapConditional(FAST_DOUBLE_ELEMENTS, + FAST_ELEMENTS, + rbx, + rdi, + &slow); + ElementsTransitionGenerator::GenerateDoubleToObject(masm, &slow); + __ movq(rbx, FieldOperand(rdx, JSObject::kElementsOffset)); + __ jmp(&finish_object_store); } // The generated code does not accept smi keys. // The generated code falls through if both probes miss. -static void GenerateMonomorphicCacheProbe(MacroAssembler* masm, - int argc, - Code::Kind kind, - Code::ExtraICState extra_ic_state) { +void CallICBase::GenerateMonomorphicCacheProbe(MacroAssembler* masm, + int argc, + Code::Kind kind, + Code::ExtraICState extra_state) { // ----------- S t a t e ------------- // rcx : function name // rdx : receiver @@ -721,7 +822,7 @@ static void GenerateMonomorphicCacheProbe(MacroAssembler* masm, // Probe the stub cache. Code::Flags flags = Code::ComputeFlags(kind, MONOMORPHIC, - extra_ic_state, + extra_state, NORMAL, argc); Isolate::Current()->stub_cache()->GenerateProbe(masm, flags, rdx, rcx, rbx, @@ -794,7 +895,7 @@ static void GenerateFunctionTailCall(MacroAssembler* masm, // The generated code falls through if the call should be handled by runtime. -static void GenerateCallNormal(MacroAssembler* masm, int argc) { +void CallICBase::GenerateNormal(MacroAssembler* masm, int argc) { // ----------- S t a t e ------------- // rcx : function name // rsp[0] : return address @@ -821,10 +922,10 @@ static void GenerateCallNormal(MacroAssembler* masm, int argc) { } -static void GenerateCallMiss(MacroAssembler* masm, - int argc, - IC::UtilityId id, - Code::ExtraICState extra_ic_state) { +void CallICBase::GenerateMiss(MacroAssembler* masm, + int argc, + IC::UtilityId id, + Code::ExtraICState extra_state) { // ----------- S t a t e ------------- // rcx : function name // rsp[0] : return address @@ -846,21 +947,22 @@ static void GenerateCallMiss(MacroAssembler* masm, __ movq(rdx, Operand(rsp, (argc + 1) * kPointerSize)); // Enter an internal frame. - __ EnterInternalFrame(); + { + FrameScope scope(masm, StackFrame::INTERNAL); - // Push the receiver and the name of the function. - __ push(rdx); - __ push(rcx); + // Push the receiver and the name of the function. + __ push(rdx); + __ push(rcx); - // Call the entry. - CEntryStub stub(1); - __ Set(rax, 2); - __ LoadAddress(rbx, ExternalReference(IC_Utility(id), masm->isolate())); - __ CallStub(&stub); + // Call the entry. + CEntryStub stub(1); + __ Set(rax, 2); + __ LoadAddress(rbx, ExternalReference(IC_Utility(id), masm->isolate())); + __ CallStub(&stub); - // Move result to rdi and exit the internal frame. - __ movq(rdi, rax); - __ LeaveInternalFrame(); + // Move result to rdi and exit the internal frame. + __ movq(rdi, rax); + } // Check if the receiver is a global object of some sort. // This can happen only for regular CallIC but not KeyedCallIC. @@ -881,7 +983,7 @@ static void GenerateCallMiss(MacroAssembler* masm, } // Invoke the function. - CallKind call_kind = CallICBase::Contextual::decode(extra_ic_state) + CallKind call_kind = CallICBase::Contextual::decode(extra_state) ? CALL_AS_FUNCTION : CALL_AS_METHOD; ParameterCount actual(argc); @@ -913,39 +1015,6 @@ void CallIC::GenerateMegamorphic(MacroAssembler* masm, } -void CallIC::GenerateNormal(MacroAssembler* masm, int argc) { - // ----------- S t a t e ------------- - // rcx : function name - // rsp[0] : return address - // rsp[8] : argument argc - // rsp[16] : argument argc - 1 - // ... - // rsp[argc * 8] : argument 1 - // rsp[(argc + 1) * 8] : argument 0 = receiver - // ----------------------------------- - - GenerateCallNormal(masm, argc); - GenerateMiss(masm, argc, Code::kNoExtraICState); -} - - -void CallIC::GenerateMiss(MacroAssembler* masm, - int argc, - Code::ExtraICState extra_ic_state) { - // ----------- S t a t e ------------- - // rcx : function name - // rsp[0] : return address - // rsp[8] : argument argc - // rsp[16] : argument argc - 1 - // ... - // rsp[argc * 8] : argument 1 - // rsp[(argc + 1) * 8] : argument 0 = receiver - // ----------------------------------- - - GenerateCallMiss(masm, argc, IC::kCallIC_Miss, extra_ic_state); -} - - void KeyedCallIC::GenerateMegamorphic(MacroAssembler* masm, int argc) { // ----------- S t a t e ------------- // rcx : function name @@ -1002,13 +1071,14 @@ void KeyedCallIC::GenerateMegamorphic(MacroAssembler* masm, int argc) { // This branch is taken when calling KeyedCallIC_Miss is neither required // nor beneficial. __ IncrementCounter(counters->keyed_call_generic_slow_load(), 1); - __ EnterInternalFrame(); - __ push(rcx); // save the key - __ push(rdx); // pass the receiver - __ push(rcx); // pass the key - __ CallRuntime(Runtime::kKeyedGetProperty, 2); - __ pop(rcx); // restore the key - __ LeaveInternalFrame(); + { + FrameScope scope(masm, StackFrame::INTERNAL); + __ push(rcx); // save the key + __ push(rdx); // pass the receiver + __ push(rcx); // pass the key + __ CallRuntime(Runtime::kKeyedGetProperty, 2); + __ pop(rcx); // restore the key + } __ movq(rdi, rax); __ jmp(&do_call); @@ -1072,27 +1142,12 @@ void KeyedCallIC::GenerateNormal(MacroAssembler* masm, int argc) { __ JumpIfSmi(rcx, &miss); Condition cond = masm->IsObjectStringType(rcx, rax, rax); __ j(NegateCondition(cond), &miss); - GenerateCallNormal(masm, argc); + CallICBase::GenerateNormal(masm, argc); __ bind(&miss); GenerateMiss(masm, argc); } -void KeyedCallIC::GenerateMiss(MacroAssembler* masm, int argc) { - // ----------- S t a t e ------------- - // rcx : function name - // rsp[0] : return address - // rsp[8] : argument argc - // rsp[16] : argument argc - 1 - // ... - // rsp[argc * 8] : argument 1 - // rsp[(argc + 1) * 8] : argument 0 = receiver - // ----------------------------------- - - GenerateCallMiss(masm, argc, IC::kKeyedCallIC_Miss, Code::kNoExtraICState); -} - - static Operand GenerateMappedArgumentsLookup(MacroAssembler* masm, Register object, Register key, @@ -1212,7 +1267,12 @@ void KeyedStoreIC::GenerateNonStrictArguments(MacroAssembler* masm) { __ movq(mapped_location, rax); __ lea(r9, mapped_location); __ movq(r8, rax); - __ RecordWrite(rbx, r9, r8); + __ RecordWrite(rbx, + r9, + r8, + kDontSaveFPRegs, + EMIT_REMEMBERED_SET, + INLINE_SMI_CHECK); __ Ret(); __ bind(¬in); // The unmapped lookup expects that the parameter map is in rbx. @@ -1221,7 +1281,12 @@ void KeyedStoreIC::GenerateNonStrictArguments(MacroAssembler* masm) { __ movq(unmapped_location, rax); __ lea(r9, unmapped_location); __ movq(r8, rax); - __ RecordWrite(rbx, r9, r8); + __ RecordWrite(rbx, + r9, + r8, + kDontSaveFPRegs, + EMIT_REMEMBERED_SET, + INLINE_SMI_CHECK); __ Ret(); __ bind(&slow); GenerateMiss(masm, false); @@ -1408,11 +1473,10 @@ void StoreIC::GenerateArrayLength(MacroAssembler* masm) { // -- rsp[0] : return address // ----------------------------------- // - // This accepts as a receiver anything JSObject::SetElementsLength accepts - // (currently anything except for external and pixel arrays which means - // anything with elements of FixedArray type.), but currently is restricted - // to JSArray. - // Value must be a number, but only smis are accepted as the most common case. + // This accepts as a receiver anything JSArray::SetElementsLength accepts + // (currently anything except for external arrays which means anything with + // elements of FixedArray type). Value must be a number, but only smis are + // accepted as the most common case. Label miss; @@ -1434,6 +1498,13 @@ void StoreIC::GenerateArrayLength(MacroAssembler* masm) { __ CmpObjectType(scratch, FIXED_ARRAY_TYPE, scratch); __ j(not_equal, &miss); + // Check that the array has fast properties, otherwise the length + // property might have been redefined. + __ movq(scratch, FieldOperand(receiver, JSArray::kPropertiesOffset)); + __ CompareRoot(FieldOperand(scratch, FixedArray::kMapOffset), + Heap::kHashTableMapRootIndex); + __ j(equal, &miss); + // Check that value is a smi. __ JumpIfNotSmi(value, &miss); @@ -1562,6 +1633,51 @@ void KeyedStoreIC::GenerateMiss(MacroAssembler* masm, bool force_generic) { } +void KeyedStoreIC::GenerateTransitionElementsSmiToDouble(MacroAssembler* masm) { + // ----------- S t a t e ------------- + // -- rbx : target map + // -- rdx : receiver + // -- rsp[0] : return address + // ----------------------------------- + // Must return the modified receiver in eax. + if (!FLAG_trace_elements_transitions) { + Label fail; + ElementsTransitionGenerator::GenerateSmiOnlyToDouble(masm, &fail); + __ movq(rax, rdx); + __ Ret(); + __ bind(&fail); + } + + __ pop(rbx); + __ push(rdx); + __ push(rbx); // return address + __ TailCallRuntime(Runtime::kTransitionElementsSmiToDouble, 1, 1); +} + + +void KeyedStoreIC::GenerateTransitionElementsDoubleToObject( + MacroAssembler* masm) { + // ----------- S t a t e ------------- + // -- rbx : target map + // -- rdx : receiver + // -- rsp[0] : return address + // ----------------------------------- + // Must return the modified receiver in eax. + if (!FLAG_trace_elements_transitions) { + Label fail; + ElementsTransitionGenerator::GenerateDoubleToObject(masm, &fail); + __ movq(rax, rdx); + __ Ret(); + __ bind(&fail); + } + + __ pop(rbx); + __ push(rdx); + __ push(rbx); // return address + __ TailCallRuntime(Runtime::kTransitionElementsDoubleToObject, 1, 1); +} + + #undef __ @@ -1573,11 +1689,9 @@ Condition CompareIC::ComputeCondition(Token::Value op) { case Token::LT: return less; case Token::GT: - // Reverse left and right operands to obtain ECMA-262 conversion order. - return less; + return greater; case Token::LTE: - // Reverse left and right operands to obtain ECMA-262 conversion order. - return greater_equal; + return less_equal; case Token::GTE: return greater_equal; default: @@ -1609,6 +1723,9 @@ void CompareIC::UpdateCaches(Handle<Object> x, Handle<Object> y) { rewritten = stub.GetCode(); } else { ICCompareStub stub(op_, state); + if (state == KNOWN_OBJECTS) { + stub.set_known_map(Handle<Map>(Handle<JSObject>::cast(x)->map())); + } rewritten = stub.GetCode(); } set_target(*rewritten); |