Lines Matching full:invoke
62 bool IntrinsicLocationsBuilderX86::TryDispatch(HInvoke* invoke) { in TryDispatch() argument
63 Dispatch(invoke); in TryDispatch()
64 LocationSummary* res = invoke->GetLocations(); in TryDispatch()
151 static void CreateFPToIntLocations(ArenaAllocator* allocator, HInvoke* invoke, bool is64bit) { in CreateFPToIntLocations() argument
153 new (allocator) LocationSummary(invoke, LocationSummary::kNoCall, kIntrinsified); in CreateFPToIntLocations()
161 static void CreateIntToFPLocations(ArenaAllocator* allocator, HInvoke* invoke, bool is64bit) { in CreateIntToFPLocations() argument
163 new (allocator) LocationSummary(invoke, LocationSummary::kNoCall, kIntrinsified); in CreateIntToFPLocations()
203 void IntrinsicLocationsBuilderX86::VisitDoubleDoubleToRawLongBits(HInvoke* invoke) { in VisitDoubleDoubleToRawLongBits() argument
204 CreateFPToIntLocations(allocator_, invoke, /* is64bit= */ true); in VisitDoubleDoubleToRawLongBits()
206 void IntrinsicLocationsBuilderX86::VisitDoubleLongBitsToDouble(HInvoke* invoke) { in VisitDoubleLongBitsToDouble() argument
207 CreateIntToFPLocations(allocator_, invoke, /* is64bit= */ true); in VisitDoubleLongBitsToDouble()
210 void IntrinsicCodeGeneratorX86::VisitDoubleDoubleToRawLongBits(HInvoke* invoke) { in VisitDoubleDoubleToRawLongBits() argument
211 MoveFPToInt(invoke->GetLocations(), /* is64bit= */ true, GetAssembler()); in VisitDoubleDoubleToRawLongBits()
213 void IntrinsicCodeGeneratorX86::VisitDoubleLongBitsToDouble(HInvoke* invoke) { in VisitDoubleLongBitsToDouble() argument
214 MoveIntToFP(invoke->GetLocations(), /* is64bit= */ true, GetAssembler()); in VisitDoubleLongBitsToDouble()
217 void IntrinsicLocationsBuilderX86::VisitFloatFloatToRawIntBits(HInvoke* invoke) { in VisitFloatFloatToRawIntBits() argument
218 CreateFPToIntLocations(allocator_, invoke, /* is64bit= */ false); in VisitFloatFloatToRawIntBits()
220 void IntrinsicLocationsBuilderX86::VisitFloatIntBitsToFloat(HInvoke* invoke) { in VisitFloatIntBitsToFloat() argument
221 CreateIntToFPLocations(allocator_, invoke, /* is64bit= */ false); in VisitFloatIntBitsToFloat()
224 void IntrinsicCodeGeneratorX86::VisitFloatFloatToRawIntBits(HInvoke* invoke) { in VisitFloatFloatToRawIntBits() argument
225 MoveFPToInt(invoke->GetLocations(), /* is64bit= */ false, GetAssembler()); in VisitFloatFloatToRawIntBits()
227 void IntrinsicCodeGeneratorX86::VisitFloatIntBitsToFloat(HInvoke* invoke) { in VisitFloatIntBitsToFloat() argument
228 MoveIntToFP(invoke->GetLocations(), /* is64bit= */ false, GetAssembler()); in VisitFloatIntBitsToFloat()
231 static void CreateIntToIntLocations(ArenaAllocator* allocator, HInvoke* invoke) { in CreateIntToIntLocations() argument
233 new (allocator) LocationSummary(invoke, LocationSummary::kNoCall, kIntrinsified); in CreateIntToIntLocations()
238 static void CreateLongToIntLocations(ArenaAllocator* allocator, HInvoke* invoke) { in CreateLongToIntLocations() argument
240 new (allocator) LocationSummary(invoke, LocationSummary::kNoCall, kIntrinsified); in CreateLongToIntLocations()
245 static void CreateLongToLongLocations(ArenaAllocator* allocator, HInvoke* invoke) { in CreateLongToLongLocations() argument
247 new (allocator) LocationSummary(invoke, LocationSummary::kNoCall, kIntrinsified); in CreateLongToLongLocations()
272 void IntrinsicLocationsBuilderX86::VisitIntegerReverseBytes(HInvoke* invoke) { in VisitIntegerReverseBytes() argument
273 CreateIntToIntLocations(allocator_, invoke); in VisitIntegerReverseBytes()
276 void IntrinsicCodeGeneratorX86::VisitIntegerReverseBytes(HInvoke* invoke) { in VisitIntegerReverseBytes() argument
277 GenReverseBytes(invoke->GetLocations(), DataType::Type::kInt32, GetAssembler()); in VisitIntegerReverseBytes()
280 void IntrinsicLocationsBuilderX86::VisitLongReverseBytes(HInvoke* invoke) { in VisitLongReverseBytes() argument
281 CreateLongToLongLocations(allocator_, invoke); in VisitLongReverseBytes()
284 void IntrinsicCodeGeneratorX86::VisitLongReverseBytes(HInvoke* invoke) { in VisitLongReverseBytes() argument
285 LocationSummary* locations = invoke->GetLocations(); in VisitLongReverseBytes()
301 void IntrinsicLocationsBuilderX86::VisitShortReverseBytes(HInvoke* invoke) { in VisitShortReverseBytes() argument
302 CreateIntToIntLocations(allocator_, invoke); in VisitShortReverseBytes()
305 void IntrinsicCodeGeneratorX86::VisitShortReverseBytes(HInvoke* invoke) { in VisitShortReverseBytes() argument
306 GenReverseBytes(invoke->GetLocations(), DataType::Type::kInt16, GetAssembler()); in VisitShortReverseBytes()
309 static void CreateFPToFPLocations(ArenaAllocator* allocator, HInvoke* invoke) { in CreateFPToFPLocations() argument
311 new (allocator) LocationSummary(invoke, LocationSummary::kNoCall, kIntrinsified); in CreateFPToFPLocations()
316 void IntrinsicLocationsBuilderX86::VisitMathSqrt(HInvoke* invoke) { in VisitMathSqrt() argument
317 CreateFPToFPLocations(allocator_, invoke); in VisitMathSqrt()
320 void IntrinsicCodeGeneratorX86::VisitMathSqrt(HInvoke* invoke) { in VisitMathSqrt() argument
321 LocationSummary* locations = invoke->GetLocations(); in VisitMathSqrt()
329 HInvoke* invoke, in CreateSSE41FPToFPLocations() argument
336 CreateFPToFPLocations(allocator, invoke); in CreateSSE41FPToFPLocations()
339 static void GenSSE41FPToFPIntrinsic(HInvoke* invoke, X86Assembler* assembler, int round_mode) { in GenSSE41FPToFPIntrinsic() argument
340 LocationSummary* locations = invoke->GetLocations(); in GenSSE41FPToFPIntrinsic()
347 void IntrinsicLocationsBuilderX86::VisitMathCeil(HInvoke* invoke) { in VisitMathCeil() argument
348 CreateSSE41FPToFPLocations(allocator_, invoke, codegen_); in VisitMathCeil()
351 void IntrinsicCodeGeneratorX86::VisitMathCeil(HInvoke* invoke) { in VisitMathCeil() argument
352 GenSSE41FPToFPIntrinsic(invoke, GetAssembler(), 2); in VisitMathCeil()
355 void IntrinsicLocationsBuilderX86::VisitMathFloor(HInvoke* invoke) { in VisitMathFloor() argument
356 CreateSSE41FPToFPLocations(allocator_, invoke, codegen_); in VisitMathFloor()
359 void IntrinsicCodeGeneratorX86::VisitMathFloor(HInvoke* invoke) { in VisitMathFloor() argument
360 GenSSE41FPToFPIntrinsic(invoke, GetAssembler(), 1); in VisitMathFloor()
363 void IntrinsicLocationsBuilderX86::VisitMathRint(HInvoke* invoke) { in VisitMathRint() argument
364 CreateSSE41FPToFPLocations(allocator_, invoke, codegen_); in VisitMathRint()
367 void IntrinsicCodeGeneratorX86::VisitMathRint(HInvoke* invoke) { in VisitMathRint() argument
368 GenSSE41FPToFPIntrinsic(invoke, GetAssembler(), 0); in VisitMathRint()
371 void IntrinsicLocationsBuilderX86::VisitMathRoundFloat(HInvoke* invoke) { in VisitMathRoundFloat() argument
377 HInvokeStaticOrDirect* static_or_direct = invoke->AsInvokeStaticOrDirect(); in VisitMathRoundFloat()
379 new (allocator_) LocationSummary(invoke, LocationSummary::kNoCall, kIntrinsified); in VisitMathRoundFloat()
382 invoke->InputAt( in VisitMathRoundFloat()
391 void IntrinsicCodeGeneratorX86::VisitMathRoundFloat(HInvoke* invoke) { in VisitMathRoundFloat() argument
392 LocationSummary* locations = invoke->GetLocations(); in VisitMathRoundFloat()
413 invoke->InputAt(1)->AsX86ComputeBaseMethodAddress(); in VisitMathRoundFloat()
447 static void CreateFPToFPCallLocations(ArenaAllocator* allocator, HInvoke* invoke) { in CreateFPToFPCallLocations() argument
449 new (allocator) LocationSummary(invoke, LocationSummary::kCallOnMainOnly, kIntrinsified); in CreateFPToFPCallLocations()
455 static void GenFPToFPCall(HInvoke* invoke, CodeGeneratorX86* codegen, QuickEntrypointEnum entry) { in GenFPToFPCall() argument
456 LocationSummary* locations = invoke->GetLocations(); in GenFPToFPCall()
458 DCHECK(invoke->IsInvokeStaticOrDirect()); in GenFPToFPCall()
469 if (invoke->GetNumberOfArguments() == 2) { in GenFPToFPCall()
474 codegen->InvokeRuntime(entry, invoke, invoke->GetDexPc()); in GenFPToFPCall()
485 static void CreateLowestOneBitLocations(ArenaAllocator* allocator, bool is_long, HInvoke* invoke) { in CreateLowestOneBitLocations() argument
487 new (allocator) LocationSummary(invoke, LocationSummary::kNoCall, kIntrinsified); in CreateLowestOneBitLocations()
499 HInvoke* invoke) { in GenLowestOneBit() argument
500 LocationSummary* locations = invoke->GetLocations(); in GenLowestOneBit()
504 if (invoke->InputAt(0)->IsConstant()) { in GenLowestOneBit()
506 int64_t value = Int64FromConstant(invoke->InputAt(0)->AsConstant()); in GenLowestOneBit()
575 void IntrinsicLocationsBuilderX86::VisitMathCos(HInvoke* invoke) { in VisitMathCos() argument
576 CreateFPToFPCallLocations(allocator_, invoke); in VisitMathCos()
579 void IntrinsicCodeGeneratorX86::VisitMathCos(HInvoke* invoke) { in VisitMathCos() argument
580 GenFPToFPCall(invoke, codegen_, kQuickCos); in VisitMathCos()
583 void IntrinsicLocationsBuilderX86::VisitMathSin(HInvoke* invoke) { in VisitMathSin() argument
584 CreateFPToFPCallLocations(allocator_, invoke); in VisitMathSin()
587 void IntrinsicCodeGeneratorX86::VisitMathSin(HInvoke* invoke) { in VisitMathSin() argument
588 GenFPToFPCall(invoke, codegen_, kQuickSin); in VisitMathSin()
591 void IntrinsicLocationsBuilderX86::VisitMathAcos(HInvoke* invoke) { in VisitMathAcos() argument
592 CreateFPToFPCallLocations(allocator_, invoke); in VisitMathAcos()
595 void IntrinsicCodeGeneratorX86::VisitMathAcos(HInvoke* invoke) { in VisitMathAcos() argument
596 GenFPToFPCall(invoke, codegen_, kQuickAcos); in VisitMathAcos()
599 void IntrinsicLocationsBuilderX86::VisitMathAsin(HInvoke* invoke) { in VisitMathAsin() argument
600 CreateFPToFPCallLocations(allocator_, invoke); in VisitMathAsin()
603 void IntrinsicCodeGeneratorX86::VisitMathAsin(HInvoke* invoke) { in VisitMathAsin() argument
604 GenFPToFPCall(invoke, codegen_, kQuickAsin); in VisitMathAsin()
607 void IntrinsicLocationsBuilderX86::VisitMathAtan(HInvoke* invoke) { in VisitMathAtan() argument
608 CreateFPToFPCallLocations(allocator_, invoke); in VisitMathAtan()
611 void IntrinsicCodeGeneratorX86::VisitMathAtan(HInvoke* invoke) { in VisitMathAtan() argument
612 GenFPToFPCall(invoke, codegen_, kQuickAtan); in VisitMathAtan()
615 void IntrinsicLocationsBuilderX86::VisitMathCbrt(HInvoke* invoke) { in VisitMathCbrt() argument
616 CreateFPToFPCallLocations(allocator_, invoke); in VisitMathCbrt()
619 void IntrinsicCodeGeneratorX86::VisitMathCbrt(HInvoke* invoke) { in VisitMathCbrt() argument
620 GenFPToFPCall(invoke, codegen_, kQuickCbrt); in VisitMathCbrt()
623 void IntrinsicLocationsBuilderX86::VisitMathCosh(HInvoke* invoke) { in VisitMathCosh() argument
624 CreateFPToFPCallLocations(allocator_, invoke); in VisitMathCosh()
627 void IntrinsicCodeGeneratorX86::VisitMathCosh(HInvoke* invoke) { in VisitMathCosh() argument
628 GenFPToFPCall(invoke, codegen_, kQuickCosh); in VisitMathCosh()
631 void IntrinsicLocationsBuilderX86::VisitMathExp(HInvoke* invoke) { in VisitMathExp() argument
632 CreateFPToFPCallLocations(allocator_, invoke); in VisitMathExp()
635 void IntrinsicCodeGeneratorX86::VisitMathExp(HInvoke* invoke) { in VisitMathExp() argument
636 GenFPToFPCall(invoke, codegen_, kQuickExp); in VisitMathExp()
639 void IntrinsicLocationsBuilderX86::VisitMathExpm1(HInvoke* invoke) { in VisitMathExpm1() argument
640 CreateFPToFPCallLocations(allocator_, invoke); in VisitMathExpm1()
643 void IntrinsicCodeGeneratorX86::VisitMathExpm1(HInvoke* invoke) { in VisitMathExpm1() argument
644 GenFPToFPCall(invoke, codegen_, kQuickExpm1); in VisitMathExpm1()
647 void IntrinsicLocationsBuilderX86::VisitMathLog(HInvoke* invoke) { in VisitMathLog() argument
648 CreateFPToFPCallLocations(allocator_, invoke); in VisitMathLog()
651 void IntrinsicCodeGeneratorX86::VisitMathLog(HInvoke* invoke) { in VisitMathLog() argument
652 GenFPToFPCall(invoke, codegen_, kQuickLog); in VisitMathLog()
655 void IntrinsicLocationsBuilderX86::VisitMathLog10(HInvoke* invoke) { in VisitMathLog10() argument
656 CreateFPToFPCallLocations(allocator_, invoke); in VisitMathLog10()
659 void IntrinsicCodeGeneratorX86::VisitMathLog10(HInvoke* invoke) { in VisitMathLog10() argument
660 GenFPToFPCall(invoke, codegen_, kQuickLog10); in VisitMathLog10()
663 void IntrinsicLocationsBuilderX86::VisitMathSinh(HInvoke* invoke) { in VisitMathSinh() argument
664 CreateFPToFPCallLocations(allocator_, invoke); in VisitMathSinh()
667 void IntrinsicCodeGeneratorX86::VisitMathSinh(HInvoke* invoke) { in VisitMathSinh() argument
668 GenFPToFPCall(invoke, codegen_, kQuickSinh); in VisitMathSinh()
671 void IntrinsicLocationsBuilderX86::VisitMathTan(HInvoke* invoke) { in VisitMathTan() argument
672 CreateFPToFPCallLocations(allocator_, invoke); in VisitMathTan()
675 void IntrinsicCodeGeneratorX86::VisitMathTan(HInvoke* invoke) { in VisitMathTan() argument
676 GenFPToFPCall(invoke, codegen_, kQuickTan); in VisitMathTan()
679 void IntrinsicLocationsBuilderX86::VisitMathTanh(HInvoke* invoke) { in VisitMathTanh() argument
680 CreateFPToFPCallLocations(allocator_, invoke); in VisitMathTanh()
683 void IntrinsicCodeGeneratorX86::VisitMathTanh(HInvoke* invoke) { in VisitMathTanh() argument
684 GenFPToFPCall(invoke, codegen_, kQuickTanh); in VisitMathTanh()
687 void IntrinsicLocationsBuilderX86::VisitIntegerLowestOneBit(HInvoke* invoke) { in VisitIntegerLowestOneBit() argument
688 CreateLowestOneBitLocations(allocator_, /*is_long=*/ false, invoke); in VisitIntegerLowestOneBit()
690 void IntrinsicCodeGeneratorX86::VisitIntegerLowestOneBit(HInvoke* invoke) { in VisitIntegerLowestOneBit() argument
691 GenLowestOneBit(GetAssembler(), codegen_, /*is_long=*/ false, invoke); in VisitIntegerLowestOneBit()
694 void IntrinsicLocationsBuilderX86::VisitLongLowestOneBit(HInvoke* invoke) { in VisitLongLowestOneBit() argument
695 CreateLowestOneBitLocations(allocator_, /*is_long=*/ true, invoke); in VisitLongLowestOneBit()
698 void IntrinsicCodeGeneratorX86::VisitLongLowestOneBit(HInvoke* invoke) { in VisitLongLowestOneBit() argument
699 GenLowestOneBit(GetAssembler(), codegen_, /*is_long=*/ true, invoke); in VisitLongLowestOneBit()
702 static void CreateFPFPToFPCallLocations(ArenaAllocator* allocator, HInvoke* invoke) { in CreateFPFPToFPCallLocations() argument
704 new (allocator) LocationSummary(invoke, LocationSummary::kCallOnMainOnly, kIntrinsified); in CreateFPFPToFPCallLocations()
711 static void CreateFPFPFPToFPCallLocations(ArenaAllocator* allocator, HInvoke* invoke) { in CreateFPFPFPToFPCallLocations() argument
712 DCHECK_EQ(invoke->GetNumberOfArguments(), 3U); in CreateFPFPFPToFPCallLocations()
714 new (allocator) LocationSummary(invoke, LocationSummary::kNoCall, kIntrinsified); in CreateFPFPFPToFPCallLocations()
722 void IntrinsicLocationsBuilderX86::VisitMathAtan2(HInvoke* invoke) { in VisitMathAtan2() argument
723 CreateFPFPToFPCallLocations(allocator_, invoke); in VisitMathAtan2()
726 void IntrinsicCodeGeneratorX86::VisitMathAtan2(HInvoke* invoke) { in VisitMathAtan2() argument
727 GenFPToFPCall(invoke, codegen_, kQuickAtan2); in VisitMathAtan2()
730 void IntrinsicLocationsBuilderX86::VisitMathPow(HInvoke* invoke) { in VisitMathPow() argument
731 CreateFPFPToFPCallLocations(allocator_, invoke); in VisitMathPow()
734 void IntrinsicCodeGeneratorX86::VisitMathPow(HInvoke* invoke) { in VisitMathPow() argument
735 GenFPToFPCall(invoke, codegen_, kQuickPow); in VisitMathPow()
738 void IntrinsicLocationsBuilderX86::VisitMathHypot(HInvoke* invoke) { in VisitMathHypot() argument
739 CreateFPFPToFPCallLocations(allocator_, invoke); in VisitMathHypot()
742 void IntrinsicCodeGeneratorX86::VisitMathHypot(HInvoke* invoke) { in VisitMathHypot() argument
743 GenFPToFPCall(invoke, codegen_, kQuickHypot); in VisitMathHypot()
746 void IntrinsicLocationsBuilderX86::VisitMathNextAfter(HInvoke* invoke) { in VisitMathNextAfter() argument
747 CreateFPFPToFPCallLocations(allocator_, invoke); in VisitMathNextAfter()
750 void IntrinsicCodeGeneratorX86::VisitMathNextAfter(HInvoke* invoke) { in VisitMathNextAfter() argument
751 GenFPToFPCall(invoke, codegen_, kQuickNextAfter); in VisitMathNextAfter()
754 static void CreateSystemArrayCopyLocations(HInvoke* invoke) { in CreateSystemArrayCopyLocations() argument
757 HIntConstant* src_pos = invoke->InputAt(1)->AsIntConstantOrNull(); in CreateSystemArrayCopyLocations()
758 HIntConstant* dest_pos = invoke->InputAt(3)->AsIntConstantOrNull(); in CreateSystemArrayCopyLocations()
759 HIntConstant* length = invoke->InputAt(4)->AsIntConstantOrNull(); in CreateSystemArrayCopyLocations()
790 new (invoke->GetBlock()->GetGraph()->GetAllocator()) in CreateSystemArrayCopyLocations()
791 LocationSummary(invoke, LocationSummary::kCallOnSlowPath, kIntrinsified); in CreateSystemArrayCopyLocations()
794 locations->SetInAt(1, Location::RegisterOrConstant(invoke->InputAt(1))); in CreateSystemArrayCopyLocations()
796 locations->SetInAt(3, Location::RegisterOrConstant(invoke->InputAt(3))); in CreateSystemArrayCopyLocations()
797 locations->SetInAt(4, Location::RegisterOrConstant(invoke->InputAt(4))); in CreateSystemArrayCopyLocations()
872 static void SystemArrayCopyPrimitive(HInvoke* invoke, in SystemArrayCopyPrimitive() argument
876 LocationSummary* locations = invoke->GetLocations(); in SystemArrayCopyPrimitive()
891 SlowPathCode* slow_path = new (codegen->GetScopedAllocator()) IntrinsicSlowPathX86(invoke); in SystemArrayCopyPrimitive()
965 void IntrinsicLocationsBuilderX86::VisitSystemArrayCopyChar(HInvoke* invoke) { in VisitSystemArrayCopyChar() argument
966 CreateSystemArrayCopyLocations(invoke); in VisitSystemArrayCopyChar()
969 void IntrinsicCodeGeneratorX86::VisitSystemArrayCopyChar(HInvoke* invoke) { in VisitSystemArrayCopyChar() argument
971 SystemArrayCopyPrimitive(invoke, assembler, codegen_, DataType::Type::kUint16); in VisitSystemArrayCopyChar()
974 void IntrinsicCodeGeneratorX86::VisitSystemArrayCopyByte(HInvoke* invoke) { in VisitSystemArrayCopyByte() argument
976 SystemArrayCopyPrimitive(invoke, assembler, codegen_, DataType::Type::kInt8); in VisitSystemArrayCopyByte()
979 void IntrinsicLocationsBuilderX86::VisitSystemArrayCopyByte(HInvoke* invoke) { in VisitSystemArrayCopyByte() argument
980 CreateSystemArrayCopyLocations(invoke); in VisitSystemArrayCopyByte()
983 void IntrinsicCodeGeneratorX86::VisitSystemArrayCopyInt(HInvoke* invoke) { in VisitSystemArrayCopyInt() argument
985 SystemArrayCopyPrimitive(invoke, assembler, codegen_, DataType::Type::kInt32); in VisitSystemArrayCopyInt()
988 void IntrinsicLocationsBuilderX86::VisitSystemArrayCopyInt(HInvoke* invoke) { in VisitSystemArrayCopyInt() argument
989 CreateSystemArrayCopyLocations(invoke); in VisitSystemArrayCopyInt()
992 void IntrinsicLocationsBuilderX86::VisitStringCompareTo(HInvoke* invoke) { in VisitStringCompareTo() argument
995 invoke, LocationSummary::kCallOnMainAndSlowPath, kIntrinsified); in VisitStringCompareTo()
1002 void IntrinsicCodeGeneratorX86::VisitStringCompareTo(HInvoke* invoke) { in VisitStringCompareTo() argument
1004 LocationSummary* locations = invoke->GetLocations(); in VisitStringCompareTo()
1007 DCHECK(!invoke->CanDoImplicitNullCheckOn(invoke->InputAt(0))); in VisitStringCompareTo()
1011 SlowPathCode* slow_path = new (codegen_->GetScopedAllocator()) IntrinsicSlowPathX86(invoke); in VisitStringCompareTo()
1015 codegen_->InvokeRuntime(kQuickStringCompareTo, invoke, invoke->GetDexPc(), slow_path); in VisitStringCompareTo()
1019 void IntrinsicLocationsBuilderX86::VisitStringEquals(HInvoke* invoke) { in VisitStringEquals() argument
1021 new (allocator_) LocationSummary(invoke, LocationSummary::kNoCall, kIntrinsified); in VisitStringEquals()
1033 void IntrinsicCodeGeneratorX86::VisitStringEquals(HInvoke* invoke) { in VisitStringEquals() argument
1035 LocationSummary* locations = invoke->GetLocations(); in VisitStringEquals()
1051 DCHECK(!invoke->CanDoImplicitNullCheckOn(invoke->InputAt(0))); in VisitStringEquals()
1053 StringEqualsOptimizations optimizations(invoke); in VisitStringEquals()
1136 static void CreateStringIndexOfLocations(HInvoke* invoke, in CreateStringIndexOfLocations() argument
1139 LocationSummary* locations = new (allocator) LocationSummary(invoke, in CreateStringIndexOfLocations()
1165 static void GenerateStringIndexOf(HInvoke* invoke, in GenerateStringIndexOf() argument
1169 LocationSummary* locations = invoke->GetLocations(); in GenerateStringIndexOf()
1172 DCHECK(!invoke->CanDoImplicitNullCheckOn(invoke->InputAt(0))); in GenerateStringIndexOf()
1191 HInstruction* code_point = invoke->InputAt(1); in GenerateStringIndexOf()
1197 slow_path = new (codegen->GetScopedAllocator()) IntrinsicSlowPathX86(invoke); in GenerateStringIndexOf()
1205 slow_path = new (codegen->GetScopedAllocator()) IntrinsicSlowPathX86(invoke); in GenerateStringIndexOf()
1317 void IntrinsicLocationsBuilderX86::VisitStringIndexOf(HInvoke* invoke) { in VisitStringIndexOf() argument
1318 CreateStringIndexOfLocations(invoke, allocator_, /* start_at_zero= */ true); in VisitStringIndexOf()
1321 void IntrinsicCodeGeneratorX86::VisitStringIndexOf(HInvoke* invoke) { in VisitStringIndexOf() argument
1322 GenerateStringIndexOf(invoke, GetAssembler(), codegen_, /* start_at_zero= */ true); in VisitStringIndexOf()
1325 void IntrinsicLocationsBuilderX86::VisitStringIndexOfAfter(HInvoke* invoke) { in VisitStringIndexOfAfter() argument
1326 CreateStringIndexOfLocations(invoke, allocator_, /* start_at_zero= */ false); in VisitStringIndexOfAfter()
1329 void IntrinsicCodeGeneratorX86::VisitStringIndexOfAfter(HInvoke* invoke) { in VisitStringIndexOfAfter() argument
1330 GenerateStringIndexOf(invoke, GetAssembler(), codegen_, /* start_at_zero= */ false); in VisitStringIndexOfAfter()
1333 void IntrinsicLocationsBuilderX86::VisitStringNewStringFromBytes(HInvoke* invoke) { in VisitStringNewStringFromBytes() argument
1335 invoke, LocationSummary::kCallOnMainAndSlowPath, kIntrinsified); in VisitStringNewStringFromBytes()
1344 void IntrinsicCodeGeneratorX86::VisitStringNewStringFromBytes(HInvoke* invoke) { in VisitStringNewStringFromBytes() argument
1346 LocationSummary* locations = invoke->GetLocations(); in VisitStringNewStringFromBytes()
1350 SlowPathCode* slow_path = new (codegen_->GetScopedAllocator()) IntrinsicSlowPathX86(invoke); in VisitStringNewStringFromBytes()
1354 codegen_->InvokeRuntime(kQuickAllocStringFromBytes, invoke, invoke->GetDexPc()); in VisitStringNewStringFromBytes()
1359 void IntrinsicLocationsBuilderX86::VisitStringNewStringFromChars(HInvoke* invoke) { in VisitStringNewStringFromChars() argument
1361 new (allocator_) LocationSummary(invoke, LocationSummary::kCallOnMainOnly, kIntrinsified); in VisitStringNewStringFromChars()
1369 void IntrinsicCodeGeneratorX86::VisitStringNewStringFromChars(HInvoke* invoke) { in VisitStringNewStringFromChars() argument
1376 codegen_->InvokeRuntime(kQuickAllocStringFromChars, invoke, invoke->GetDexPc()); in VisitStringNewStringFromChars()
1380 void IntrinsicLocationsBuilderX86::VisitStringNewStringFromString(HInvoke* invoke) { in VisitStringNewStringFromString() argument
1382 invoke, LocationSummary::kCallOnMainAndSlowPath, kIntrinsified); in VisitStringNewStringFromString()
1388 void IntrinsicCodeGeneratorX86::VisitStringNewStringFromString(HInvoke* invoke) { in VisitStringNewStringFromString() argument
1390 LocationSummary* locations = invoke->GetLocations(); in VisitStringNewStringFromString()
1394 SlowPathCode* slow_path = new (codegen_->GetScopedAllocator()) IntrinsicSlowPathX86(invoke); in VisitStringNewStringFromString()
1398 codegen_->InvokeRuntime(kQuickAllocStringFromString, invoke, invoke->GetDexPc()); in VisitStringNewStringFromString()
1403 void IntrinsicLocationsBuilderX86::VisitStringGetCharsNoCheck(HInvoke* invoke) { in VisitStringGetCharsNoCheck() argument
1406 new (allocator_) LocationSummary(invoke, LocationSummary::kNoCall, kIntrinsified); in VisitStringGetCharsNoCheck()
1408 locations->SetInAt(1, Location::RegisterOrConstant(invoke->InputAt(1))); in VisitStringGetCharsNoCheck()
1420 void IntrinsicCodeGeneratorX86::VisitStringGetCharsNoCheck(HInvoke* invoke) { in VisitStringGetCharsNoCheck() argument
1422 LocationSummary* locations = invoke->GetLocations(); in VisitStringGetCharsNoCheck()
1532 void IntrinsicLocationsBuilderX86::VisitMemoryPeekByte(HInvoke* invoke) { in VisitMemoryPeekByte() argument
1533 CreateLongToIntLocations(allocator_, invoke); in VisitMemoryPeekByte()
1536 void IntrinsicCodeGeneratorX86::VisitMemoryPeekByte(HInvoke* invoke) { in VisitMemoryPeekByte() argument
1537 GenPeek(invoke->GetLocations(), DataType::Type::kInt8, GetAssembler()); in VisitMemoryPeekByte()
1540 void IntrinsicLocationsBuilderX86::VisitMemoryPeekIntNative(HInvoke* invoke) { in VisitMemoryPeekIntNative() argument
1541 CreateLongToIntLocations(allocator_, invoke); in VisitMemoryPeekIntNative()
1544 void IntrinsicCodeGeneratorX86::VisitMemoryPeekIntNative(HInvoke* invoke) { in VisitMemoryPeekIntNative() argument
1545 GenPeek(invoke->GetLocations(), DataType::Type::kInt32, GetAssembler()); in VisitMemoryPeekIntNative()
1548 void IntrinsicLocationsBuilderX86::VisitMemoryPeekLongNative(HInvoke* invoke) { in VisitMemoryPeekLongNative() argument
1549 CreateLongToLongLocations(allocator_, invoke); in VisitMemoryPeekLongNative()
1552 void IntrinsicCodeGeneratorX86::VisitMemoryPeekLongNative(HInvoke* invoke) { in VisitMemoryPeekLongNative() argument
1553 GenPeek(invoke->GetLocations(), DataType::Type::kInt64, GetAssembler()); in VisitMemoryPeekLongNative()
1556 void IntrinsicLocationsBuilderX86::VisitMemoryPeekShortNative(HInvoke* invoke) { in VisitMemoryPeekShortNative() argument
1557 CreateLongToIntLocations(allocator_, invoke); in VisitMemoryPeekShortNative()
1560 void IntrinsicCodeGeneratorX86::VisitMemoryPeekShortNative(HInvoke* invoke) { in VisitMemoryPeekShortNative() argument
1561 GenPeek(invoke->GetLocations(), DataType::Type::kInt16, GetAssembler()); in VisitMemoryPeekShortNative()
1566 HInvoke* invoke) { in CreateLongIntToVoidLocations() argument
1568 new (allocator) LocationSummary(invoke, LocationSummary::kNoCall, kIntrinsified); in CreateLongIntToVoidLocations()
1570 HInstruction* value = invoke->InputAt(1); in CreateLongIntToVoidLocations()
1624 void IntrinsicLocationsBuilderX86::VisitMemoryPokeByte(HInvoke* invoke) { in VisitMemoryPokeByte() argument
1625 CreateLongIntToVoidLocations(allocator_, DataType::Type::kInt8, invoke); in VisitMemoryPokeByte()
1628 void IntrinsicCodeGeneratorX86::VisitMemoryPokeByte(HInvoke* invoke) { in VisitMemoryPokeByte() argument
1629 GenPoke(invoke->GetLocations(), DataType::Type::kInt8, GetAssembler()); in VisitMemoryPokeByte()
1632 void IntrinsicLocationsBuilderX86::VisitMemoryPokeIntNative(HInvoke* invoke) { in VisitMemoryPokeIntNative() argument
1633 CreateLongIntToVoidLocations(allocator_, DataType::Type::kInt32, invoke); in VisitMemoryPokeIntNative()
1636 void IntrinsicCodeGeneratorX86::VisitMemoryPokeIntNative(HInvoke* invoke) { in VisitMemoryPokeIntNative() argument
1637 GenPoke(invoke->GetLocations(), DataType::Type::kInt32, GetAssembler()); in VisitMemoryPokeIntNative()
1640 void IntrinsicLocationsBuilderX86::VisitMemoryPokeLongNative(HInvoke* invoke) { in VisitMemoryPokeLongNative() argument
1641 CreateLongIntToVoidLocations(allocator_, DataType::Type::kInt64, invoke); in VisitMemoryPokeLongNative()
1644 void IntrinsicCodeGeneratorX86::VisitMemoryPokeLongNative(HInvoke* invoke) { in VisitMemoryPokeLongNative() argument
1645 GenPoke(invoke->GetLocations(), DataType::Type::kInt64, GetAssembler()); in VisitMemoryPokeLongNative()
1648 void IntrinsicLocationsBuilderX86::VisitMemoryPokeShortNative(HInvoke* invoke) { in VisitMemoryPokeShortNative() argument
1649 CreateLongIntToVoidLocations(allocator_, DataType::Type::kInt16, invoke); in VisitMemoryPokeShortNative()
1652 void IntrinsicCodeGeneratorX86::VisitMemoryPokeShortNative(HInvoke* invoke) { in VisitMemoryPokeShortNative() argument
1653 GenPoke(invoke->GetLocations(), DataType::Type::kInt16, GetAssembler()); in VisitMemoryPokeShortNative()
1656 void IntrinsicLocationsBuilderX86::VisitThreadCurrentThread(HInvoke* invoke) { in VisitThreadCurrentThread() argument
1658 new (allocator_) LocationSummary(invoke, LocationSummary::kNoCall, kIntrinsified); in VisitThreadCurrentThread()
1662 void IntrinsicCodeGeneratorX86::VisitThreadCurrentThread(HInvoke* invoke) { in VisitThreadCurrentThread() argument
1663 Register out = invoke->GetLocations()->Out().AsRegister<Register>(); in VisitThreadCurrentThread()
1667 static void GenUnsafeGet(HInvoke* invoke, in GenUnsafeGet() argument
1672 LocationSummary* locations = invoke->GetLocations(); in GenUnsafeGet()
1698 invoke, output_loc, base, src, /* needs_null_check= */ false); in GenUnsafeGet()
1702 invoke, output_loc, output_loc, base_loc, 0U, offset_loc); in GenUnsafeGet()
1734 static void GenUnsafeGetAbsolute(HInvoke* invoke, in GenUnsafeGetAbsolute() argument
1739 LocationSummary* locations = invoke->GetLocations(); in GenUnsafeGetAbsolute()
1782 HInvoke* invoke, in CreateIntIntToIntLocations() argument
1786 new (allocator) LocationSummary(invoke, LocationSummary::kNoCall, kIntrinsified); in CreateIntIntToIntLocations()
1803 HInvoke* invoke, in CreateIntIntIntToIntLocations() argument
1807 bool can_call = codegen->EmitReadBarrier() && IsUnsafeGetReference(invoke); in CreateIntIntIntToIntLocations()
1809 new (allocator) LocationSummary(invoke, in CreateIntIntIntToIntLocations()
1834 void IntrinsicLocationsBuilderX86::VisitUnsafeGet(HInvoke* invoke) { in VisitUnsafeGet() argument
1835 VisitJdkUnsafeGet(invoke); in VisitUnsafeGet()
1837 void IntrinsicLocationsBuilderX86::VisitUnsafeGetAbsolute(HInvoke* invoke) { in VisitUnsafeGetAbsolute() argument
1838 VisitJdkUnsafeGetAbsolute(invoke); in VisitUnsafeGetAbsolute()
1840 void IntrinsicLocationsBuilderX86::VisitUnsafeGetVolatile(HInvoke* invoke) { in VisitUnsafeGetVolatile() argument
1841 VisitJdkUnsafeGetVolatile(invoke); in VisitUnsafeGetVolatile()
1843 void IntrinsicLocationsBuilderX86::VisitUnsafeGetLong(HInvoke* invoke) { in VisitUnsafeGetLong() argument
1844 VisitJdkUnsafeGetLong(invoke); in VisitUnsafeGetLong()
1846 void IntrinsicLocationsBuilderX86::VisitUnsafeGetLongVolatile(HInvoke* invoke) { in VisitUnsafeGetLongVolatile() argument
1847 VisitJdkUnsafeGetLongVolatile(invoke); in VisitUnsafeGetLongVolatile()
1849 void IntrinsicLocationsBuilderX86::VisitUnsafeGetObject(HInvoke* invoke) { in VisitUnsafeGetObject() argument
1850 VisitJdkUnsafeGetReference(invoke); in VisitUnsafeGetObject()
1852 void IntrinsicLocationsBuilderX86::VisitUnsafeGetObjectVolatile(HInvoke* invoke) { in VisitUnsafeGetObjectVolatile() argument
1853 VisitJdkUnsafeGetReferenceVolatile(invoke); in VisitUnsafeGetObjectVolatile()
1855 void IntrinsicLocationsBuilderX86::VisitUnsafeGetByte(HInvoke* invoke) { in VisitUnsafeGetByte() argument
1856 VisitJdkUnsafeGetByte(invoke); in VisitUnsafeGetByte()
1859 void IntrinsicCodeGeneratorX86::VisitUnsafeGet(HInvoke* invoke) { in VisitUnsafeGet() argument
1860 VisitJdkUnsafeGet(invoke); in VisitUnsafeGet()
1862 void IntrinsicCodeGeneratorX86::VisitUnsafeGetAbsolute(HInvoke* invoke) { in VisitUnsafeGetAbsolute() argument
1863 VisitJdkUnsafeGetAbsolute(invoke); in VisitUnsafeGetAbsolute()
1865 void IntrinsicCodeGeneratorX86::VisitUnsafeGetVolatile(HInvoke* invoke) { in VisitUnsafeGetVolatile() argument
1866 VisitJdkUnsafeGetVolatile(invoke); in VisitUnsafeGetVolatile()
1868 void IntrinsicCodeGeneratorX86::VisitUnsafeGetLong(HInvoke* invoke) { in VisitUnsafeGetLong() argument
1869 VisitJdkUnsafeGetLong(invoke); in VisitUnsafeGetLong()
1871 void IntrinsicCodeGeneratorX86::VisitUnsafeGetLongVolatile(HInvoke* invoke) { in VisitUnsafeGetLongVolatile() argument
1872 VisitJdkUnsafeGetLongVolatile(invoke); in VisitUnsafeGetLongVolatile()
1874 void IntrinsicCodeGeneratorX86::VisitUnsafeGetObject(HInvoke* invoke) { in VisitUnsafeGetObject() argument
1875 VisitJdkUnsafeGetReference(invoke); in VisitUnsafeGetObject()
1877 void IntrinsicCodeGeneratorX86::VisitUnsafeGetObjectVolatile(HInvoke* invoke) { in VisitUnsafeGetObjectVolatile() argument
1878 VisitJdkUnsafeGetReferenceVolatile(invoke); in VisitUnsafeGetObjectVolatile()
1880 void IntrinsicCodeGeneratorX86::VisitUnsafeGetByte(HInvoke* invoke) { in VisitUnsafeGetByte() argument
1881 VisitJdkUnsafeGetByte(invoke); in VisitUnsafeGetByte()
1884 void IntrinsicLocationsBuilderX86::VisitJdkUnsafeGet(HInvoke* invoke) { in VisitJdkUnsafeGet() argument
1886 allocator_, invoke, codegen_, DataType::Type::kInt32, /*is_volatile=*/ false); in VisitJdkUnsafeGet()
1888 void IntrinsicLocationsBuilderX86::VisitJdkUnsafeGetAbsolute(HInvoke* invoke) { in VisitJdkUnsafeGetAbsolute() argument
1889 CreateIntIntToIntLocations(allocator_, invoke, DataType::Type::kInt32, /*is_volatile=*/false); in VisitJdkUnsafeGetAbsolute()
1891 void IntrinsicLocationsBuilderX86::VisitJdkUnsafeGetVolatile(HInvoke* invoke) { in VisitJdkUnsafeGetVolatile() argument
1893 allocator_, invoke, codegen_, DataType::Type::kInt32, /*is_volatile=*/ true); in VisitJdkUnsafeGetVolatile()
1895 void IntrinsicLocationsBuilderX86::VisitJdkUnsafeGetAcquire(HInvoke* invoke) { in VisitJdkUnsafeGetAcquire() argument
1897 allocator_, invoke, codegen_, DataType::Type::kInt32, /*is_volatile=*/ true); in VisitJdkUnsafeGetAcquire()
1899 void IntrinsicLocationsBuilderX86::VisitJdkUnsafeGetLong(HInvoke* invoke) { in VisitJdkUnsafeGetLong() argument
1901 allocator_, invoke, codegen_, DataType::Type::kInt64, /*is_volatile=*/ false); in VisitJdkUnsafeGetLong()
1903 void IntrinsicLocationsBuilderX86::VisitJdkUnsafeGetLongVolatile(HInvoke* invoke) { in VisitJdkUnsafeGetLongVolatile() argument
1905 allocator_, invoke, codegen_, DataType::Type::kInt64, /*is_volatile=*/ true); in VisitJdkUnsafeGetLongVolatile()
1907 void IntrinsicLocationsBuilderX86::VisitJdkUnsafeGetLongAcquire(HInvoke* invoke) { in VisitJdkUnsafeGetLongAcquire() argument
1909 allocator_, invoke, codegen_, DataType::Type::kInt64, /*is_volatile=*/ true); in VisitJdkUnsafeGetLongAcquire()
1911 void IntrinsicLocationsBuilderX86::VisitJdkUnsafeGetReference(HInvoke* invoke) { in VisitJdkUnsafeGetReference() argument
1913 allocator_, invoke, codegen_, DataType::Type::kReference, /*is_volatile=*/ false); in VisitJdkUnsafeGetReference()
1915 void IntrinsicLocationsBuilderX86::VisitJdkUnsafeGetReferenceVolatile(HInvoke* invoke) { in VisitJdkUnsafeGetReferenceVolatile() argument
1917 allocator_, invoke, codegen_, DataType::Type::kReference, /*is_volatile=*/ true); in VisitJdkUnsafeGetReferenceVolatile()
1919 void IntrinsicLocationsBuilderX86::VisitJdkUnsafeGetReferenceAcquire(HInvoke* invoke) { in VisitJdkUnsafeGetReferenceAcquire() argument
1921 allocator_, invoke, codegen_, DataType::Type::kReference, /*is_volatile=*/ true); in VisitJdkUnsafeGetReferenceAcquire()
1923 void IntrinsicLocationsBuilderX86::VisitJdkUnsafeGetByte(HInvoke* invoke) { in VisitJdkUnsafeGetByte() argument
1925 allocator_, invoke, codegen_, DataType::Type::kInt8, /*is_volatile=*/ false); in VisitJdkUnsafeGetByte()
1928 void IntrinsicCodeGeneratorX86::VisitJdkUnsafeGet(HInvoke* invoke) { in VisitJdkUnsafeGet() argument
1929 GenUnsafeGet(invoke, DataType::Type::kInt32, /*is_volatile=*/ false, codegen_); in VisitJdkUnsafeGet()
1931 void IntrinsicCodeGeneratorX86::VisitJdkUnsafeGetAbsolute(HInvoke* invoke) { in VisitJdkUnsafeGetAbsolute() argument
1932 GenUnsafeGetAbsolute(invoke, DataType::Type::kInt32, /*is_volatile=*/ false, codegen_); in VisitJdkUnsafeGetAbsolute()
1934 void IntrinsicCodeGeneratorX86::VisitJdkUnsafeGetVolatile(HInvoke* invoke) { in VisitJdkUnsafeGetVolatile() argument
1935 GenUnsafeGet(invoke, DataType::Type::kInt32, /*is_volatile=*/ true, codegen_); in VisitJdkUnsafeGetVolatile()
1937 void IntrinsicCodeGeneratorX86::VisitJdkUnsafeGetAcquire(HInvoke* invoke) { in VisitJdkUnsafeGetAcquire() argument
1938 GenUnsafeGet(invoke, DataType::Type::kInt32, /*is_volatile=*/ true, codegen_); in VisitJdkUnsafeGetAcquire()
1940 void IntrinsicCodeGeneratorX86::VisitJdkUnsafeGetLong(HInvoke* invoke) { in VisitJdkUnsafeGetLong() argument
1941 GenUnsafeGet(invoke, DataType::Type::kInt64, /*is_volatile=*/ false, codegen_); in VisitJdkUnsafeGetLong()
1943 void IntrinsicCodeGeneratorX86::VisitJdkUnsafeGetLongVolatile(HInvoke* invoke) { in VisitJdkUnsafeGetLongVolatile() argument
1944 GenUnsafeGet(invoke, DataType::Type::kInt64, /*is_volatile=*/ true, codegen_); in VisitJdkUnsafeGetLongVolatile()
1946 void IntrinsicCodeGeneratorX86::VisitJdkUnsafeGetLongAcquire(HInvoke* invoke) { in VisitJdkUnsafeGetLongAcquire() argument
1947 GenUnsafeGet(invoke, DataType::Type::kInt64, /*is_volatile=*/ true, codegen_); in VisitJdkUnsafeGetLongAcquire()
1949 void IntrinsicCodeGeneratorX86::VisitJdkUnsafeGetReference(HInvoke* invoke) { in VisitJdkUnsafeGetReference() argument
1950 GenUnsafeGet(invoke, DataType::Type::kReference, /*is_volatile=*/ false, codegen_); in VisitJdkUnsafeGetReference()
1952 void IntrinsicCodeGeneratorX86::VisitJdkUnsafeGetReferenceVolatile(HInvoke* invoke) { in VisitJdkUnsafeGetReferenceVolatile() argument
1953 GenUnsafeGet(invoke, DataType::Type::kReference, /*is_volatile=*/ true, codegen_); in VisitJdkUnsafeGetReferenceVolatile()
1955 void IntrinsicCodeGeneratorX86::VisitJdkUnsafeGetReferenceAcquire(HInvoke* invoke) { in VisitJdkUnsafeGetReferenceAcquire() argument
1956 GenUnsafeGet(invoke, DataType::Type::kReference, /*is_volatile=*/ true, codegen_); in VisitJdkUnsafeGetReferenceAcquire()
1958 void IntrinsicCodeGeneratorX86::VisitJdkUnsafeGetByte(HInvoke* invoke) { in VisitJdkUnsafeGetByte() argument
1959 GenUnsafeGet(invoke, DataType::Type::kInt8, /*is_volatile=*/ false, codegen_); in VisitJdkUnsafeGetByte()
1964 HInvoke* invoke, in CreateIntIntIntToVoidPlusTempsLocations() argument
1967 new (allocator) LocationSummary(invoke, LocationSummary::kNoCall, kIntrinsified); in CreateIntIntIntToVoidPlusTempsLocations()
1972 locations->SetInAt(2, Location::ByteRegisterOrConstant(EAX, invoke->InputAt(3))); in CreateIntIntIntToVoidPlusTempsLocations()
1984 HInvoke* invoke, in CreateIntIntIntIntToVoidPlusTempsLocations() argument
1987 new (allocator) LocationSummary(invoke, LocationSummary::kNoCall, kIntrinsified); in CreateIntIntIntIntToVoidPlusTempsLocations()
1993 locations->SetInAt(3, Location::ByteRegisterOrConstant(EAX, invoke->InputAt(3))); in CreateIntIntIntIntToVoidPlusTempsLocations()
2008 void IntrinsicLocationsBuilderX86::VisitUnsafePut(HInvoke* invoke) { in VisitUnsafePut() argument
2009 VisitJdkUnsafePut(invoke); in VisitUnsafePut()
2011 void IntrinsicLocationsBuilderX86::VisitUnsafePutAbsolute(HInvoke* invoke) { in VisitUnsafePutAbsolute() argument
2012 VisitJdkUnsafePutAbsolute(invoke); in VisitUnsafePutAbsolute()
2014 void IntrinsicLocationsBuilderX86::VisitUnsafePutOrdered(HInvoke* invoke) { in VisitUnsafePutOrdered() argument
2015 VisitJdkUnsafePutOrdered(invoke); in VisitUnsafePutOrdered()
2017 void IntrinsicLocationsBuilderX86::VisitUnsafePutVolatile(HInvoke* invoke) { in VisitUnsafePutVolatile() argument
2018 VisitJdkUnsafePutVolatile(invoke); in VisitUnsafePutVolatile()
2020 void IntrinsicLocationsBuilderX86::VisitUnsafePutObject(HInvoke* invoke) { in VisitUnsafePutObject() argument
2021 VisitJdkUnsafePutReference(invoke); in VisitUnsafePutObject()
2023 void IntrinsicLocationsBuilderX86::VisitUnsafePutObjectOrdered(HInvoke* invoke) { in VisitUnsafePutObjectOrdered() argument
2024 VisitJdkUnsafePutObjectOrdered(invoke); in VisitUnsafePutObjectOrdered()
2026 void IntrinsicLocationsBuilderX86::VisitUnsafePutObjectVolatile(HInvoke* invoke) { in VisitUnsafePutObjectVolatile() argument
2027 VisitJdkUnsafePutReferenceVolatile(invoke); in VisitUnsafePutObjectVolatile()
2029 void IntrinsicLocationsBuilderX86::VisitUnsafePutLong(HInvoke* invoke) { in VisitUnsafePutLong() argument
2030 VisitJdkUnsafePutLong(invoke); in VisitUnsafePutLong()
2032 void IntrinsicLocationsBuilderX86::VisitUnsafePutLongOrdered(HInvoke* invoke) { in VisitUnsafePutLongOrdered() argument
2033 VisitJdkUnsafePutLongOrdered(invoke); in VisitUnsafePutLongOrdered()
2035 void IntrinsicLocationsBuilderX86::VisitUnsafePutLongVolatile(HInvoke* invoke) { in VisitUnsafePutLongVolatile() argument
2036 VisitJdkUnsafePutLongVolatile(invoke); in VisitUnsafePutLongVolatile()
2038 void IntrinsicLocationsBuilderX86::VisitUnsafePutByte(HInvoke* invoke) { in VisitUnsafePutByte() argument
2039 VisitJdkUnsafePutByte(invoke); in VisitUnsafePutByte()
2042 void IntrinsicLocationsBuilderX86::VisitJdkUnsafePut(HInvoke* invoke) { in VisitJdkUnsafePut() argument
2044 allocator_, DataType::Type::kInt32, invoke, /*is_volatile=*/ false); in VisitJdkUnsafePut()
2046 void IntrinsicLocationsBuilderX86::VisitJdkUnsafePutAbsolute(HInvoke* invoke) { in VisitJdkUnsafePutAbsolute() argument
2048 allocator_, DataType::Type::kInt64, invoke, /*is_volatile=*/ false); in VisitJdkUnsafePutAbsolute()
2050 void IntrinsicLocationsBuilderX86::VisitJdkUnsafePutOrdered(HInvoke* invoke) { in VisitJdkUnsafePutOrdered() argument
2052 allocator_, DataType::Type::kInt32, invoke, /*is_volatile=*/ false); in VisitJdkUnsafePutOrdered()
2054 void IntrinsicLocationsBuilderX86::VisitJdkUnsafePutVolatile(HInvoke* invoke) { in VisitJdkUnsafePutVolatile() argument
2056 allocator_, DataType::Type::kInt32, invoke, /*is_volatile=*/ true); in VisitJdkUnsafePutVolatile()
2058 void IntrinsicLocationsBuilderX86::VisitJdkUnsafePutRelease(HInvoke* invoke) { in VisitJdkUnsafePutRelease() argument
2060 allocator_, DataType::Type::kInt32, invoke, /*is_volatile=*/ true); in VisitJdkUnsafePutRelease()
2062 void IntrinsicLocationsBuilderX86::VisitJdkUnsafePutReference(HInvoke* invoke) { in VisitJdkUnsafePutReference() argument
2064 allocator_, DataType::Type::kReference, invoke, /*is_volatile=*/ false); in VisitJdkUnsafePutReference()
2066 void IntrinsicLocationsBuilderX86::VisitJdkUnsafePutObjectOrdered(HInvoke* invoke) { in VisitJdkUnsafePutObjectOrdered() argument
2068 allocator_, DataType::Type::kReference, invoke, /*is_volatile=*/ false); in VisitJdkUnsafePutObjectOrdered()
2070 void IntrinsicLocationsBuilderX86::VisitJdkUnsafePutReferenceVolatile(HInvoke* invoke) { in VisitJdkUnsafePutReferenceVolatile() argument
2072 allocator_, DataType::Type::kReference, invoke, /*is_volatile=*/ true); in VisitJdkUnsafePutReferenceVolatile()
2074 void IntrinsicLocationsBuilderX86::VisitJdkUnsafePutReferenceRelease(HInvoke* invoke) { in VisitJdkUnsafePutReferenceRelease() argument
2076 allocator_, DataType::Type::kReference, invoke, /*is_volatile=*/ true); in VisitJdkUnsafePutReferenceRelease()
2078 void IntrinsicLocationsBuilderX86::VisitJdkUnsafePutLong(HInvoke* invoke) { in VisitJdkUnsafePutLong() argument
2080 allocator_, DataType::Type::kInt64, invoke, /*is_volatile=*/ false); in VisitJdkUnsafePutLong()
2082 void IntrinsicLocationsBuilderX86::VisitJdkUnsafePutLongOrdered(HInvoke* invoke) { in VisitJdkUnsafePutLongOrdered() argument
2084 allocator_, DataType::Type::kInt64, invoke, /*is_volatile=*/ false); in VisitJdkUnsafePutLongOrdered()
2086 void IntrinsicLocationsBuilderX86::VisitJdkUnsafePutLongVolatile(HInvoke* invoke) { in VisitJdkUnsafePutLongVolatile() argument
2088 allocator_, DataType::Type::kInt64, invoke, /*is_volatile=*/ true); in VisitJdkUnsafePutLongVolatile()
2090 void IntrinsicLocationsBuilderX86::VisitJdkUnsafePutLongRelease(HInvoke* invoke) { in VisitJdkUnsafePutLongRelease() argument
2092 allocator_, DataType::Type::kInt64, invoke, /*is_volatile=*/ true); in VisitJdkUnsafePutLongRelease()
2094 void IntrinsicLocationsBuilderX86::VisitJdkUnsafePutByte(HInvoke* invoke) { in VisitJdkUnsafePutByte() argument
2096 allocator_, DataType::Type::kInt8, invoke, /*is_volatile=*/ false); in VisitJdkUnsafePutByte()
2197 void IntrinsicCodeGeneratorX86::VisitUnsafePut(HInvoke* invoke) { in VisitUnsafePut() argument
2198 VisitJdkUnsafePut(invoke); in VisitUnsafePut()
2200 void IntrinsicCodeGeneratorX86::VisitUnsafePutAbsolute(HInvoke* invoke) { in VisitUnsafePutAbsolute() argument
2201 VisitJdkUnsafePutAbsolute(invoke); in VisitUnsafePutAbsolute()
2203 void IntrinsicCodeGeneratorX86::VisitUnsafePutOrdered(HInvoke* invoke) { in VisitUnsafePutOrdered() argument
2204 VisitJdkUnsafePutOrdered(invoke); in VisitUnsafePutOrdered()
2206 void IntrinsicCodeGeneratorX86::VisitUnsafePutVolatile(HInvoke* invoke) { in VisitUnsafePutVolatile() argument
2207 VisitJdkUnsafePutVolatile(invoke); in VisitUnsafePutVolatile()
2209 void IntrinsicCodeGeneratorX86::VisitUnsafePutObject(HInvoke* invoke) { in VisitUnsafePutObject() argument
2210 VisitJdkUnsafePutReference(invoke); in VisitUnsafePutObject()
2212 void IntrinsicCodeGeneratorX86::VisitUnsafePutObjectOrdered(HInvoke* invoke) { in VisitUnsafePutObjectOrdered() argument
2213 VisitJdkUnsafePutObjectOrdered(invoke); in VisitUnsafePutObjectOrdered()
2215 void IntrinsicCodeGeneratorX86::VisitUnsafePutObjectVolatile(HInvoke* invoke) { in VisitUnsafePutObjectVolatile() argument
2216 VisitJdkUnsafePutReferenceVolatile(invoke); in VisitUnsafePutObjectVolatile()
2218 void IntrinsicCodeGeneratorX86::VisitUnsafePutLong(HInvoke* invoke) { in VisitUnsafePutLong() argument
2219 VisitJdkUnsafePutLong(invoke); in VisitUnsafePutLong()
2221 void IntrinsicCodeGeneratorX86::VisitUnsafePutLongOrdered(HInvoke* invoke) { in VisitUnsafePutLongOrdered() argument
2222 VisitJdkUnsafePutLongOrdered(invoke); in VisitUnsafePutLongOrdered()
2224 void IntrinsicCodeGeneratorX86::VisitUnsafePutLongVolatile(HInvoke* invoke) { in VisitUnsafePutLongVolatile() argument
2225 VisitJdkUnsafePutLongVolatile(invoke); in VisitUnsafePutLongVolatile()
2227 void IntrinsicCodeGeneratorX86::VisitUnsafePutByte(HInvoke* invoke) { in VisitUnsafePutByte() argument
2228 VisitJdkUnsafePutByte(invoke); in VisitUnsafePutByte()
2231 void IntrinsicCodeGeneratorX86::VisitJdkUnsafePut(HInvoke* invoke) { in VisitJdkUnsafePut() argument
2232 GenUnsafePut(invoke->GetLocations(), DataType::Type::kInt32, /*is_volatile=*/ false, codegen_); in VisitJdkUnsafePut()
2234 void IntrinsicCodeGeneratorX86::VisitJdkUnsafePutAbsolute(HInvoke* invoke) { in VisitJdkUnsafePutAbsolute() argument
2236 invoke->GetLocations(), DataType::Type::kInt32, /*is_volatile=*/false, codegen_); in VisitJdkUnsafePutAbsolute()
2238 void IntrinsicCodeGeneratorX86::VisitJdkUnsafePutOrdered(HInvoke* invoke) { in VisitJdkUnsafePutOrdered() argument
2239 GenUnsafePut(invoke->GetLocations(), DataType::Type::kInt32, /*is_volatile=*/ false, codegen_); in VisitJdkUnsafePutOrdered()
2241 void IntrinsicCodeGeneratorX86::VisitJdkUnsafePutVolatile(HInvoke* invoke) { in VisitJdkUnsafePutVolatile() argument
2242 GenUnsafePut(invoke->GetLocations(), DataType::Type::kInt32, /*is_volatile=*/ true, codegen_); in VisitJdkUnsafePutVolatile()
2244 void IntrinsicCodeGeneratorX86::VisitJdkUnsafePutRelease(HInvoke* invoke) { in VisitJdkUnsafePutRelease() argument
2245 GenUnsafePut(invoke->GetLocations(), DataType::Type::kInt32, /*is_volatile=*/ true, codegen_); in VisitJdkUnsafePutRelease()
2247 void IntrinsicCodeGeneratorX86::VisitJdkUnsafePutReference(HInvoke* invoke) { in VisitJdkUnsafePutReference() argument
2249 invoke->GetLocations(), DataType::Type::kReference, /*is_volatile=*/ false, codegen_); in VisitJdkUnsafePutReference()
2251 void IntrinsicCodeGeneratorX86::VisitJdkUnsafePutObjectOrdered(HInvoke* invoke) { in VisitJdkUnsafePutObjectOrdered() argument
2253 invoke->GetLocations(), DataType::Type::kReference, /*is_volatile=*/ false, codegen_); in VisitJdkUnsafePutObjectOrdered()
2255 void IntrinsicCodeGeneratorX86::VisitJdkUnsafePutReferenceVolatile(HInvoke* invoke) { in VisitJdkUnsafePutReferenceVolatile() argument
2257 invoke->GetLocations(), DataType::Type::kReference, /*is_volatile=*/ true, codegen_); in VisitJdkUnsafePutReferenceVolatile()
2259 void IntrinsicCodeGeneratorX86::VisitJdkUnsafePutReferenceRelease(HInvoke* invoke) { in VisitJdkUnsafePutReferenceRelease() argument
2261 invoke->GetLocations(), DataType::Type::kReference, /*is_volatile=*/ true, codegen_); in VisitJdkUnsafePutReferenceRelease()
2263 void IntrinsicCodeGeneratorX86::VisitJdkUnsafePutLong(HInvoke* invoke) { in VisitJdkUnsafePutLong() argument
2264 GenUnsafePut(invoke->GetLocations(), DataType::Type::kInt64, /*is_volatile=*/ false, codegen_); in VisitJdkUnsafePutLong()
2266 void IntrinsicCodeGeneratorX86::VisitJdkUnsafePutLongOrdered(HInvoke* invoke) { in VisitJdkUnsafePutLongOrdered() argument
2267 GenUnsafePut(invoke->GetLocations(), DataType::Type::kInt64, /*is_volatile=*/ false, codegen_); in VisitJdkUnsafePutLongOrdered()
2269 void IntrinsicCodeGeneratorX86::VisitJdkUnsafePutLongVolatile(HInvoke* invoke) { in VisitJdkUnsafePutLongVolatile() argument
2270 GenUnsafePut(invoke->GetLocations(), DataType::Type::kInt64, /*is_volatile=*/ true, codegen_); in VisitJdkUnsafePutLongVolatile()
2272 void IntrinsicCodeGeneratorX86::VisitJdkUnsafePutLongRelease(HInvoke* invoke) { in VisitJdkUnsafePutLongRelease() argument
2273 GenUnsafePut(invoke->GetLocations(), DataType::Type::kInt64, /*is_volatile=*/ true, codegen_); in VisitJdkUnsafePutLongRelease()
2275 void IntrinsicCodeGeneratorX86::VisitJdkUnsafePutByte(HInvoke* invoke) { in VisitJdkUnsafePutByte() argument
2276 GenUnsafePut(invoke->GetLocations(), DataType::Type::kInt8, /*is_volatile=*/ false, codegen_); in VisitJdkUnsafePutByte()
2282 HInvoke* invoke) { in CreateIntIntIntIntIntToInt() argument
2283 const bool can_call = codegen->EmitBakerReadBarrier() && IsUnsafeCASReference(invoke); in CreateIntIntIntIntIntToInt()
2285 new (allocator) LocationSummary(invoke, in CreateIntIntIntIntIntToInt()
2293 // Can we update the invoke here to remove a TypeConvert to Long? in CreateIntIntIntIntIntToInt()
2316 void IntrinsicLocationsBuilderX86::VisitUnsafeCASInt(HInvoke* invoke) { in VisitUnsafeCASInt() argument
2317 VisitJdkUnsafeCASInt(invoke); in VisitUnsafeCASInt()
2320 void IntrinsicLocationsBuilderX86::VisitUnsafeCASLong(HInvoke* invoke) { in VisitUnsafeCASLong() argument
2321 VisitJdkUnsafeCASLong(invoke); in VisitUnsafeCASLong()
2324 void IntrinsicLocationsBuilderX86::VisitUnsafeCASObject(HInvoke* invoke) { in VisitUnsafeCASObject() argument
2325 VisitJdkUnsafeCASObject(invoke); in VisitUnsafeCASObject()
2328 void IntrinsicLocationsBuilderX86::VisitJdkUnsafeCASInt(HInvoke* invoke) { in VisitJdkUnsafeCASInt() argument
2330 VisitJdkUnsafeCompareAndSetInt(invoke); in VisitJdkUnsafeCASInt()
2333 void IntrinsicLocationsBuilderX86::VisitJdkUnsafeCASLong(HInvoke* invoke) { in VisitJdkUnsafeCASLong() argument
2335 VisitJdkUnsafeCompareAndSetLong(invoke); in VisitJdkUnsafeCASLong()
2338 void IntrinsicLocationsBuilderX86::VisitJdkUnsafeCASObject(HInvoke* invoke) { in VisitJdkUnsafeCASObject() argument
2340 VisitJdkUnsafeCompareAndSetReference(invoke); in VisitJdkUnsafeCASObject()
2343 void IntrinsicLocationsBuilderX86::VisitJdkUnsafeCompareAndSetInt(HInvoke* invoke) { in VisitJdkUnsafeCompareAndSetInt() argument
2344 CreateIntIntIntIntIntToInt(allocator_, codegen_, DataType::Type::kInt32, invoke); in VisitJdkUnsafeCompareAndSetInt()
2347 void IntrinsicLocationsBuilderX86::VisitJdkUnsafeCompareAndSetLong(HInvoke* invoke) { in VisitJdkUnsafeCompareAndSetLong() argument
2348 CreateIntIntIntIntIntToInt(allocator_, codegen_, DataType::Type::kInt64, invoke); in VisitJdkUnsafeCompareAndSetLong()
2351 void IntrinsicLocationsBuilderX86::VisitJdkUnsafeCompareAndSetReference(HInvoke* invoke) { in VisitJdkUnsafeCompareAndSetReference() argument
2357 CreateIntIntIntIntIntToInt(allocator_, codegen_, DataType::Type::kReference, invoke); in VisitJdkUnsafeCompareAndSetReference()
2461 static void GenReferenceCAS(HInvoke* invoke, in GenReferenceCAS() argument
2471 LocationSummary* locations = invoke->GetLocations(); in GenReferenceCAS()
2486 invoke, in GenReferenceCAS()
2556 static void GenCAS(DataType::Type type, HInvoke* invoke, CodeGeneratorX86* codegen) { in GenCAS() argument
2557 LocationSummary* locations = invoke->GetLocations(); in GenCAS()
2573 GenReferenceCAS(invoke, codegen, expected_value, new_value, base, offset, temp, temp2); in GenCAS()
2580 void IntrinsicCodeGeneratorX86::VisitUnsafeCASInt(HInvoke* invoke) { in VisitUnsafeCASInt() argument
2581 VisitJdkUnsafeCASInt(invoke); in VisitUnsafeCASInt()
2584 void IntrinsicCodeGeneratorX86::VisitUnsafeCASLong(HInvoke* invoke) { in VisitUnsafeCASLong() argument
2585 VisitJdkUnsafeCASLong(invoke); in VisitUnsafeCASLong()
2588 void IntrinsicCodeGeneratorX86::VisitUnsafeCASObject(HInvoke* invoke) { in VisitUnsafeCASObject() argument
2593 GenCAS(DataType::Type::kReference, invoke, codegen_); in VisitUnsafeCASObject()
2596 void IntrinsicCodeGeneratorX86::VisitJdkUnsafeCASInt(HInvoke* invoke) { in VisitJdkUnsafeCASInt() argument
2598 VisitJdkUnsafeCompareAndSetInt(invoke); in VisitJdkUnsafeCASInt()
2601 void IntrinsicCodeGeneratorX86::VisitJdkUnsafeCASLong(HInvoke* invoke) { in VisitJdkUnsafeCASLong() argument
2603 VisitJdkUnsafeCompareAndSetLong(invoke); in VisitJdkUnsafeCASLong()
2606 void IntrinsicCodeGeneratorX86::VisitJdkUnsafeCASObject(HInvoke* invoke) { in VisitJdkUnsafeCASObject() argument
2608 VisitJdkUnsafeCompareAndSetReference(invoke); in VisitJdkUnsafeCASObject()
2611 void IntrinsicCodeGeneratorX86::VisitJdkUnsafeCompareAndSetInt(HInvoke* invoke) { in VisitJdkUnsafeCompareAndSetInt() argument
2612 GenCAS(DataType::Type::kInt32, invoke, codegen_); in VisitJdkUnsafeCompareAndSetInt()
2615 void IntrinsicCodeGeneratorX86::VisitJdkUnsafeCompareAndSetLong(HInvoke* invoke) { in VisitJdkUnsafeCompareAndSetLong() argument
2616 GenCAS(DataType::Type::kInt64, invoke, codegen_); in VisitJdkUnsafeCompareAndSetLong()
2619 void IntrinsicCodeGeneratorX86::VisitJdkUnsafeCompareAndSetReference(HInvoke* invoke) { in VisitJdkUnsafeCompareAndSetReference() argument
2623 GenCAS(DataType::Type::kReference, invoke, codegen_); in VisitJdkUnsafeCompareAndSetReference()
2634 HInvoke* invoke, in CreateUnsafeGetAndUpdateLocations() argument
2638 const bool can_call = codegen->EmitReadBarrier() && IsUnsafeGetAndSetReference(invoke); in CreateUnsafeGetAndUpdateLocations()
2640 new (allocator) LocationSummary(invoke, in CreateUnsafeGetAndUpdateLocations()
2649 const bool is_void = invoke->GetType() == DataType::Type::kVoid; in CreateUnsafeGetAndUpdateLocations()
2685 void IntrinsicLocationsBuilderX86::VisitUnsafeGetAndAddInt(HInvoke* invoke) { in VisitUnsafeGetAndAddInt() argument
2686 VisitJdkUnsafeGetAndAddInt(invoke); in VisitUnsafeGetAndAddInt()
2689 void IntrinsicLocationsBuilderX86::VisitUnsafeGetAndAddLong(HInvoke* invoke) { in VisitUnsafeGetAndAddLong() argument
2690 VisitJdkUnsafeGetAndAddLong(invoke); in VisitUnsafeGetAndAddLong()
2693 void IntrinsicLocationsBuilderX86::VisitUnsafeGetAndSetInt(HInvoke* invoke) { in VisitUnsafeGetAndSetInt() argument
2694 VisitJdkUnsafeGetAndSetInt(invoke); in VisitUnsafeGetAndSetInt()
2697 void IntrinsicLocationsBuilderX86::VisitUnsafeGetAndSetLong(HInvoke* invoke) { in VisitUnsafeGetAndSetLong() argument
2698 VisitJdkUnsafeGetAndSetLong(invoke); in VisitUnsafeGetAndSetLong()
2701 void IntrinsicLocationsBuilderX86::VisitUnsafeGetAndSetObject(HInvoke* invoke) { in VisitUnsafeGetAndSetObject() argument
2702 VisitJdkUnsafeGetAndSetReference(invoke); in VisitUnsafeGetAndSetObject()
2705 void IntrinsicLocationsBuilderX86::VisitJdkUnsafeGetAndAddInt(HInvoke* invoke) { in VisitJdkUnsafeGetAndAddInt() argument
2707 allocator_, invoke, codegen_, DataType::Type::kInt32, GetAndUpdateOp::kAdd); in VisitJdkUnsafeGetAndAddInt()
2710 void IntrinsicLocationsBuilderX86::VisitJdkUnsafeGetAndAddLong(HInvoke* invoke) { in VisitJdkUnsafeGetAndAddLong() argument
2712 allocator_, invoke, codegen_, DataType::Type::kInt64, GetAndUpdateOp::kAdd); in VisitJdkUnsafeGetAndAddLong()
2715 void IntrinsicLocationsBuilderX86::VisitJdkUnsafeGetAndSetInt(HInvoke* invoke) { in VisitJdkUnsafeGetAndSetInt() argument
2717 allocator_, invoke, codegen_, DataType::Type::kInt32, GetAndUpdateOp::kSet); in VisitJdkUnsafeGetAndSetInt()
2720 void IntrinsicLocationsBuilderX86::VisitJdkUnsafeGetAndSetLong(HInvoke* invoke) { in VisitJdkUnsafeGetAndSetLong() argument
2722 allocator_, invoke, codegen_, DataType::Type::kInt64, GetAndUpdateOp::kSet); in VisitJdkUnsafeGetAndSetLong()
2725 void IntrinsicLocationsBuilderX86::VisitJdkUnsafeGetAndSetReference(HInvoke* invoke) { in VisitJdkUnsafeGetAndSetReference() argument
2732 allocator_, invoke, codegen_, DataType::Type::kReference, GetAndUpdateOp::kSet); in VisitJdkUnsafeGetAndSetReference()
2733 LocationSummary* locations = invoke->GetLocations(); in VisitJdkUnsafeGetAndSetReference()
2738 static void GenUnsafeGetAndUpdate(HInvoke* invoke, in GenUnsafeGetAndUpdate() argument
2743 LocationSummary* locations = invoke->GetLocations(); in GenUnsafeGetAndUpdate()
2745 const bool is_void = invoke->GetType() == DataType::Type::kVoid; in GenUnsafeGetAndUpdate()
2808 invoke, in GenUnsafeGetAndUpdate()
2838 void IntrinsicCodeGeneratorX86::VisitUnsafeGetAndAddInt(HInvoke* invoke) { in VisitUnsafeGetAndAddInt() argument
2839 VisitJdkUnsafeGetAndAddInt(invoke); in VisitUnsafeGetAndAddInt()
2842 void IntrinsicCodeGeneratorX86::VisitUnsafeGetAndAddLong(HInvoke* invoke) { in VisitUnsafeGetAndAddLong() argument
2843 VisitJdkUnsafeGetAndAddLong(invoke); in VisitUnsafeGetAndAddLong()
2846 void IntrinsicCodeGeneratorX86::VisitUnsafeGetAndSetInt(HInvoke* invoke) { in VisitUnsafeGetAndSetInt() argument
2847 VisitJdkUnsafeGetAndSetInt(invoke); in VisitUnsafeGetAndSetInt()
2850 void IntrinsicCodeGeneratorX86::VisitUnsafeGetAndSetLong(HInvoke* invoke) { in VisitUnsafeGetAndSetLong() argument
2851 VisitJdkUnsafeGetAndSetLong(invoke); in VisitUnsafeGetAndSetLong()
2854 void IntrinsicCodeGeneratorX86::VisitUnsafeGetAndSetObject(HInvoke* invoke) { in VisitUnsafeGetAndSetObject() argument
2855 VisitJdkUnsafeGetAndSetReference(invoke); in VisitUnsafeGetAndSetObject()
2858 void IntrinsicCodeGeneratorX86::VisitJdkUnsafeGetAndAddInt(HInvoke* invoke) { in VisitJdkUnsafeGetAndAddInt() argument
2859 GenUnsafeGetAndUpdate(invoke, DataType::Type::kInt32, codegen_, GetAndUpdateOp::kAdd); in VisitJdkUnsafeGetAndAddInt()
2862 void IntrinsicCodeGeneratorX86::VisitJdkUnsafeGetAndAddLong(HInvoke* invoke) { in VisitJdkUnsafeGetAndAddLong() argument
2863 GenUnsafeGetAndUpdate(invoke, DataType::Type::kInt64, codegen_, GetAndUpdateOp::kAdd); in VisitJdkUnsafeGetAndAddLong()
2866 void IntrinsicCodeGeneratorX86::VisitJdkUnsafeGetAndSetInt(HInvoke* invoke) { in VisitJdkUnsafeGetAndSetInt() argument
2867 GenUnsafeGetAndUpdate(invoke, DataType::Type::kInt32, codegen_, GetAndUpdateOp::kSet); in VisitJdkUnsafeGetAndSetInt()
2870 void IntrinsicCodeGeneratorX86::VisitJdkUnsafeGetAndSetLong(HInvoke* invoke) { in VisitJdkUnsafeGetAndSetLong() argument
2871 GenUnsafeGetAndUpdate(invoke, DataType::Type::kInt64, codegen_, GetAndUpdateOp::kSet); in VisitJdkUnsafeGetAndSetLong()
2874 void IntrinsicCodeGeneratorX86::VisitJdkUnsafeGetAndSetReference(HInvoke* invoke) { in VisitJdkUnsafeGetAndSetReference() argument
2875 GenUnsafeGetAndUpdate(invoke, DataType::Type::kReference, codegen_, GetAndUpdateOp::kSet); in VisitJdkUnsafeGetAndSetReference()
2878 void IntrinsicLocationsBuilderX86::VisitIntegerReverse(HInvoke* invoke) { in VisitIntegerReverse() argument
2880 new (allocator_) LocationSummary(invoke, LocationSummary::kNoCall, kIntrinsified); in VisitIntegerReverse()
2898 void IntrinsicCodeGeneratorX86::VisitIntegerReverse(HInvoke* invoke) { in VisitIntegerReverse() argument
2900 LocationSummary* locations = invoke->GetLocations(); in VisitIntegerReverse()
2920 void IntrinsicLocationsBuilderX86::VisitLongReverse(HInvoke* invoke) { in VisitLongReverse() argument
2922 new (allocator_) LocationSummary(invoke, LocationSummary::kNoCall, kIntrinsified); in VisitLongReverse()
2928 void IntrinsicCodeGeneratorX86::VisitLongReverse(HInvoke* invoke) { in VisitLongReverse() argument
2930 LocationSummary* locations = invoke->GetLocations(); in VisitLongReverse()
2957 ArenaAllocator* allocator, CodeGeneratorX86* codegen, HInvoke* invoke, bool is_long) { in CreateBitCountLocations() argument
2964 new (allocator) LocationSummary(invoke, LocationSummary::kNoCall, kIntrinsified); in CreateBitCountLocations()
2974 HInvoke* invoke, bool is_long) { in GenBitCount() argument
2975 LocationSummary* locations = invoke->GetLocations(); in GenBitCount()
2979 if (invoke->InputAt(0)->IsConstant()) { in GenBitCount()
2981 int64_t value = Int64FromConstant(invoke->InputAt(0)->AsConstant()); in GenBitCount()
3012 void IntrinsicLocationsBuilderX86::VisitIntegerBitCount(HInvoke* invoke) { in VisitIntegerBitCount() argument
3013 CreateBitCountLocations(allocator_, codegen_, invoke, /* is_long= */ false); in VisitIntegerBitCount()
3016 void IntrinsicCodeGeneratorX86::VisitIntegerBitCount(HInvoke* invoke) { in VisitIntegerBitCount() argument
3017 GenBitCount(GetAssembler(), codegen_, invoke, /* is_long= */ false); in VisitIntegerBitCount()
3020 void IntrinsicLocationsBuilderX86::VisitLongBitCount(HInvoke* invoke) { in VisitLongBitCount() argument
3021 CreateBitCountLocations(allocator_, codegen_, invoke, /* is_long= */ true); in VisitLongBitCount()
3024 void IntrinsicCodeGeneratorX86::VisitLongBitCount(HInvoke* invoke) { in VisitLongBitCount() argument
3025 GenBitCount(GetAssembler(), codegen_, invoke, /* is_long= */ true); in VisitLongBitCount()
3028 static void CreateLeadingZeroLocations(ArenaAllocator* allocator, HInvoke* invoke, bool is_long) { in CreateLeadingZeroLocations() argument
3030 new (allocator) LocationSummary(invoke, LocationSummary::kNoCall, kIntrinsified); in CreateLeadingZeroLocations()
3041 HInvoke* invoke, bool is_long) { in GenLeadingZeros() argument
3042 LocationSummary* locations = invoke->GetLocations(); in GenLeadingZeros()
3046 if (invoke->InputAt(0)->IsConstant()) { in GenLeadingZeros()
3048 int64_t value = Int64FromConstant(invoke->InputAt(0)->AsConstant()); in GenLeadingZeros()
3116 void IntrinsicLocationsBuilderX86::VisitIntegerNumberOfLeadingZeros(HInvoke* invoke) { in VisitIntegerNumberOfLeadingZeros() argument
3117 CreateLeadingZeroLocations(allocator_, invoke, /* is_long= */ false); in VisitIntegerNumberOfLeadingZeros()
3120 void IntrinsicCodeGeneratorX86::VisitIntegerNumberOfLeadingZeros(HInvoke* invoke) { in VisitIntegerNumberOfLeadingZeros() argument
3121 GenLeadingZeros(GetAssembler(), codegen_, invoke, /* is_long= */ false); in VisitIntegerNumberOfLeadingZeros()
3124 void IntrinsicLocationsBuilderX86::VisitLongNumberOfLeadingZeros(HInvoke* invoke) { in VisitLongNumberOfLeadingZeros() argument
3125 CreateLeadingZeroLocations(allocator_, invoke, /* is_long= */ true); in VisitLongNumberOfLeadingZeros()
3128 void IntrinsicCodeGeneratorX86::VisitLongNumberOfLeadingZeros(HInvoke* invoke) { in VisitLongNumberOfLeadingZeros() argument
3129 GenLeadingZeros(GetAssembler(), codegen_, invoke, /* is_long= */ true); in VisitLongNumberOfLeadingZeros()
3132 static void CreateTrailingZeroLocations(ArenaAllocator* allocator, HInvoke* invoke, bool is_long) { in CreateTrailingZeroLocations() argument
3134 new (allocator) LocationSummary(invoke, LocationSummary::kNoCall, kIntrinsified); in CreateTrailingZeroLocations()
3145 HInvoke* invoke, bool is_long) { in GenTrailingZeros() argument
3146 LocationSummary* locations = invoke->GetLocations(); in GenTrailingZeros()
3150 if (invoke->InputAt(0)->IsConstant()) { in GenTrailingZeros()
3152 int64_t value = Int64FromConstant(invoke->InputAt(0)->AsConstant()); in GenTrailingZeros()
3207 void IntrinsicLocationsBuilderX86::VisitIntegerNumberOfTrailingZeros(HInvoke* invoke) { in VisitIntegerNumberOfTrailingZeros() argument
3208 CreateTrailingZeroLocations(allocator_, invoke, /* is_long= */ false); in VisitIntegerNumberOfTrailingZeros()
3211 void IntrinsicCodeGeneratorX86::VisitIntegerNumberOfTrailingZeros(HInvoke* invoke) { in VisitIntegerNumberOfTrailingZeros() argument
3212 GenTrailingZeros(GetAssembler(), codegen_, invoke, /* is_long= */ false); in VisitIntegerNumberOfTrailingZeros()
3215 void IntrinsicLocationsBuilderX86::VisitLongNumberOfTrailingZeros(HInvoke* invoke) { in VisitLongNumberOfTrailingZeros() argument
3216 CreateTrailingZeroLocations(allocator_, invoke, /* is_long= */ true); in VisitLongNumberOfTrailingZeros()
3219 void IntrinsicCodeGeneratorX86::VisitLongNumberOfTrailingZeros(HInvoke* invoke) { in VisitLongNumberOfTrailingZeros() argument
3220 GenTrailingZeros(GetAssembler(), codegen_, invoke, /* is_long= */ true); in VisitLongNumberOfTrailingZeros()
3227 void IntrinsicLocationsBuilderX86::VisitSystemArrayCopy(HInvoke* invoke) { in VisitSystemArrayCopy() argument
3237 invoke, kLengthThreshold, kInitialNumTemps); in VisitSystemArrayCopy()
3261 IsSameInput(invoke, kSrc, kDest) || in VisitSystemArrayCopy()
3262 IsSameInput(invoke, kSrcPos, kDestPos)) { in VisitSystemArrayCopy()
3275 void IntrinsicCodeGeneratorX86::VisitSystemArrayCopy(HInvoke* invoke) { in VisitSystemArrayCopy() argument
3281 LocationSummary* locations = invoke->GetLocations(); in VisitSystemArrayCopy()
3300 new (codegen_->GetScopedAllocator()) IntrinsicSlowPathX86(invoke); in VisitSystemArrayCopy()
3304 SystemArrayCopyOptimizations optimizations(invoke); in VisitSystemArrayCopy()
3405 invoke, temp1_loc, dest, class_offset, /* needs_null_check= */ false); in VisitSystemArrayCopy()
3413 invoke, temp2_loc, src, class_offset, /* needs_null_check= */ false); in VisitSystemArrayCopy()
3524 new (codegen_->GetScopedAllocator()) ReadBarrierSystemArrayCopySlowPathX86(invoke); in VisitSystemArrayCopy()
3558 static void RequestBaseMethodAddressInRegister(HInvoke* invoke) { in RequestBaseMethodAddressInRegister() argument
3559 LocationSummary* locations = invoke->GetLocations(); in RequestBaseMethodAddressInRegister()
3561 HInvokeStaticOrDirect* invoke_static_or_direct = invoke->AsInvokeStaticOrDirect(); in RequestBaseMethodAddressInRegister()
3574 void IntrinsicLocationsBuilderX86::Visit##name##ValueOf(HInvoke* invoke) { \
3577 invoke, \
3583 RequestBaseMethodAddressInRegister(invoke); \
3585 void IntrinsicCodeGeneratorX86::Visit##name##ValueOf(HInvoke* invoke) { \
3587 IntrinsicVisitor::ComputeValueOfInfo(invoke, \
3593 HandleValueOf(invoke, info, type); \
3598 void IntrinsicCodeGeneratorX86::HandleValueOf(HInvoke* invoke, in BOXED_TYPES()
3601 DCHECK(invoke->IsInvokeStaticOrDirect()); in BOXED_TYPES()
3602 LocationSummary* locations = invoke->GetLocations(); in BOXED_TYPES()
3608 codegen_->LoadIntrinsicDeclaringClass(out, invoke->AsInvokeStaticOrDirect()); in BOXED_TYPES()
3609 codegen_->InvokeRuntime(kQuickAllocObjectInitialized, invoke, invoke->GetDexPc()); in BOXED_TYPES()
3612 if (invoke->InputAt(0)->IsIntConstant()) { in BOXED_TYPES()
3613 int32_t value = invoke->InputAt(0)->AsIntConstant()->GetValue(); in BOXED_TYPES()
3618 out, info.value_boot_image_reference, invoke->AsInvokeStaticOrDirect()); in BOXED_TYPES()
3626 Location::ConstantLocation(invoke->InputAt(0)->AsIntConstant()), in BOXED_TYPES()
3645 DCHECK_EQ(invoke->InputCount(), invoke->GetNumberOfArguments() + 1u); in BOXED_TYPES()
3646 size_t method_address_index = invoke->AsInvokeStaticOrDirect()->GetSpecialInputIndex(); in BOXED_TYPES()
3648 invoke->InputAt(method_address_index)->AsX86ComputeBaseMethodAddress(); in BOXED_TYPES()
3651 invoke->GetLocations()->InAt(method_address_index).AsRegister<Register>(); in BOXED_TYPES()
3661 out, mid_array_boot_image_offset, invoke->AsInvokeStaticOrDirect()); in BOXED_TYPES()
3680 void IntrinsicLocationsBuilderX86::VisitReferenceGetReferent(HInvoke* invoke) { in VisitReferenceGetReferent() argument
3681 IntrinsicVisitor::CreateReferenceGetReferentLocations(invoke, codegen_); in VisitReferenceGetReferent()
3682 RequestBaseMethodAddressInRegister(invoke); in VisitReferenceGetReferent()
3685 void IntrinsicCodeGeneratorX86::VisitReferenceGetReferent(HInvoke* invoke) { in VisitReferenceGetReferent() argument
3687 LocationSummary* locations = invoke->GetLocations(); in VisitReferenceGetReferent()
3692 SlowPathCode* slow_path = new (GetAllocator()) IntrinsicSlowPathX86(invoke); in VisitReferenceGetReferent()
3705 invoke->AsInvokeStaticOrDirect()); in VisitReferenceGetReferent()
3719 codegen_->GenerateFieldLoadWithBakerReadBarrier(invoke, in VisitReferenceGetReferent()
3728 codegen_->MaybeRecordImplicitNullCheck(invoke); in VisitReferenceGetReferent()
3731 codegen_->MaybeGenerateReadBarrierSlow(invoke, out, out, obj, referent_offset); in VisitReferenceGetReferent()
3736 void IntrinsicLocationsBuilderX86::VisitReferenceRefersTo(HInvoke* invoke) { in VisitReferenceRefersTo() argument
3737 IntrinsicVisitor::CreateReferenceRefersToLocations(invoke, codegen_); in VisitReferenceRefersTo()
3740 void IntrinsicCodeGeneratorX86::VisitReferenceRefersTo(HInvoke* invoke) { in VisitReferenceRefersTo() argument
3742 LocationSummary* locations = invoke->GetLocations(); in VisitReferenceRefersTo()
3752 codegen_->MaybeRecordImplicitNullCheck(invoke); in VisitReferenceRefersTo()
3797 void IntrinsicLocationsBuilderX86::VisitThreadInterrupted(HInvoke* invoke) { in VisitThreadInterrupted() argument
3799 new (allocator_) LocationSummary(invoke, LocationSummary::kNoCall, kIntrinsified); in VisitThreadInterrupted()
3803 void IntrinsicCodeGeneratorX86::VisitThreadInterrupted(HInvoke* invoke) { in VisitThreadInterrupted() argument
3805 Register out = invoke->GetLocations()->Out().AsRegister<Register>(); in VisitThreadInterrupted()
3816 void IntrinsicLocationsBuilderX86::VisitReachabilityFence(HInvoke* invoke) { in VisitReachabilityFence() argument
3818 new (allocator_) LocationSummary(invoke, LocationSummary::kNoCall, kIntrinsified); in VisitReachabilityFence()
3822 void IntrinsicCodeGeneratorX86::VisitReachabilityFence([[maybe_unused]] HInvoke* invoke) {} in VisitReachabilityFence() argument
3824 void IntrinsicLocationsBuilderX86::VisitIntegerDivideUnsigned(HInvoke* invoke) { in VisitIntegerDivideUnsigned() argument
3825 LocationSummary* locations = new (allocator_) LocationSummary(invoke, in VisitIntegerDivideUnsigned()
3835 void IntrinsicCodeGeneratorX86::VisitIntegerDivideUnsigned(HInvoke* invoke) { in VisitIntegerDivideUnsigned() argument
3837 LocationSummary* locations = invoke->GetLocations(); in VisitIntegerDivideUnsigned()
3850 SlowPathCode* slow_path = new (codegen_->GetScopedAllocator()) IntrinsicSlowPathX86(invoke); in VisitIntegerDivideUnsigned()
3860 static bool HasVarHandleIntrinsicImplementation(HInvoke* invoke) { in HasVarHandleIntrinsicImplementation() argument
3861 VarHandleOptimizations optimizations(invoke); in HasVarHandleIntrinsicImplementation()
3866 size_t expected_coordinates_count = GetExpectedVarHandleCoordinatesCount(invoke); in HasVarHandleIntrinsicImplementation()
3934 static void GenerateVarHandleInstanceFieldChecks(HInvoke* invoke, in GenerateVarHandleInstanceFieldChecks() argument
3938 VarHandleOptimizations optimizations(invoke); in GenerateVarHandleInstanceFieldChecks()
3939 LocationSummary* locations = invoke->GetLocations(); in GenerateVarHandleInstanceFieldChecks()
3984 static void GenerateVarHandleCommonChecks(HInvoke *invoke, in GenerateVarHandleCommonChecks() argument
3988 LocationSummary* locations = invoke->GetLocations(); in GenerateVarHandleCommonChecks()
3991 mirror::VarHandle::GetAccessModeByIntrinsic(invoke->GetIntrinsic()); in GenerateVarHandleCommonChecks()
3998 size_t expected_coordinates_count = GetExpectedVarHandleCoordinatesCount(invoke); in GenerateVarHandleCommonChecks()
4004 GenerateVarHandleInstanceFieldChecks(invoke, temp, slow_path, assembler); in GenerateVarHandleCommonChecks()
4015 DataType::Type type = invoke->GetType(); in GenerateVarHandleCommonChecks()
4025 uint32_t value_index = invoke->GetNumberOfArguments() - 1; in GenerateVarHandleCommonChecks()
4026 DataType::Type value_type = GetDataTypeFromShorty(invoke, value_index); in GenerateVarHandleCommonChecks()
4044 uint32_t new_value_index = invoke->GetNumberOfArguments() - 1; in GenerateVarHandleCommonChecks()
4045 uint32_t expected_value_index = invoke->GetNumberOfArguments() - 2; in GenerateVarHandleCommonChecks()
4046 DataType::Type value_type = GetDataTypeFromShorty(invoke, new_value_index); in GenerateVarHandleCommonChecks()
4047 DCHECK_EQ(value_type, GetDataTypeFromShorty(invoke, expected_value_index)); in GenerateVarHandleCommonChecks()
4076 static Register GenerateVarHandleFieldReference(HInvoke* invoke, in GenerateVarHandleFieldReference() argument
4081 LocationSummary* locations = invoke->GetLocations(); in GenerateVarHandleFieldReference()
4090 size_t expected_coordinates_count = GetExpectedVarHandleCoordinatesCount(invoke); in GenerateVarHandleFieldReference()
4095 instr_codegen->GenerateGcRootFieldLoad(invoke, in GenerateVarHandleFieldReference()
4109 static void CreateVarHandleGetLocations(HInvoke* invoke, CodeGeneratorX86* codegen) { in CreateVarHandleGetLocations() argument
4116 if (!HasVarHandleIntrinsicImplementation(invoke)) { in CreateVarHandleGetLocations()
4120 ArenaAllocator* allocator = invoke->GetBlock()->GetGraph()->GetAllocator(); in CreateVarHandleGetLocations()
4122 invoke, LocationSummary::kCallOnSlowPath, kIntrinsified); in CreateVarHandleGetLocations()
4124 size_t expected_coordinates_count = GetExpectedVarHandleCoordinatesCount(invoke); in CreateVarHandleGetLocations()
4131 DataType::Type type = invoke->GetType(); in CreateVarHandleGetLocations()
4135 if (invoke->GetIntrinsic() != Intrinsics::kVarHandleGet) { in CreateVarHandleGetLocations()
4151 static void GenerateVarHandleGet(HInvoke* invoke, CodeGeneratorX86* codegen) { in GenerateVarHandleGet() argument
4157 LocationSummary* locations = invoke->GetLocations(); in GenerateVarHandleGet()
4158 DataType::Type type = invoke->GetType(); in GenerateVarHandleGet()
4161 SlowPathCode* slow_path = new (codegen->GetScopedAllocator()) IntrinsicSlowPathX86(invoke); in GenerateVarHandleGet()
4164 GenerateVarHandleCommonChecks(invoke, temp, slow_path, assembler); in GenerateVarHandleGet()
4174 Register ref = GenerateVarHandleFieldReference(invoke, codegen, temp, offset); in GenerateVarHandleGet()
4180 invoke, out, ref, field_addr, /* needs_null_check= */ false); in GenerateVarHandleGet()
4182 invoke->GetIntrinsic() != Intrinsics::kVarHandleGet) { in GenerateVarHandleGet()
4190 if (invoke->GetIntrinsic() == Intrinsics::kVarHandleGetVolatile || in GenerateVarHandleGet()
4191 invoke->GetIntrinsic() == Intrinsics::kVarHandleGetAcquire) { in GenerateVarHandleGet()
4200 void IntrinsicLocationsBuilderX86::VisitVarHandleGet(HInvoke* invoke) { in VisitVarHandleGet() argument
4201 CreateVarHandleGetLocations(invoke, codegen_); in VisitVarHandleGet()
4204 void IntrinsicCodeGeneratorX86::VisitVarHandleGet(HInvoke* invoke) { in VisitVarHandleGet() argument
4205 GenerateVarHandleGet(invoke, codegen_); in VisitVarHandleGet()
4208 void IntrinsicLocationsBuilderX86::VisitVarHandleGetVolatile(HInvoke* invoke) { in VisitVarHandleGetVolatile() argument
4209 CreateVarHandleGetLocations(invoke, codegen_); in VisitVarHandleGetVolatile()
4212 void IntrinsicCodeGeneratorX86::VisitVarHandleGetVolatile(HInvoke* invoke) { in VisitVarHandleGetVolatile() argument
4213 GenerateVarHandleGet(invoke, codegen_); in VisitVarHandleGetVolatile()
4216 void IntrinsicLocationsBuilderX86::VisitVarHandleGetAcquire(HInvoke* invoke) { in VisitVarHandleGetAcquire() argument
4217 CreateVarHandleGetLocations(invoke, codegen_); in VisitVarHandleGetAcquire()
4220 void IntrinsicCodeGeneratorX86::VisitVarHandleGetAcquire(HInvoke* invoke) { in VisitVarHandleGetAcquire() argument
4221 GenerateVarHandleGet(invoke, codegen_); in VisitVarHandleGetAcquire()
4224 void IntrinsicLocationsBuilderX86::VisitVarHandleGetOpaque(HInvoke* invoke) { in VisitVarHandleGetOpaque() argument
4225 CreateVarHandleGetLocations(invoke, codegen_); in VisitVarHandleGetOpaque()
4228 void IntrinsicCodeGeneratorX86::VisitVarHandleGetOpaque(HInvoke* invoke) { in VisitVarHandleGetOpaque() argument
4229 GenerateVarHandleGet(invoke, codegen_); in VisitVarHandleGetOpaque()
4232 static void CreateVarHandleSetLocations(HInvoke* invoke, CodeGeneratorX86* codegen) { in CreateVarHandleSetLocations() argument
4239 if (!HasVarHandleIntrinsicImplementation(invoke)) { in CreateVarHandleSetLocations()
4244 uint32_t value_index = invoke->GetNumberOfArguments() - 1; in CreateVarHandleSetLocations()
4245 HInstruction* value = invoke->InputAt(value_index); in CreateVarHandleSetLocations()
4246 DataType::Type value_type = GetDataTypeFromShorty(invoke, value_index); in CreateVarHandleSetLocations()
4247 bool needs_atomicity = invoke->GetIntrinsic() != Intrinsics::kVarHandleSet; in CreateVarHandleSetLocations()
4256 ArenaAllocator* allocator = invoke->GetBlock()->GetGraph()->GetAllocator(); in CreateVarHandleSetLocations()
4258 invoke, LocationSummary::kCallOnSlowPath, kIntrinsified); in CreateVarHandleSetLocations()
4260 size_t expected_coordinates_count = GetExpectedVarHandleCoordinatesCount(invoke); in CreateVarHandleSetLocations()
4305 static void GenerateVarHandleSet(HInvoke* invoke, CodeGeneratorX86* codegen) { in GenerateVarHandleSet() argument
4311 LocationSummary* locations = invoke->GetLocations(); in GenerateVarHandleSet()
4313 uint32_t value_index = invoke->GetNumberOfArguments() - 1; in GenerateVarHandleSet()
4314 DataType::Type value_type = GetDataTypeFromShorty(invoke, value_index); in GenerateVarHandleSet()
4317 SlowPathCode* slow_path = new (codegen->GetScopedAllocator()) IntrinsicSlowPathX86(invoke); in GenerateVarHandleSet()
4320 GenerateVarHandleCommonChecks(invoke, temp, slow_path, assembler); in GenerateVarHandleSet()
4325 size_t expected_coordinates_count = GetExpectedVarHandleCoordinatesCount(invoke); in GenerateVarHandleSet()
4334 Register reference = GenerateVarHandleFieldReference(invoke, codegen, temp, offset); in GenerateVarHandleSet()
4337 switch (invoke->GetIntrinsic()) { in GenerateVarHandleSet()
4353 LOG(FATAL) << "GenerateVarHandleSet received non-set intrinsic " << invoke->GetIntrinsic(); in GenerateVarHandleSet()
4360 invoke, in GenerateVarHandleSet()
4374 void IntrinsicLocationsBuilderX86::VisitVarHandleSet(HInvoke* invoke) { in VisitVarHandleSet() argument
4375 CreateVarHandleSetLocations(invoke, codegen_); in VisitVarHandleSet()
4378 void IntrinsicCodeGeneratorX86::VisitVarHandleSet(HInvoke* invoke) { in VisitVarHandleSet() argument
4379 GenerateVarHandleSet(invoke, codegen_); in VisitVarHandleSet()
4382 void IntrinsicLocationsBuilderX86::VisitVarHandleSetVolatile(HInvoke* invoke) { in VisitVarHandleSetVolatile() argument
4383 CreateVarHandleSetLocations(invoke, codegen_); in VisitVarHandleSetVolatile()
4386 void IntrinsicCodeGeneratorX86::VisitVarHandleSetVolatile(HInvoke* invoke) { in VisitVarHandleSetVolatile() argument
4387 GenerateVarHandleSet(invoke, codegen_); in VisitVarHandleSetVolatile()
4390 void IntrinsicLocationsBuilderX86::VisitVarHandleSetRelease(HInvoke* invoke) { in VisitVarHandleSetRelease() argument
4391 CreateVarHandleSetLocations(invoke, codegen_); in VisitVarHandleSetRelease()
4394 void IntrinsicCodeGeneratorX86::VisitVarHandleSetRelease(HInvoke* invoke) { in VisitVarHandleSetRelease() argument
4395 GenerateVarHandleSet(invoke, codegen_); in VisitVarHandleSetRelease()
4398 void IntrinsicLocationsBuilderX86::VisitVarHandleSetOpaque(HInvoke* invoke) { in VisitVarHandleSetOpaque() argument
4399 CreateVarHandleSetLocations(invoke, codegen_); in VisitVarHandleSetOpaque()
4402 void IntrinsicCodeGeneratorX86::VisitVarHandleSetOpaque(HInvoke* invoke) { in VisitVarHandleSetOpaque() argument
4403 GenerateVarHandleSet(invoke, codegen_); in VisitVarHandleSetOpaque()
4406 static void CreateVarHandleGetAndSetLocations(HInvoke* invoke, CodeGeneratorX86* codegen) { in CreateVarHandleGetAndSetLocations() argument
4413 if (!HasVarHandleIntrinsicImplementation(invoke)) { in CreateVarHandleGetAndSetLocations()
4418 uint32_t number_of_arguments = invoke->GetNumberOfArguments(); in CreateVarHandleGetAndSetLocations()
4420 DataType::Type value_type = GetDataTypeFromShorty(invoke, value_index); in CreateVarHandleGetAndSetLocations()
4421 DataType::Type return_type = invoke->GetType(); in CreateVarHandleGetAndSetLocations()
4433 ArenaAllocator* allocator = invoke->GetBlock()->GetGraph()->GetAllocator(); in CreateVarHandleGetAndSetLocations()
4435 invoke, LocationSummary::kCallOnSlowPath, kIntrinsified); in CreateVarHandleGetAndSetLocations()
4440 if (GetExpectedVarHandleCoordinatesCount(invoke) == 1u) { in CreateVarHandleGetAndSetLocations()
4449 locations->SetInAt(value_index, Location::FpuRegisterOrConstant(invoke->InputAt(value_index))); in CreateVarHandleGetAndSetLocations()
4466 static void GenerateVarHandleGetAndSet(HInvoke* invoke, CodeGeneratorX86* codegen) { in GenerateVarHandleGetAndSet() argument
4472 LocationSummary* locations = invoke->GetLocations(); in GenerateVarHandleGetAndSet()
4474 uint32_t value_index = invoke->GetNumberOfArguments() - 1; in GenerateVarHandleGetAndSet()
4477 DataType::Type value_type = GetDataTypeFromShorty(invoke, value_index); in GenerateVarHandleGetAndSet()
4480 SlowPathCode* slow_path = new (codegen->GetScopedAllocator()) IntrinsicSlowPathX86(invoke); in GenerateVarHandleGetAndSet()
4483 GenerateVarHandleCommonChecks(invoke, temp, slow_path, assembler); in GenerateVarHandleGetAndSet()
4489 Register reference = GenerateVarHandleFieldReference(invoke, codegen, temp, offset); in GenerateVarHandleGetAndSet()
4492 if (invoke->GetIntrinsic() == Intrinsics::kVarHandleGetAndSetRelease) { in GenerateVarHandleGetAndSet()
4496 size_t expected_coordinates_count = GetExpectedVarHandleCoordinatesCount(invoke); in GenerateVarHandleGetAndSet()
4502 DataType::Type return_type = invoke->GetType(); in GenerateVarHandleGetAndSet()
4547 invoke, in GenerateVarHandleGetAndSet()
4576 if (invoke->GetIntrinsic() == Intrinsics::kVarHandleGetAndSetAcquire) { in GenerateVarHandleGetAndSet()
4583 void IntrinsicLocationsBuilderX86::VisitVarHandleGetAndSet(HInvoke* invoke) { in VisitVarHandleGetAndSet() argument
4584 CreateVarHandleGetAndSetLocations(invoke, codegen_); in VisitVarHandleGetAndSet()
4587 void IntrinsicCodeGeneratorX86::VisitVarHandleGetAndSet(HInvoke* invoke) { in VisitVarHandleGetAndSet() argument
4588 GenerateVarHandleGetAndSet(invoke, codegen_); in VisitVarHandleGetAndSet()
4591 void IntrinsicLocationsBuilderX86::VisitVarHandleGetAndSetAcquire(HInvoke* invoke) { in VisitVarHandleGetAndSetAcquire() argument
4592 CreateVarHandleGetAndSetLocations(invoke, codegen_); in VisitVarHandleGetAndSetAcquire()
4595 void IntrinsicCodeGeneratorX86::VisitVarHandleGetAndSetAcquire(HInvoke* invoke) { in VisitVarHandleGetAndSetAcquire() argument
4596 GenerateVarHandleGetAndSet(invoke, codegen_); in VisitVarHandleGetAndSetAcquire()
4599 void IntrinsicLocationsBuilderX86::VisitVarHandleGetAndSetRelease(HInvoke* invoke) { in VisitVarHandleGetAndSetRelease() argument
4600 CreateVarHandleGetAndSetLocations(invoke, codegen_); in VisitVarHandleGetAndSetRelease()
4603 void IntrinsicCodeGeneratorX86::VisitVarHandleGetAndSetRelease(HInvoke* invoke) { in VisitVarHandleGetAndSetRelease() argument
4604 GenerateVarHandleGetAndSet(invoke, codegen_); in VisitVarHandleGetAndSetRelease()
4607 static void CreateVarHandleCompareAndSetOrExchangeLocations(HInvoke* invoke, in CreateVarHandleCompareAndSetOrExchangeLocations() argument
4615 if (!HasVarHandleIntrinsicImplementation(invoke)) { in CreateVarHandleCompareAndSetOrExchangeLocations()
4619 uint32_t number_of_arguments = invoke->GetNumberOfArguments(); in CreateVarHandleCompareAndSetOrExchangeLocations()
4622 DataType::Type value_type = GetDataTypeFromShorty(invoke, expected_value_index); in CreateVarHandleCompareAndSetOrExchangeLocations()
4623 DCHECK_EQ(value_type, GetDataTypeFromShorty(invoke, new_value_index)); in CreateVarHandleCompareAndSetOrExchangeLocations()
4633 ArenaAllocator* allocator = invoke->GetBlock()->GetGraph()->GetAllocator(); in CreateVarHandleCompareAndSetOrExchangeLocations()
4635 invoke, LocationSummary::kCallOnSlowPath, kIntrinsified); in CreateVarHandleCompareAndSetOrExchangeLocations()
4640 if (GetExpectedVarHandleCoordinatesCount(invoke) == 1u) { in CreateVarHandleCompareAndSetOrExchangeLocations()
4651 Location::FpuRegisterOrConstant(invoke->InputAt(new_value_index))); in CreateVarHandleCompareAndSetOrExchangeLocations()
4653 Location::FpuRegisterOrConstant(invoke->InputAt(expected_value_index))); in CreateVarHandleCompareAndSetOrExchangeLocations()
4661 mirror::VarHandle::GetAccessModeTemplateByIntrinsic(invoke->GetIntrinsic()); in CreateVarHandleCompareAndSetOrExchangeLocations()
4671 static void GenerateVarHandleCompareAndSetOrExchange(HInvoke* invoke, CodeGeneratorX86* codegen) { in GenerateVarHandleCompareAndSetOrExchange() argument
4677 LocationSummary* locations = invoke->GetLocations(); in GenerateVarHandleCompareAndSetOrExchange()
4678 uint32_t number_of_arguments = invoke->GetNumberOfArguments(); in GenerateVarHandleCompareAndSetOrExchange()
4681 DataType::Type type = GetDataTypeFromShorty(invoke, expected_value_index); in GenerateVarHandleCompareAndSetOrExchange()
4682 DCHECK_EQ(type, GetDataTypeFromShorty(invoke, new_value_index)); in GenerateVarHandleCompareAndSetOrExchange()
4688 SlowPathCode* slow_path = new (codegen->GetScopedAllocator()) IntrinsicSlowPathX86(invoke); in GenerateVarHandleCompareAndSetOrExchange()
4691 GenerateVarHandleCommonChecks(invoke, temp, slow_path, assembler); in GenerateVarHandleCompareAndSetOrExchange()
4696 Register reference = GenerateVarHandleFieldReference(invoke, codegen, temp, offset); in GenerateVarHandleCompareAndSetOrExchange()
4698 uint32_t expected_coordinates_count = GetExpectedVarHandleCoordinatesCount(invoke); in GenerateVarHandleCompareAndSetOrExchange()
4710 mirror::VarHandle::GetAccessModeTemplateByIntrinsic(invoke->GetIntrinsic()); in GenerateVarHandleCompareAndSetOrExchange()
4716 invoke, codegen, expected_value, new_value, reference, offset, temp, temp2, is_cmpxchg); in GenerateVarHandleCompareAndSetOrExchange()
4726 void IntrinsicLocationsBuilderX86::VisitVarHandleCompareAndSet(HInvoke* invoke) { in VisitVarHandleCompareAndSet() argument
4727 CreateVarHandleCompareAndSetOrExchangeLocations(invoke, codegen_); in VisitVarHandleCompareAndSet()
4730 void IntrinsicCodeGeneratorX86::VisitVarHandleCompareAndSet(HInvoke* invoke) { in VisitVarHandleCompareAndSet() argument
4731 GenerateVarHandleCompareAndSetOrExchange(invoke, codegen_); in VisitVarHandleCompareAndSet()
4734 void IntrinsicLocationsBuilderX86::VisitVarHandleWeakCompareAndSet(HInvoke* invoke) { in VisitVarHandleWeakCompareAndSet() argument
4735 CreateVarHandleCompareAndSetOrExchangeLocations(invoke, codegen_); in VisitVarHandleWeakCompareAndSet()
4738 void IntrinsicCodeGeneratorX86::VisitVarHandleWeakCompareAndSet(HInvoke* invoke) { in VisitVarHandleWeakCompareAndSet() argument
4739 GenerateVarHandleCompareAndSetOrExchange(invoke, codegen_); in VisitVarHandleWeakCompareAndSet()
4742 void IntrinsicLocationsBuilderX86::VisitVarHandleWeakCompareAndSetPlain(HInvoke* invoke) { in VisitVarHandleWeakCompareAndSetPlain() argument
4743 CreateVarHandleCompareAndSetOrExchangeLocations(invoke, codegen_); in VisitVarHandleWeakCompareAndSetPlain()
4746 void IntrinsicCodeGeneratorX86::VisitVarHandleWeakCompareAndSetPlain(HInvoke* invoke) { in VisitVarHandleWeakCompareAndSetPlain() argument
4747 GenerateVarHandleCompareAndSetOrExchange(invoke, codegen_); in VisitVarHandleWeakCompareAndSetPlain()
4750 void IntrinsicLocationsBuilderX86::VisitVarHandleWeakCompareAndSetAcquire(HInvoke* invoke) { in VisitVarHandleWeakCompareAndSetAcquire() argument
4751 CreateVarHandleCompareAndSetOrExchangeLocations(invoke, codegen_); in VisitVarHandleWeakCompareAndSetAcquire()
4754 void IntrinsicCodeGeneratorX86::VisitVarHandleWeakCompareAndSetAcquire(HInvoke* invoke) { in VisitVarHandleWeakCompareAndSetAcquire() argument
4755 GenerateVarHandleCompareAndSetOrExchange(invoke, codegen_); in VisitVarHandleWeakCompareAndSetAcquire()
4758 void IntrinsicLocationsBuilderX86::VisitVarHandleWeakCompareAndSetRelease(HInvoke* invoke) { in VisitVarHandleWeakCompareAndSetRelease() argument
4759 CreateVarHandleCompareAndSetOrExchangeLocations(invoke, codegen_); in VisitVarHandleWeakCompareAndSetRelease()
4762 void IntrinsicCodeGeneratorX86::VisitVarHandleWeakCompareAndSetRelease(HInvoke* invoke) { in VisitVarHandleWeakCompareAndSetRelease() argument
4763 GenerateVarHandleCompareAndSetOrExchange(invoke, codegen_); in VisitVarHandleWeakCompareAndSetRelease()
4766 void IntrinsicLocationsBuilderX86::VisitVarHandleCompareAndExchange(HInvoke* invoke) { in VisitVarHandleCompareAndExchange() argument
4767 CreateVarHandleCompareAndSetOrExchangeLocations(invoke, codegen_); in VisitVarHandleCompareAndExchange()
4770 void IntrinsicCodeGeneratorX86::VisitVarHandleCompareAndExchange(HInvoke* invoke) { in VisitVarHandleCompareAndExchange() argument
4771 GenerateVarHandleCompareAndSetOrExchange(invoke, codegen_); in VisitVarHandleCompareAndExchange()
4774 void IntrinsicLocationsBuilderX86::VisitVarHandleCompareAndExchangeAcquire(HInvoke* invoke) { in VisitVarHandleCompareAndExchangeAcquire() argument
4775 CreateVarHandleCompareAndSetOrExchangeLocations(invoke, codegen_); in VisitVarHandleCompareAndExchangeAcquire()
4778 void IntrinsicCodeGeneratorX86::VisitVarHandleCompareAndExchangeAcquire(HInvoke* invoke) { in VisitVarHandleCompareAndExchangeAcquire() argument
4779 GenerateVarHandleCompareAndSetOrExchange(invoke, codegen_); in VisitVarHandleCompareAndExchangeAcquire()
4782 void IntrinsicLocationsBuilderX86::VisitVarHandleCompareAndExchangeRelease(HInvoke* invoke) { in VisitVarHandleCompareAndExchangeRelease() argument
4783 CreateVarHandleCompareAndSetOrExchangeLocations(invoke, codegen_); in VisitVarHandleCompareAndExchangeRelease()
4786 void IntrinsicCodeGeneratorX86::VisitVarHandleCompareAndExchangeRelease(HInvoke* invoke) { in VisitVarHandleCompareAndExchangeRelease() argument
4787 GenerateVarHandleCompareAndSetOrExchange(invoke, codegen_); in VisitVarHandleCompareAndExchangeRelease()
4790 static void CreateVarHandleGetAndAddLocations(HInvoke* invoke, CodeGeneratorX86* codegen) { in CreateVarHandleGetAndAddLocations() argument
4797 if (!HasVarHandleIntrinsicImplementation(invoke)) { in CreateVarHandleGetAndAddLocations()
4803 uint32_t value_index = invoke->GetNumberOfArguments() - 1; in CreateVarHandleGetAndAddLocations()
4804 DataType::Type value_type = GetDataTypeFromShorty(invoke, value_index); in CreateVarHandleGetAndAddLocations()
4813 ArenaAllocator* allocator = invoke->GetBlock()->GetGraph()->GetAllocator(); in CreateVarHandleGetAndAddLocations()
4815 invoke, LocationSummary::kCallOnSlowPath, kIntrinsified); in CreateVarHandleGetAndAddLocations()
4818 size_t expected_coordinates_count = GetExpectedVarHandleCoordinatesCount(invoke); in CreateVarHandleGetAndAddLocations()
4827 DataType::Type return_type = invoke->GetType(); in CreateVarHandleGetAndAddLocations()
4852 static void GenerateVarHandleGetAndAdd(HInvoke* invoke, CodeGeneratorX86* codegen) { in GenerateVarHandleGetAndAdd() argument
4858 LocationSummary* locations = invoke->GetLocations(); in GenerateVarHandleGetAndAdd()
4859 uint32_t number_of_arguments = invoke->GetNumberOfArguments(); in GenerateVarHandleGetAndAdd()
4862 DataType::Type type = GetDataTypeFromShorty(invoke, value_index); in GenerateVarHandleGetAndAdd()
4863 DataType::Type return_type = invoke->GetType(); in GenerateVarHandleGetAndAdd()
4868 SlowPathCode* slow_path = new (codegen->GetScopedAllocator()) IntrinsicSlowPathX86(invoke); in GenerateVarHandleGetAndAdd()
4871 GenerateVarHandleCommonChecks(invoke, temp, slow_path, assembler); in GenerateVarHandleGetAndAdd()
4877 Register reference = GenerateVarHandleFieldReference(invoke, codegen, temp, offset); in GenerateVarHandleGetAndAdd()
4879 size_t expected_coordinates_count = GetExpectedVarHandleCoordinatesCount(invoke); in GenerateVarHandleGetAndAdd()
4941 void IntrinsicLocationsBuilderX86::VisitVarHandleGetAndAdd(HInvoke* invoke) { in VisitVarHandleGetAndAdd() argument
4942 CreateVarHandleGetAndAddLocations(invoke, codegen_); in VisitVarHandleGetAndAdd()
4945 void IntrinsicCodeGeneratorX86::VisitVarHandleGetAndAdd(HInvoke* invoke) { in VisitVarHandleGetAndAdd() argument
4946 GenerateVarHandleGetAndAdd(invoke, codegen_); in VisitVarHandleGetAndAdd()
4949 void IntrinsicLocationsBuilderX86::VisitVarHandleGetAndAddAcquire(HInvoke* invoke) { in VisitVarHandleGetAndAddAcquire() argument
4950 CreateVarHandleGetAndAddLocations(invoke, codegen_); in VisitVarHandleGetAndAddAcquire()
4953 void IntrinsicCodeGeneratorX86::VisitVarHandleGetAndAddAcquire(HInvoke* invoke) { in VisitVarHandleGetAndAddAcquire() argument
4954 GenerateVarHandleGetAndAdd(invoke, codegen_); in VisitVarHandleGetAndAddAcquire()
4957 void IntrinsicLocationsBuilderX86::VisitVarHandleGetAndAddRelease(HInvoke* invoke) { in VisitVarHandleGetAndAddRelease() argument
4958 CreateVarHandleGetAndAddLocations(invoke, codegen_); in VisitVarHandleGetAndAddRelease()
4961 void IntrinsicCodeGeneratorX86::VisitVarHandleGetAndAddRelease(HInvoke* invoke) { in VisitVarHandleGetAndAddRelease() argument
4962 GenerateVarHandleGetAndAdd(invoke, codegen_); in VisitVarHandleGetAndAddRelease()
4965 static void CreateVarHandleGetAndBitwiseOpLocations(HInvoke* invoke, CodeGeneratorX86* codegen) { in CreateVarHandleGetAndBitwiseOpLocations() argument
4972 if (!HasVarHandleIntrinsicImplementation(invoke)) { in CreateVarHandleGetAndBitwiseOpLocations()
4978 uint32_t value_index = invoke->GetNumberOfArguments() - 1; in CreateVarHandleGetAndBitwiseOpLocations()
4979 DataType::Type value_type = GetDataTypeFromShorty(invoke, value_index); in CreateVarHandleGetAndBitwiseOpLocations()
4988 ArenaAllocator* allocator = invoke->GetBlock()->GetGraph()->GetAllocator(); in CreateVarHandleGetAndBitwiseOpLocations()
4990 invoke, LocationSummary::kCallOnSlowPath, kIntrinsified); in CreateVarHandleGetAndBitwiseOpLocations()
4995 size_t expected_coordinates_count = GetExpectedVarHandleCoordinatesCount(invoke); in CreateVarHandleGetAndBitwiseOpLocations()
5004 locations->SetInAt(value_index, Location::RegisterOrConstant(invoke->InputAt(value_index))); in CreateVarHandleGetAndBitwiseOpLocations()
5006 DataType::Type return_type = invoke->GetType(); in CreateVarHandleGetAndBitwiseOpLocations()
5019 static void GenerateBitwiseOp(HInvoke* invoke, in GenerateBitwiseOp() argument
5025 switch (invoke->GetIntrinsic()) { in GenerateBitwiseOp()
5042 LOG(FATAL) << "Unexpected intrinsic: " << invoke->GetIntrinsic(); in GenerateBitwiseOp()
5047 static void GenerateVarHandleGetAndBitwiseOp(HInvoke* invoke, CodeGeneratorX86* codegen) { in GenerateVarHandleGetAndBitwiseOp() argument
5053 LocationSummary* locations = invoke->GetLocations(); in GenerateVarHandleGetAndBitwiseOp()
5055 uint32_t value_index = invoke->GetNumberOfArguments() - 1; in GenerateVarHandleGetAndBitwiseOp()
5056 DataType::Type type = GetDataTypeFromShorty(invoke, value_index); in GenerateVarHandleGetAndBitwiseOp()
5057 DataType::Type return_type = invoke->GetType(); in GenerateVarHandleGetAndBitwiseOp()
5061 SlowPathCode* slow_path = new (codegen->GetScopedAllocator()) IntrinsicSlowPathX86(invoke); in GenerateVarHandleGetAndBitwiseOp()
5064 GenerateVarHandleCommonChecks(invoke, temp, slow_path, assembler); in GenerateVarHandleGetAndBitwiseOp()
5067 size_t expected_coordinates_count = GetExpectedVarHandleCoordinatesCount(invoke); in GenerateVarHandleGetAndBitwiseOp()
5074 reference = GenerateVarHandleFieldReference(invoke, codegen, reference, offset); in GenerateVarHandleGetAndBitwiseOp()
5082 if (invoke->GetIntrinsic() == Intrinsics::kVarHandleGetAndBitwiseOrRelease || in GenerateVarHandleGetAndBitwiseOp()
5083 invoke->GetIntrinsic() == Intrinsics::kVarHandleGetAndBitwiseXorRelease || in GenerateVarHandleGetAndBitwiseOp()
5084 invoke->GetIntrinsic() == Intrinsics::kVarHandleGetAndBitwiseAndRelease) { in GenerateVarHandleGetAndBitwiseOp()
5093 GenerateBitwiseOp(invoke, codegen, temp, eax); in GenerateVarHandleGetAndBitwiseOp()
5105 if (invoke->GetIntrinsic() == Intrinsics::kVarHandleGetAndBitwiseOrAcquire || in GenerateVarHandleGetAndBitwiseOp()
5106 invoke->GetIntrinsic() == Intrinsics::kVarHandleGetAndBitwiseXorAcquire || in GenerateVarHandleGetAndBitwiseOp()
5107 invoke->GetIntrinsic() == Intrinsics::kVarHandleGetAndBitwiseAndAcquire) { in GenerateVarHandleGetAndBitwiseOp()
5114 void IntrinsicLocationsBuilderX86::VisitVarHandleGetAndBitwiseOr(HInvoke* invoke) { in VisitVarHandleGetAndBitwiseOr() argument
5115 CreateVarHandleGetAndBitwiseOpLocations(invoke, codegen_); in VisitVarHandleGetAndBitwiseOr()
5118 void IntrinsicCodeGeneratorX86::VisitVarHandleGetAndBitwiseOr(HInvoke* invoke) { in VisitVarHandleGetAndBitwiseOr() argument
5119 GenerateVarHandleGetAndBitwiseOp(invoke, codegen_); in VisitVarHandleGetAndBitwiseOr()
5122 void IntrinsicLocationsBuilderX86::VisitVarHandleGetAndBitwiseOrAcquire(HInvoke* invoke) { in VisitVarHandleGetAndBitwiseOrAcquire() argument
5123 CreateVarHandleGetAndBitwiseOpLocations(invoke, codegen_); in VisitVarHandleGetAndBitwiseOrAcquire()
5126 void IntrinsicCodeGeneratorX86::VisitVarHandleGetAndBitwiseOrAcquire(HInvoke* invoke) { in VisitVarHandleGetAndBitwiseOrAcquire() argument
5127 GenerateVarHandleGetAndBitwiseOp(invoke, codegen_); in VisitVarHandleGetAndBitwiseOrAcquire()
5130 void IntrinsicLocationsBuilderX86::VisitVarHandleGetAndBitwiseOrRelease(HInvoke* invoke) { in VisitVarHandleGetAndBitwiseOrRelease() argument
5131 CreateVarHandleGetAndBitwiseOpLocations(invoke, codegen_); in VisitVarHandleGetAndBitwiseOrRelease()
5134 void IntrinsicCodeGeneratorX86::VisitVarHandleGetAndBitwiseOrRelease(HInvoke* invoke) { in VisitVarHandleGetAndBitwiseOrRelease() argument
5135 GenerateVarHandleGetAndBitwiseOp(invoke, codegen_); in VisitVarHandleGetAndBitwiseOrRelease()
5138 void IntrinsicLocationsBuilderX86::VisitVarHandleGetAndBitwiseXor(HInvoke* invoke) { in VisitVarHandleGetAndBitwiseXor() argument
5139 CreateVarHandleGetAndBitwiseOpLocations(invoke, codegen_); in VisitVarHandleGetAndBitwiseXor()
5142 void IntrinsicCodeGeneratorX86::VisitVarHandleGetAndBitwiseXor(HInvoke* invoke) { in VisitVarHandleGetAndBitwiseXor() argument
5143 GenerateVarHandleGetAndBitwiseOp(invoke, codegen_); in VisitVarHandleGetAndBitwiseXor()
5146 void IntrinsicLocationsBuilderX86::VisitVarHandleGetAndBitwiseXorAcquire(HInvoke* invoke) { in VisitVarHandleGetAndBitwiseXorAcquire() argument
5147 CreateVarHandleGetAndBitwiseOpLocations(invoke, codegen_); in VisitVarHandleGetAndBitwiseXorAcquire()
5150 void IntrinsicCodeGeneratorX86::VisitVarHandleGetAndBitwiseXorAcquire(HInvoke* invoke) { in VisitVarHandleGetAndBitwiseXorAcquire() argument
5151 GenerateVarHandleGetAndBitwiseOp(invoke, codegen_); in VisitVarHandleGetAndBitwiseXorAcquire()
5154 void IntrinsicLocationsBuilderX86::VisitVarHandleGetAndBitwiseXorRelease(HInvoke* invoke) { in VisitVarHandleGetAndBitwiseXorRelease() argument
5155 CreateVarHandleGetAndBitwiseOpLocations(invoke, codegen_); in VisitVarHandleGetAndBitwiseXorRelease()
5158 void IntrinsicCodeGeneratorX86::VisitVarHandleGetAndBitwiseXorRelease(HInvoke* invoke) { in VisitVarHandleGetAndBitwiseXorRelease() argument
5159 GenerateVarHandleGetAndBitwiseOp(invoke, codegen_); in VisitVarHandleGetAndBitwiseXorRelease()
5162 void IntrinsicLocationsBuilderX86::VisitVarHandleGetAndBitwiseAnd(HInvoke* invoke) { in VisitVarHandleGetAndBitwiseAnd() argument
5163 CreateVarHandleGetAndBitwiseOpLocations(invoke, codegen_); in VisitVarHandleGetAndBitwiseAnd()
5166 void IntrinsicCodeGeneratorX86::VisitVarHandleGetAndBitwiseAnd(HInvoke* invoke) { in VisitVarHandleGetAndBitwiseAnd() argument
5167 GenerateVarHandleGetAndBitwiseOp(invoke, codegen_); in VisitVarHandleGetAndBitwiseAnd()
5170 void IntrinsicLocationsBuilderX86::VisitVarHandleGetAndBitwiseAndAcquire(HInvoke* invoke) { in VisitVarHandleGetAndBitwiseAndAcquire() argument
5171 CreateVarHandleGetAndBitwiseOpLocations(invoke, codegen_); in VisitVarHandleGetAndBitwiseAndAcquire()
5174 void IntrinsicCodeGeneratorX86::VisitVarHandleGetAndBitwiseAndAcquire(HInvoke* invoke) { in VisitVarHandleGetAndBitwiseAndAcquire() argument
5175 GenerateVarHandleGetAndBitwiseOp(invoke, codegen_); in VisitVarHandleGetAndBitwiseAndAcquire()
5178 void IntrinsicLocationsBuilderX86::VisitVarHandleGetAndBitwiseAndRelease(HInvoke* invoke) { in VisitVarHandleGetAndBitwiseAndRelease() argument
5179 CreateVarHandleGetAndBitwiseOpLocations(invoke, codegen_); in VisitVarHandleGetAndBitwiseAndRelease()
5182 void IntrinsicCodeGeneratorX86::VisitVarHandleGetAndBitwiseAndRelease(HInvoke* invoke) { in VisitVarHandleGetAndBitwiseAndRelease() argument
5183 GenerateVarHandleGetAndBitwiseOp(invoke, codegen_); in VisitVarHandleGetAndBitwiseAndRelease()
5186 static void GenerateMathFma(HInvoke* invoke, CodeGeneratorX86* codegen) { in GenerateMathFma() argument
5187 DCHECK(DataType::IsFloatingPointType(invoke->GetType())); in GenerateMathFma()
5188 LocationSummary* locations = invoke->GetLocations(); in GenerateMathFma()
5194 if (invoke->GetType() == DataType::Type::kFloat32) { in GenerateMathFma()
5197 DCHECK_EQ(invoke->GetType(), DataType::Type::kFloat64); in GenerateMathFma()
5202 void IntrinsicCodeGeneratorX86::VisitMathFmaDouble(HInvoke* invoke) { in VisitMathFmaDouble() argument
5204 GenerateMathFma(invoke, codegen_); in VisitMathFmaDouble()
5207 void IntrinsicLocationsBuilderX86::VisitMathFmaDouble(HInvoke* invoke) { in VisitMathFmaDouble() argument
5209 CreateFPFPFPToFPCallLocations(allocator_, invoke); in VisitMathFmaDouble()
5213 void IntrinsicCodeGeneratorX86::VisitMathFmaFloat(HInvoke* invoke) { in VisitMathFmaFloat() argument
5215 GenerateMathFma(invoke, codegen_); in VisitMathFmaFloat()
5218 void IntrinsicLocationsBuilderX86::VisitMathFmaFloat(HInvoke* invoke) { in VisitMathFmaFloat() argument
5220 CreateFPFPFPToFPCallLocations(allocator_, invoke); in VisitMathFmaFloat()