Lines Matching full:invoke
63 bool IntrinsicLocationsBuilderX86_64::TryDispatch(HInvoke* invoke) { in TryDispatch() argument
64 Dispatch(invoke); in TryDispatch()
65 LocationSummary* res = invoke->GetLocations(); in TryDispatch()
183 static void CreateFPToIntLocations(ArenaAllocator* allocator, HInvoke* invoke) { in CreateFPToIntLocations() argument
185 new (allocator) LocationSummary(invoke, LocationSummary::kNoCall, kIntrinsified); in CreateFPToIntLocations()
190 static void CreateIntToFPLocations(ArenaAllocator* allocator, HInvoke* invoke) { in CreateIntToFPLocations() argument
192 new (allocator) LocationSummary(invoke, LocationSummary::kNoCall, kIntrinsified); in CreateIntToFPLocations()
209 void IntrinsicLocationsBuilderX86_64::VisitDoubleDoubleToRawLongBits(HInvoke* invoke) { in VisitDoubleDoubleToRawLongBits() argument
210 CreateFPToIntLocations(allocator_, invoke); in VisitDoubleDoubleToRawLongBits()
212 void IntrinsicLocationsBuilderX86_64::VisitDoubleLongBitsToDouble(HInvoke* invoke) { in VisitDoubleLongBitsToDouble() argument
213 CreateIntToFPLocations(allocator_, invoke); in VisitDoubleLongBitsToDouble()
216 void IntrinsicCodeGeneratorX86_64::VisitDoubleDoubleToRawLongBits(HInvoke* invoke) { in VisitDoubleDoubleToRawLongBits() argument
217 MoveFPToInt(invoke->GetLocations(), /* is64bit= */ true, GetAssembler()); in VisitDoubleDoubleToRawLongBits()
219 void IntrinsicCodeGeneratorX86_64::VisitDoubleLongBitsToDouble(HInvoke* invoke) { in VisitDoubleLongBitsToDouble() argument
220 MoveIntToFP(invoke->GetLocations(), /* is64bit= */ true, GetAssembler()); in VisitDoubleLongBitsToDouble()
223 void IntrinsicLocationsBuilderX86_64::VisitFloatFloatToRawIntBits(HInvoke* invoke) { in VisitFloatFloatToRawIntBits() argument
224 CreateFPToIntLocations(allocator_, invoke); in VisitFloatFloatToRawIntBits()
226 void IntrinsicLocationsBuilderX86_64::VisitFloatIntBitsToFloat(HInvoke* invoke) { in VisitFloatIntBitsToFloat() argument
227 CreateIntToFPLocations(allocator_, invoke); in VisitFloatIntBitsToFloat()
230 void IntrinsicCodeGeneratorX86_64::VisitFloatFloatToRawIntBits(HInvoke* invoke) { in VisitFloatFloatToRawIntBits() argument
231 MoveFPToInt(invoke->GetLocations(), /* is64bit= */ false, GetAssembler()); in VisitFloatFloatToRawIntBits()
233 void IntrinsicCodeGeneratorX86_64::VisitFloatIntBitsToFloat(HInvoke* invoke) { in VisitFloatIntBitsToFloat() argument
234 MoveIntToFP(invoke->GetLocations(), /* is64bit= */ false, GetAssembler()); in VisitFloatIntBitsToFloat()
237 static void CreateIntToIntLocations(ArenaAllocator* allocator, HInvoke* invoke) { in CreateIntToIntLocations() argument
239 new (allocator) LocationSummary(invoke, LocationSummary::kNoCall, kIntrinsified); in CreateIntToIntLocations()
244 void IntrinsicLocationsBuilderX86_64::VisitIntegerReverseBytes(HInvoke* invoke) { in VisitIntegerReverseBytes() argument
245 CreateIntToIntLocations(allocator_, invoke); in VisitIntegerReverseBytes()
248 void IntrinsicCodeGeneratorX86_64::VisitIntegerReverseBytes(HInvoke* invoke) { in VisitIntegerReverseBytes() argument
249 codegen_->GetInstructionCodegen()->Bswap(invoke->GetLocations()->Out(), DataType::Type::kInt32); in VisitIntegerReverseBytes()
252 void IntrinsicLocationsBuilderX86_64::VisitLongReverseBytes(HInvoke* invoke) { in VisitLongReverseBytes() argument
253 CreateIntToIntLocations(allocator_, invoke); in VisitLongReverseBytes()
256 void IntrinsicCodeGeneratorX86_64::VisitLongReverseBytes(HInvoke* invoke) { in VisitLongReverseBytes() argument
257 codegen_->GetInstructionCodegen()->Bswap(invoke->GetLocations()->Out(), DataType::Type::kInt64); in VisitLongReverseBytes()
260 void IntrinsicLocationsBuilderX86_64::VisitShortReverseBytes(HInvoke* invoke) { in VisitShortReverseBytes() argument
261 CreateIntToIntLocations(allocator_, invoke); in VisitShortReverseBytes()
264 void IntrinsicCodeGeneratorX86_64::VisitShortReverseBytes(HInvoke* invoke) { in VisitShortReverseBytes() argument
265 codegen_->GetInstructionCodegen()->Bswap(invoke->GetLocations()->Out(), DataType::Type::kInt16); in VisitShortReverseBytes()
313 void IntrinsicLocationsBuilderX86_64::VisitFloatIsInfinite(HInvoke* invoke) { in VisitFloatIsInfinite() argument
314 CreateFPToIntLocations(allocator_, invoke); in VisitFloatIsInfinite()
317 void IntrinsicCodeGeneratorX86_64::VisitFloatIsInfinite(HInvoke* invoke) { in VisitFloatIsInfinite() argument
318 GenIsInfinite(invoke->GetLocations(), /* is64bit=*/ false, codegen_); in VisitFloatIsInfinite()
321 void IntrinsicLocationsBuilderX86_64::VisitDoubleIsInfinite(HInvoke* invoke) { in VisitDoubleIsInfinite() argument
322 CreateFPToIntLocations(allocator_, invoke); in VisitDoubleIsInfinite()
325 void IntrinsicCodeGeneratorX86_64::VisitDoubleIsInfinite(HInvoke* invoke) { in VisitDoubleIsInfinite() argument
326 GenIsInfinite(invoke->GetLocations(), /* is64bit=*/ true, codegen_); in VisitDoubleIsInfinite()
329 static void CreateFPToFPLocations(ArenaAllocator* allocator, HInvoke* invoke) { in CreateFPToFPLocations() argument
331 new (allocator) LocationSummary(invoke, LocationSummary::kNoCall, kIntrinsified); in CreateFPToFPLocations()
336 void IntrinsicLocationsBuilderX86_64::VisitMathSqrt(HInvoke* invoke) { in VisitMathSqrt() argument
337 CreateFPToFPLocations(allocator_, invoke); in VisitMathSqrt()
340 void IntrinsicCodeGeneratorX86_64::VisitMathSqrt(HInvoke* invoke) { in VisitMathSqrt() argument
341 LocationSummary* locations = invoke->GetLocations(); in VisitMathSqrt()
349 HInvoke* invoke, in CreateSSE41FPToFPLocations() argument
356 CreateFPToFPLocations(allocator, invoke); in CreateSSE41FPToFPLocations()
359 static void GenSSE41FPToFPIntrinsic(HInvoke* invoke, X86_64Assembler* assembler, int round_mode) { in GenSSE41FPToFPIntrinsic() argument
360 LocationSummary* locations = invoke->GetLocations(); in GenSSE41FPToFPIntrinsic()
367 void IntrinsicLocationsBuilderX86_64::VisitMathCeil(HInvoke* invoke) { in VisitMathCeil() argument
368 CreateSSE41FPToFPLocations(allocator_, invoke, codegen_); in VisitMathCeil()
371 void IntrinsicCodeGeneratorX86_64::VisitMathCeil(HInvoke* invoke) { in VisitMathCeil() argument
372 GenSSE41FPToFPIntrinsic(invoke, GetAssembler(), 2); in VisitMathCeil()
375 void IntrinsicLocationsBuilderX86_64::VisitMathFloor(HInvoke* invoke) { in VisitMathFloor() argument
376 CreateSSE41FPToFPLocations(allocator_, invoke, codegen_); in VisitMathFloor()
379 void IntrinsicCodeGeneratorX86_64::VisitMathFloor(HInvoke* invoke) { in VisitMathFloor() argument
380 GenSSE41FPToFPIntrinsic(invoke, GetAssembler(), 1); in VisitMathFloor()
383 void IntrinsicLocationsBuilderX86_64::VisitMathRint(HInvoke* invoke) { in VisitMathRint() argument
384 CreateSSE41FPToFPLocations(allocator_, invoke, codegen_); in VisitMathRint()
387 void IntrinsicCodeGeneratorX86_64::VisitMathRint(HInvoke* invoke) { in VisitMathRint() argument
388 GenSSE41FPToFPIntrinsic(invoke, GetAssembler(), 0); in VisitMathRint()
392 HInvoke* invoke, in CreateSSE41FPToIntLocations() argument
400 new (allocator) LocationSummary(invoke, LocationSummary::kNoCall, kIntrinsified); in CreateSSE41FPToIntLocations()
407 void IntrinsicLocationsBuilderX86_64::VisitMathRoundFloat(HInvoke* invoke) { in VisitMathRoundFloat() argument
408 CreateSSE41FPToIntLocations(allocator_, invoke, codegen_); in VisitMathRoundFloat()
411 void IntrinsicCodeGeneratorX86_64::VisitMathRoundFloat(HInvoke* invoke) { in VisitMathRoundFloat() argument
412 LocationSummary* locations = invoke->GetLocations(); in VisitMathRoundFloat()
448 void IntrinsicLocationsBuilderX86_64::VisitMathRoundDouble(HInvoke* invoke) { in VisitMathRoundDouble() argument
449 CreateSSE41FPToIntLocations(allocator_, invoke, codegen_); in VisitMathRoundDouble()
452 void IntrinsicCodeGeneratorX86_64::VisitMathRoundDouble(HInvoke* invoke) { in VisitMathRoundDouble() argument
453 LocationSummary* locations = invoke->GetLocations(); in VisitMathRoundDouble()
489 static void CreateFPToFPCallLocations(ArenaAllocator* allocator, HInvoke* invoke) { in CreateFPToFPCallLocations() argument
491 new (allocator) LocationSummary(invoke, LocationSummary::kCallOnMainOnly, kIntrinsified); in CreateFPToFPCallLocations()
499 static void GenFPToFPCall(HInvoke* invoke, CodeGeneratorX86_64* codegen, in GenFPToFPCall() argument
501 LocationSummary* locations = invoke->GetLocations(); in GenFPToFPCall()
503 DCHECK(invoke->IsInvokeStaticOrDirect()); in GenFPToFPCall()
505 codegen->InvokeRuntime(entry, invoke, invoke->GetDexPc()); in GenFPToFPCall()
508 void IntrinsicLocationsBuilderX86_64::VisitMathCos(HInvoke* invoke) { in VisitMathCos() argument
509 CreateFPToFPCallLocations(allocator_, invoke); in VisitMathCos()
512 void IntrinsicCodeGeneratorX86_64::VisitMathCos(HInvoke* invoke) { in VisitMathCos() argument
513 GenFPToFPCall(invoke, codegen_, kQuickCos); in VisitMathCos()
516 void IntrinsicLocationsBuilderX86_64::VisitMathSin(HInvoke* invoke) { in VisitMathSin() argument
517 CreateFPToFPCallLocations(allocator_, invoke); in VisitMathSin()
520 void IntrinsicCodeGeneratorX86_64::VisitMathSin(HInvoke* invoke) { in VisitMathSin() argument
521 GenFPToFPCall(invoke, codegen_, kQuickSin); in VisitMathSin()
524 void IntrinsicLocationsBuilderX86_64::VisitMathAcos(HInvoke* invoke) { in VisitMathAcos() argument
525 CreateFPToFPCallLocations(allocator_, invoke); in VisitMathAcos()
528 void IntrinsicCodeGeneratorX86_64::VisitMathAcos(HInvoke* invoke) { in VisitMathAcos() argument
529 GenFPToFPCall(invoke, codegen_, kQuickAcos); in VisitMathAcos()
532 void IntrinsicLocationsBuilderX86_64::VisitMathAsin(HInvoke* invoke) { in VisitMathAsin() argument
533 CreateFPToFPCallLocations(allocator_, invoke); in VisitMathAsin()
536 void IntrinsicCodeGeneratorX86_64::VisitMathAsin(HInvoke* invoke) { in VisitMathAsin() argument
537 GenFPToFPCall(invoke, codegen_, kQuickAsin); in VisitMathAsin()
540 void IntrinsicLocationsBuilderX86_64::VisitMathAtan(HInvoke* invoke) { in VisitMathAtan() argument
541 CreateFPToFPCallLocations(allocator_, invoke); in VisitMathAtan()
544 void IntrinsicCodeGeneratorX86_64::VisitMathAtan(HInvoke* invoke) { in VisitMathAtan() argument
545 GenFPToFPCall(invoke, codegen_, kQuickAtan); in VisitMathAtan()
548 void IntrinsicLocationsBuilderX86_64::VisitMathCbrt(HInvoke* invoke) { in VisitMathCbrt() argument
549 CreateFPToFPCallLocations(allocator_, invoke); in VisitMathCbrt()
552 void IntrinsicCodeGeneratorX86_64::VisitMathCbrt(HInvoke* invoke) { in VisitMathCbrt() argument
553 GenFPToFPCall(invoke, codegen_, kQuickCbrt); in VisitMathCbrt()
556 void IntrinsicLocationsBuilderX86_64::VisitMathCosh(HInvoke* invoke) { in VisitMathCosh() argument
557 CreateFPToFPCallLocations(allocator_, invoke); in VisitMathCosh()
560 void IntrinsicCodeGeneratorX86_64::VisitMathCosh(HInvoke* invoke) { in VisitMathCosh() argument
561 GenFPToFPCall(invoke, codegen_, kQuickCosh); in VisitMathCosh()
564 void IntrinsicLocationsBuilderX86_64::VisitMathExp(HInvoke* invoke) { in VisitMathExp() argument
565 CreateFPToFPCallLocations(allocator_, invoke); in VisitMathExp()
568 void IntrinsicCodeGeneratorX86_64::VisitMathExp(HInvoke* invoke) { in VisitMathExp() argument
569 GenFPToFPCall(invoke, codegen_, kQuickExp); in VisitMathExp()
572 void IntrinsicLocationsBuilderX86_64::VisitMathExpm1(HInvoke* invoke) { in VisitMathExpm1() argument
573 CreateFPToFPCallLocations(allocator_, invoke); in VisitMathExpm1()
576 void IntrinsicCodeGeneratorX86_64::VisitMathExpm1(HInvoke* invoke) { in VisitMathExpm1() argument
577 GenFPToFPCall(invoke, codegen_, kQuickExpm1); in VisitMathExpm1()
580 void IntrinsicLocationsBuilderX86_64::VisitMathLog(HInvoke* invoke) { in VisitMathLog() argument
581 CreateFPToFPCallLocations(allocator_, invoke); in VisitMathLog()
584 void IntrinsicCodeGeneratorX86_64::VisitMathLog(HInvoke* invoke) { in VisitMathLog() argument
585 GenFPToFPCall(invoke, codegen_, kQuickLog); in VisitMathLog()
588 void IntrinsicLocationsBuilderX86_64::VisitMathLog10(HInvoke* invoke) { in VisitMathLog10() argument
589 CreateFPToFPCallLocations(allocator_, invoke); in VisitMathLog10()
592 void IntrinsicCodeGeneratorX86_64::VisitMathLog10(HInvoke* invoke) { in VisitMathLog10() argument
593 GenFPToFPCall(invoke, codegen_, kQuickLog10); in VisitMathLog10()
596 void IntrinsicLocationsBuilderX86_64::VisitMathSinh(HInvoke* invoke) { in VisitMathSinh() argument
597 CreateFPToFPCallLocations(allocator_, invoke); in VisitMathSinh()
600 void IntrinsicCodeGeneratorX86_64::VisitMathSinh(HInvoke* invoke) { in VisitMathSinh() argument
601 GenFPToFPCall(invoke, codegen_, kQuickSinh); in VisitMathSinh()
604 void IntrinsicLocationsBuilderX86_64::VisitMathTan(HInvoke* invoke) { in VisitMathTan() argument
605 CreateFPToFPCallLocations(allocator_, invoke); in VisitMathTan()
608 void IntrinsicCodeGeneratorX86_64::VisitMathTan(HInvoke* invoke) { in VisitMathTan() argument
609 GenFPToFPCall(invoke, codegen_, kQuickTan); in VisitMathTan()
612 void IntrinsicLocationsBuilderX86_64::VisitMathTanh(HInvoke* invoke) { in VisitMathTanh() argument
613 CreateFPToFPCallLocations(allocator_, invoke); in VisitMathTanh()
616 void IntrinsicCodeGeneratorX86_64::VisitMathTanh(HInvoke* invoke) { in VisitMathTanh() argument
617 GenFPToFPCall(invoke, codegen_, kQuickTanh); in VisitMathTanh()
620 static void CreateFPFPToFPCallLocations(ArenaAllocator* allocator, HInvoke* invoke) { in CreateFPFPToFPCallLocations() argument
622 new (allocator) LocationSummary(invoke, LocationSummary::kCallOnMainOnly, kIntrinsified); in CreateFPFPToFPCallLocations()
631 static void CreateFPFPFPToFPCallLocations(ArenaAllocator* allocator, HInvoke* invoke) { in CreateFPFPFPToFPCallLocations() argument
632 DCHECK_EQ(invoke->GetNumberOfArguments(), 3U); in CreateFPFPFPToFPCallLocations()
634 new (allocator) LocationSummary(invoke, LocationSummary::kNoCall, kIntrinsified); in CreateFPFPFPToFPCallLocations()
642 void IntrinsicLocationsBuilderX86_64::VisitMathAtan2(HInvoke* invoke) { in VisitMathAtan2() argument
643 CreateFPFPToFPCallLocations(allocator_, invoke); in VisitMathAtan2()
646 void IntrinsicCodeGeneratorX86_64::VisitMathAtan2(HInvoke* invoke) { in VisitMathAtan2() argument
647 GenFPToFPCall(invoke, codegen_, kQuickAtan2); in VisitMathAtan2()
650 void IntrinsicLocationsBuilderX86_64::VisitMathPow(HInvoke* invoke) { in VisitMathPow() argument
651 CreateFPFPToFPCallLocations(allocator_, invoke); in VisitMathPow()
654 void IntrinsicCodeGeneratorX86_64::VisitMathPow(HInvoke* invoke) { in VisitMathPow() argument
655 GenFPToFPCall(invoke, codegen_, kQuickPow); in VisitMathPow()
658 void IntrinsicLocationsBuilderX86_64::VisitMathHypot(HInvoke* invoke) { in VisitMathHypot() argument
659 CreateFPFPToFPCallLocations(allocator_, invoke); in VisitMathHypot()
662 void IntrinsicCodeGeneratorX86_64::VisitMathHypot(HInvoke* invoke) { in VisitMathHypot() argument
663 GenFPToFPCall(invoke, codegen_, kQuickHypot); in VisitMathHypot()
666 void IntrinsicLocationsBuilderX86_64::VisitMathNextAfter(HInvoke* invoke) { in VisitMathNextAfter() argument
667 CreateFPFPToFPCallLocations(allocator_, invoke); in VisitMathNextAfter()
670 void IntrinsicCodeGeneratorX86_64::VisitMathNextAfter(HInvoke* invoke) { in VisitMathNextAfter() argument
671 GenFPToFPCall(invoke, codegen_, kQuickNextAfter); in VisitMathNextAfter()
674 static void CreateSystemArrayCopyLocations(HInvoke* invoke) { in CreateSystemArrayCopyLocations() argument
677 HIntConstant* src_pos = invoke->InputAt(1)->AsIntConstantOrNull(); in CreateSystemArrayCopyLocations()
678 HIntConstant* dest_pos = invoke->InputAt(3)->AsIntConstantOrNull(); in CreateSystemArrayCopyLocations()
688 HIntConstant* length = invoke->InputAt(4)->AsIntConstantOrNull(); in CreateSystemArrayCopyLocations()
697 new (invoke->GetBlock()->GetGraph()->GetAllocator()) LocationSummary in CreateSystemArrayCopyLocations()
698 (invoke, LocationSummary::kCallOnSlowPath, kIntrinsified); in CreateSystemArrayCopyLocations()
701 locations->SetInAt(1, Location::RegisterOrConstant(invoke->InputAt(1))); in CreateSystemArrayCopyLocations()
703 locations->SetInAt(3, Location::RegisterOrConstant(invoke->InputAt(3))); in CreateSystemArrayCopyLocations()
704 locations->SetInAt(4, Location::RegisterOrConstant(invoke->InputAt(4))); in CreateSystemArrayCopyLocations()
779 static void SystemArrayCopyPrimitive(HInvoke* invoke, in SystemArrayCopyPrimitive() argument
783 LocationSummary* locations = invoke->GetLocations(); in SystemArrayCopyPrimitive()
798 SlowPathCode* slow_path = new (codegen->GetScopedAllocator()) IntrinsicSlowPathX86_64(invoke); in SystemArrayCopyPrimitive()
872 void IntrinsicLocationsBuilderX86_64::VisitSystemArrayCopyChar(HInvoke* invoke) { in VisitSystemArrayCopyChar() argument
873 CreateSystemArrayCopyLocations(invoke); in VisitSystemArrayCopyChar()
875 void IntrinsicCodeGeneratorX86_64::VisitSystemArrayCopyChar(HInvoke* invoke) { in VisitSystemArrayCopyChar() argument
877 SystemArrayCopyPrimitive(invoke, assembler, codegen_, DataType::Type::kUint16); in VisitSystemArrayCopyChar()
880 void IntrinsicCodeGeneratorX86_64::VisitSystemArrayCopyByte(HInvoke* invoke) { in VisitSystemArrayCopyByte() argument
882 SystemArrayCopyPrimitive(invoke, assembler, codegen_, DataType::Type::kInt8); in VisitSystemArrayCopyByte()
885 void IntrinsicLocationsBuilderX86_64::VisitSystemArrayCopyByte(HInvoke* invoke) { in VisitSystemArrayCopyByte() argument
886 CreateSystemArrayCopyLocations(invoke); in VisitSystemArrayCopyByte()
889 void IntrinsicCodeGeneratorX86_64::VisitSystemArrayCopyInt(HInvoke* invoke) { in VisitSystemArrayCopyInt() argument
891 SystemArrayCopyPrimitive(invoke, assembler, codegen_, DataType::Type::kInt32); in VisitSystemArrayCopyInt()
894 void IntrinsicLocationsBuilderX86_64::VisitSystemArrayCopyInt(HInvoke* invoke) { in VisitSystemArrayCopyInt() argument
895 CreateSystemArrayCopyLocations(invoke); in VisitSystemArrayCopyInt()
898 void IntrinsicLocationsBuilderX86_64::VisitSystemArrayCopy(HInvoke* invoke) { in VisitSystemArrayCopy() argument
908 invoke, kLengthThreshold, kInitialNumTemps); in VisitSystemArrayCopy()
918 void IntrinsicCodeGeneratorX86_64::VisitSystemArrayCopy(HInvoke* invoke) { in VisitSystemArrayCopy() argument
924 LocationSummary* locations = invoke->GetLocations(); in VisitSystemArrayCopy()
945 new (codegen_->GetScopedAllocator()) IntrinsicSlowPathX86_64(invoke); in VisitSystemArrayCopy()
949 SystemArrayCopyOptimizations optimizations(invoke); in VisitSystemArrayCopy()
1050 invoke, temp1_loc, dest, class_offset, /* needs_null_check= */ false); in VisitSystemArrayCopy()
1058 invoke, temp2_loc, src, class_offset, /* needs_null_check= */ false); in VisitSystemArrayCopy()
1170 new (codegen_->GetScopedAllocator()) ReadBarrierSystemArrayCopySlowPathX86_64(invoke); in VisitSystemArrayCopy()
1203 void IntrinsicLocationsBuilderX86_64::VisitStringCompareTo(HInvoke* invoke) { in VisitStringCompareTo() argument
1205 invoke, LocationSummary::kCallOnMainAndSlowPath, kIntrinsified); in VisitStringCompareTo()
1212 void IntrinsicCodeGeneratorX86_64::VisitStringCompareTo(HInvoke* invoke) { in VisitStringCompareTo() argument
1214 LocationSummary* locations = invoke->GetLocations(); in VisitStringCompareTo()
1217 DCHECK(!invoke->CanDoImplicitNullCheckOn(invoke->InputAt(0))); in VisitStringCompareTo()
1221 SlowPathCode* slow_path = new (codegen_->GetScopedAllocator()) IntrinsicSlowPathX86_64(invoke); in VisitStringCompareTo()
1225 codegen_->InvokeRuntime(kQuickStringCompareTo, invoke, invoke->GetDexPc(), slow_path); in VisitStringCompareTo()
1229 void IntrinsicLocationsBuilderX86_64::VisitStringEquals(HInvoke* invoke) { in VisitStringEquals() argument
1231 new (allocator_) LocationSummary(invoke, LocationSummary::kNoCall, kIntrinsified); in VisitStringEquals()
1243 void IntrinsicCodeGeneratorX86_64::VisitStringEquals(HInvoke* invoke) { in VisitStringEquals() argument
1245 LocationSummary* locations = invoke->GetLocations(); in VisitStringEquals()
1261 DCHECK(!invoke->CanDoImplicitNullCheckOn(invoke->InputAt(0))); in VisitStringEquals()
1263 StringEqualsOptimizations optimizations(invoke); in VisitStringEquals()
1346 static void CreateStringIndexOfLocations(HInvoke* invoke, in CreateStringIndexOfLocations() argument
1349 LocationSummary* locations = new (allocator) LocationSummary(invoke, in CreateStringIndexOfLocations()
1371 static void GenerateStringIndexOf(HInvoke* invoke, in GenerateStringIndexOf() argument
1375 LocationSummary* locations = invoke->GetLocations(); in GenerateStringIndexOf()
1378 DCHECK(!invoke->CanDoImplicitNullCheckOn(invoke->InputAt(0))); in GenerateStringIndexOf()
1395 HInstruction* code_point = invoke->InputAt(1); in GenerateStringIndexOf()
1401 slow_path = new (codegen->GetScopedAllocator()) IntrinsicSlowPathX86_64(invoke); in GenerateStringIndexOf()
1409 slow_path = new (codegen->GetScopedAllocator()) IntrinsicSlowPathX86_64(invoke); in GenerateStringIndexOf()
1513 void IntrinsicLocationsBuilderX86_64::VisitStringIndexOf(HInvoke* invoke) { in VisitStringIndexOf() argument
1514 CreateStringIndexOfLocations(invoke, allocator_, /* start_at_zero= */ true); in VisitStringIndexOf()
1517 void IntrinsicCodeGeneratorX86_64::VisitStringIndexOf(HInvoke* invoke) { in VisitStringIndexOf() argument
1518 GenerateStringIndexOf(invoke, GetAssembler(), codegen_, /* start_at_zero= */ true); in VisitStringIndexOf()
1521 void IntrinsicLocationsBuilderX86_64::VisitStringIndexOfAfter(HInvoke* invoke) { in VisitStringIndexOfAfter() argument
1522 CreateStringIndexOfLocations(invoke, allocator_, /* start_at_zero= */ false); in VisitStringIndexOfAfter()
1525 void IntrinsicCodeGeneratorX86_64::VisitStringIndexOfAfter(HInvoke* invoke) { in VisitStringIndexOfAfter() argument
1526 GenerateStringIndexOf(invoke, GetAssembler(), codegen_, /* start_at_zero= */ false); in VisitStringIndexOfAfter()
1529 void IntrinsicLocationsBuilderX86_64::VisitStringNewStringFromBytes(HInvoke* invoke) { in VisitStringNewStringFromBytes() argument
1531 invoke, LocationSummary::kCallOnMainAndSlowPath, kIntrinsified); in VisitStringNewStringFromBytes()
1540 void IntrinsicCodeGeneratorX86_64::VisitStringNewStringFromBytes(HInvoke* invoke) { in VisitStringNewStringFromBytes() argument
1542 LocationSummary* locations = invoke->GetLocations(); in VisitStringNewStringFromBytes()
1546 SlowPathCode* slow_path = new (codegen_->GetScopedAllocator()) IntrinsicSlowPathX86_64(invoke); in VisitStringNewStringFromBytes()
1550 codegen_->InvokeRuntime(kQuickAllocStringFromBytes, invoke, invoke->GetDexPc()); in VisitStringNewStringFromBytes()
1555 void IntrinsicLocationsBuilderX86_64::VisitStringNewStringFromChars(HInvoke* invoke) { in VisitStringNewStringFromChars() argument
1557 new (allocator_) LocationSummary(invoke, LocationSummary::kCallOnMainOnly, kIntrinsified); in VisitStringNewStringFromChars()
1565 void IntrinsicCodeGeneratorX86_64::VisitStringNewStringFromChars(HInvoke* invoke) { in VisitStringNewStringFromChars() argument
1572 codegen_->InvokeRuntime(kQuickAllocStringFromChars, invoke, invoke->GetDexPc()); in VisitStringNewStringFromChars()
1576 void IntrinsicLocationsBuilderX86_64::VisitStringNewStringFromString(HInvoke* invoke) { in VisitStringNewStringFromString() argument
1578 invoke, LocationSummary::kCallOnMainAndSlowPath, kIntrinsified); in VisitStringNewStringFromString()
1584 void IntrinsicCodeGeneratorX86_64::VisitStringNewStringFromString(HInvoke* invoke) { in VisitStringNewStringFromString() argument
1586 LocationSummary* locations = invoke->GetLocations(); in VisitStringNewStringFromString()
1590 SlowPathCode* slow_path = new (codegen_->GetScopedAllocator()) IntrinsicSlowPathX86_64(invoke); in VisitStringNewStringFromString()
1594 codegen_->InvokeRuntime(kQuickAllocStringFromString, invoke, invoke->GetDexPc()); in VisitStringNewStringFromString()
1599 void IntrinsicLocationsBuilderX86_64::VisitStringGetCharsNoCheck(HInvoke* invoke) { in VisitStringGetCharsNoCheck() argument
1602 new (allocator_) LocationSummary(invoke, LocationSummary::kNoCall, kIntrinsified); in VisitStringGetCharsNoCheck()
1604 locations->SetInAt(1, Location::RegisterOrConstant(invoke->InputAt(1))); in VisitStringGetCharsNoCheck()
1615 void IntrinsicCodeGeneratorX86_64::VisitStringGetCharsNoCheck(HInvoke* invoke) { in VisitStringGetCharsNoCheck() argument
1617 LocationSummary* locations = invoke->GetLocations(); in VisitStringGetCharsNoCheck()
1714 void IntrinsicLocationsBuilderX86_64::VisitMemoryPeekByte(HInvoke* invoke) { in VisitMemoryPeekByte() argument
1715 CreateIntToIntLocations(allocator_, invoke); in VisitMemoryPeekByte()
1718 void IntrinsicCodeGeneratorX86_64::VisitMemoryPeekByte(HInvoke* invoke) { in VisitMemoryPeekByte() argument
1719 GenPeek(invoke->GetLocations(), DataType::Type::kInt8, GetAssembler()); in VisitMemoryPeekByte()
1722 void IntrinsicLocationsBuilderX86_64::VisitMemoryPeekIntNative(HInvoke* invoke) { in VisitMemoryPeekIntNative() argument
1723 CreateIntToIntLocations(allocator_, invoke); in VisitMemoryPeekIntNative()
1726 void IntrinsicCodeGeneratorX86_64::VisitMemoryPeekIntNative(HInvoke* invoke) { in VisitMemoryPeekIntNative() argument
1727 GenPeek(invoke->GetLocations(), DataType::Type::kInt32, GetAssembler()); in VisitMemoryPeekIntNative()
1730 void IntrinsicLocationsBuilderX86_64::VisitMemoryPeekLongNative(HInvoke* invoke) { in VisitMemoryPeekLongNative() argument
1731 CreateIntToIntLocations(allocator_, invoke); in VisitMemoryPeekLongNative()
1734 void IntrinsicCodeGeneratorX86_64::VisitMemoryPeekLongNative(HInvoke* invoke) { in VisitMemoryPeekLongNative() argument
1735 GenPeek(invoke->GetLocations(), DataType::Type::kInt64, GetAssembler()); in VisitMemoryPeekLongNative()
1738 void IntrinsicLocationsBuilderX86_64::VisitMemoryPeekShortNative(HInvoke* invoke) { in VisitMemoryPeekShortNative() argument
1739 CreateIntToIntLocations(allocator_, invoke); in VisitMemoryPeekShortNative()
1742 void IntrinsicCodeGeneratorX86_64::VisitMemoryPeekShortNative(HInvoke* invoke) { in VisitMemoryPeekShortNative() argument
1743 GenPeek(invoke->GetLocations(), DataType::Type::kInt16, GetAssembler()); in VisitMemoryPeekShortNative()
1746 static void CreateIntIntToVoidLocations(ArenaAllocator* allocator, HInvoke* invoke) { in CreateIntIntToVoidLocations() argument
1748 new (allocator) LocationSummary(invoke, LocationSummary::kNoCall, kIntrinsified); in CreateIntIntToVoidLocations()
1750 locations->SetInAt(1, Location::RegisterOrInt32Constant(invoke->InputAt(1))); in CreateIntIntToVoidLocations()
1799 void IntrinsicLocationsBuilderX86_64::VisitMemoryPokeByte(HInvoke* invoke) { in VisitMemoryPokeByte() argument
1800 CreateIntIntToVoidLocations(allocator_, invoke); in VisitMemoryPokeByte()
1803 void IntrinsicCodeGeneratorX86_64::VisitMemoryPokeByte(HInvoke* invoke) { in VisitMemoryPokeByte() argument
1804 GenPoke(invoke->GetLocations(), DataType::Type::kInt8, GetAssembler()); in VisitMemoryPokeByte()
1807 void IntrinsicLocationsBuilderX86_64::VisitMemoryPokeIntNative(HInvoke* invoke) { in VisitMemoryPokeIntNative() argument
1808 CreateIntIntToVoidLocations(allocator_, invoke); in VisitMemoryPokeIntNative()
1811 void IntrinsicCodeGeneratorX86_64::VisitMemoryPokeIntNative(HInvoke* invoke) { in VisitMemoryPokeIntNative() argument
1812 GenPoke(invoke->GetLocations(), DataType::Type::kInt32, GetAssembler()); in VisitMemoryPokeIntNative()
1815 void IntrinsicLocationsBuilderX86_64::VisitMemoryPokeLongNative(HInvoke* invoke) { in VisitMemoryPokeLongNative() argument
1816 CreateIntIntToVoidLocations(allocator_, invoke); in VisitMemoryPokeLongNative()
1819 void IntrinsicCodeGeneratorX86_64::VisitMemoryPokeLongNative(HInvoke* invoke) { in VisitMemoryPokeLongNative() argument
1820 GenPoke(invoke->GetLocations(), DataType::Type::kInt64, GetAssembler()); in VisitMemoryPokeLongNative()
1823 void IntrinsicLocationsBuilderX86_64::VisitMemoryPokeShortNative(HInvoke* invoke) { in VisitMemoryPokeShortNative() argument
1824 CreateIntIntToVoidLocations(allocator_, invoke); in VisitMemoryPokeShortNative()
1827 void IntrinsicCodeGeneratorX86_64::VisitMemoryPokeShortNative(HInvoke* invoke) { in VisitMemoryPokeShortNative() argument
1828 GenPoke(invoke->GetLocations(), DataType::Type::kInt16, GetAssembler()); in VisitMemoryPokeShortNative()
1831 void IntrinsicLocationsBuilderX86_64::VisitThreadCurrentThread(HInvoke* invoke) { in VisitThreadCurrentThread() argument
1833 new (allocator_) LocationSummary(invoke, LocationSummary::kNoCall, kIntrinsified); in VisitThreadCurrentThread()
1837 void IntrinsicCodeGeneratorX86_64::VisitThreadCurrentThread(HInvoke* invoke) { in VisitThreadCurrentThread() argument
1838 CpuRegister out = invoke->GetLocations()->Out().AsRegister<CpuRegister>(); in VisitThreadCurrentThread()
1843 static void GenUnsafeGet(HInvoke* invoke, in GenUnsafeGet() argument
1848 LocationSummary* locations = invoke->GetLocations(); in GenUnsafeGet()
1870 invoke, output_loc, base, src, /* needs_null_check= */ false); in GenUnsafeGet()
1874 invoke, output_loc, output_loc, base_loc, 0U, offset_loc); in GenUnsafeGet()
1893 static void GenUnsafeGetAbsolute(HInvoke* invoke, in GenUnsafeGetAbsolute() argument
1897 LocationSummary* locations = invoke->GetLocations(); in GenUnsafeGetAbsolute()
1922 static void CreateIntIntToIntLocations(ArenaAllocator* allocator, HInvoke* invoke) { in CreateIntIntToIntLocations() argument
1924 new (allocator) LocationSummary(invoke, LocationSummary::kNoCall, kIntrinsified); in CreateIntIntToIntLocations()
1931 HInvoke* invoke, in CreateIntIntIntToIntLocations() argument
1933 bool can_call = codegen->EmitReadBarrier() && IsUnsafeGetReference(invoke); in CreateIntIntIntToIntLocations()
1935 new (allocator) LocationSummary(invoke, in CreateIntIntIntToIntLocations()
1950 void IntrinsicLocationsBuilderX86_64::VisitUnsafeGet(HInvoke* invoke) { in VisitUnsafeGet() argument
1951 VisitJdkUnsafeGet(invoke); in VisitUnsafeGet()
1953 void IntrinsicLocationsBuilderX86_64::VisitUnsafeGetAbsolute(HInvoke* invoke) { in VisitUnsafeGetAbsolute() argument
1954 VisitJdkUnsafeGetAbsolute(invoke); in VisitUnsafeGetAbsolute()
1956 void IntrinsicLocationsBuilderX86_64::VisitUnsafeGetVolatile(HInvoke* invoke) { in VisitUnsafeGetVolatile() argument
1957 VisitJdkUnsafeGetVolatile(invoke); in VisitUnsafeGetVolatile()
1959 void IntrinsicLocationsBuilderX86_64::VisitUnsafeGetLong(HInvoke* invoke) { in VisitUnsafeGetLong() argument
1960 VisitJdkUnsafeGetLong(invoke); in VisitUnsafeGetLong()
1962 void IntrinsicLocationsBuilderX86_64::VisitUnsafeGetLongVolatile(HInvoke* invoke) { in VisitUnsafeGetLongVolatile() argument
1963 VisitJdkUnsafeGetLongVolatile(invoke); in VisitUnsafeGetLongVolatile()
1965 void IntrinsicLocationsBuilderX86_64::VisitUnsafeGetObject(HInvoke* invoke) { in VisitUnsafeGetObject() argument
1966 VisitJdkUnsafeGetReference(invoke); in VisitUnsafeGetObject()
1968 void IntrinsicLocationsBuilderX86_64::VisitUnsafeGetObjectVolatile(HInvoke* invoke) { in VisitUnsafeGetObjectVolatile() argument
1969 VisitJdkUnsafeGetReferenceVolatile(invoke); in VisitUnsafeGetObjectVolatile()
1971 void IntrinsicLocationsBuilderX86_64::VisitUnsafeGetByte(HInvoke* invoke) { in VisitUnsafeGetByte() argument
1972 VisitJdkUnsafeGetByte(invoke); in VisitUnsafeGetByte()
1975 void IntrinsicLocationsBuilderX86_64::VisitJdkUnsafeGet(HInvoke* invoke) { in VisitJdkUnsafeGet() argument
1976 CreateIntIntIntToIntLocations(allocator_, invoke, codegen_); in VisitJdkUnsafeGet()
1978 void IntrinsicLocationsBuilderX86_64::VisitJdkUnsafeGetAbsolute(HInvoke* invoke) { in VisitJdkUnsafeGetAbsolute() argument
1979 CreateIntIntToIntLocations(allocator_, invoke); in VisitJdkUnsafeGetAbsolute()
1981 void IntrinsicLocationsBuilderX86_64::VisitJdkUnsafeGetVolatile(HInvoke* invoke) { in VisitJdkUnsafeGetVolatile() argument
1982 CreateIntIntIntToIntLocations(allocator_, invoke, codegen_); in VisitJdkUnsafeGetVolatile()
1984 void IntrinsicLocationsBuilderX86_64::VisitJdkUnsafeGetAcquire(HInvoke* invoke) { in VisitJdkUnsafeGetAcquire() argument
1985 CreateIntIntIntToIntLocations(allocator_, invoke, codegen_); in VisitJdkUnsafeGetAcquire()
1987 void IntrinsicLocationsBuilderX86_64::VisitJdkUnsafeGetLong(HInvoke* invoke) { in VisitJdkUnsafeGetLong() argument
1988 CreateIntIntIntToIntLocations(allocator_, invoke, codegen_); in VisitJdkUnsafeGetLong()
1990 void IntrinsicLocationsBuilderX86_64::VisitJdkUnsafeGetLongVolatile(HInvoke* invoke) { in VisitJdkUnsafeGetLongVolatile() argument
1991 CreateIntIntIntToIntLocations(allocator_, invoke, codegen_); in VisitJdkUnsafeGetLongVolatile()
1993 void IntrinsicLocationsBuilderX86_64::VisitJdkUnsafeGetLongAcquire(HInvoke* invoke) { in VisitJdkUnsafeGetLongAcquire() argument
1994 CreateIntIntIntToIntLocations(allocator_, invoke, codegen_); in VisitJdkUnsafeGetLongAcquire()
1996 void IntrinsicLocationsBuilderX86_64::VisitJdkUnsafeGetReference(HInvoke* invoke) { in VisitJdkUnsafeGetReference() argument
1997 CreateIntIntIntToIntLocations(allocator_, invoke, codegen_); in VisitJdkUnsafeGetReference()
1999 void IntrinsicLocationsBuilderX86_64::VisitJdkUnsafeGetReferenceVolatile(HInvoke* invoke) { in VisitJdkUnsafeGetReferenceVolatile() argument
2000 CreateIntIntIntToIntLocations(allocator_, invoke, codegen_); in VisitJdkUnsafeGetReferenceVolatile()
2002 void IntrinsicLocationsBuilderX86_64::VisitJdkUnsafeGetReferenceAcquire(HInvoke* invoke) { in VisitJdkUnsafeGetReferenceAcquire() argument
2003 CreateIntIntIntToIntLocations(allocator_, invoke, codegen_); in VisitJdkUnsafeGetReferenceAcquire()
2005 void IntrinsicLocationsBuilderX86_64::VisitJdkUnsafeGetByte(HInvoke* invoke) { in VisitJdkUnsafeGetByte() argument
2006 CreateIntIntIntToIntLocations(allocator_, invoke, codegen_); in VisitJdkUnsafeGetByte()
2009 void IntrinsicCodeGeneratorX86_64::VisitUnsafeGet(HInvoke* invoke) { in VisitUnsafeGet() argument
2010 VisitJdkUnsafeGet(invoke); in VisitUnsafeGet()
2012 void IntrinsicCodeGeneratorX86_64::VisitUnsafeGetAbsolute(HInvoke* invoke) { in VisitUnsafeGetAbsolute() argument
2013 VisitJdkUnsafeGetAbsolute(invoke); in VisitUnsafeGetAbsolute()
2015 void IntrinsicCodeGeneratorX86_64::VisitUnsafeGetVolatile(HInvoke* invoke) { in VisitUnsafeGetVolatile() argument
2016 VisitJdkUnsafeGetVolatile(invoke); in VisitUnsafeGetVolatile()
2018 void IntrinsicCodeGeneratorX86_64::VisitUnsafeGetLong(HInvoke* invoke) { in VisitUnsafeGetLong() argument
2019 VisitJdkUnsafeGetLong(invoke); in VisitUnsafeGetLong()
2021 void IntrinsicCodeGeneratorX86_64::VisitUnsafeGetLongVolatile(HInvoke* invoke) { in VisitUnsafeGetLongVolatile() argument
2022 VisitJdkUnsafeGetLongVolatile(invoke); in VisitUnsafeGetLongVolatile()
2024 void IntrinsicCodeGeneratorX86_64::VisitUnsafeGetObject(HInvoke* invoke) { in VisitUnsafeGetObject() argument
2025 VisitJdkUnsafeGetReference(invoke); in VisitUnsafeGetObject()
2027 void IntrinsicCodeGeneratorX86_64::VisitUnsafeGetObjectVolatile(HInvoke* invoke) { in VisitUnsafeGetObjectVolatile() argument
2028 VisitJdkUnsafeGetReferenceVolatile(invoke); in VisitUnsafeGetObjectVolatile()
2030 void IntrinsicCodeGeneratorX86_64::VisitUnsafeGetByte(HInvoke* invoke) { in VisitUnsafeGetByte() argument
2031 VisitJdkUnsafeGetByte(invoke); in VisitUnsafeGetByte()
2034 void IntrinsicCodeGeneratorX86_64::VisitJdkUnsafeGet(HInvoke* invoke) { in VisitJdkUnsafeGet() argument
2035 GenUnsafeGet(invoke, DataType::Type::kInt32, /*is_volatile=*/ false, codegen_); in VisitJdkUnsafeGet()
2037 void IntrinsicCodeGeneratorX86_64::VisitJdkUnsafeGetAbsolute(HInvoke* invoke) { in VisitJdkUnsafeGetAbsolute() argument
2038 GenUnsafeGetAbsolute(invoke, DataType::Type::kInt32, codegen_); in VisitJdkUnsafeGetAbsolute()
2040 void IntrinsicCodeGeneratorX86_64::VisitJdkUnsafeGetVolatile(HInvoke* invoke) { in VisitJdkUnsafeGetVolatile() argument
2041 GenUnsafeGet(invoke, DataType::Type::kInt32, /*is_volatile=*/ true, codegen_); in VisitJdkUnsafeGetVolatile()
2043 void IntrinsicCodeGeneratorX86_64::VisitJdkUnsafeGetAcquire(HInvoke* invoke) { in VisitJdkUnsafeGetAcquire() argument
2044 GenUnsafeGet(invoke, DataType::Type::kInt32, /*is_volatile=*/ true, codegen_); in VisitJdkUnsafeGetAcquire()
2046 void IntrinsicCodeGeneratorX86_64::VisitJdkUnsafeGetLong(HInvoke* invoke) { in VisitJdkUnsafeGetLong() argument
2047 GenUnsafeGet(invoke, DataType::Type::kInt64, /*is_volatile=*/ false, codegen_); in VisitJdkUnsafeGetLong()
2049 void IntrinsicCodeGeneratorX86_64::VisitJdkUnsafeGetLongVolatile(HInvoke* invoke) { in VisitJdkUnsafeGetLongVolatile() argument
2050 GenUnsafeGet(invoke, DataType::Type::kInt64, /*is_volatile=*/ true, codegen_); in VisitJdkUnsafeGetLongVolatile()
2052 void IntrinsicCodeGeneratorX86_64::VisitJdkUnsafeGetLongAcquire(HInvoke* invoke) { in VisitJdkUnsafeGetLongAcquire() argument
2053 GenUnsafeGet(invoke, DataType::Type::kInt64, /*is_volatile=*/ true, codegen_); in VisitJdkUnsafeGetLongAcquire()
2055 void IntrinsicCodeGeneratorX86_64::VisitJdkUnsafeGetReference(HInvoke* invoke) { in VisitJdkUnsafeGetReference() argument
2056 GenUnsafeGet(invoke, DataType::Type::kReference, /*is_volatile=*/ false, codegen_); in VisitJdkUnsafeGetReference()
2058 void IntrinsicCodeGeneratorX86_64::VisitJdkUnsafeGetReferenceVolatile(HInvoke* invoke) { in VisitJdkUnsafeGetReferenceVolatile() argument
2059 GenUnsafeGet(invoke, DataType::Type::kReference, /*is_volatile=*/ true, codegen_); in VisitJdkUnsafeGetReferenceVolatile()
2061 void IntrinsicCodeGeneratorX86_64::VisitJdkUnsafeGetReferenceAcquire(HInvoke* invoke) { in VisitJdkUnsafeGetReferenceAcquire() argument
2062 GenUnsafeGet(invoke, DataType::Type::kReference, /*is_volatile=*/ true, codegen_); in VisitJdkUnsafeGetReferenceAcquire()
2064 void IntrinsicCodeGeneratorX86_64::VisitJdkUnsafeGetByte(HInvoke* invoke) { in VisitJdkUnsafeGetByte() argument
2065 GenUnsafeGet(invoke, DataType::Type::kInt8, /*is_volatile=*/false, codegen_); in VisitJdkUnsafeGetByte()
2070 HInvoke* invoke) { in CreateIntIntIntToVoidPlusTempsLocations() argument
2072 new (allocator) LocationSummary(invoke, LocationSummary::kNoCall, kIntrinsified); in CreateIntIntIntToVoidPlusTempsLocations()
2080 HInvoke* invoke) { in CreateIntIntIntIntToVoidPlusTempsLocations() argument
2082 new (allocator) LocationSummary(invoke, LocationSummary::kNoCall, kIntrinsified); in CreateIntIntIntIntToVoidPlusTempsLocations()
2094 void IntrinsicLocationsBuilderX86_64::VisitUnsafePut(HInvoke* invoke) { in VisitUnsafePut() argument
2095 VisitJdkUnsafePut(invoke); in VisitUnsafePut()
2097 void IntrinsicLocationsBuilderX86_64::VisitUnsafePutAbsolute(HInvoke* invoke) { in VisitUnsafePutAbsolute() argument
2098 VisitJdkUnsafePutAbsolute(invoke); in VisitUnsafePutAbsolute()
2100 void IntrinsicLocationsBuilderX86_64::VisitUnsafePutOrdered(HInvoke* invoke) { in VisitUnsafePutOrdered() argument
2101 VisitJdkUnsafePutOrdered(invoke); in VisitUnsafePutOrdered()
2103 void IntrinsicLocationsBuilderX86_64::VisitUnsafePutVolatile(HInvoke* invoke) { in VisitUnsafePutVolatile() argument
2104 VisitJdkUnsafePutVolatile(invoke); in VisitUnsafePutVolatile()
2106 void IntrinsicLocationsBuilderX86_64::VisitUnsafePutObject(HInvoke* invoke) { in VisitUnsafePutObject() argument
2107 VisitJdkUnsafePutReference(invoke); in VisitUnsafePutObject()
2109 void IntrinsicLocationsBuilderX86_64::VisitUnsafePutObjectOrdered(HInvoke* invoke) { in VisitUnsafePutObjectOrdered() argument
2110 VisitJdkUnsafePutObjectOrdered(invoke); in VisitUnsafePutObjectOrdered()
2112 void IntrinsicLocationsBuilderX86_64::VisitUnsafePutObjectVolatile(HInvoke* invoke) { in VisitUnsafePutObjectVolatile() argument
2113 VisitJdkUnsafePutReferenceVolatile(invoke); in VisitUnsafePutObjectVolatile()
2115 void IntrinsicLocationsBuilderX86_64::VisitUnsafePutLong(HInvoke* invoke) { in VisitUnsafePutLong() argument
2116 VisitJdkUnsafePutLong(invoke); in VisitUnsafePutLong()
2118 void IntrinsicLocationsBuilderX86_64::VisitUnsafePutLongOrdered(HInvoke* invoke) { in VisitUnsafePutLongOrdered() argument
2119 VisitJdkUnsafePutLongOrdered(invoke); in VisitUnsafePutLongOrdered()
2121 void IntrinsicLocationsBuilderX86_64::VisitUnsafePutLongVolatile(HInvoke* invoke) { in VisitUnsafePutLongVolatile() argument
2122 VisitJdkUnsafePutLongVolatile(invoke); in VisitUnsafePutLongVolatile()
2124 void IntrinsicLocationsBuilderX86_64::VisitUnsafePutByte(HInvoke* invoke) { in VisitUnsafePutByte() argument
2125 VisitJdkUnsafePut(invoke); in VisitUnsafePutByte()
2128 void IntrinsicLocationsBuilderX86_64::VisitJdkUnsafePut(HInvoke* invoke) { in VisitJdkUnsafePut() argument
2129 CreateIntIntIntIntToVoidPlusTempsLocations(allocator_, DataType::Type::kInt32, invoke); in VisitJdkUnsafePut()
2131 void IntrinsicLocationsBuilderX86_64::VisitJdkUnsafePutAbsolute(HInvoke* invoke) { in VisitJdkUnsafePutAbsolute() argument
2132 CreateIntIntIntToVoidPlusTempsLocations(allocator_, DataType::Type::kInt32, invoke); in VisitJdkUnsafePutAbsolute()
2134 void IntrinsicLocationsBuilderX86_64::VisitJdkUnsafePutOrdered(HInvoke* invoke) { in VisitJdkUnsafePutOrdered() argument
2135 CreateIntIntIntIntToVoidPlusTempsLocations(allocator_, DataType::Type::kInt32, invoke); in VisitJdkUnsafePutOrdered()
2137 void IntrinsicLocationsBuilderX86_64::VisitJdkUnsafePutVolatile(HInvoke* invoke) { in VisitJdkUnsafePutVolatile() argument
2138 CreateIntIntIntIntToVoidPlusTempsLocations(allocator_, DataType::Type::kInt32, invoke); in VisitJdkUnsafePutVolatile()
2140 void IntrinsicLocationsBuilderX86_64::VisitJdkUnsafePutRelease(HInvoke* invoke) { in VisitJdkUnsafePutRelease() argument
2141 CreateIntIntIntIntToVoidPlusTempsLocations(allocator_, DataType::Type::kInt32, invoke); in VisitJdkUnsafePutRelease()
2143 void IntrinsicLocationsBuilderX86_64::VisitJdkUnsafePutReference(HInvoke* invoke) { in VisitJdkUnsafePutReference() argument
2144 CreateIntIntIntIntToVoidPlusTempsLocations(allocator_, DataType::Type::kReference, invoke); in VisitJdkUnsafePutReference()
2146 void IntrinsicLocationsBuilderX86_64::VisitJdkUnsafePutObjectOrdered(HInvoke* invoke) { in VisitJdkUnsafePutObjectOrdered() argument
2147 CreateIntIntIntIntToVoidPlusTempsLocations(allocator_, DataType::Type::kReference, invoke); in VisitJdkUnsafePutObjectOrdered()
2149 void IntrinsicLocationsBuilderX86_64::VisitJdkUnsafePutReferenceVolatile(HInvoke* invoke) { in VisitJdkUnsafePutReferenceVolatile() argument
2150 CreateIntIntIntIntToVoidPlusTempsLocations(allocator_, DataType::Type::kReference, invoke); in VisitJdkUnsafePutReferenceVolatile()
2152 void IntrinsicLocationsBuilderX86_64::VisitJdkUnsafePutReferenceRelease(HInvoke* invoke) { in VisitJdkUnsafePutReferenceRelease() argument
2153 CreateIntIntIntIntToVoidPlusTempsLocations(allocator_, DataType::Type::kReference, invoke); in VisitJdkUnsafePutReferenceRelease()
2155 void IntrinsicLocationsBuilderX86_64::VisitJdkUnsafePutLong(HInvoke* invoke) { in VisitJdkUnsafePutLong() argument
2156 CreateIntIntIntIntToVoidPlusTempsLocations(allocator_, DataType::Type::kInt64, invoke); in VisitJdkUnsafePutLong()
2158 void IntrinsicLocationsBuilderX86_64::VisitJdkUnsafePutLongOrdered(HInvoke* invoke) { in VisitJdkUnsafePutLongOrdered() argument
2159 CreateIntIntIntIntToVoidPlusTempsLocations(allocator_, DataType::Type::kInt64, invoke); in VisitJdkUnsafePutLongOrdered()
2161 void IntrinsicLocationsBuilderX86_64::VisitJdkUnsafePutLongVolatile(HInvoke* invoke) { in VisitJdkUnsafePutLongVolatile() argument
2162 CreateIntIntIntIntToVoidPlusTempsLocations(allocator_, DataType::Type::kInt64, invoke); in VisitJdkUnsafePutLongVolatile()
2164 void IntrinsicLocationsBuilderX86_64::VisitJdkUnsafePutLongRelease(HInvoke* invoke) { in VisitJdkUnsafePutLongRelease() argument
2165 CreateIntIntIntIntToVoidPlusTempsLocations(allocator_, DataType::Type::kInt64, invoke); in VisitJdkUnsafePutLongRelease()
2167 void IntrinsicLocationsBuilderX86_64::VisitJdkUnsafePutByte(HInvoke* invoke) { in VisitJdkUnsafePutByte() argument
2168 CreateIntIntIntIntToVoidPlusTempsLocations(allocator_, DataType::Type::kInt8, invoke); in VisitJdkUnsafePutByte()
2233 void IntrinsicCodeGeneratorX86_64::VisitUnsafePut(HInvoke* invoke) { in VisitUnsafePut() argument
2234 VisitJdkUnsafePut(invoke); in VisitUnsafePut()
2236 void IntrinsicCodeGeneratorX86_64::VisitUnsafePutAbsolute(HInvoke* invoke) { in VisitUnsafePutAbsolute() argument
2237 VisitJdkUnsafePutAbsolute(invoke); in VisitUnsafePutAbsolute()
2239 void IntrinsicCodeGeneratorX86_64::VisitUnsafePutOrdered(HInvoke* invoke) { in VisitUnsafePutOrdered() argument
2240 VisitJdkUnsafePutOrdered(invoke); in VisitUnsafePutOrdered()
2242 void IntrinsicCodeGeneratorX86_64::VisitUnsafePutVolatile(HInvoke* invoke) { in VisitUnsafePutVolatile() argument
2243 VisitJdkUnsafePutVolatile(invoke); in VisitUnsafePutVolatile()
2245 void IntrinsicCodeGeneratorX86_64::VisitUnsafePutObject(HInvoke* invoke) { in VisitUnsafePutObject() argument
2246 VisitJdkUnsafePutReference(invoke); in VisitUnsafePutObject()
2248 void IntrinsicCodeGeneratorX86_64::VisitUnsafePutObjectOrdered(HInvoke* invoke) { in VisitUnsafePutObjectOrdered() argument
2249 VisitJdkUnsafePutObjectOrdered(invoke); in VisitUnsafePutObjectOrdered()
2251 void IntrinsicCodeGeneratorX86_64::VisitUnsafePutObjectVolatile(HInvoke* invoke) { in VisitUnsafePutObjectVolatile() argument
2252 VisitJdkUnsafePutReferenceVolatile(invoke); in VisitUnsafePutObjectVolatile()
2254 void IntrinsicCodeGeneratorX86_64::VisitUnsafePutLong(HInvoke* invoke) { in VisitUnsafePutLong() argument
2255 VisitJdkUnsafePutLong(invoke); in VisitUnsafePutLong()
2257 void IntrinsicCodeGeneratorX86_64::VisitUnsafePutLongOrdered(HInvoke* invoke) { in VisitUnsafePutLongOrdered() argument
2258 VisitJdkUnsafePutLongOrdered(invoke); in VisitUnsafePutLongOrdered()
2260 void IntrinsicCodeGeneratorX86_64::VisitUnsafePutLongVolatile(HInvoke* invoke) { in VisitUnsafePutLongVolatile() argument
2261 VisitJdkUnsafePutLongVolatile(invoke); in VisitUnsafePutLongVolatile()
2263 void IntrinsicCodeGeneratorX86_64::VisitUnsafePutByte(HInvoke* invoke) { in VisitUnsafePutByte() argument
2264 VisitJdkUnsafePutByte(invoke); in VisitUnsafePutByte()
2267 void IntrinsicCodeGeneratorX86_64::VisitJdkUnsafePut(HInvoke* invoke) { in VisitJdkUnsafePut() argument
2268 GenUnsafePut(invoke->GetLocations(), DataType::Type::kInt32, /*is_volatile=*/ false, codegen_); in VisitJdkUnsafePut()
2270 void IntrinsicCodeGeneratorX86_64::VisitJdkUnsafePutAbsolute(HInvoke* invoke) { in VisitJdkUnsafePutAbsolute() argument
2272 invoke->GetLocations(), DataType::Type::kInt32, /*is_volatile=*/false, codegen_); in VisitJdkUnsafePutAbsolute()
2274 void IntrinsicCodeGeneratorX86_64::VisitJdkUnsafePutOrdered(HInvoke* invoke) { in VisitJdkUnsafePutOrdered() argument
2275 GenUnsafePut(invoke->GetLocations(), DataType::Type::kInt32, /*is_volatile=*/ false, codegen_); in VisitJdkUnsafePutOrdered()
2277 void IntrinsicCodeGeneratorX86_64::VisitJdkUnsafePutVolatile(HInvoke* invoke) { in VisitJdkUnsafePutVolatile() argument
2278 GenUnsafePut(invoke->GetLocations(), DataType::Type::kInt32, /*is_volatile=*/ true, codegen_); in VisitJdkUnsafePutVolatile()
2280 void IntrinsicCodeGeneratorX86_64::VisitJdkUnsafePutRelease(HInvoke* invoke) { in VisitJdkUnsafePutRelease() argument
2281 GenUnsafePut(invoke->GetLocations(), DataType::Type::kInt32, /* is_volatile= */ true, codegen_); in VisitJdkUnsafePutRelease()
2283 void IntrinsicCodeGeneratorX86_64::VisitJdkUnsafePutReference(HInvoke* invoke) { in VisitJdkUnsafePutReference() argument
2285 invoke->GetLocations(), DataType::Type::kReference, /*is_volatile=*/ false, codegen_); in VisitJdkUnsafePutReference()
2287 void IntrinsicCodeGeneratorX86_64::VisitJdkUnsafePutObjectOrdered(HInvoke* invoke) { in VisitJdkUnsafePutObjectOrdered() argument
2289 invoke->GetLocations(), DataType::Type::kReference, /*is_volatile=*/ false, codegen_); in VisitJdkUnsafePutObjectOrdered()
2291 void IntrinsicCodeGeneratorX86_64::VisitJdkUnsafePutReferenceVolatile(HInvoke* invoke) { in VisitJdkUnsafePutReferenceVolatile() argument
2293 invoke->GetLocations(), DataType::Type::kReference, /*is_volatile=*/ true, codegen_); in VisitJdkUnsafePutReferenceVolatile()
2295 void IntrinsicCodeGeneratorX86_64::VisitJdkUnsafePutReferenceRelease(HInvoke* invoke) { in VisitJdkUnsafePutReferenceRelease() argument
2297 invoke->GetLocations(), DataType::Type::kReference, /*is_volatile=*/ true, codegen_); in VisitJdkUnsafePutReferenceRelease()
2299 void IntrinsicCodeGeneratorX86_64::VisitJdkUnsafePutLong(HInvoke* invoke) { in VisitJdkUnsafePutLong() argument
2300 GenUnsafePut(invoke->GetLocations(), DataType::Type::kInt64, /*is_volatile=*/ false, codegen_); in VisitJdkUnsafePutLong()
2302 void IntrinsicCodeGeneratorX86_64::VisitJdkUnsafePutLongOrdered(HInvoke* invoke) { in VisitJdkUnsafePutLongOrdered() argument
2303 GenUnsafePut(invoke->GetLocations(), DataType::Type::kInt64, /*is_volatile=*/ false, codegen_); in VisitJdkUnsafePutLongOrdered()
2305 void IntrinsicCodeGeneratorX86_64::VisitJdkUnsafePutLongVolatile(HInvoke* invoke) { in VisitJdkUnsafePutLongVolatile() argument
2306 GenUnsafePut(invoke->GetLocations(), DataType::Type::kInt64, /*is_volatile=*/ true, codegen_); in VisitJdkUnsafePutLongVolatile()
2308 void IntrinsicCodeGeneratorX86_64::VisitJdkUnsafePutLongRelease(HInvoke* invoke) { in VisitJdkUnsafePutLongRelease() argument
2309 GenUnsafePut(invoke->GetLocations(), DataType::Type::kInt64, /*is_volatile=*/ true, codegen_); in VisitJdkUnsafePutLongRelease()
2311 void IntrinsicCodeGeneratorX86_64::VisitJdkUnsafePutByte(HInvoke* invoke) { in VisitJdkUnsafePutByte() argument
2312 GenUnsafePut(invoke->GetLocations(), DataType::Type::kInt8, /*is_volatile=*/false, codegen_); in VisitJdkUnsafePutByte()
2316 HInvoke* invoke, in CreateUnsafeCASLocations() argument
2319 const bool can_call = codegen->EmitBakerReadBarrier() && IsUnsafeCASReference(invoke); in CreateUnsafeCASLocations()
2321 new (allocator) LocationSummary(invoke, in CreateUnsafeCASLocations()
2348 void IntrinsicLocationsBuilderX86_64::VisitUnsafeCASInt(HInvoke* invoke) { in VisitUnsafeCASInt() argument
2349 VisitJdkUnsafeCASInt(invoke); in VisitUnsafeCASInt()
2352 void IntrinsicLocationsBuilderX86_64::VisitUnsafeCASLong(HInvoke* invoke) { in VisitUnsafeCASLong() argument
2353 VisitJdkUnsafeCASLong(invoke); in VisitUnsafeCASLong()
2356 void IntrinsicLocationsBuilderX86_64::VisitUnsafeCASObject(HInvoke* invoke) { in VisitUnsafeCASObject() argument
2357 VisitJdkUnsafeCASObject(invoke); in VisitUnsafeCASObject()
2360 void IntrinsicLocationsBuilderX86_64::VisitJdkUnsafeCASInt(HInvoke* invoke) { in VisitJdkUnsafeCASInt() argument
2362 VisitJdkUnsafeCompareAndSetInt(invoke); in VisitJdkUnsafeCASInt()
2365 void IntrinsicLocationsBuilderX86_64::VisitJdkUnsafeCASLong(HInvoke* invoke) { in VisitJdkUnsafeCASLong() argument
2367 VisitJdkUnsafeCompareAndSetLong(invoke); in VisitJdkUnsafeCASLong()
2370 void IntrinsicLocationsBuilderX86_64::VisitJdkUnsafeCASObject(HInvoke* invoke) { in VisitJdkUnsafeCASObject() argument
2372 VisitJdkUnsafeCompareAndSetReference(invoke); in VisitJdkUnsafeCASObject()
2375 void IntrinsicLocationsBuilderX86_64::VisitJdkUnsafeCompareAndSetInt(HInvoke* invoke) { in VisitJdkUnsafeCompareAndSetInt() argument
2376 CreateUnsafeCASLocations(allocator_, invoke, codegen_, DataType::Type::kInt32); in VisitJdkUnsafeCompareAndSetInt()
2379 void IntrinsicLocationsBuilderX86_64::VisitJdkUnsafeCompareAndSetLong(HInvoke* invoke) { in VisitJdkUnsafeCompareAndSetLong() argument
2380 CreateUnsafeCASLocations(allocator_, invoke, codegen_, DataType::Type::kInt64); in VisitJdkUnsafeCompareAndSetLong()
2383 void IntrinsicLocationsBuilderX86_64::VisitJdkUnsafeCompareAndSetReference(HInvoke* invoke) { in VisitJdkUnsafeCompareAndSetReference() argument
2389 CreateUnsafeCASLocations(allocator_, invoke, codegen_, DataType::Type::kReference); in VisitJdkUnsafeCompareAndSetReference()
2516 HInvoke* invoke, in GenCompareAndSetOrExchangeRef() argument
2538 invoke, in GenCompareAndSetOrExchangeRef()
2612 HInvoke* invoke, in GenCompareAndSetOrExchange() argument
2624 LocationSummary* locations = invoke->GetLocations(); in GenCompareAndSetOrExchange()
2650 codegen, invoke, base, offset, new_value_reg, temp1, temp2, temp3, is_cmpxchg); in GenCompareAndSetOrExchange()
2657 static void GenCAS(DataType::Type type, HInvoke* invoke, CodeGeneratorX86_64* codegen) { in GenCAS() argument
2658 LocationSummary* locations = invoke->GetLocations(); in GenCAS()
2660 invoke, in GenCAS()
2674 void IntrinsicCodeGeneratorX86_64::VisitUnsafeCASInt(HInvoke* invoke) { in VisitUnsafeCASInt() argument
2675 VisitJdkUnsafeCASInt(invoke); in VisitUnsafeCASInt()
2678 void IntrinsicCodeGeneratorX86_64::VisitUnsafeCASLong(HInvoke* invoke) { in VisitUnsafeCASLong() argument
2679 VisitJdkUnsafeCASLong(invoke); in VisitUnsafeCASLong()
2682 void IntrinsicCodeGeneratorX86_64::VisitUnsafeCASObject(HInvoke* invoke) { in VisitUnsafeCASObject() argument
2683 VisitJdkUnsafeCASObject(invoke); in VisitUnsafeCASObject()
2686 void IntrinsicCodeGeneratorX86_64::VisitJdkUnsafeCASInt(HInvoke* invoke) { in VisitJdkUnsafeCASInt() argument
2688 VisitJdkUnsafeCompareAndSetInt(invoke); in VisitJdkUnsafeCASInt()
2691 void IntrinsicCodeGeneratorX86_64::VisitJdkUnsafeCASLong(HInvoke* invoke) { in VisitJdkUnsafeCASLong() argument
2693 VisitJdkUnsafeCompareAndSetLong(invoke); in VisitJdkUnsafeCASLong()
2696 void IntrinsicCodeGeneratorX86_64::VisitJdkUnsafeCASObject(HInvoke* invoke) { in VisitJdkUnsafeCASObject() argument
2698 VisitJdkUnsafeCompareAndSetReference(invoke); in VisitJdkUnsafeCASObject()
2701 void IntrinsicCodeGeneratorX86_64::VisitJdkUnsafeCompareAndSetInt(HInvoke* invoke) { in VisitJdkUnsafeCompareAndSetInt() argument
2702 GenCAS(DataType::Type::kInt32, invoke, codegen_); in VisitJdkUnsafeCompareAndSetInt()
2705 void IntrinsicCodeGeneratorX86_64::VisitJdkUnsafeCompareAndSetLong(HInvoke* invoke) { in VisitJdkUnsafeCompareAndSetLong() argument
2706 GenCAS(DataType::Type::kInt64, invoke, codegen_); in VisitJdkUnsafeCompareAndSetLong()
2709 void IntrinsicCodeGeneratorX86_64::VisitJdkUnsafeCompareAndSetReference(HInvoke* invoke) { in VisitJdkUnsafeCompareAndSetReference() argument
2713 GenCAS(DataType::Type::kReference, invoke, codegen_); in VisitJdkUnsafeCompareAndSetReference()
2717 HInvoke* invoke, in CreateUnsafeGetAndUpdateLocations() argument
2719 const bool can_call = codegen->EmitReadBarrier() && IsUnsafeGetAndSetReference(invoke); in CreateUnsafeGetAndUpdateLocations()
2721 new (allocator) LocationSummary(invoke, in CreateUnsafeGetAndUpdateLocations()
2737 if (invoke->GetType() == DataType::Type::kVoid) { in CreateUnsafeGetAndUpdateLocations()
2744 void IntrinsicLocationsBuilderX86_64::VisitUnsafeGetAndAddInt(HInvoke* invoke) { in VisitUnsafeGetAndAddInt() argument
2745 VisitJdkUnsafeGetAndAddInt(invoke); in VisitUnsafeGetAndAddInt()
2748 void IntrinsicLocationsBuilderX86_64::VisitUnsafeGetAndAddLong(HInvoke* invoke) { in VisitUnsafeGetAndAddLong() argument
2749 VisitJdkUnsafeGetAndAddLong(invoke); in VisitUnsafeGetAndAddLong()
2752 void IntrinsicLocationsBuilderX86_64::VisitUnsafeGetAndSetInt(HInvoke* invoke) { in VisitUnsafeGetAndSetInt() argument
2753 VisitJdkUnsafeGetAndSetInt(invoke); in VisitUnsafeGetAndSetInt()
2756 void IntrinsicLocationsBuilderX86_64::VisitUnsafeGetAndSetLong(HInvoke* invoke) { in VisitUnsafeGetAndSetLong() argument
2757 VisitJdkUnsafeGetAndSetLong(invoke); in VisitUnsafeGetAndSetLong()
2760 void IntrinsicLocationsBuilderX86_64::VisitUnsafeGetAndSetObject(HInvoke* invoke) { in VisitUnsafeGetAndSetObject() argument
2761 VisitJdkUnsafeGetAndSetReference(invoke); in VisitUnsafeGetAndSetObject()
2764 void IntrinsicLocationsBuilderX86_64::VisitJdkUnsafeGetAndAddInt(HInvoke* invoke) { in VisitJdkUnsafeGetAndAddInt() argument
2765 CreateUnsafeGetAndUpdateLocations(allocator_, invoke, codegen_); in VisitJdkUnsafeGetAndAddInt()
2768 void IntrinsicLocationsBuilderX86_64::VisitJdkUnsafeGetAndAddLong(HInvoke* invoke) { in VisitJdkUnsafeGetAndAddLong() argument
2769 CreateUnsafeGetAndUpdateLocations(allocator_, invoke, codegen_); in VisitJdkUnsafeGetAndAddLong()
2772 void IntrinsicLocationsBuilderX86_64::VisitJdkUnsafeGetAndSetInt(HInvoke* invoke) { in VisitJdkUnsafeGetAndSetInt() argument
2773 CreateUnsafeGetAndUpdateLocations(allocator_, invoke, codegen_); in VisitJdkUnsafeGetAndSetInt()
2776 void IntrinsicLocationsBuilderX86_64::VisitJdkUnsafeGetAndSetLong(HInvoke* invoke) { in VisitJdkUnsafeGetAndSetLong() argument
2777 CreateUnsafeGetAndUpdateLocations(allocator_, invoke, codegen_); in VisitJdkUnsafeGetAndSetLong()
2780 void IntrinsicLocationsBuilderX86_64::VisitJdkUnsafeGetAndSetReference(HInvoke* invoke) { in VisitJdkUnsafeGetAndSetReference() argument
2786 CreateUnsafeGetAndUpdateLocations(allocator_, invoke, codegen_); in VisitJdkUnsafeGetAndSetReference()
2787 invoke->GetLocations()->AddRegisterTemps(3); in VisitJdkUnsafeGetAndSetReference()
2798 static void GenUnsafeGetAndUpdate(HInvoke* invoke, in GenUnsafeGetAndUpdate() argument
2803 LocationSummary* locations = invoke->GetLocations(); in GenUnsafeGetAndUpdate()
2805 const bool is_void = invoke->GetType() == DataType::Type::kVoid; in GenUnsafeGetAndUpdate()
2847 invoke, in GenUnsafeGetAndUpdate()
2877 void IntrinsicCodeGeneratorX86_64::VisitUnsafeGetAndAddInt(HInvoke* invoke) { in VisitUnsafeGetAndAddInt() argument
2878 VisitJdkUnsafeGetAndAddInt(invoke); in VisitUnsafeGetAndAddInt()
2881 void IntrinsicCodeGeneratorX86_64::VisitUnsafeGetAndAddLong(HInvoke* invoke) { in VisitUnsafeGetAndAddLong() argument
2882 VisitJdkUnsafeGetAndAddLong(invoke); in VisitUnsafeGetAndAddLong()
2885 void IntrinsicCodeGeneratorX86_64::VisitUnsafeGetAndSetInt(HInvoke* invoke) { in VisitUnsafeGetAndSetInt() argument
2886 VisitJdkUnsafeGetAndSetInt(invoke); in VisitUnsafeGetAndSetInt()
2889 void IntrinsicCodeGeneratorX86_64::VisitUnsafeGetAndSetLong(HInvoke* invoke) { in VisitUnsafeGetAndSetLong() argument
2890 VisitJdkUnsafeGetAndSetLong(invoke); in VisitUnsafeGetAndSetLong()
2893 void IntrinsicCodeGeneratorX86_64::VisitUnsafeGetAndSetObject(HInvoke* invoke) { in VisitUnsafeGetAndSetObject() argument
2894 VisitJdkUnsafeGetAndSetReference(invoke); in VisitUnsafeGetAndSetObject()
2897 void IntrinsicCodeGeneratorX86_64::VisitJdkUnsafeGetAndAddInt(HInvoke* invoke) { in VisitJdkUnsafeGetAndAddInt() argument
2898 GenUnsafeGetAndUpdate(invoke, DataType::Type::kInt32, codegen_, GetAndUpdateOp::kAdd); in VisitJdkUnsafeGetAndAddInt()
2901 void IntrinsicCodeGeneratorX86_64::VisitJdkUnsafeGetAndAddLong(HInvoke* invoke) { in VisitJdkUnsafeGetAndAddLong() argument
2902 GenUnsafeGetAndUpdate(invoke, DataType::Type::kInt64, codegen_, GetAndUpdateOp::kAdd); in VisitJdkUnsafeGetAndAddLong()
2905 void IntrinsicCodeGeneratorX86_64::VisitJdkUnsafeGetAndSetInt(HInvoke* invoke) { in VisitJdkUnsafeGetAndSetInt() argument
2906 GenUnsafeGetAndUpdate(invoke, DataType::Type::kInt32, codegen_, GetAndUpdateOp::kSet); in VisitJdkUnsafeGetAndSetInt()
2909 void IntrinsicCodeGeneratorX86_64::VisitJdkUnsafeGetAndSetLong(HInvoke* invoke) { in VisitJdkUnsafeGetAndSetLong() argument
2910 GenUnsafeGetAndUpdate(invoke, DataType::Type::kInt64, codegen_, GetAndUpdateOp::kSet); in VisitJdkUnsafeGetAndSetLong()
2913 void IntrinsicCodeGeneratorX86_64::VisitJdkUnsafeGetAndSetReference(HInvoke* invoke) { in VisitJdkUnsafeGetAndSetReference() argument
2914 GenUnsafeGetAndUpdate(invoke, DataType::Type::kReference, codegen_, GetAndUpdateOp::kSet); in VisitJdkUnsafeGetAndSetReference()
2917 void IntrinsicLocationsBuilderX86_64::VisitIntegerReverse(HInvoke* invoke) { in VisitIntegerReverse() argument
2919 new (allocator_) LocationSummary(invoke, LocationSummary::kNoCall, kIntrinsified); in VisitIntegerReverse()
2937 void IntrinsicCodeGeneratorX86_64::VisitIntegerReverse(HInvoke* invoke) { in VisitIntegerReverse() argument
2939 LocationSummary* locations = invoke->GetLocations(); in VisitIntegerReverse()
2959 void IntrinsicLocationsBuilderX86_64::VisitLongReverse(HInvoke* invoke) { in VisitLongReverse() argument
2961 new (allocator_) LocationSummary(invoke, LocationSummary::kNoCall, kIntrinsified); in VisitLongReverse()
2979 void IntrinsicCodeGeneratorX86_64::VisitLongReverse(HInvoke* invoke) { in VisitLongReverse() argument
2981 LocationSummary* locations = invoke->GetLocations(); in VisitLongReverse()
3003 ArenaAllocator* allocator, CodeGeneratorX86_64* codegen, HInvoke* invoke) { in CreateBitCountLocations() argument
3010 new (allocator) LocationSummary(invoke, LocationSummary::kNoCall, kIntrinsified); in CreateBitCountLocations()
3017 HInvoke* invoke, in GenBitCount() argument
3019 LocationSummary* locations = invoke->GetLocations(); in GenBitCount()
3023 if (invoke->InputAt(0)->IsConstant()) { in GenBitCount()
3025 int64_t value = Int64FromConstant(invoke->InputAt(0)->AsConstant()); in GenBitCount()
3048 void IntrinsicLocationsBuilderX86_64::VisitIntegerBitCount(HInvoke* invoke) { in VisitIntegerBitCount() argument
3049 CreateBitCountLocations(allocator_, codegen_, invoke); in VisitIntegerBitCount()
3052 void IntrinsicCodeGeneratorX86_64::VisitIntegerBitCount(HInvoke* invoke) { in VisitIntegerBitCount() argument
3053 GenBitCount(GetAssembler(), codegen_, invoke, /* is_long= */ false); in VisitIntegerBitCount()
3056 void IntrinsicLocationsBuilderX86_64::VisitLongBitCount(HInvoke* invoke) { in VisitLongBitCount() argument
3057 CreateBitCountLocations(allocator_, codegen_, invoke); in VisitLongBitCount()
3060 void IntrinsicCodeGeneratorX86_64::VisitLongBitCount(HInvoke* invoke) { in VisitLongBitCount() argument
3061 GenBitCount(GetAssembler(), codegen_, invoke, /* is_long= */ true); in VisitLongBitCount()
3064 static void CreateOneBitLocations(ArenaAllocator* allocator, HInvoke* invoke, bool is_high) { in CreateOneBitLocations() argument
3066 new (allocator) LocationSummary(invoke, LocationSummary::kNoCall, kIntrinsified); in CreateOneBitLocations()
3075 HInvoke* invoke, in GenOneBit() argument
3077 LocationSummary* locations = invoke->GetLocations(); in GenOneBit()
3081 if (invoke->InputAt(0)->IsConstant()) { in GenOneBit()
3083 int64_t value = Int64FromConstant(invoke->InputAt(0)->AsConstant()); in GenOneBit()
3167 void IntrinsicLocationsBuilderX86_64::VisitIntegerHighestOneBit(HInvoke* invoke) { in VisitIntegerHighestOneBit() argument
3168 CreateOneBitLocations(allocator_, invoke, /* is_high= */ true); in VisitIntegerHighestOneBit()
3171 void IntrinsicCodeGeneratorX86_64::VisitIntegerHighestOneBit(HInvoke* invoke) { in VisitIntegerHighestOneBit() argument
3172 GenOneBit(GetAssembler(), codegen_, invoke, /* is_high= */ true, /* is_long= */ false); in VisitIntegerHighestOneBit()
3175 void IntrinsicLocationsBuilderX86_64::VisitLongHighestOneBit(HInvoke* invoke) { in VisitLongHighestOneBit() argument
3176 CreateOneBitLocations(allocator_, invoke, /* is_high= */ true); in VisitLongHighestOneBit()
3179 void IntrinsicCodeGeneratorX86_64::VisitLongHighestOneBit(HInvoke* invoke) { in VisitLongHighestOneBit() argument
3180 GenOneBit(GetAssembler(), codegen_, invoke, /* is_high= */ true, /* is_long= */ true); in VisitLongHighestOneBit()
3183 void IntrinsicLocationsBuilderX86_64::VisitIntegerLowestOneBit(HInvoke* invoke) { in VisitIntegerLowestOneBit() argument
3184 CreateOneBitLocations(allocator_, invoke, /* is_high= */ false); in VisitIntegerLowestOneBit()
3187 void IntrinsicCodeGeneratorX86_64::VisitIntegerLowestOneBit(HInvoke* invoke) { in VisitIntegerLowestOneBit() argument
3188 GenOneBit(GetAssembler(), codegen_, invoke, /* is_high= */ false, /* is_long= */ false); in VisitIntegerLowestOneBit()
3191 void IntrinsicLocationsBuilderX86_64::VisitLongLowestOneBit(HInvoke* invoke) { in VisitLongLowestOneBit() argument
3192 CreateOneBitLocations(allocator_, invoke, /* is_high= */ false); in VisitLongLowestOneBit()
3195 void IntrinsicCodeGeneratorX86_64::VisitLongLowestOneBit(HInvoke* invoke) { in VisitLongLowestOneBit() argument
3196 GenOneBit(GetAssembler(), codegen_, invoke, /* is_high= */ false, /* is_long= */ true); in VisitLongLowestOneBit()
3199 static void CreateLeadingZeroLocations(ArenaAllocator* allocator, HInvoke* invoke) { in CreateLeadingZeroLocations() argument
3201 new (allocator) LocationSummary(invoke, LocationSummary::kNoCall, kIntrinsified); in CreateLeadingZeroLocations()
3208 HInvoke* invoke, bool is_long) { in GenLeadingZeros() argument
3209 LocationSummary* locations = invoke->GetLocations(); in GenLeadingZeros()
3214 if (invoke->InputAt(0)->IsConstant()) { in GenLeadingZeros()
3216 int64_t value = Int64FromConstant(invoke->InputAt(0)->AsConstant()); in GenLeadingZeros()
3256 void IntrinsicLocationsBuilderX86_64::VisitIntegerNumberOfLeadingZeros(HInvoke* invoke) { in VisitIntegerNumberOfLeadingZeros() argument
3257 CreateLeadingZeroLocations(allocator_, invoke); in VisitIntegerNumberOfLeadingZeros()
3260 void IntrinsicCodeGeneratorX86_64::VisitIntegerNumberOfLeadingZeros(HInvoke* invoke) { in VisitIntegerNumberOfLeadingZeros() argument
3261 GenLeadingZeros(GetAssembler(), codegen_, invoke, /* is_long= */ false); in VisitIntegerNumberOfLeadingZeros()
3264 void IntrinsicLocationsBuilderX86_64::VisitLongNumberOfLeadingZeros(HInvoke* invoke) { in VisitLongNumberOfLeadingZeros() argument
3265 CreateLeadingZeroLocations(allocator_, invoke); in VisitLongNumberOfLeadingZeros()
3268 void IntrinsicCodeGeneratorX86_64::VisitLongNumberOfLeadingZeros(HInvoke* invoke) { in VisitLongNumberOfLeadingZeros() argument
3269 GenLeadingZeros(GetAssembler(), codegen_, invoke, /* is_long= */ true); in VisitLongNumberOfLeadingZeros()
3272 static void CreateTrailingZeroLocations(ArenaAllocator* allocator, HInvoke* invoke) { in CreateTrailingZeroLocations() argument
3274 new (allocator) LocationSummary(invoke, LocationSummary::kNoCall, kIntrinsified); in CreateTrailingZeroLocations()
3281 HInvoke* invoke, bool is_long) { in GenTrailingZeros() argument
3282 LocationSummary* locations = invoke->GetLocations(); in GenTrailingZeros()
3287 if (invoke->InputAt(0)->IsConstant()) { in GenTrailingZeros()
3289 int64_t value = Int64FromConstant(invoke->InputAt(0)->AsConstant()); in GenTrailingZeros()
3324 void IntrinsicLocationsBuilderX86_64::VisitIntegerNumberOfTrailingZeros(HInvoke* invoke) { in VisitIntegerNumberOfTrailingZeros() argument
3325 CreateTrailingZeroLocations(allocator_, invoke); in VisitIntegerNumberOfTrailingZeros()
3328 void IntrinsicCodeGeneratorX86_64::VisitIntegerNumberOfTrailingZeros(HInvoke* invoke) { in VisitIntegerNumberOfTrailingZeros() argument
3329 GenTrailingZeros(GetAssembler(), codegen_, invoke, /* is_long= */ false); in VisitIntegerNumberOfTrailingZeros()
3332 void IntrinsicLocationsBuilderX86_64::VisitLongNumberOfTrailingZeros(HInvoke* invoke) { in VisitLongNumberOfTrailingZeros() argument
3333 CreateTrailingZeroLocations(allocator_, invoke); in VisitLongNumberOfTrailingZeros()
3336 void IntrinsicCodeGeneratorX86_64::VisitLongNumberOfTrailingZeros(HInvoke* invoke) { in VisitLongNumberOfTrailingZeros() argument
3337 GenTrailingZeros(GetAssembler(), codegen_, invoke, /* is_long= */ true); in VisitLongNumberOfTrailingZeros()
3341 void IntrinsicLocationsBuilderX86_64::Visit##name##ValueOf(HInvoke* invoke) { \
3344 invoke, \
3351 void IntrinsicCodeGeneratorX86_64::Visit##name##ValueOf(HInvoke* invoke) { \
3353 IntrinsicVisitor::ComputeValueOfInfo(invoke, \
3359 HandleValueOf(invoke, info, type); \
3390 void IntrinsicCodeGeneratorX86_64::HandleValueOf(HInvoke* invoke, in HandleValueOf() argument
3393 LocationSummary* locations = invoke->GetLocations(); in HandleValueOf()
3400 codegen_->LoadIntrinsicDeclaringClass(argument, invoke); in HandleValueOf()
3401 codegen_->InvokeRuntime(kQuickAllocObjectInitialized, invoke, invoke->GetDexPc()); in HandleValueOf()
3404 if (invoke->InputAt(0)->IsIntConstant()) { in HandleValueOf()
3405 int32_t value = invoke->InputAt(0)->AsIntConstant()->GetValue(); in HandleValueOf()
3442 void IntrinsicLocationsBuilderX86_64::VisitReferenceGetReferent(HInvoke* invoke) { in VisitReferenceGetReferent() argument
3443 IntrinsicVisitor::CreateReferenceGetReferentLocations(invoke, codegen_); in VisitReferenceGetReferent()
3446 void IntrinsicCodeGeneratorX86_64::VisitReferenceGetReferent(HInvoke* invoke) { in VisitReferenceGetReferent() argument
3448 LocationSummary* locations = invoke->GetLocations(); in VisitReferenceGetReferent()
3453 SlowPathCode* slow_path = new (GetAllocator()) IntrinsicSlowPathX86_64(invoke); in VisitReferenceGetReferent()
3465 codegen_->LoadIntrinsicDeclaringClass(out.AsRegister<CpuRegister>(), invoke); in VisitReferenceGetReferent()
3479 codegen_->GenerateFieldLoadWithBakerReadBarrier(invoke, in VisitReferenceGetReferent()
3488 codegen_->MaybeRecordImplicitNullCheck(invoke); in VisitReferenceGetReferent()
3491 codegen_->MaybeGenerateReadBarrierSlow(invoke, out, out, obj, referent_offset); in VisitReferenceGetReferent()
3496 void IntrinsicLocationsBuilderX86_64::VisitReferenceRefersTo(HInvoke* invoke) { in VisitReferenceRefersTo() argument
3497 IntrinsicVisitor::CreateReferenceRefersToLocations(invoke, codegen_); in VisitReferenceRefersTo()
3500 void IntrinsicCodeGeneratorX86_64::VisitReferenceRefersTo(HInvoke* invoke) { in VisitReferenceRefersTo() argument
3502 LocationSummary* locations = invoke->GetLocations(); in VisitReferenceRefersTo()
3512 codegen_->MaybeRecordImplicitNullCheck(invoke); in VisitReferenceRefersTo()
3551 void IntrinsicLocationsBuilderX86_64::VisitThreadInterrupted(HInvoke* invoke) { in VisitThreadInterrupted() argument
3553 new (allocator_) LocationSummary(invoke, LocationSummary::kNoCall, kIntrinsified); in VisitThreadInterrupted()
3557 void IntrinsicCodeGeneratorX86_64::VisitThreadInterrupted(HInvoke* invoke) { in VisitThreadInterrupted() argument
3559 CpuRegister out = invoke->GetLocations()->Out().AsRegister<CpuRegister>(); in VisitThreadInterrupted()
3571 void IntrinsicLocationsBuilderX86_64::VisitReachabilityFence(HInvoke* invoke) { in VisitReachabilityFence() argument
3573 new (allocator_) LocationSummary(invoke, LocationSummary::kNoCall, kIntrinsified); in VisitReachabilityFence()
3577 void IntrinsicCodeGeneratorX86_64::VisitReachabilityFence([[maybe_unused]] HInvoke* invoke) {} in VisitReachabilityFence() argument
3579 static void CreateDivideUnsignedLocations(HInvoke* invoke, ArenaAllocator* allocator) { in CreateDivideUnsignedLocations() argument
3581 new (allocator) LocationSummary(invoke, LocationSummary::kCallOnSlowPath, kIntrinsified); in CreateDivideUnsignedLocations()
3589 static void GenerateDivideUnsigned(HInvoke* invoke, in GenerateDivideUnsigned() argument
3592 LocationSummary* locations = invoke->GetLocations(); in GenerateDivideUnsigned()
3604 auto* slow_path = new (codegen->GetScopedAllocator()) IntrinsicSlowPathX86_64(invoke); in GenerateDivideUnsigned()
3623 void IntrinsicLocationsBuilderX86_64::VisitIntegerDivideUnsigned(HInvoke* invoke) { in VisitIntegerDivideUnsigned() argument
3624 CreateDivideUnsignedLocations(invoke, allocator_); in VisitIntegerDivideUnsigned()
3627 void IntrinsicCodeGeneratorX86_64::VisitIntegerDivideUnsigned(HInvoke* invoke) { in VisitIntegerDivideUnsigned() argument
3628 GenerateDivideUnsigned(invoke, codegen_, DataType::Type::kInt32); in VisitIntegerDivideUnsigned()
3631 void IntrinsicLocationsBuilderX86_64::VisitLongDivideUnsigned(HInvoke* invoke) { in VisitLongDivideUnsigned() argument
3632 CreateDivideUnsignedLocations(invoke, allocator_); in VisitLongDivideUnsigned()
3635 void IntrinsicCodeGeneratorX86_64::VisitLongDivideUnsigned(HInvoke* invoke) { in VisitLongDivideUnsigned() argument
3636 GenerateDivideUnsigned(invoke, codegen_, DataType::Type::kInt64); in VisitLongDivideUnsigned()
3639 void IntrinsicLocationsBuilderX86_64::VisitMathMultiplyHigh(HInvoke* invoke) { in VisitMathMultiplyHigh() argument
3641 new (allocator_) LocationSummary(invoke, LocationSummary::kNoCall, kIntrinsified); in VisitMathMultiplyHigh()
3648 void IntrinsicCodeGeneratorX86_64::VisitMathMultiplyHigh(HInvoke* invoke) { in VisitMathMultiplyHigh() argument
3650 LocationSummary* locations = invoke->GetLocations(); in VisitMathMultiplyHigh()
3662 explicit VarHandleSlowPathX86_64(HInvoke* invoke) in VarHandleSlowPathX86_64() argument
3663 : IntrinsicSlowPathX86_64(invoke) { in VarHandleSlowPathX86_64()
3723 static void GenerateMathFma(HInvoke* invoke, CodeGeneratorX86_64* codegen) { in GenerateMathFma() argument
3724 DCHECK(DataType::IsFloatingPointType(invoke->GetType())); in GenerateMathFma()
3726 LocationSummary* locations = invoke->GetLocations(); in GenerateMathFma()
3731 if (invoke->GetType() == DataType::Type::kFloat32) { in GenerateMathFma()
3734 DCHECK_EQ(invoke->GetType(), DataType::Type::kFloat64); in GenerateMathFma()
3739 void IntrinsicCodeGeneratorX86_64::VisitMathFmaDouble(HInvoke* invoke) { in VisitMathFmaDouble() argument
3741 GenerateMathFma(invoke, codegen_); in VisitMathFmaDouble()
3744 void IntrinsicLocationsBuilderX86_64::VisitMathFmaDouble(HInvoke* invoke) { in VisitMathFmaDouble() argument
3746 CreateFPFPFPToFPCallLocations(allocator_, invoke); in VisitMathFmaDouble()
3750 void IntrinsicCodeGeneratorX86_64::VisitMathFmaFloat(HInvoke* invoke) { in VisitMathFmaFloat() argument
3752 GenerateMathFma(invoke, codegen_); in VisitMathFmaFloat()
3755 void IntrinsicLocationsBuilderX86_64::VisitMathFmaFloat(HInvoke* invoke) { in VisitMathFmaFloat() argument
3757 CreateFPFPFPToFPCallLocations(allocator_, invoke); in VisitMathFmaFloat()
3801 static void GenerateVarHandleAccessModeAndVarTypeChecks(HInvoke* invoke, in GenerateVarHandleAccessModeAndVarTypeChecks() argument
3807 LocationSummary* locations = invoke->GetLocations(); in GenerateVarHandleAccessModeAndVarTypeChecks()
3812 mirror::VarHandle::GetAccessModeByIntrinsic(invoke->GetIntrinsic()); in GenerateVarHandleAccessModeAndVarTypeChecks()
3838 size_t expected_coordinates_count = GetExpectedVarHandleCoordinatesCount(invoke); in GenerateVarHandleAccessModeAndVarTypeChecks()
3840 uint32_t number_of_arguments = invoke->GetNumberOfArguments(); in GenerateVarHandleAccessModeAndVarTypeChecks()
3842 HInstruction* arg = invoke->InputAt(arg_index); in GenerateVarHandleAccessModeAndVarTypeChecks()
3845 CpuRegister arg_reg = invoke->GetLocations()->InAt(arg_index).AsRegister<CpuRegister>(); in GenerateVarHandleAccessModeAndVarTypeChecks()
3853 static void GenerateVarHandleStaticFieldCheck(HInvoke* invoke, in GenerateVarHandleStaticFieldCheck() argument
3858 LocationSummary* locations = invoke->GetLocations(); in GenerateVarHandleStaticFieldCheck()
3869 static void GenerateVarHandleInstanceFieldChecks(HInvoke* invoke, in GenerateVarHandleInstanceFieldChecks() argument
3872 VarHandleOptimizations optimizations(invoke); in GenerateVarHandleInstanceFieldChecks()
3875 LocationSummary* locations = invoke->GetLocations(); in GenerateVarHandleInstanceFieldChecks()
3907 static void GenerateVarHandleArrayChecks(HInvoke* invoke, in GenerateVarHandleArrayChecks() argument
3910 VarHandleOptimizations optimizations(invoke); in GenerateVarHandleArrayChecks()
3912 LocationSummary* locations = invoke->GetLocations(); in GenerateVarHandleArrayChecks()
3918 GetVarHandleExpectedValueType(invoke, /*expected_coordinates_count=*/ 2u); in GenerateVarHandleArrayChecks()
3990 static void GenerateVarHandleCoordinateChecks(HInvoke* invoke, in GenerateVarHandleCoordinateChecks() argument
3993 size_t expected_coordinates_count = GetExpectedVarHandleCoordinatesCount(invoke); in GenerateVarHandleCoordinateChecks()
3995 GenerateVarHandleStaticFieldCheck(invoke, codegen, slow_path); in GenerateVarHandleCoordinateChecks()
3997 GenerateVarHandleInstanceFieldChecks(invoke, codegen, slow_path); in GenerateVarHandleCoordinateChecks()
4000 GenerateVarHandleArrayChecks(invoke, codegen, slow_path); in GenerateVarHandleCoordinateChecks()
4004 static VarHandleSlowPathX86_64* GenerateVarHandleChecks(HInvoke* invoke, in GenerateVarHandleChecks() argument
4007 size_t expected_coordinates_count = GetExpectedVarHandleCoordinatesCount(invoke); in GenerateVarHandleChecks()
4008 VarHandleOptimizations optimizations(invoke); in GenerateVarHandleChecks()
4017 new (codegen->GetScopedAllocator()) VarHandleSlowPathX86_64(invoke); in GenerateVarHandleChecks()
4021 GenerateVarHandleAccessModeAndVarTypeChecks(invoke, codegen, slow_path, type); in GenerateVarHandleChecks()
4023 GenerateVarHandleCoordinateChecks(invoke, codegen, slow_path); in GenerateVarHandleChecks()
4033 static VarHandleTarget GetVarHandleTarget(HInvoke* invoke) { in GetVarHandleTarget() argument
4034 size_t expected_coordinates_count = GetExpectedVarHandleCoordinatesCount(invoke); in GetVarHandleTarget()
4035 LocationSummary* locations = invoke->GetLocations(); in GetVarHandleTarget()
4047 static void GenerateVarHandleTarget(HInvoke* invoke, in GenerateVarHandleTarget() argument
4050 LocationSummary* locations = invoke->GetLocations(); in GenerateVarHandleTarget()
4052 size_t expected_coordinates_count = GetExpectedVarHandleCoordinatesCount(invoke); in GenerateVarHandleTarget()
4057 if (VarHandleOptimizations(invoke).GetUseKnownImageVarHandle()) { in GenerateVarHandleTarget()
4059 ArtField* target_field = GetBootImageVarHandleField(invoke); in GenerateVarHandleTarget()
4087 instr_codegen->GenerateGcRootFieldLoad(invoke, in GenerateVarHandleTarget()
4098 GetVarHandleExpectedValueType(invoke, /*expected_coordinates_count=*/ 2u); in GenerateVarHandleTarget()
4108 static bool HasVarHandleIntrinsicImplementation(HInvoke* invoke, CodeGeneratorX86_64* codegen) { in HasVarHandleIntrinsicImplementation() argument
4114 VarHandleOptimizations optimizations(invoke); in HasVarHandleIntrinsicImplementation()
4119 size_t expected_coordinates_count = GetExpectedVarHandleCoordinatesCount(invoke); in HasVarHandleIntrinsicImplementation()
4124 static LocationSummary* CreateVarHandleCommonLocations(HInvoke* invoke) { in CreateVarHandleCommonLocations() argument
4125 size_t expected_coordinates_count = GetExpectedVarHandleCoordinatesCount(invoke); in CreateVarHandleCommonLocations()
4126 ArenaAllocator* allocator = invoke->GetBlock()->GetGraph()->GetAllocator(); in CreateVarHandleCommonLocations()
4128 invoke, LocationSummary::kCallOnSlowPath, kIntrinsified); in CreateVarHandleCommonLocations()
4138 uint32_t number_of_arguments = invoke->GetNumberOfArguments(); in CreateVarHandleCommonLocations()
4140 HInstruction* arg = invoke->InputAt(arg_index); in CreateVarHandleCommonLocations()
4159 static void CreateVarHandleGetLocations(HInvoke* invoke, CodeGeneratorX86_64* codegen) { in CreateVarHandleGetLocations() argument
4160 if (!HasVarHandleIntrinsicImplementation(invoke, codegen)) { in CreateVarHandleGetLocations()
4164 LocationSummary* locations = CreateVarHandleCommonLocations(invoke); in CreateVarHandleGetLocations()
4165 if (DataType::IsFloatingPointType(invoke->GetType())) { in CreateVarHandleGetLocations()
4172 static void GenerateVarHandleGet(HInvoke* invoke, in GenerateVarHandleGet() argument
4175 DataType::Type type = invoke->GetType(); in GenerateVarHandleGet()
4178 LocationSummary* locations = invoke->GetLocations(); in GenerateVarHandleGet()
4181 VarHandleTarget target = GetVarHandleTarget(invoke); in GenerateVarHandleGet()
4184 slow_path = GenerateVarHandleChecks(invoke, codegen, type); in GenerateVarHandleGet()
4185 GenerateVarHandleTarget(invoke, target, codegen); in GenerateVarHandleGet()
4199 invoke, out, CpuRegister(target.object), src, /* needs_null_check= */ false); in GenerateVarHandleGet()
4219 void IntrinsicLocationsBuilderX86_64::VisitMethodHandleInvokeExact(HInvoke* invoke) { in VisitMethodHandleInvokeExact() argument
4220 ArenaAllocator* allocator = invoke->GetBlock()->GetGraph()->GetAllocator(); in VisitMethodHandleInvokeExact()
4222 LocationSummary(invoke, LocationSummary::kCallOnMainAndSlowPath, kIntrinsified); in VisitMethodHandleInvokeExact()
4225 locations->SetOut(calling_convention.GetReturnLocation(invoke->GetType())); in VisitMethodHandleInvokeExact()
4229 // Accomodating LocationSummary for underlying invoke-* call. in VisitMethodHandleInvokeExact()
4230 uint32_t number_of_args = invoke->GetNumberOfArguments(); in VisitMethodHandleInvokeExact()
4232 locations->SetInAt(i, calling_convention.GetNextLocation(invoke->InputAt(i)->GetType())); in VisitMethodHandleInvokeExact()
4239 // Hidden arg for invoke-interface. in VisitMethodHandleInvokeExact()
4243 void IntrinsicCodeGeneratorX86_64::VisitMethodHandleInvokeExact(HInvoke* invoke) { in VisitMethodHandleInvokeExact() argument
4244 LocationSummary* locations = invoke->GetLocations(); in VisitMethodHandleInvokeExact()
4249 new (codegen_->GetScopedAllocator()) InvokePolymorphicSlowPathX86_64(invoke, method_handle); in VisitMethodHandleInvokeExact()
4254 locations->InAt(invoke->GetNumberOfArguments()).AsRegister<CpuRegister>(); in VisitMethodHandleInvokeExact()
4268 if (invoke->AsInvokePolymorphic()->CanTargetInstanceMethod()) { in VisitMethodHandleInvokeExact()
4276 // No dispatch is needed for invoke-direct. in VisitMethodHandleInvokeExact()
4280 // Handle invoke-virtual case. in VisitMethodHandleInvokeExact()
4354 // MH's kind is invoke-static. The method can be called directly, hence fall-through. in VisitMethodHandleInvokeExact()
4360 codegen_->RecordPcInfo(invoke, invoke->GetDexPc(), slow_path); in VisitMethodHandleInvokeExact()
4364 void IntrinsicLocationsBuilderX86_64::VisitVarHandleGet(HInvoke* invoke) { in VisitVarHandleGet() argument
4365 CreateVarHandleGetLocations(invoke, codegen_); in VisitVarHandleGet()
4368 void IntrinsicCodeGeneratorX86_64::VisitVarHandleGet(HInvoke* invoke) { in VisitVarHandleGet() argument
4369 GenerateVarHandleGet(invoke, codegen_); in VisitVarHandleGet()
4372 void IntrinsicLocationsBuilderX86_64::VisitVarHandleGetAcquire(HInvoke* invoke) { in VisitVarHandleGetAcquire() argument
4373 CreateVarHandleGetLocations(invoke, codegen_); in VisitVarHandleGetAcquire()
4376 void IntrinsicCodeGeneratorX86_64::VisitVarHandleGetAcquire(HInvoke* invoke) { in VisitVarHandleGetAcquire() argument
4378 GenerateVarHandleGet(invoke, codegen_); in VisitVarHandleGetAcquire()
4381 void IntrinsicLocationsBuilderX86_64::VisitVarHandleGetOpaque(HInvoke* invoke) { in VisitVarHandleGetOpaque() argument
4382 CreateVarHandleGetLocations(invoke, codegen_); in VisitVarHandleGetOpaque()
4385 void IntrinsicCodeGeneratorX86_64::VisitVarHandleGetOpaque(HInvoke* invoke) { in VisitVarHandleGetOpaque() argument
4387 GenerateVarHandleGet(invoke, codegen_); in VisitVarHandleGetOpaque()
4390 void IntrinsicLocationsBuilderX86_64::VisitVarHandleGetVolatile(HInvoke* invoke) { in VisitVarHandleGetVolatile() argument
4391 CreateVarHandleGetLocations(invoke, codegen_); in VisitVarHandleGetVolatile()
4394 void IntrinsicCodeGeneratorX86_64::VisitVarHandleGetVolatile(HInvoke* invoke) { in VisitVarHandleGetVolatile() argument
4396 GenerateVarHandleGet(invoke, codegen_); in VisitVarHandleGetVolatile()
4399 static void CreateVarHandleSetLocations(HInvoke* invoke, CodeGeneratorX86_64* codegen) { in CreateVarHandleSetLocations() argument
4400 if (!HasVarHandleIntrinsicImplementation(invoke, codegen)) { in CreateVarHandleSetLocations()
4404 LocationSummary* locations = CreateVarHandleCommonLocations(invoke); in CreateVarHandleSetLocations()
4410 static void GenerateVarHandleSet(HInvoke* invoke, in GenerateVarHandleSet() argument
4417 LocationSummary* locations = invoke->GetLocations(); in GenerateVarHandleSet()
4420 uint32_t value_index = invoke->GetNumberOfArguments() - 1; in GenerateVarHandleSet()
4421 DataType::Type value_type = GetDataTypeFromShorty(invoke, value_index); in GenerateVarHandleSet()
4423 VarHandleTarget target = GetVarHandleTarget(invoke); in GenerateVarHandleSet()
4426 slow_path = GenerateVarHandleChecks(invoke, codegen, value_type); in GenerateVarHandleSet()
4427 GenerateVarHandleTarget(invoke, target, codegen); in GenerateVarHandleSet()
4435 switch (invoke->GetIntrinsic()) { in GenerateVarHandleSet()
4451 invoke, in GenerateVarHandleSet()
4473 void IntrinsicLocationsBuilderX86_64::VisitVarHandleSet(HInvoke* invoke) { in VisitVarHandleSet() argument
4474 CreateVarHandleSetLocations(invoke, codegen_); in VisitVarHandleSet()
4477 void IntrinsicCodeGeneratorX86_64::VisitVarHandleSet(HInvoke* invoke) { in VisitVarHandleSet() argument
4478 GenerateVarHandleSet(invoke, codegen_, /*is_volatile=*/ false, /*is_atomic=*/ true); in VisitVarHandleSet()
4481 void IntrinsicLocationsBuilderX86_64::VisitVarHandleSetOpaque(HInvoke* invoke) { in VisitVarHandleSetOpaque() argument
4482 CreateVarHandleSetLocations(invoke, codegen_); in VisitVarHandleSetOpaque()
4485 void IntrinsicCodeGeneratorX86_64::VisitVarHandleSetOpaque(HInvoke* invoke) { in VisitVarHandleSetOpaque() argument
4486 GenerateVarHandleSet(invoke, codegen_, /*is_volatile=*/ false, /*is_atomic=*/ true); in VisitVarHandleSetOpaque()
4489 void IntrinsicLocationsBuilderX86_64::VisitVarHandleSetRelease(HInvoke* invoke) { in VisitVarHandleSetRelease() argument
4490 CreateVarHandleSetLocations(invoke, codegen_); in VisitVarHandleSetRelease()
4493 void IntrinsicCodeGeneratorX86_64::VisitVarHandleSetRelease(HInvoke* invoke) { in VisitVarHandleSetRelease() argument
4494 GenerateVarHandleSet(invoke, codegen_, /*is_volatile=*/ false, /*is_atomic=*/ true); in VisitVarHandleSetRelease()
4497 void IntrinsicLocationsBuilderX86_64::VisitVarHandleSetVolatile(HInvoke* invoke) { in VisitVarHandleSetVolatile() argument
4498 CreateVarHandleSetLocations(invoke, codegen_); in VisitVarHandleSetVolatile()
4501 void IntrinsicCodeGeneratorX86_64::VisitVarHandleSetVolatile(HInvoke* invoke) { in VisitVarHandleSetVolatile() argument
4502 GenerateVarHandleSet(invoke, codegen_, /*is_volatile=*/ true, /*is_atomic=*/ true); in VisitVarHandleSetVolatile()
4505 static void CreateVarHandleCompareAndSetOrExchangeLocations(HInvoke* invoke, in CreateVarHandleCompareAndSetOrExchangeLocations() argument
4507 if (!HasVarHandleIntrinsicImplementation(invoke, codegen)) { in CreateVarHandleCompareAndSetOrExchangeLocations()
4511 uint32_t number_of_arguments = invoke->GetNumberOfArguments(); in CreateVarHandleCompareAndSetOrExchangeLocations()
4514 DataType::Type return_type = invoke->GetType(); in CreateVarHandleCompareAndSetOrExchangeLocations()
4515 DataType::Type expected_type = GetDataTypeFromShorty(invoke, expected_value_index); in CreateVarHandleCompareAndSetOrExchangeLocations()
4516 DCHECK_EQ(expected_type, GetDataTypeFromShorty(invoke, new_value_index)); in CreateVarHandleCompareAndSetOrExchangeLocations()
4518 LocationSummary* locations = CreateVarHandleCommonLocations(invoke); in CreateVarHandleCompareAndSetOrExchangeLocations()
4550 static void GenerateVarHandleCompareAndSetOrExchange(HInvoke* invoke, in GenerateVarHandleCompareAndSetOrExchange() argument
4557 LocationSummary* locations = invoke->GetLocations(); in GenerateVarHandleCompareAndSetOrExchange()
4559 uint32_t number_of_arguments = invoke->GetNumberOfArguments(); in GenerateVarHandleCompareAndSetOrExchange()
4562 DataType::Type type = GetDataTypeFromShorty(invoke, expected_value_index); in GenerateVarHandleCompareAndSetOrExchange()
4565 VarHandleTarget target = GetVarHandleTarget(invoke); in GenerateVarHandleCompareAndSetOrExchange()
4567 slow_path = GenerateVarHandleChecks(invoke, codegen, type); in GenerateVarHandleCompareAndSetOrExchange()
4568 GenerateVarHandleTarget(invoke, target, codegen); in GenerateVarHandleCompareAndSetOrExchange()
4576 invoke, in GenerateVarHandleCompareAndSetOrExchange()
4598 void IntrinsicLocationsBuilderX86_64::VisitVarHandleCompareAndSet(HInvoke* invoke) { in VisitVarHandleCompareAndSet() argument
4599 CreateVarHandleCompareAndSetOrExchangeLocations(invoke, codegen_); in VisitVarHandleCompareAndSet()
4602 void IntrinsicCodeGeneratorX86_64::VisitVarHandleCompareAndSet(HInvoke* invoke) { in VisitVarHandleCompareAndSet() argument
4603 GenerateVarHandleCompareAndSetOrExchange(invoke, codegen_, /*is_cmpxchg=*/ false); in VisitVarHandleCompareAndSet()
4606 void IntrinsicLocationsBuilderX86_64::VisitVarHandleWeakCompareAndSet(HInvoke* invoke) { in VisitVarHandleWeakCompareAndSet() argument
4607 CreateVarHandleCompareAndSetOrExchangeLocations(invoke, codegen_); in VisitVarHandleWeakCompareAndSet()
4610 void IntrinsicCodeGeneratorX86_64::VisitVarHandleWeakCompareAndSet(HInvoke* invoke) { in VisitVarHandleWeakCompareAndSet() argument
4611 GenerateVarHandleCompareAndSetOrExchange(invoke, codegen_, /*is_cmpxchg=*/ false); in VisitVarHandleWeakCompareAndSet()
4614 void IntrinsicLocationsBuilderX86_64::VisitVarHandleWeakCompareAndSetPlain(HInvoke* invoke) { in VisitVarHandleWeakCompareAndSetPlain() argument
4615 CreateVarHandleCompareAndSetOrExchangeLocations(invoke, codegen_); in VisitVarHandleWeakCompareAndSetPlain()
4618 void IntrinsicCodeGeneratorX86_64::VisitVarHandleWeakCompareAndSetPlain(HInvoke* invoke) { in VisitVarHandleWeakCompareAndSetPlain() argument
4619 GenerateVarHandleCompareAndSetOrExchange(invoke, codegen_, /*is_cmpxchg=*/ false); in VisitVarHandleWeakCompareAndSetPlain()
4622 void IntrinsicLocationsBuilderX86_64::VisitVarHandleWeakCompareAndSetAcquire(HInvoke* invoke) { in VisitVarHandleWeakCompareAndSetAcquire() argument
4623 CreateVarHandleCompareAndSetOrExchangeLocations(invoke, codegen_); in VisitVarHandleWeakCompareAndSetAcquire()
4626 void IntrinsicCodeGeneratorX86_64::VisitVarHandleWeakCompareAndSetAcquire(HInvoke* invoke) { in VisitVarHandleWeakCompareAndSetAcquire() argument
4627 GenerateVarHandleCompareAndSetOrExchange(invoke, codegen_, /*is_cmpxchg=*/ false); in VisitVarHandleWeakCompareAndSetAcquire()
4630 void IntrinsicLocationsBuilderX86_64::VisitVarHandleWeakCompareAndSetRelease(HInvoke* invoke) { in VisitVarHandleWeakCompareAndSetRelease() argument
4631 CreateVarHandleCompareAndSetOrExchangeLocations(invoke, codegen_); in VisitVarHandleWeakCompareAndSetRelease()
4634 void IntrinsicCodeGeneratorX86_64::VisitVarHandleWeakCompareAndSetRelease(HInvoke* invoke) { in VisitVarHandleWeakCompareAndSetRelease() argument
4635 GenerateVarHandleCompareAndSetOrExchange(invoke, codegen_, /*is_cmpxchg=*/ false); in VisitVarHandleWeakCompareAndSetRelease()
4638 void IntrinsicLocationsBuilderX86_64::VisitVarHandleCompareAndExchange(HInvoke* invoke) { in VisitVarHandleCompareAndExchange() argument
4639 CreateVarHandleCompareAndSetOrExchangeLocations(invoke, codegen_); in VisitVarHandleCompareAndExchange()
4642 void IntrinsicCodeGeneratorX86_64::VisitVarHandleCompareAndExchange(HInvoke* invoke) { in VisitVarHandleCompareAndExchange() argument
4643 GenerateVarHandleCompareAndSetOrExchange(invoke, codegen_, /*is_cmpxchg=*/ true); in VisitVarHandleCompareAndExchange()
4646 void IntrinsicLocationsBuilderX86_64::VisitVarHandleCompareAndExchangeAcquire(HInvoke* invoke) { in VisitVarHandleCompareAndExchangeAcquire() argument
4647 CreateVarHandleCompareAndSetOrExchangeLocations(invoke, codegen_); in VisitVarHandleCompareAndExchangeAcquire()
4650 void IntrinsicCodeGeneratorX86_64::VisitVarHandleCompareAndExchangeAcquire(HInvoke* invoke) { in VisitVarHandleCompareAndExchangeAcquire() argument
4651 GenerateVarHandleCompareAndSetOrExchange(invoke, codegen_, /*is_cmpxchg=*/ true); in VisitVarHandleCompareAndExchangeAcquire()
4654 void IntrinsicLocationsBuilderX86_64::VisitVarHandleCompareAndExchangeRelease(HInvoke* invoke) { in VisitVarHandleCompareAndExchangeRelease() argument
4655 CreateVarHandleCompareAndSetOrExchangeLocations(invoke, codegen_); in VisitVarHandleCompareAndExchangeRelease()
4658 void IntrinsicCodeGeneratorX86_64::VisitVarHandleCompareAndExchangeRelease(HInvoke* invoke) { in VisitVarHandleCompareAndExchangeRelease() argument
4659 GenerateVarHandleCompareAndSetOrExchange(invoke, codegen_, /*is_cmpxchg=*/ true); in VisitVarHandleCompareAndExchangeRelease()
4662 static void CreateVarHandleGetAndSetLocations(HInvoke* invoke, CodeGeneratorX86_64* codegen) { in CreateVarHandleGetAndSetLocations() argument
4663 if (!HasVarHandleIntrinsicImplementation(invoke, codegen)) { in CreateVarHandleGetAndSetLocations()
4668 uint32_t number_of_arguments = invoke->GetNumberOfArguments(); in CreateVarHandleGetAndSetLocations()
4670 DataType::Type value_type = GetDataTypeFromShorty(invoke, new_value_index); in CreateVarHandleGetAndSetLocations()
4671 DataType::Type return_type = invoke->GetType(); in CreateVarHandleGetAndSetLocations()
4675 LocationSummary* locations = CreateVarHandleCommonLocations(invoke); in CreateVarHandleGetAndSetLocations()
4707 static void GenerateVarHandleGetAndSet(HInvoke* invoke, in GenerateVarHandleGetAndSet() argument
4715 LocationSummary* locations = invoke->GetLocations(); in GenerateVarHandleGetAndSet()
4718 DataType::Type return_type = invoke->GetType(); in GenerateVarHandleGetAndSet()
4760 invoke, in GenerateVarHandleGetAndSet()
4837 static void CreateVarHandleGetAndBitwiseOpLocations(HInvoke* invoke, CodeGeneratorX86_64* codegen) { in CreateVarHandleGetAndBitwiseOpLocations() argument
4838 if (!HasVarHandleIntrinsicImplementation(invoke, codegen)) { in CreateVarHandleGetAndBitwiseOpLocations()
4843 uint32_t number_of_arguments = invoke->GetNumberOfArguments(); in CreateVarHandleGetAndBitwiseOpLocations()
4845 DataType::Type value_type = GetDataTypeFromShorty(invoke, new_value_index); in CreateVarHandleGetAndBitwiseOpLocations()
4846 DataType::Type return_type = invoke->GetType(); in CreateVarHandleGetAndBitwiseOpLocations()
4850 LocationSummary* locations = CreateVarHandleCommonLocations(invoke); in CreateVarHandleGetAndBitwiseOpLocations()
4861 Location::RegisterOrConstant(invoke->InputAt(new_value_index))); in CreateVarHandleGetAndBitwiseOpLocations()
4873 static void GenerateVarHandleGetAndOp(HInvoke* invoke, in GenerateVarHandleGetAndOp() argument
4881 LocationSummary* locations = invoke->GetLocations(); in GenerateVarHandleGetAndOp()
4884 const bool is_void = invoke->GetType() == DataType::Type::kVoid; in GenerateVarHandleGetAndOp()
4992 static void CreateVarHandleGetAndAddLocations(HInvoke* invoke, CodeGeneratorX86_64* codegen) { in CreateVarHandleGetAndAddLocations() argument
4993 if (!HasVarHandleIntrinsicImplementation(invoke, codegen)) { in CreateVarHandleGetAndAddLocations()
4998 uint32_t number_of_arguments = invoke->GetNumberOfArguments(); in CreateVarHandleGetAndAddLocations()
5000 DataType::Type value_type = GetDataTypeFromShorty(invoke, new_value_index); in CreateVarHandleGetAndAddLocations()
5001 DataType::Type return_type = invoke->GetType(); in CreateVarHandleGetAndAddLocations()
5005 LocationSummary* locations = CreateVarHandleCommonLocations(invoke); in CreateVarHandleGetAndAddLocations()
5023 if (GetExpectedVarHandleCoordinatesCount(invoke) == 2) { in CreateVarHandleGetAndAddLocations()
5044 static void GenerateVarHandleGetAndAdd(HInvoke* invoke, in GenerateVarHandleGetAndAdd() argument
5051 LocationSummary* locations = invoke->GetLocations(); in GenerateVarHandleGetAndAdd()
5055 DataType::Type return_type = invoke->GetType(); in GenerateVarHandleGetAndAdd()
5064 // java.lang.invoke.MethodHandles.byteArrayViewVarHandle()). But ART varhandle implementation in GenerateVarHandleGetAndAdd()
5130 DCHECK_EQ(GetExpectedVarHandleCoordinatesCount(invoke), 2u); in GenerateVarHandleGetAndAdd()
5139 invoke, codegen, temp, type, field_addr, GetAndUpdateOp::kAdd, byte_swap); in GenerateVarHandleGetAndAdd()
5188 static void GenerateVarHandleGetAndUpdate(HInvoke* invoke, in GenerateVarHandleGetAndUpdate() argument
5197 LocationSummary* locations = invoke->GetLocations(); in GenerateVarHandleGetAndUpdate()
5200 uint32_t number_of_arguments = invoke->GetNumberOfArguments(); in GenerateVarHandleGetAndUpdate()
5202 DataType::Type type = GetDataTypeFromShorty(invoke, number_of_arguments - 1); in GenerateVarHandleGetAndUpdate()
5205 VarHandleTarget target = GetVarHandleTarget(invoke); in GenerateVarHandleGetAndUpdate()
5207 slow_path = GenerateVarHandleChecks(invoke, codegen, type); in GenerateVarHandleGetAndUpdate()
5208 GenerateVarHandleTarget(invoke, target, codegen); in GenerateVarHandleGetAndUpdate()
5226 GenerateVarHandleGetAndSet(invoke, codegen, value, type, field_addr, ref, byte_swap); in GenerateVarHandleGetAndUpdate()
5229 GenerateVarHandleGetAndAdd(invoke, codegen, value, type, field_addr, byte_swap); in GenerateVarHandleGetAndUpdate()
5235 invoke, codegen, value, type, field_addr, get_and_update_op, byte_swap); in GenerateVarHandleGetAndUpdate()
5249 void IntrinsicLocationsBuilderX86_64::VisitVarHandleGetAndSet(HInvoke* invoke) { in VisitVarHandleGetAndSet() argument
5250 CreateVarHandleGetAndSetLocations(invoke, codegen_); in VisitVarHandleGetAndSet()
5253 void IntrinsicCodeGeneratorX86_64::VisitVarHandleGetAndSet(HInvoke* invoke) { in VisitVarHandleGetAndSet() argument
5255 GenerateVarHandleGetAndUpdate(invoke, in VisitVarHandleGetAndSet()
5262 void IntrinsicLocationsBuilderX86_64::VisitVarHandleGetAndSetAcquire(HInvoke* invoke) { in VisitVarHandleGetAndSetAcquire() argument
5263 CreateVarHandleGetAndSetLocations(invoke, codegen_); in VisitVarHandleGetAndSetAcquire()
5266 void IntrinsicCodeGeneratorX86_64::VisitVarHandleGetAndSetAcquire(HInvoke* invoke) { in VisitVarHandleGetAndSetAcquire() argument
5268 GenerateVarHandleGetAndUpdate(invoke, in VisitVarHandleGetAndSetAcquire()
5275 void IntrinsicLocationsBuilderX86_64::VisitVarHandleGetAndSetRelease(HInvoke* invoke) { in VisitVarHandleGetAndSetRelease() argument
5276 CreateVarHandleGetAndSetLocations(invoke, codegen_); in VisitVarHandleGetAndSetRelease()
5279 void IntrinsicCodeGeneratorX86_64::VisitVarHandleGetAndSetRelease(HInvoke* invoke) { in VisitVarHandleGetAndSetRelease() argument
5281 GenerateVarHandleGetAndUpdate(invoke, in VisitVarHandleGetAndSetRelease()
5288 void IntrinsicLocationsBuilderX86_64::VisitVarHandleGetAndAdd(HInvoke* invoke) { in VisitVarHandleGetAndAdd() argument
5289 CreateVarHandleGetAndAddLocations(invoke, codegen_); in VisitVarHandleGetAndAdd()
5292 void IntrinsicCodeGeneratorX86_64::VisitVarHandleGetAndAdd(HInvoke* invoke) { in VisitVarHandleGetAndAdd() argument
5294 GenerateVarHandleGetAndUpdate(invoke, in VisitVarHandleGetAndAdd()
5301 void IntrinsicLocationsBuilderX86_64::VisitVarHandleGetAndAddAcquire(HInvoke* invoke) { in VisitVarHandleGetAndAddAcquire() argument
5302 CreateVarHandleGetAndAddLocations(invoke, codegen_); in VisitVarHandleGetAndAddAcquire()
5305 void IntrinsicCodeGeneratorX86_64::VisitVarHandleGetAndAddAcquire(HInvoke* invoke) { in VisitVarHandleGetAndAddAcquire() argument
5307 GenerateVarHandleGetAndUpdate(invoke, in VisitVarHandleGetAndAddAcquire()
5314 void IntrinsicLocationsBuilderX86_64::VisitVarHandleGetAndAddRelease(HInvoke* invoke) { in VisitVarHandleGetAndAddRelease() argument
5315 CreateVarHandleGetAndAddLocations(invoke, codegen_); in VisitVarHandleGetAndAddRelease()
5318 void IntrinsicCodeGeneratorX86_64::VisitVarHandleGetAndAddRelease(HInvoke* invoke) { in VisitVarHandleGetAndAddRelease() argument
5320 GenerateVarHandleGetAndUpdate(invoke, in VisitVarHandleGetAndAddRelease()
5327 void IntrinsicLocationsBuilderX86_64::VisitVarHandleGetAndBitwiseAnd(HInvoke* invoke) { in VisitVarHandleGetAndBitwiseAnd() argument
5328 CreateVarHandleGetAndBitwiseOpLocations(invoke, codegen_); in VisitVarHandleGetAndBitwiseAnd()
5331 void IntrinsicCodeGeneratorX86_64::VisitVarHandleGetAndBitwiseAnd(HInvoke* invoke) { in VisitVarHandleGetAndBitwiseAnd() argument
5333 GenerateVarHandleGetAndUpdate(invoke, in VisitVarHandleGetAndBitwiseAnd()
5340 void IntrinsicLocationsBuilderX86_64::VisitVarHandleGetAndBitwiseAndAcquire(HInvoke* invoke) { in VisitVarHandleGetAndBitwiseAndAcquire() argument
5341 CreateVarHandleGetAndBitwiseOpLocations(invoke, codegen_); in VisitVarHandleGetAndBitwiseAndAcquire()
5344 void IntrinsicCodeGeneratorX86_64::VisitVarHandleGetAndBitwiseAndAcquire(HInvoke* invoke) { in VisitVarHandleGetAndBitwiseAndAcquire() argument
5346 GenerateVarHandleGetAndUpdate(invoke, in VisitVarHandleGetAndBitwiseAndAcquire()
5353 void IntrinsicLocationsBuilderX86_64::VisitVarHandleGetAndBitwiseAndRelease(HInvoke* invoke) { in VisitVarHandleGetAndBitwiseAndRelease() argument
5354 CreateVarHandleGetAndBitwiseOpLocations(invoke, codegen_); in VisitVarHandleGetAndBitwiseAndRelease()
5357 void IntrinsicCodeGeneratorX86_64::VisitVarHandleGetAndBitwiseAndRelease(HInvoke* invoke) { in VisitVarHandleGetAndBitwiseAndRelease() argument
5359 GenerateVarHandleGetAndUpdate(invoke, in VisitVarHandleGetAndBitwiseAndRelease()
5366 void IntrinsicLocationsBuilderX86_64::VisitVarHandleGetAndBitwiseOr(HInvoke* invoke) { in VisitVarHandleGetAndBitwiseOr() argument
5367 CreateVarHandleGetAndBitwiseOpLocations(invoke, codegen_); in VisitVarHandleGetAndBitwiseOr()
5370 void IntrinsicCodeGeneratorX86_64::VisitVarHandleGetAndBitwiseOr(HInvoke* invoke) { in VisitVarHandleGetAndBitwiseOr() argument
5372 GenerateVarHandleGetAndUpdate(invoke, in VisitVarHandleGetAndBitwiseOr()
5379 void IntrinsicLocationsBuilderX86_64::VisitVarHandleGetAndBitwiseOrAcquire(HInvoke* invoke) { in VisitVarHandleGetAndBitwiseOrAcquire() argument
5380 CreateVarHandleGetAndBitwiseOpLocations(invoke, codegen_); in VisitVarHandleGetAndBitwiseOrAcquire()
5383 void IntrinsicCodeGeneratorX86_64::VisitVarHandleGetAndBitwiseOrAcquire(HInvoke* invoke) { in VisitVarHandleGetAndBitwiseOrAcquire() argument
5385 GenerateVarHandleGetAndUpdate(invoke, in VisitVarHandleGetAndBitwiseOrAcquire()
5392 void IntrinsicLocationsBuilderX86_64::VisitVarHandleGetAndBitwiseOrRelease(HInvoke* invoke) { in VisitVarHandleGetAndBitwiseOrRelease() argument
5393 CreateVarHandleGetAndBitwiseOpLocations(invoke, codegen_); in VisitVarHandleGetAndBitwiseOrRelease()
5396 void IntrinsicCodeGeneratorX86_64::VisitVarHandleGetAndBitwiseOrRelease(HInvoke* invoke) { in VisitVarHandleGetAndBitwiseOrRelease() argument
5398 GenerateVarHandleGetAndUpdate(invoke, in VisitVarHandleGetAndBitwiseOrRelease()
5405 void IntrinsicLocationsBuilderX86_64::VisitVarHandleGetAndBitwiseXor(HInvoke* invoke) { in VisitVarHandleGetAndBitwiseXor() argument
5406 CreateVarHandleGetAndBitwiseOpLocations(invoke, codegen_); in VisitVarHandleGetAndBitwiseXor()
5409 void IntrinsicCodeGeneratorX86_64::VisitVarHandleGetAndBitwiseXor(HInvoke* invoke) { in VisitVarHandleGetAndBitwiseXor() argument
5411 GenerateVarHandleGetAndUpdate(invoke, in VisitVarHandleGetAndBitwiseXor()
5418 void IntrinsicLocationsBuilderX86_64::VisitVarHandleGetAndBitwiseXorAcquire(HInvoke* invoke) { in VisitVarHandleGetAndBitwiseXorAcquire() argument
5419 CreateVarHandleGetAndBitwiseOpLocations(invoke, codegen_); in VisitVarHandleGetAndBitwiseXorAcquire()
5422 void IntrinsicCodeGeneratorX86_64::VisitVarHandleGetAndBitwiseXorAcquire(HInvoke* invoke) { in VisitVarHandleGetAndBitwiseXorAcquire() argument
5424 GenerateVarHandleGetAndUpdate(invoke, in VisitVarHandleGetAndBitwiseXorAcquire()
5431 void IntrinsicLocationsBuilderX86_64::VisitVarHandleGetAndBitwiseXorRelease(HInvoke* invoke) { in VisitVarHandleGetAndBitwiseXorRelease() argument
5432 CreateVarHandleGetAndBitwiseOpLocations(invoke, codegen_); in VisitVarHandleGetAndBitwiseXorRelease()
5435 void IntrinsicCodeGeneratorX86_64::VisitVarHandleGetAndBitwiseXorRelease(HInvoke* invoke) { in VisitVarHandleGetAndBitwiseXorRelease() argument
5437 GenerateVarHandleGetAndUpdate(invoke, in VisitVarHandleGetAndBitwiseXorRelease()
5448 HInvoke* invoke = GetInvoke(); in EmitByteArrayViewCode() local
5449 LocationSummary* locations = invoke->GetLocations(); in EmitByteArrayViewCode()
5452 GetVarHandleExpectedValueType(invoke, /*expected_coordinates_count=*/ 2u); in EmitByteArrayViewCode()
5467 VarHandleTarget target = GetVarHandleTarget(invoke); in EmitByteArrayViewCode()
5498 IsZeroBitPattern(invoke->InputAt(invoke->GetNumberOfArguments() - 1u))) { in EmitByteArrayViewCode()
5509 GenerateVarHandleGet(invoke, codegen, /*byte_swap=*/ true); in EmitByteArrayViewCode()
5512 GenerateVarHandleSet(invoke, codegen, is_volatile_, is_atomic_, /*byte_swap=*/ true); in EmitByteArrayViewCode()
5516 invoke, codegen, /*is_cmpxchg=*/ false, /*byte_swap=*/ true); in EmitByteArrayViewCode()
5520 invoke, codegen, /*is_cmpxchg=*/ true, /*byte_swap=*/ true); in EmitByteArrayViewCode()
5523 GenerateVarHandleGetAndUpdate(invoke, in EmitByteArrayViewCode()