Lines Matching full:invoke
200 bool IntrinsicLocationsBuilderARM64::TryDispatch(HInvoke* invoke) { in TryDispatch() argument
201 Dispatch(invoke); in TryDispatch()
202 LocationSummary* res = invoke->GetLocations(); in TryDispatch()
211 static void CreateFPToIntLocations(ArenaAllocator* allocator, HInvoke* invoke) { in CreateFPToIntLocations() argument
213 new (allocator) LocationSummary(invoke, LocationSummary::kNoCall, kIntrinsified); in CreateFPToIntLocations()
218 static void CreateIntToFPLocations(ArenaAllocator* allocator, HInvoke* invoke) { in CreateIntToFPLocations() argument
220 new (allocator) LocationSummary(invoke, LocationSummary::kNoCall, kIntrinsified); in CreateIntToFPLocations()
239 void IntrinsicLocationsBuilderARM64::VisitDoubleDoubleToRawLongBits(HInvoke* invoke) { in VisitDoubleDoubleToRawLongBits() argument
240 CreateFPToIntLocations(allocator_, invoke); in VisitDoubleDoubleToRawLongBits()
242 void IntrinsicLocationsBuilderARM64::VisitDoubleLongBitsToDouble(HInvoke* invoke) { in VisitDoubleLongBitsToDouble() argument
243 CreateIntToFPLocations(allocator_, invoke); in VisitDoubleLongBitsToDouble()
246 void IntrinsicCodeGeneratorARM64::VisitDoubleDoubleToRawLongBits(HInvoke* invoke) { in VisitDoubleDoubleToRawLongBits() argument
247 MoveFPToInt(invoke->GetLocations(), /* is64bit= */ true, GetVIXLAssembler()); in VisitDoubleDoubleToRawLongBits()
249 void IntrinsicCodeGeneratorARM64::VisitDoubleLongBitsToDouble(HInvoke* invoke) { in VisitDoubleLongBitsToDouble() argument
250 MoveIntToFP(invoke->GetLocations(), /* is64bit= */ true, GetVIXLAssembler()); in VisitDoubleLongBitsToDouble()
253 void IntrinsicLocationsBuilderARM64::VisitFloatFloatToRawIntBits(HInvoke* invoke) { in VisitFloatFloatToRawIntBits() argument
254 CreateFPToIntLocations(allocator_, invoke); in VisitFloatFloatToRawIntBits()
256 void IntrinsicLocationsBuilderARM64::VisitFloatIntBitsToFloat(HInvoke* invoke) { in VisitFloatIntBitsToFloat() argument
257 CreateIntToFPLocations(allocator_, invoke); in VisitFloatIntBitsToFloat()
260 void IntrinsicCodeGeneratorARM64::VisitFloatFloatToRawIntBits(HInvoke* invoke) { in VisitFloatFloatToRawIntBits() argument
261 MoveFPToInt(invoke->GetLocations(), /* is64bit= */ false, GetVIXLAssembler()); in VisitFloatFloatToRawIntBits()
263 void IntrinsicCodeGeneratorARM64::VisitFloatIntBitsToFloat(HInvoke* invoke) { in VisitFloatIntBitsToFloat() argument
264 MoveIntToFP(invoke->GetLocations(), /* is64bit= */ false, GetVIXLAssembler()); in VisitFloatIntBitsToFloat()
267 static void CreateIntToIntLocations(ArenaAllocator* allocator, HInvoke* invoke) { in CreateIntToIntLocations() argument
269 new (allocator) LocationSummary(invoke, LocationSummary::kNoCall, kIntrinsified); in CreateIntToIntLocations()
274 static void CreateIntIntToIntLocations(ArenaAllocator* allocator, HInvoke* invoke) { in CreateIntIntToIntLocations() argument
276 new (allocator) LocationSummary(invoke, LocationSummary::kNoCall, kIntrinsified); in CreateIntIntToIntLocations()
282 static void CreateIntIntToIntSlowPathCallLocations(ArenaAllocator* allocator, HInvoke* invoke) { in CreateIntIntToIntSlowPathCallLocations() argument
284 new (allocator) LocationSummary(invoke, LocationSummary::kCallOnSlowPath, kIntrinsified); in CreateIntIntToIntSlowPathCallLocations()
331 void IntrinsicLocationsBuilderARM64::VisitIntegerReverseBytes(HInvoke* invoke) { in VisitIntegerReverseBytes() argument
332 CreateIntToIntLocations(allocator_, invoke); in VisitIntegerReverseBytes()
335 void IntrinsicCodeGeneratorARM64::VisitIntegerReverseBytes(HInvoke* invoke) { in VisitIntegerReverseBytes() argument
336 GenReverseBytes(invoke->GetLocations(), DataType::Type::kInt32, GetVIXLAssembler()); in VisitIntegerReverseBytes()
339 void IntrinsicLocationsBuilderARM64::VisitLongReverseBytes(HInvoke* invoke) { in VisitLongReverseBytes() argument
340 CreateIntToIntLocations(allocator_, invoke); in VisitLongReverseBytes()
343 void IntrinsicCodeGeneratorARM64::VisitLongReverseBytes(HInvoke* invoke) { in VisitLongReverseBytes() argument
344 GenReverseBytes(invoke->GetLocations(), DataType::Type::kInt64, GetVIXLAssembler()); in VisitLongReverseBytes()
347 void IntrinsicLocationsBuilderARM64::VisitShortReverseBytes(HInvoke* invoke) { in VisitShortReverseBytes() argument
348 CreateIntToIntLocations(allocator_, invoke); in VisitShortReverseBytes()
351 void IntrinsicCodeGeneratorARM64::VisitShortReverseBytes(HInvoke* invoke) { in VisitShortReverseBytes() argument
352 GenReverseBytes(invoke->GetLocations(), DataType::Type::kInt16, GetVIXLAssembler()); in VisitShortReverseBytes()
366 void IntrinsicLocationsBuilderARM64::VisitIntegerNumberOfLeadingZeros(HInvoke* invoke) { in VisitIntegerNumberOfLeadingZeros() argument
367 CreateIntToIntLocations(allocator_, invoke); in VisitIntegerNumberOfLeadingZeros()
370 void IntrinsicCodeGeneratorARM64::VisitIntegerNumberOfLeadingZeros(HInvoke* invoke) { in VisitIntegerNumberOfLeadingZeros() argument
371 GenNumberOfLeadingZeros(invoke->GetLocations(), DataType::Type::kInt32, GetVIXLAssembler()); in VisitIntegerNumberOfLeadingZeros()
374 void IntrinsicLocationsBuilderARM64::VisitLongNumberOfLeadingZeros(HInvoke* invoke) { in VisitLongNumberOfLeadingZeros() argument
375 CreateIntToIntLocations(allocator_, invoke); in VisitLongNumberOfLeadingZeros()
378 void IntrinsicCodeGeneratorARM64::VisitLongNumberOfLeadingZeros(HInvoke* invoke) { in VisitLongNumberOfLeadingZeros() argument
379 GenNumberOfLeadingZeros(invoke->GetLocations(), DataType::Type::kInt64, GetVIXLAssembler()); in VisitLongNumberOfLeadingZeros()
394 void IntrinsicLocationsBuilderARM64::VisitIntegerNumberOfTrailingZeros(HInvoke* invoke) { in VisitIntegerNumberOfTrailingZeros() argument
395 CreateIntToIntLocations(allocator_, invoke); in VisitIntegerNumberOfTrailingZeros()
398 void IntrinsicCodeGeneratorARM64::VisitIntegerNumberOfTrailingZeros(HInvoke* invoke) { in VisitIntegerNumberOfTrailingZeros() argument
399 GenNumberOfTrailingZeros(invoke->GetLocations(), DataType::Type::kInt32, GetVIXLAssembler()); in VisitIntegerNumberOfTrailingZeros()
402 void IntrinsicLocationsBuilderARM64::VisitLongNumberOfTrailingZeros(HInvoke* invoke) { in VisitLongNumberOfTrailingZeros() argument
403 CreateIntToIntLocations(allocator_, invoke); in VisitLongNumberOfTrailingZeros()
406 void IntrinsicCodeGeneratorARM64::VisitLongNumberOfTrailingZeros(HInvoke* invoke) { in VisitLongNumberOfTrailingZeros() argument
407 GenNumberOfTrailingZeros(invoke->GetLocations(), DataType::Type::kInt64, GetVIXLAssembler()); in VisitLongNumberOfTrailingZeros()
421 void IntrinsicLocationsBuilderARM64::VisitIntegerReverse(HInvoke* invoke) { in VisitIntegerReverse() argument
422 CreateIntToIntLocations(allocator_, invoke); in VisitIntegerReverse()
425 void IntrinsicCodeGeneratorARM64::VisitIntegerReverse(HInvoke* invoke) { in VisitIntegerReverse() argument
426 GenReverse(invoke->GetLocations(), DataType::Type::kInt32, GetVIXLAssembler()); in VisitIntegerReverse()
429 void IntrinsicLocationsBuilderARM64::VisitLongReverse(HInvoke* invoke) { in VisitLongReverse() argument
430 CreateIntToIntLocations(allocator_, invoke); in VisitLongReverse()
433 void IntrinsicCodeGeneratorARM64::VisitLongReverse(HInvoke* invoke) { in VisitLongReverse() argument
434 GenReverse(invoke->GetLocations(), DataType::Type::kInt64, GetVIXLAssembler()); in VisitLongReverse()
454 void IntrinsicLocationsBuilderARM64::VisitLongBitCount(HInvoke* invoke) { in VisitLongBitCount() argument
455 CreateIntToIntLocations(allocator_, invoke); in VisitLongBitCount()
458 void IntrinsicCodeGeneratorARM64::VisitLongBitCount(HInvoke* invoke) { in VisitLongBitCount() argument
459 GenBitCount(invoke, DataType::Type::kInt64, GetVIXLAssembler()); in VisitLongBitCount()
462 void IntrinsicLocationsBuilderARM64::VisitIntegerBitCount(HInvoke* invoke) { in VisitIntegerBitCount() argument
463 CreateIntToIntLocations(allocator_, invoke); in VisitIntegerBitCount()
466 void IntrinsicCodeGeneratorARM64::VisitIntegerBitCount(HInvoke* invoke) { in VisitIntegerBitCount() argument
467 GenBitCount(invoke, DataType::Type::kInt32, GetVIXLAssembler()); in VisitIntegerBitCount()
470 static void GenHighestOneBit(HInvoke* invoke, DataType::Type type, MacroAssembler* masm) { in GenHighestOneBit() argument
475 Register src = InputRegisterAt(invoke, 0); in GenHighestOneBit()
476 Register dst = RegisterFrom(invoke->GetLocations()->Out(), type); in GenHighestOneBit()
487 void IntrinsicLocationsBuilderARM64::VisitIntegerHighestOneBit(HInvoke* invoke) { in VisitIntegerHighestOneBit() argument
488 CreateIntToIntLocations(allocator_, invoke); in VisitIntegerHighestOneBit()
491 void IntrinsicCodeGeneratorARM64::VisitIntegerHighestOneBit(HInvoke* invoke) { in VisitIntegerHighestOneBit() argument
492 GenHighestOneBit(invoke, DataType::Type::kInt32, GetVIXLAssembler()); in VisitIntegerHighestOneBit()
495 void IntrinsicLocationsBuilderARM64::VisitLongHighestOneBit(HInvoke* invoke) { in VisitLongHighestOneBit() argument
496 CreateIntToIntLocations(allocator_, invoke); in VisitLongHighestOneBit()
499 void IntrinsicCodeGeneratorARM64::VisitLongHighestOneBit(HInvoke* invoke) { in VisitLongHighestOneBit() argument
500 GenHighestOneBit(invoke, DataType::Type::kInt64, GetVIXLAssembler()); in VisitLongHighestOneBit()
503 static void GenLowestOneBit(HInvoke* invoke, DataType::Type type, MacroAssembler* masm) { in GenLowestOneBit() argument
508 Register src = InputRegisterAt(invoke, 0); in GenLowestOneBit()
509 Register dst = RegisterFrom(invoke->GetLocations()->Out(), type); in GenLowestOneBit()
516 void IntrinsicLocationsBuilderARM64::VisitIntegerLowestOneBit(HInvoke* invoke) { in VisitIntegerLowestOneBit() argument
517 CreateIntToIntLocations(allocator_, invoke); in VisitIntegerLowestOneBit()
520 void IntrinsicCodeGeneratorARM64::VisitIntegerLowestOneBit(HInvoke* invoke) { in VisitIntegerLowestOneBit() argument
521 GenLowestOneBit(invoke, DataType::Type::kInt32, GetVIXLAssembler()); in VisitIntegerLowestOneBit()
524 void IntrinsicLocationsBuilderARM64::VisitLongLowestOneBit(HInvoke* invoke) { in VisitLongLowestOneBit() argument
525 CreateIntToIntLocations(allocator_, invoke); in VisitLongLowestOneBit()
528 void IntrinsicCodeGeneratorARM64::VisitLongLowestOneBit(HInvoke* invoke) { in VisitLongLowestOneBit() argument
529 GenLowestOneBit(invoke, DataType::Type::kInt64, GetVIXLAssembler()); in VisitLongLowestOneBit()
532 static void CreateFPToFPLocations(ArenaAllocator* allocator, HInvoke* invoke) { in CreateFPToFPLocations() argument
534 new (allocator) LocationSummary(invoke, LocationSummary::kNoCall, kIntrinsified); in CreateFPToFPLocations()
539 void IntrinsicLocationsBuilderARM64::VisitMathSqrt(HInvoke* invoke) { in VisitMathSqrt() argument
540 CreateFPToFPLocations(allocator_, invoke); in VisitMathSqrt()
543 void IntrinsicCodeGeneratorARM64::VisitMathSqrt(HInvoke* invoke) { in VisitMathSqrt() argument
544 LocationSummary* locations = invoke->GetLocations(); in VisitMathSqrt()
549 void IntrinsicLocationsBuilderARM64::VisitMathCeil(HInvoke* invoke) { in VisitMathCeil() argument
550 CreateFPToFPLocations(allocator_, invoke); in VisitMathCeil()
553 void IntrinsicCodeGeneratorARM64::VisitMathCeil(HInvoke* invoke) { in VisitMathCeil() argument
554 LocationSummary* locations = invoke->GetLocations(); in VisitMathCeil()
559 void IntrinsicLocationsBuilderARM64::VisitMathFloor(HInvoke* invoke) { in VisitMathFloor() argument
560 CreateFPToFPLocations(allocator_, invoke); in VisitMathFloor()
563 void IntrinsicCodeGeneratorARM64::VisitMathFloor(HInvoke* invoke) { in VisitMathFloor() argument
564 LocationSummary* locations = invoke->GetLocations(); in VisitMathFloor()
569 void IntrinsicLocationsBuilderARM64::VisitMathRint(HInvoke* invoke) { in VisitMathRint() argument
570 CreateFPToFPLocations(allocator_, invoke); in VisitMathRint()
573 void IntrinsicCodeGeneratorARM64::VisitMathRint(HInvoke* invoke) { in VisitMathRint() argument
574 LocationSummary* locations = invoke->GetLocations(); in VisitMathRint()
579 static void CreateFPToIntPlusFPTempLocations(ArenaAllocator* allocator, HInvoke* invoke) { in CreateFPToIntPlusFPTempLocations() argument
581 new (allocator) LocationSummary(invoke, LocationSummary::kNoCall, kIntrinsified); in CreateFPToIntPlusFPTempLocations()
587 static void GenMathRound(HInvoke* invoke, bool is_double, vixl::aarch64::MacroAssembler* masm) { in GenMathRound() argument
601 LocationSummary* l = invoke->GetLocations(); in GenMathRound()
624 void IntrinsicLocationsBuilderARM64::VisitMathRoundDouble(HInvoke* invoke) { in VisitMathRoundDouble() argument
625 CreateFPToIntPlusFPTempLocations(allocator_, invoke); in VisitMathRoundDouble()
628 void IntrinsicCodeGeneratorARM64::VisitMathRoundDouble(HInvoke* invoke) { in VisitMathRoundDouble() argument
629 GenMathRound(invoke, /* is_double= */ true, GetVIXLAssembler()); in VisitMathRoundDouble()
632 void IntrinsicLocationsBuilderARM64::VisitMathRoundFloat(HInvoke* invoke) { in VisitMathRoundFloat() argument
633 CreateFPToIntPlusFPTempLocations(allocator_, invoke); in VisitMathRoundFloat()
636 void IntrinsicCodeGeneratorARM64::VisitMathRoundFloat(HInvoke* invoke) { in VisitMathRoundFloat() argument
637 GenMathRound(invoke, /* is_double= */ false, GetVIXLAssembler()); in VisitMathRoundFloat()
640 void IntrinsicLocationsBuilderARM64::VisitMemoryPeekByte(HInvoke* invoke) { in VisitMemoryPeekByte() argument
641 CreateIntToIntLocations(allocator_, invoke); in VisitMemoryPeekByte()
644 void IntrinsicCodeGeneratorARM64::VisitMemoryPeekByte(HInvoke* invoke) { in VisitMemoryPeekByte() argument
646 __ Ldrsb(WRegisterFrom(invoke->GetLocations()->Out()), in VisitMemoryPeekByte()
647 AbsoluteHeapOperandFrom(invoke->GetLocations()->InAt(0), 0)); in VisitMemoryPeekByte()
650 void IntrinsicLocationsBuilderARM64::VisitMemoryPeekIntNative(HInvoke* invoke) { in VisitMemoryPeekIntNative() argument
651 CreateIntToIntLocations(allocator_, invoke); in VisitMemoryPeekIntNative()
654 void IntrinsicCodeGeneratorARM64::VisitMemoryPeekIntNative(HInvoke* invoke) { in VisitMemoryPeekIntNative() argument
656 __ Ldr(WRegisterFrom(invoke->GetLocations()->Out()), in VisitMemoryPeekIntNative()
657 AbsoluteHeapOperandFrom(invoke->GetLocations()->InAt(0), 0)); in VisitMemoryPeekIntNative()
660 void IntrinsicLocationsBuilderARM64::VisitMemoryPeekLongNative(HInvoke* invoke) { in VisitMemoryPeekLongNative() argument
661 CreateIntToIntLocations(allocator_, invoke); in VisitMemoryPeekLongNative()
664 void IntrinsicCodeGeneratorARM64::VisitMemoryPeekLongNative(HInvoke* invoke) { in VisitMemoryPeekLongNative() argument
666 __ Ldr(XRegisterFrom(invoke->GetLocations()->Out()), in VisitMemoryPeekLongNative()
667 AbsoluteHeapOperandFrom(invoke->GetLocations()->InAt(0), 0)); in VisitMemoryPeekLongNative()
670 void IntrinsicLocationsBuilderARM64::VisitMemoryPeekShortNative(HInvoke* invoke) { in VisitMemoryPeekShortNative() argument
671 CreateIntToIntLocations(allocator_, invoke); in VisitMemoryPeekShortNative()
674 void IntrinsicCodeGeneratorARM64::VisitMemoryPeekShortNative(HInvoke* invoke) { in VisitMemoryPeekShortNative() argument
676 __ Ldrsh(WRegisterFrom(invoke->GetLocations()->Out()), in VisitMemoryPeekShortNative()
677 AbsoluteHeapOperandFrom(invoke->GetLocations()->InAt(0), 0)); in VisitMemoryPeekShortNative()
680 static void CreateIntIntToVoidLocations(ArenaAllocator* allocator, HInvoke* invoke) { in CreateIntIntToVoidLocations() argument
682 new (allocator) LocationSummary(invoke, LocationSummary::kNoCall, kIntrinsified); in CreateIntIntToVoidLocations()
687 void IntrinsicLocationsBuilderARM64::VisitMemoryPokeByte(HInvoke* invoke) { in VisitMemoryPokeByte() argument
688 CreateIntIntToVoidLocations(allocator_, invoke); in VisitMemoryPokeByte()
691 void IntrinsicCodeGeneratorARM64::VisitMemoryPokeByte(HInvoke* invoke) { in VisitMemoryPokeByte() argument
693 __ Strb(WRegisterFrom(invoke->GetLocations()->InAt(1)), in VisitMemoryPokeByte()
694 AbsoluteHeapOperandFrom(invoke->GetLocations()->InAt(0), 0)); in VisitMemoryPokeByte()
697 void IntrinsicLocationsBuilderARM64::VisitMemoryPokeIntNative(HInvoke* invoke) { in VisitMemoryPokeIntNative() argument
698 CreateIntIntToVoidLocations(allocator_, invoke); in VisitMemoryPokeIntNative()
701 void IntrinsicCodeGeneratorARM64::VisitMemoryPokeIntNative(HInvoke* invoke) { in VisitMemoryPokeIntNative() argument
703 __ Str(WRegisterFrom(invoke->GetLocations()->InAt(1)), in VisitMemoryPokeIntNative()
704 AbsoluteHeapOperandFrom(invoke->GetLocations()->InAt(0), 0)); in VisitMemoryPokeIntNative()
707 void IntrinsicLocationsBuilderARM64::VisitMemoryPokeLongNative(HInvoke* invoke) { in VisitMemoryPokeLongNative() argument
708 CreateIntIntToVoidLocations(allocator_, invoke); in VisitMemoryPokeLongNative()
711 void IntrinsicCodeGeneratorARM64::VisitMemoryPokeLongNative(HInvoke* invoke) { in VisitMemoryPokeLongNative() argument
713 __ Str(XRegisterFrom(invoke->GetLocations()->InAt(1)), in VisitMemoryPokeLongNative()
714 AbsoluteHeapOperandFrom(invoke->GetLocations()->InAt(0), 0)); in VisitMemoryPokeLongNative()
717 void IntrinsicLocationsBuilderARM64::VisitMemoryPokeShortNative(HInvoke* invoke) { in VisitMemoryPokeShortNative() argument
718 CreateIntIntToVoidLocations(allocator_, invoke); in VisitMemoryPokeShortNative()
721 void IntrinsicCodeGeneratorARM64::VisitMemoryPokeShortNative(HInvoke* invoke) { in VisitMemoryPokeShortNative() argument
723 __ Strh(WRegisterFrom(invoke->GetLocations()->InAt(1)), in VisitMemoryPokeShortNative()
724 AbsoluteHeapOperandFrom(invoke->GetLocations()->InAt(0), 0)); in VisitMemoryPokeShortNative()
727 void IntrinsicLocationsBuilderARM64::VisitThreadCurrentThread(HInvoke* invoke) { in VisitThreadCurrentThread() argument
729 new (allocator_) LocationSummary(invoke, LocationSummary::kNoCall, kIntrinsified); in VisitThreadCurrentThread()
733 void IntrinsicCodeGeneratorARM64::VisitThreadCurrentThread(HInvoke* invoke) { in VisitThreadCurrentThread() argument
734 codegen_->Load(DataType::Type::kReference, WRegisterFrom(invoke->GetLocations()->Out()), in VisitThreadCurrentThread()
738 static bool ReadBarrierNeedsTemp(bool is_volatile, HInvoke* invoke) { in ReadBarrierNeedsTemp() argument
740 !invoke->InputAt(2)->IsLongConstant() || in ReadBarrierNeedsTemp()
741 invoke->InputAt(2)->AsLongConstant()->GetValue() >= kReferenceLoadMinFarOffset; in ReadBarrierNeedsTemp()
744 static void GenUnsafeGet(HInvoke* invoke, in GenUnsafeGet() argument
748 LocationSummary* locations = invoke->GetLocations(); in GenUnsafeGet()
766 Location maybe_temp = ReadBarrierNeedsTemp(is_volatile, invoke) in GenUnsafeGet()
768 DCHECK_EQ(locations->GetTempCount(), ReadBarrierNeedsTemp(is_volatile, invoke)); in GenUnsafeGet()
769 codegen->GenerateFieldLoadWithBakerReadBarrier(invoke, in GenUnsafeGet()
778 codegen->GenerateFieldLoadWithBakerReadBarrier(invoke, in GenUnsafeGet()
794 codegen->LoadAcquire(invoke, type, trg, mem_op, /* needs_null_check= */ true); in GenUnsafeGet()
801 codegen->MaybeGenerateReadBarrierSlow(invoke, trg_loc, trg_loc, base_loc, 0u, offset_loc); in GenUnsafeGet()
806 static void GenUnsafeGetAbsolute(HInvoke* invoke, in GenUnsafeGetAbsolute() argument
810 LocationSummary* locations = invoke->GetLocations(); in GenUnsafeGetAbsolute()
820 codegen->LoadAcquire(invoke, type, trg, mem_op, /* needs_null_check= */ true); in GenUnsafeGetAbsolute()
827 HInvoke* invoke, in CreateUnsafeGetLocations() argument
830 bool can_call = codegen->EmitReadBarrier() && IsUnsafeGetReference(invoke); in CreateUnsafeGetLocations()
832 new (allocator) LocationSummary(invoke, in CreateUnsafeGetLocations()
839 if (ReadBarrierNeedsTemp(is_volatile, invoke)) { in CreateUnsafeGetLocations()
847 locations->SetInAt(2, Location::RegisterOrConstant(invoke->InputAt(2))); in CreateUnsafeGetLocations()
853 HInvoke* invoke) { in CreateUnsafeGetAbsoluteLocations() argument
855 new (allocator) LocationSummary(invoke, LocationSummary::kNoCall, kIntrinsified); in CreateUnsafeGetAbsoluteLocations()
861 void IntrinsicLocationsBuilderARM64::VisitUnsafeGet(HInvoke* invoke) { in VisitUnsafeGet() argument
862 VisitJdkUnsafeGet(invoke); in VisitUnsafeGet()
864 void IntrinsicLocationsBuilderARM64::VisitUnsafeGetAbsolute(HInvoke* invoke) { in VisitUnsafeGetAbsolute() argument
865 VisitJdkUnsafeGetAbsolute(invoke); in VisitUnsafeGetAbsolute()
867 void IntrinsicLocationsBuilderARM64::VisitUnsafeGetVolatile(HInvoke* invoke) { in VisitUnsafeGetVolatile() argument
868 VisitJdkUnsafeGetVolatile(invoke); in VisitUnsafeGetVolatile()
870 void IntrinsicLocationsBuilderARM64::VisitUnsafeGetLong(HInvoke* invoke) { in VisitUnsafeGetLong() argument
871 VisitJdkUnsafeGetLong(invoke); in VisitUnsafeGetLong()
873 void IntrinsicLocationsBuilderARM64::VisitUnsafeGetLongVolatile(HInvoke* invoke) { in VisitUnsafeGetLongVolatile() argument
874 VisitJdkUnsafeGetLongVolatile(invoke); in VisitUnsafeGetLongVolatile()
876 void IntrinsicLocationsBuilderARM64::VisitUnsafeGetObject(HInvoke* invoke) { in VisitUnsafeGetObject() argument
877 VisitJdkUnsafeGetReference(invoke); in VisitUnsafeGetObject()
879 void IntrinsicLocationsBuilderARM64::VisitUnsafeGetObjectVolatile(HInvoke* invoke) { in VisitUnsafeGetObjectVolatile() argument
880 VisitJdkUnsafeGetReferenceVolatile(invoke); in VisitUnsafeGetObjectVolatile()
882 void IntrinsicLocationsBuilderARM64::VisitUnsafeGetByte(HInvoke* invoke) { in VisitUnsafeGetByte() argument
883 VisitJdkUnsafeGetByte(invoke); in VisitUnsafeGetByte()
885 void IntrinsicLocationsBuilderARM64::VisitJdkUnsafeGet(HInvoke* invoke) { in VisitJdkUnsafeGet() argument
886 CreateUnsafeGetLocations(allocator_, invoke, codegen_); in VisitJdkUnsafeGet()
888 void IntrinsicLocationsBuilderARM64::VisitJdkUnsafeGetAbsolute(HInvoke* invoke) { in VisitJdkUnsafeGetAbsolute() argument
889 CreateUnsafeGetAbsoluteLocations(allocator_, invoke); in VisitJdkUnsafeGetAbsolute()
891 void IntrinsicLocationsBuilderARM64::VisitJdkUnsafeGetVolatile(HInvoke* invoke) { in VisitJdkUnsafeGetVolatile() argument
892 CreateUnsafeGetLocations(allocator_, invoke, codegen_, /* is_volatile= */ true); in VisitJdkUnsafeGetVolatile()
894 void IntrinsicLocationsBuilderARM64::VisitJdkUnsafeGetAcquire(HInvoke* invoke) { in VisitJdkUnsafeGetAcquire() argument
895 CreateUnsafeGetLocations(allocator_, invoke, codegen_, /* is_volatile= */ true); in VisitJdkUnsafeGetAcquire()
897 void IntrinsicLocationsBuilderARM64::VisitJdkUnsafeGetLong(HInvoke* invoke) { in VisitJdkUnsafeGetLong() argument
898 CreateUnsafeGetLocations(allocator_, invoke, codegen_); in VisitJdkUnsafeGetLong()
900 void IntrinsicLocationsBuilderARM64::VisitJdkUnsafeGetLongVolatile(HInvoke* invoke) { in VisitJdkUnsafeGetLongVolatile() argument
901 CreateUnsafeGetLocations(allocator_, invoke, codegen_, /* is_volatile= */ true); in VisitJdkUnsafeGetLongVolatile()
903 void IntrinsicLocationsBuilderARM64::VisitJdkUnsafeGetLongAcquire(HInvoke* invoke) { in VisitJdkUnsafeGetLongAcquire() argument
904 CreateUnsafeGetLocations(allocator_, invoke, codegen_, /* is_volatile= */ true); in VisitJdkUnsafeGetLongAcquire()
906 void IntrinsicLocationsBuilderARM64::VisitJdkUnsafeGetReference(HInvoke* invoke) { in VisitJdkUnsafeGetReference() argument
907 CreateUnsafeGetLocations(allocator_, invoke, codegen_); in VisitJdkUnsafeGetReference()
909 void IntrinsicLocationsBuilderARM64::VisitJdkUnsafeGetReferenceVolatile(HInvoke* invoke) { in VisitJdkUnsafeGetReferenceVolatile() argument
910 CreateUnsafeGetLocations(allocator_, invoke, codegen_, /* is_volatile= */ true); in VisitJdkUnsafeGetReferenceVolatile()
912 void IntrinsicLocationsBuilderARM64::VisitJdkUnsafeGetReferenceAcquire(HInvoke* invoke) { in VisitJdkUnsafeGetReferenceAcquire() argument
913 CreateUnsafeGetLocations(allocator_, invoke, codegen_, /* is_volatile= */ true); in VisitJdkUnsafeGetReferenceAcquire()
915 void IntrinsicLocationsBuilderARM64::VisitJdkUnsafeGetByte(HInvoke* invoke) { in VisitJdkUnsafeGetByte() argument
916 CreateUnsafeGetLocations(allocator_, invoke, codegen_); in VisitJdkUnsafeGetByte()
919 void IntrinsicCodeGeneratorARM64::VisitUnsafeGet(HInvoke* invoke) { in VisitUnsafeGet() argument
920 VisitJdkUnsafeGet(invoke); in VisitUnsafeGet()
922 void IntrinsicCodeGeneratorARM64::VisitUnsafeGetAbsolute(HInvoke* invoke) { in VisitUnsafeGetAbsolute() argument
923 VisitJdkUnsafeGetAbsolute(invoke); in VisitUnsafeGetAbsolute()
925 void IntrinsicCodeGeneratorARM64::VisitUnsafeGetVolatile(HInvoke* invoke) { in VisitUnsafeGetVolatile() argument
926 VisitJdkUnsafeGetVolatile(invoke); in VisitUnsafeGetVolatile()
928 void IntrinsicCodeGeneratorARM64::VisitUnsafeGetLong(HInvoke* invoke) { in VisitUnsafeGetLong() argument
929 VisitJdkUnsafeGetLong(invoke); in VisitUnsafeGetLong()
931 void IntrinsicCodeGeneratorARM64::VisitUnsafeGetLongVolatile(HInvoke* invoke) { in VisitUnsafeGetLongVolatile() argument
932 VisitJdkUnsafeGetLongVolatile(invoke); in VisitUnsafeGetLongVolatile()
934 void IntrinsicCodeGeneratorARM64::VisitUnsafeGetObject(HInvoke* invoke) { in VisitUnsafeGetObject() argument
935 VisitJdkUnsafeGetReference(invoke); in VisitUnsafeGetObject()
937 void IntrinsicCodeGeneratorARM64::VisitUnsafeGetObjectVolatile(HInvoke* invoke) { in VisitUnsafeGetObjectVolatile() argument
938 VisitJdkUnsafeGetReferenceVolatile(invoke); in VisitUnsafeGetObjectVolatile()
940 void IntrinsicCodeGeneratorARM64::VisitUnsafeGetByte(HInvoke* invoke) { in VisitUnsafeGetByte() argument
941 VisitJdkUnsafeGetByte(invoke); in VisitUnsafeGetByte()
944 void IntrinsicCodeGeneratorARM64::VisitJdkUnsafeGet(HInvoke* invoke) { in VisitJdkUnsafeGet() argument
945 GenUnsafeGet(invoke, DataType::Type::kInt32, /*is_volatile=*/ false, codegen_); in VisitJdkUnsafeGet()
947 void IntrinsicCodeGeneratorARM64::VisitJdkUnsafeGetAbsolute(HInvoke* invoke) { in VisitJdkUnsafeGetAbsolute() argument
948 GenUnsafeGetAbsolute(invoke, DataType::Type::kInt32, /*is_volatile=*/ false, codegen_); in VisitJdkUnsafeGetAbsolute()
950 void IntrinsicCodeGeneratorARM64::VisitJdkUnsafeGetVolatile(HInvoke* invoke) { in VisitJdkUnsafeGetVolatile() argument
951 GenUnsafeGet(invoke, DataType::Type::kInt32, /*is_volatile=*/ true, codegen_); in VisitJdkUnsafeGetVolatile()
953 void IntrinsicCodeGeneratorARM64::VisitJdkUnsafeGetAcquire(HInvoke* invoke) { in VisitJdkUnsafeGetAcquire() argument
954 GenUnsafeGet(invoke, DataType::Type::kInt32, /*is_volatile=*/ true, codegen_); in VisitJdkUnsafeGetAcquire()
956 void IntrinsicCodeGeneratorARM64::VisitJdkUnsafeGetLong(HInvoke* invoke) { in VisitJdkUnsafeGetLong() argument
957 GenUnsafeGet(invoke, DataType::Type::kInt64, /*is_volatile=*/ false, codegen_); in VisitJdkUnsafeGetLong()
959 void IntrinsicCodeGeneratorARM64::VisitJdkUnsafeGetLongVolatile(HInvoke* invoke) { in VisitJdkUnsafeGetLongVolatile() argument
960 GenUnsafeGet(invoke, DataType::Type::kInt64, /*is_volatile=*/ true, codegen_); in VisitJdkUnsafeGetLongVolatile()
962 void IntrinsicCodeGeneratorARM64::VisitJdkUnsafeGetLongAcquire(HInvoke* invoke) { in VisitJdkUnsafeGetLongAcquire() argument
963 GenUnsafeGet(invoke, DataType::Type::kInt64, /*is_volatile=*/ true, codegen_); in VisitJdkUnsafeGetLongAcquire()
965 void IntrinsicCodeGeneratorARM64::VisitJdkUnsafeGetReference(HInvoke* invoke) { in VisitJdkUnsafeGetReference() argument
966 GenUnsafeGet(invoke, DataType::Type::kReference, /*is_volatile=*/ false, codegen_); in VisitJdkUnsafeGetReference()
968 void IntrinsicCodeGeneratorARM64::VisitJdkUnsafeGetReferenceVolatile(HInvoke* invoke) { in VisitJdkUnsafeGetReferenceVolatile() argument
969 GenUnsafeGet(invoke, DataType::Type::kReference, /*is_volatile=*/ true, codegen_); in VisitJdkUnsafeGetReferenceVolatile()
971 void IntrinsicCodeGeneratorARM64::VisitJdkUnsafeGetReferenceAcquire(HInvoke* invoke) { in VisitJdkUnsafeGetReferenceAcquire() argument
972 GenUnsafeGet(invoke, DataType::Type::kReference, /*is_volatile=*/ true, codegen_); in VisitJdkUnsafeGetReferenceAcquire()
974 void IntrinsicCodeGeneratorARM64::VisitJdkUnsafeGetByte(HInvoke* invoke) { in VisitJdkUnsafeGetByte() argument
975 GenUnsafeGet(invoke, DataType::Type::kInt8, /*is_volatile=*/ false, codegen_); in VisitJdkUnsafeGetByte()
978 static void CreateUnsafePutLocations(ArenaAllocator* allocator, HInvoke* invoke) { in CreateUnsafePutLocations() argument
980 new (allocator) LocationSummary(invoke, LocationSummary::kNoCall, kIntrinsified); in CreateUnsafePutLocations()
989 kOffsetIndex, Location::RegisterOrConstant(invoke->InputAt(kOffsetIndex))); in CreateUnsafePutLocations()
991 if (IsZeroBitPattern(invoke->InputAt(kValueIndex))) { in CreateUnsafePutLocations()
992 locations->SetInAt(kValueIndex, Location::ConstantLocation(invoke->InputAt(kValueIndex))); in CreateUnsafePutLocations()
998 static void CreateUnsafePutAbsoluteLocations(ArenaAllocator* allocator, HInvoke* invoke) { in CreateUnsafePutAbsoluteLocations() argument
1000 new (allocator) LocationSummary(invoke, LocationSummary::kNoCall, kIntrinsified); in CreateUnsafePutAbsoluteLocations()
1008 if (IsZeroBitPattern(invoke->InputAt(kValueIndex))) { in CreateUnsafePutAbsoluteLocations()
1009 locations->SetInAt(kValueIndex, Location::ConstantLocation(invoke->InputAt(kValueIndex))); in CreateUnsafePutAbsoluteLocations()
1015 void IntrinsicLocationsBuilderARM64::VisitUnsafePut(HInvoke* invoke) { in VisitUnsafePut() argument
1016 VisitJdkUnsafePut(invoke); in VisitUnsafePut()
1018 void IntrinsicLocationsBuilderARM64::VisitUnsafePutAbsolute(HInvoke* invoke) { in VisitUnsafePutAbsolute() argument
1019 VisitJdkUnsafePutAbsolute(invoke); in VisitUnsafePutAbsolute()
1021 void IntrinsicLocationsBuilderARM64::VisitUnsafePutOrdered(HInvoke* invoke) { in VisitUnsafePutOrdered() argument
1022 VisitJdkUnsafePutOrdered(invoke); in VisitUnsafePutOrdered()
1024 void IntrinsicLocationsBuilderARM64::VisitUnsafePutVolatile(HInvoke* invoke) { in VisitUnsafePutVolatile() argument
1025 VisitJdkUnsafePutVolatile(invoke); in VisitUnsafePutVolatile()
1027 void IntrinsicLocationsBuilderARM64::VisitUnsafePutObject(HInvoke* invoke) { in VisitUnsafePutObject() argument
1028 VisitJdkUnsafePutReference(invoke); in VisitUnsafePutObject()
1030 void IntrinsicLocationsBuilderARM64::VisitUnsafePutObjectOrdered(HInvoke* invoke) { in VisitUnsafePutObjectOrdered() argument
1031 VisitJdkUnsafePutObjectOrdered(invoke); in VisitUnsafePutObjectOrdered()
1033 void IntrinsicLocationsBuilderARM64::VisitUnsafePutObjectVolatile(HInvoke* invoke) { in VisitUnsafePutObjectVolatile() argument
1034 VisitJdkUnsafePutReferenceVolatile(invoke); in VisitUnsafePutObjectVolatile()
1036 void IntrinsicLocationsBuilderARM64::VisitUnsafePutLong(HInvoke* invoke) { in VisitUnsafePutLong() argument
1037 VisitJdkUnsafePutLong(invoke); in VisitUnsafePutLong()
1039 void IntrinsicLocationsBuilderARM64::VisitUnsafePutLongOrdered(HInvoke* invoke) { in VisitUnsafePutLongOrdered() argument
1040 VisitJdkUnsafePutLongOrdered(invoke); in VisitUnsafePutLongOrdered()
1042 void IntrinsicLocationsBuilderARM64::VisitUnsafePutLongVolatile(HInvoke* invoke) { in VisitUnsafePutLongVolatile() argument
1043 VisitJdkUnsafePutLongVolatile(invoke); in VisitUnsafePutLongVolatile()
1045 void IntrinsicLocationsBuilderARM64::VisitUnsafePutByte(HInvoke* invoke) { in VisitUnsafePutByte() argument
1046 VisitJdkUnsafePutByte(invoke); in VisitUnsafePutByte()
1049 void IntrinsicLocationsBuilderARM64::VisitJdkUnsafePut(HInvoke* invoke) { in VisitJdkUnsafePut() argument
1050 CreateUnsafePutLocations(allocator_, invoke); in VisitJdkUnsafePut()
1052 void IntrinsicLocationsBuilderARM64::VisitJdkUnsafePutAbsolute(HInvoke* invoke) { in VisitJdkUnsafePutAbsolute() argument
1053 CreateUnsafePutAbsoluteLocations(allocator_, invoke); in VisitJdkUnsafePutAbsolute()
1055 void IntrinsicLocationsBuilderARM64::VisitJdkUnsafePutOrdered(HInvoke* invoke) { in VisitJdkUnsafePutOrdered() argument
1056 CreateUnsafePutLocations(allocator_, invoke); in VisitJdkUnsafePutOrdered()
1058 void IntrinsicLocationsBuilderARM64::VisitJdkUnsafePutVolatile(HInvoke* invoke) { in VisitJdkUnsafePutVolatile() argument
1059 CreateUnsafePutLocations(allocator_, invoke); in VisitJdkUnsafePutVolatile()
1061 void IntrinsicLocationsBuilderARM64::VisitJdkUnsafePutRelease(HInvoke* invoke) { in VisitJdkUnsafePutRelease() argument
1062 CreateUnsafePutLocations(allocator_, invoke); in VisitJdkUnsafePutRelease()
1064 void IntrinsicLocationsBuilderARM64::VisitJdkUnsafePutReference(HInvoke* invoke) { in VisitJdkUnsafePutReference() argument
1065 CreateUnsafePutLocations(allocator_, invoke); in VisitJdkUnsafePutReference()
1067 void IntrinsicLocationsBuilderARM64::VisitJdkUnsafePutObjectOrdered(HInvoke* invoke) { in VisitJdkUnsafePutObjectOrdered() argument
1068 CreateUnsafePutLocations(allocator_, invoke); in VisitJdkUnsafePutObjectOrdered()
1070 void IntrinsicLocationsBuilderARM64::VisitJdkUnsafePutReferenceVolatile(HInvoke* invoke) { in VisitJdkUnsafePutReferenceVolatile() argument
1071 CreateUnsafePutLocations(allocator_, invoke); in VisitJdkUnsafePutReferenceVolatile()
1073 void IntrinsicLocationsBuilderARM64::VisitJdkUnsafePutReferenceRelease(HInvoke* invoke) { in VisitJdkUnsafePutReferenceRelease() argument
1074 CreateUnsafePutLocations(allocator_, invoke); in VisitJdkUnsafePutReferenceRelease()
1076 void IntrinsicLocationsBuilderARM64::VisitJdkUnsafePutLong(HInvoke* invoke) { in VisitJdkUnsafePutLong() argument
1077 CreateUnsafePutLocations(allocator_, invoke); in VisitJdkUnsafePutLong()
1079 void IntrinsicLocationsBuilderARM64::VisitJdkUnsafePutLongOrdered(HInvoke* invoke) { in VisitJdkUnsafePutLongOrdered() argument
1080 CreateUnsafePutLocations(allocator_, invoke); in VisitJdkUnsafePutLongOrdered()
1082 void IntrinsicLocationsBuilderARM64::VisitJdkUnsafePutLongVolatile(HInvoke* invoke) { in VisitJdkUnsafePutLongVolatile() argument
1083 CreateUnsafePutLocations(allocator_, invoke); in VisitJdkUnsafePutLongVolatile()
1085 void IntrinsicLocationsBuilderARM64::VisitJdkUnsafePutLongRelease(HInvoke* invoke) { in VisitJdkUnsafePutLongRelease() argument
1086 CreateUnsafePutLocations(allocator_, invoke); in VisitJdkUnsafePutLongRelease()
1088 void IntrinsicLocationsBuilderARM64::VisitJdkUnsafePutByte(HInvoke* invoke) { in VisitJdkUnsafePutByte() argument
1089 CreateUnsafePutLocations(allocator_, invoke); in VisitJdkUnsafePutByte()
1092 static void GenUnsafePut(HInvoke* invoke, in GenUnsafePut() argument
1097 LocationSummary* locations = invoke->GetLocations(); in GenUnsafePut()
1104 CPURegister value = InputCPURegisterOrZeroRegAt(invoke, kValueIndex); in GenUnsafePut()
1120 !IsZeroBitPattern(invoke->InputAt(kValueIndex))) { in GenUnsafePut()
1129 codegen->StoreRelease(invoke, type, source, mem_op, /* needs_null_check= */ false); in GenUnsafePut()
1135 if (type == DataType::Type::kReference && !IsZeroBitPattern(invoke->InputAt(kValueIndex))) { in GenUnsafePut()
1141 static void GenUnsafePutAbsolute(HInvoke* invoke, in GenUnsafePutAbsolute() argument
1146 LocationSummary* locations = invoke->GetLocations(); in GenUnsafePutAbsolute()
1152 CPURegister value = InputCPURegisterOrZeroRegAt(invoke, kValueIndex); in GenUnsafePutAbsolute()
1155 codegen->StoreRelease(invoke, type, value, mem_op, /* needs_null_check= */ false); in GenUnsafePutAbsolute()
1161 void IntrinsicCodeGeneratorARM64::VisitUnsafePut(HInvoke* invoke) { in VisitUnsafePut() argument
1162 VisitJdkUnsafePut(invoke); in VisitUnsafePut()
1164 void IntrinsicCodeGeneratorARM64::VisitUnsafePutAbsolute(HInvoke* invoke) { in VisitUnsafePutAbsolute() argument
1165 VisitJdkUnsafePutAbsolute(invoke); in VisitUnsafePutAbsolute()
1167 void IntrinsicCodeGeneratorARM64::VisitUnsafePutOrdered(HInvoke* invoke) { in VisitUnsafePutOrdered() argument
1168 VisitJdkUnsafePutOrdered(invoke); in VisitUnsafePutOrdered()
1170 void IntrinsicCodeGeneratorARM64::VisitUnsafePutVolatile(HInvoke* invoke) { in VisitUnsafePutVolatile() argument
1171 VisitJdkUnsafePutVolatile(invoke); in VisitUnsafePutVolatile()
1173 void IntrinsicCodeGeneratorARM64::VisitUnsafePutObject(HInvoke* invoke) { in VisitUnsafePutObject() argument
1174 VisitJdkUnsafePutReference(invoke); in VisitUnsafePutObject()
1176 void IntrinsicCodeGeneratorARM64::VisitUnsafePutObjectOrdered(HInvoke* invoke) { in VisitUnsafePutObjectOrdered() argument
1177 VisitJdkUnsafePutObjectOrdered(invoke); in VisitUnsafePutObjectOrdered()
1179 void IntrinsicCodeGeneratorARM64::VisitUnsafePutObjectVolatile(HInvoke* invoke) { in VisitUnsafePutObjectVolatile() argument
1180 VisitJdkUnsafePutReferenceVolatile(invoke); in VisitUnsafePutObjectVolatile()
1182 void IntrinsicCodeGeneratorARM64::VisitUnsafePutLong(HInvoke* invoke) { in VisitUnsafePutLong() argument
1183 VisitJdkUnsafePutLong(invoke); in VisitUnsafePutLong()
1185 void IntrinsicCodeGeneratorARM64::VisitUnsafePutLongOrdered(HInvoke* invoke) { in VisitUnsafePutLongOrdered() argument
1186 VisitJdkUnsafePutLongOrdered(invoke); in VisitUnsafePutLongOrdered()
1188 void IntrinsicCodeGeneratorARM64::VisitUnsafePutLongVolatile(HInvoke* invoke) { in VisitUnsafePutLongVolatile() argument
1189 VisitJdkUnsafePutLongVolatile(invoke); in VisitUnsafePutLongVolatile()
1191 void IntrinsicCodeGeneratorARM64::VisitUnsafePutByte(HInvoke* invoke) { in VisitUnsafePutByte() argument
1192 VisitJdkUnsafePutByte(invoke); in VisitUnsafePutByte()
1195 void IntrinsicCodeGeneratorARM64::VisitJdkUnsafePut(HInvoke* invoke) { in VisitJdkUnsafePut() argument
1196 GenUnsafePut(invoke, in VisitJdkUnsafePut()
1202 void IntrinsicCodeGeneratorARM64::VisitJdkUnsafePutAbsolute(HInvoke* invoke) { in VisitJdkUnsafePutAbsolute() argument
1203 GenUnsafePutAbsolute(invoke, in VisitJdkUnsafePutAbsolute()
1209 void IntrinsicCodeGeneratorARM64::VisitJdkUnsafePutOrdered(HInvoke* invoke) { in VisitJdkUnsafePutOrdered() argument
1210 GenUnsafePut(invoke, in VisitJdkUnsafePutOrdered()
1216 void IntrinsicCodeGeneratorARM64::VisitJdkUnsafePutVolatile(HInvoke* invoke) { in VisitJdkUnsafePutVolatile() argument
1217 GenUnsafePut(invoke, in VisitJdkUnsafePutVolatile()
1223 void IntrinsicCodeGeneratorARM64::VisitJdkUnsafePutRelease(HInvoke* invoke) { in VisitJdkUnsafePutRelease() argument
1224 GenUnsafePut(invoke, in VisitJdkUnsafePutRelease()
1230 void IntrinsicCodeGeneratorARM64::VisitJdkUnsafePutReference(HInvoke* invoke) { in VisitJdkUnsafePutReference() argument
1231 GenUnsafePut(invoke, in VisitJdkUnsafePutReference()
1237 void IntrinsicCodeGeneratorARM64::VisitJdkUnsafePutObjectOrdered(HInvoke* invoke) { in VisitJdkUnsafePutObjectOrdered() argument
1238 GenUnsafePut(invoke, in VisitJdkUnsafePutObjectOrdered()
1244 void IntrinsicCodeGeneratorARM64::VisitJdkUnsafePutReferenceVolatile(HInvoke* invoke) { in VisitJdkUnsafePutReferenceVolatile() argument
1245 GenUnsafePut(invoke, in VisitJdkUnsafePutReferenceVolatile()
1251 void IntrinsicCodeGeneratorARM64::VisitJdkUnsafePutReferenceRelease(HInvoke* invoke) { in VisitJdkUnsafePutReferenceRelease() argument
1252 GenUnsafePut(invoke, in VisitJdkUnsafePutReferenceRelease()
1258 void IntrinsicCodeGeneratorARM64::VisitJdkUnsafePutLong(HInvoke* invoke) { in VisitJdkUnsafePutLong() argument
1259 GenUnsafePut(invoke, in VisitJdkUnsafePutLong()
1265 void IntrinsicCodeGeneratorARM64::VisitJdkUnsafePutLongOrdered(HInvoke* invoke) { in VisitJdkUnsafePutLongOrdered() argument
1266 GenUnsafePut(invoke, in VisitJdkUnsafePutLongOrdered()
1272 void IntrinsicCodeGeneratorARM64::VisitJdkUnsafePutLongVolatile(HInvoke* invoke) { in VisitJdkUnsafePutLongVolatile() argument
1273 GenUnsafePut(invoke, in VisitJdkUnsafePutLongVolatile()
1279 void IntrinsicCodeGeneratorARM64::VisitJdkUnsafePutLongRelease(HInvoke* invoke) { in VisitJdkUnsafePutLongRelease() argument
1280 GenUnsafePut(invoke, in VisitJdkUnsafePutLongRelease()
1286 void IntrinsicCodeGeneratorARM64::VisitJdkUnsafePutByte(HInvoke* invoke) { in VisitJdkUnsafePutByte() argument
1287 GenUnsafePut(invoke, in VisitJdkUnsafePutByte()
1295 HInvoke* invoke, in CreateUnsafeCASLocations() argument
1297 const bool can_call = codegen->EmitReadBarrier() && IsUnsafeCASReference(invoke); in CreateUnsafeCASLocations()
1299 new (allocator) LocationSummary(invoke, in CreateUnsafeCASLocations()
1485 ReadBarrierCasSlowPathARM64(HInvoke* invoke, in ReadBarrierCasSlowPathARM64() argument
1497 : SlowPathCodeARM64(invoke), in ReadBarrierCasSlowPathARM64()
1513 invoke, in ReadBarrierCasSlowPathARM64()
1521 invoke, in ReadBarrierCasSlowPathARM64()
1624 static void GenUnsafeCas(HInvoke* invoke, DataType::Type type, CodeGeneratorARM64* codegen) { in GenUnsafeCas() argument
1626 LocationSummary* locations = invoke->GetLocations(); in GenUnsafeCas()
1658 invoke, in GenUnsafeCas()
1693 void IntrinsicLocationsBuilderARM64::VisitUnsafeCASInt(HInvoke* invoke) { in VisitUnsafeCASInt() argument
1694 VisitJdkUnsafeCASInt(invoke); in VisitUnsafeCASInt()
1696 void IntrinsicLocationsBuilderARM64::VisitUnsafeCASLong(HInvoke* invoke) { in VisitUnsafeCASLong() argument
1697 VisitJdkUnsafeCASLong(invoke); in VisitUnsafeCASLong()
1699 void IntrinsicLocationsBuilderARM64::VisitUnsafeCASObject(HInvoke* invoke) { in VisitUnsafeCASObject() argument
1700 VisitJdkUnsafeCASObject(invoke); in VisitUnsafeCASObject()
1703 void IntrinsicLocationsBuilderARM64::VisitJdkUnsafeCASInt(HInvoke* invoke) { in VisitJdkUnsafeCASInt() argument
1705 VisitJdkUnsafeCompareAndSetInt(invoke); in VisitJdkUnsafeCASInt()
1707 void IntrinsicLocationsBuilderARM64::VisitJdkUnsafeCASLong(HInvoke* invoke) { in VisitJdkUnsafeCASLong() argument
1709 VisitJdkUnsafeCompareAndSetLong(invoke); in VisitJdkUnsafeCASLong()
1711 void IntrinsicLocationsBuilderARM64::VisitJdkUnsafeCASObject(HInvoke* invoke) { in VisitJdkUnsafeCASObject() argument
1713 VisitJdkUnsafeCompareAndSetReference(invoke); in VisitJdkUnsafeCASObject()
1716 void IntrinsicLocationsBuilderARM64::VisitJdkUnsafeCompareAndSetInt(HInvoke* invoke) { in VisitJdkUnsafeCompareAndSetInt() argument
1717 CreateUnsafeCASLocations(allocator_, invoke, codegen_); in VisitJdkUnsafeCompareAndSetInt()
1719 void IntrinsicLocationsBuilderARM64::VisitJdkUnsafeCompareAndSetLong(HInvoke* invoke) { in VisitJdkUnsafeCompareAndSetLong() argument
1720 CreateUnsafeCASLocations(allocator_, invoke, codegen_); in VisitJdkUnsafeCompareAndSetLong()
1722 void IntrinsicLocationsBuilderARM64::VisitJdkUnsafeCompareAndSetReference(HInvoke* invoke) { in VisitJdkUnsafeCompareAndSetReference() argument
1728 CreateUnsafeCASLocations(allocator_, invoke, codegen_); in VisitJdkUnsafeCompareAndSetReference()
1731 LocationSummary* locations = invoke->GetLocations(); in VisitJdkUnsafeCompareAndSetReference()
1747 void IntrinsicCodeGeneratorARM64::VisitUnsafeCASInt(HInvoke* invoke) { in VisitUnsafeCASInt() argument
1748 VisitJdkUnsafeCASInt(invoke); in VisitUnsafeCASInt()
1750 void IntrinsicCodeGeneratorARM64::VisitUnsafeCASLong(HInvoke* invoke) { in VisitUnsafeCASLong() argument
1751 VisitJdkUnsafeCASLong(invoke); in VisitUnsafeCASLong()
1753 void IntrinsicCodeGeneratorARM64::VisitUnsafeCASObject(HInvoke* invoke) { in VisitUnsafeCASObject() argument
1754 VisitJdkUnsafeCASObject(invoke); in VisitUnsafeCASObject()
1757 void IntrinsicCodeGeneratorARM64::VisitJdkUnsafeCASInt(HInvoke* invoke) { in VisitJdkUnsafeCASInt() argument
1759 VisitJdkUnsafeCompareAndSetInt(invoke); in VisitJdkUnsafeCASInt()
1761 void IntrinsicCodeGeneratorARM64::VisitJdkUnsafeCASLong(HInvoke* invoke) { in VisitJdkUnsafeCASLong() argument
1763 VisitJdkUnsafeCompareAndSetLong(invoke); in VisitJdkUnsafeCASLong()
1765 void IntrinsicCodeGeneratorARM64::VisitJdkUnsafeCASObject(HInvoke* invoke) { in VisitJdkUnsafeCASObject() argument
1767 VisitJdkUnsafeCompareAndSetReference(invoke); in VisitJdkUnsafeCASObject()
1770 void IntrinsicCodeGeneratorARM64::VisitJdkUnsafeCompareAndSetInt(HInvoke* invoke) { in VisitJdkUnsafeCompareAndSetInt() argument
1771 GenUnsafeCas(invoke, DataType::Type::kInt32, codegen_); in VisitJdkUnsafeCompareAndSetInt()
1773 void IntrinsicCodeGeneratorARM64::VisitJdkUnsafeCompareAndSetLong(HInvoke* invoke) { in VisitJdkUnsafeCompareAndSetLong() argument
1774 GenUnsafeCas(invoke, DataType::Type::kInt64, codegen_); in VisitJdkUnsafeCompareAndSetLong()
1776 void IntrinsicCodeGeneratorARM64::VisitJdkUnsafeCompareAndSetReference(HInvoke* invoke) { in VisitJdkUnsafeCompareAndSetReference() argument
1780 GenUnsafeCas(invoke, DataType::Type::kReference, codegen_); in VisitJdkUnsafeCompareAndSetReference()
1874 HInvoke* invoke, in CreateUnsafeGetAndUpdateLocations() argument
1876 const bool can_call = codegen->EmitReadBarrier() && IsUnsafeGetAndSetReference(invoke); in CreateUnsafeGetAndUpdateLocations()
1878 new (allocator) LocationSummary(invoke, in CreateUnsafeGetAndUpdateLocations()
1893 DataType::Type return_type = invoke->GetType(); in CreateUnsafeGetAndUpdateLocations()
1902 static void GenUnsafeGetAndUpdate(HInvoke* invoke, in GenUnsafeGetAndUpdate() argument
1911 LocationSummary* locations = invoke->GetLocations(); in GenUnsafeGetAndUpdate()
1913 DataType::Type return_type = invoke->GetType(); in GenUnsafeGetAndUpdate()
1946 codegen->GenerateReadBarrierSlow(invoke, in GenUnsafeGetAndUpdate()
1956 void IntrinsicLocationsBuilderARM64::VisitUnsafeGetAndAddInt(HInvoke* invoke) { in VisitUnsafeGetAndAddInt() argument
1957 VisitJdkUnsafeGetAndAddInt(invoke); in VisitUnsafeGetAndAddInt()
1959 void IntrinsicLocationsBuilderARM64::VisitUnsafeGetAndAddLong(HInvoke* invoke) { in VisitUnsafeGetAndAddLong() argument
1960 VisitJdkUnsafeGetAndAddLong(invoke); in VisitUnsafeGetAndAddLong()
1962 void IntrinsicLocationsBuilderARM64::VisitUnsafeGetAndSetInt(HInvoke* invoke) { in VisitUnsafeGetAndSetInt() argument
1963 VisitJdkUnsafeGetAndSetInt(invoke); in VisitUnsafeGetAndSetInt()
1965 void IntrinsicLocationsBuilderARM64::VisitUnsafeGetAndSetLong(HInvoke* invoke) { in VisitUnsafeGetAndSetLong() argument
1966 VisitJdkUnsafeGetAndSetLong(invoke); in VisitUnsafeGetAndSetLong()
1968 void IntrinsicLocationsBuilderARM64::VisitUnsafeGetAndSetObject(HInvoke* invoke) { in VisitUnsafeGetAndSetObject() argument
1969 VisitJdkUnsafeGetAndSetReference(invoke); in VisitUnsafeGetAndSetObject()
1972 void IntrinsicLocationsBuilderARM64::VisitJdkUnsafeGetAndAddInt(HInvoke* invoke) { in VisitJdkUnsafeGetAndAddInt() argument
1973 CreateUnsafeGetAndUpdateLocations(allocator_, invoke, codegen_); in VisitJdkUnsafeGetAndAddInt()
1975 void IntrinsicLocationsBuilderARM64::VisitJdkUnsafeGetAndAddLong(HInvoke* invoke) { in VisitJdkUnsafeGetAndAddLong() argument
1976 CreateUnsafeGetAndUpdateLocations(allocator_, invoke, codegen_); in VisitJdkUnsafeGetAndAddLong()
1978 void IntrinsicLocationsBuilderARM64::VisitJdkUnsafeGetAndSetInt(HInvoke* invoke) { in VisitJdkUnsafeGetAndSetInt() argument
1979 CreateUnsafeGetAndUpdateLocations(allocator_, invoke, codegen_); in VisitJdkUnsafeGetAndSetInt()
1981 void IntrinsicLocationsBuilderARM64::VisitJdkUnsafeGetAndSetLong(HInvoke* invoke) { in VisitJdkUnsafeGetAndSetLong() argument
1982 CreateUnsafeGetAndUpdateLocations(allocator_, invoke, codegen_); in VisitJdkUnsafeGetAndSetLong()
1984 void IntrinsicLocationsBuilderARM64::VisitJdkUnsafeGetAndSetReference(HInvoke* invoke) { in VisitJdkUnsafeGetAndSetReference() argument
1985 CreateUnsafeGetAndUpdateLocations(allocator_, invoke, codegen_); in VisitJdkUnsafeGetAndSetReference()
1988 void IntrinsicCodeGeneratorARM64::VisitUnsafeGetAndAddInt(HInvoke* invoke) { in VisitUnsafeGetAndAddInt() argument
1989 VisitJdkUnsafeGetAndAddInt(invoke); in VisitUnsafeGetAndAddInt()
1991 void IntrinsicCodeGeneratorARM64::VisitUnsafeGetAndAddLong(HInvoke* invoke) { in VisitUnsafeGetAndAddLong() argument
1992 VisitJdkUnsafeGetAndAddLong(invoke); in VisitUnsafeGetAndAddLong()
1994 void IntrinsicCodeGeneratorARM64::VisitUnsafeGetAndSetInt(HInvoke* invoke) { in VisitUnsafeGetAndSetInt() argument
1995 VisitJdkUnsafeGetAndSetInt(invoke); in VisitUnsafeGetAndSetInt()
1997 void IntrinsicCodeGeneratorARM64::VisitUnsafeGetAndSetLong(HInvoke* invoke) { in VisitUnsafeGetAndSetLong() argument
1998 VisitJdkUnsafeGetAndSetLong(invoke); in VisitUnsafeGetAndSetLong()
2000 void IntrinsicCodeGeneratorARM64::VisitUnsafeGetAndSetObject(HInvoke* invoke) { in VisitUnsafeGetAndSetObject() argument
2001 VisitJdkUnsafeGetAndSetReference(invoke); in VisitUnsafeGetAndSetObject()
2004 void IntrinsicCodeGeneratorARM64::VisitJdkUnsafeGetAndAddInt(HInvoke* invoke) { in VisitJdkUnsafeGetAndAddInt() argument
2005 GenUnsafeGetAndUpdate(invoke, DataType::Type::kInt32, codegen_, GetAndUpdateOp::kAdd); in VisitJdkUnsafeGetAndAddInt()
2007 void IntrinsicCodeGeneratorARM64::VisitJdkUnsafeGetAndAddLong(HInvoke* invoke) { in VisitJdkUnsafeGetAndAddLong() argument
2008 GenUnsafeGetAndUpdate(invoke, DataType::Type::kInt64, codegen_, GetAndUpdateOp::kAdd); in VisitJdkUnsafeGetAndAddLong()
2010 void IntrinsicCodeGeneratorARM64::VisitJdkUnsafeGetAndSetInt(HInvoke* invoke) { in VisitJdkUnsafeGetAndSetInt() argument
2011 GenUnsafeGetAndUpdate(invoke, DataType::Type::kInt32, codegen_, GetAndUpdateOp::kSet); in VisitJdkUnsafeGetAndSetInt()
2013 void IntrinsicCodeGeneratorARM64::VisitJdkUnsafeGetAndSetLong(HInvoke* invoke) { in VisitJdkUnsafeGetAndSetLong() argument
2014 GenUnsafeGetAndUpdate(invoke, DataType::Type::kInt64, codegen_, GetAndUpdateOp::kSet); in VisitJdkUnsafeGetAndSetLong()
2016 void IntrinsicCodeGeneratorARM64::VisitJdkUnsafeGetAndSetReference(HInvoke* invoke) { in VisitJdkUnsafeGetAndSetReference() argument
2017 GenUnsafeGetAndUpdate(invoke, DataType::Type::kReference, codegen_, GetAndUpdateOp::kSet); in VisitJdkUnsafeGetAndSetReference()
2020 void IntrinsicLocationsBuilderARM64::VisitStringCompareTo(HInvoke* invoke) { in VisitStringCompareTo() argument
2022 new (allocator_) LocationSummary(invoke, in VisitStringCompareTo()
2023 invoke->InputAt(1)->CanBeNull() in VisitStringCompareTo()
2037 void IntrinsicCodeGeneratorARM64::VisitStringCompareTo(HInvoke* invoke) { in VisitStringCompareTo() argument
2039 LocationSummary* locations = invoke->GetLocations(); in VisitStringCompareTo()
2041 Register str = InputRegisterAt(invoke, 0); in VisitStringCompareTo()
2042 Register arg = InputRegisterAt(invoke, 1); in VisitStringCompareTo()
2045 Register out = OutputRegister(invoke); in VisitStringCompareTo()
2065 DCHECK(!invoke->CanDoImplicitNullCheckOn(invoke->InputAt(0))); in VisitStringCompareTo()
2069 const bool can_slow_path = invoke->InputAt(1)->CanBeNull(); in VisitStringCompareTo()
2071 slow_path = new (codegen_->GetScopedAllocator()) IntrinsicSlowPathARM64(invoke); in VisitStringCompareTo()
2250 void IntrinsicLocationsBuilderARM64::VisitStringEquals(HInvoke* invoke) { in VisitStringEquals() argument
2252 new (allocator_) LocationSummary(invoke, LocationSummary::kNoCall, kIntrinsified); in VisitStringEquals()
2259 const char* const_string = GetConstString(invoke->InputAt(0), &const_string_length); in VisitStringEquals()
2261 const_string = GetConstString(invoke->InputAt(1), &const_string_length); in VisitStringEquals()
2277 void IntrinsicCodeGeneratorARM64::VisitStringEquals(HInvoke* invoke) { in VisitStringEquals() argument
2279 LocationSummary* locations = invoke->GetLocations(); in VisitStringEquals()
2300 DCHECK(!invoke->CanDoImplicitNullCheckOn(invoke->InputAt(0))); in VisitStringEquals()
2302 StringEqualsOptimizations optimizations(invoke); in VisitStringEquals()
2334 const char* const_string = GetConstString(invoke->InputAt(0), &const_string_length); in VisitStringEquals()
2336 const_string = GetConstString(invoke->InputAt(1), &const_string_length); in VisitStringEquals()
2439 static void GenerateVisitStringIndexOf(HInvoke* invoke, in GenerateVisitStringIndexOf() argument
2443 LocationSummary* locations = invoke->GetLocations(); in GenerateVisitStringIndexOf()
2446 DCHECK(!invoke->CanDoImplicitNullCheckOn(invoke->InputAt(0))); in GenerateVisitStringIndexOf()
2451 HInstruction* code_point = invoke->InputAt(1); in GenerateVisitStringIndexOf()
2456 slow_path = new (codegen->GetScopedAllocator()) IntrinsicSlowPathARM64(invoke); in GenerateVisitStringIndexOf()
2465 slow_path = new (codegen->GetScopedAllocator()) IntrinsicSlowPathARM64(invoke); in GenerateVisitStringIndexOf()
2476 codegen->InvokeRuntime(kQuickIndexOf, invoke, invoke->GetDexPc(), slow_path); in GenerateVisitStringIndexOf()
2484 void IntrinsicLocationsBuilderARM64::VisitStringIndexOf(HInvoke* invoke) { in VisitStringIndexOf() argument
2486 invoke, LocationSummary::kCallOnMainAndSlowPath, kIntrinsified); in VisitStringIndexOf()
2498 void IntrinsicCodeGeneratorARM64::VisitStringIndexOf(HInvoke* invoke) { in VisitStringIndexOf() argument
2499 GenerateVisitStringIndexOf(invoke, GetVIXLAssembler(), codegen_, /* start_at_zero= */ true); in VisitStringIndexOf()
2502 void IntrinsicLocationsBuilderARM64::VisitStringIndexOfAfter(HInvoke* invoke) { in VisitStringIndexOfAfter() argument
2504 invoke, LocationSummary::kCallOnMainAndSlowPath, kIntrinsified); in VisitStringIndexOfAfter()
2514 void IntrinsicCodeGeneratorARM64::VisitStringIndexOfAfter(HInvoke* invoke) { in VisitStringIndexOfAfter() argument
2515 GenerateVisitStringIndexOf(invoke, GetVIXLAssembler(), codegen_, /* start_at_zero= */ false); in VisitStringIndexOfAfter()
2518 void IntrinsicLocationsBuilderARM64::VisitStringNewStringFromBytes(HInvoke* invoke) { in VisitStringNewStringFromBytes() argument
2520 invoke, LocationSummary::kCallOnMainAndSlowPath, kIntrinsified); in VisitStringNewStringFromBytes()
2529 void IntrinsicCodeGeneratorARM64::VisitStringNewStringFromBytes(HInvoke* invoke) { in VisitStringNewStringFromBytes() argument
2531 LocationSummary* locations = invoke->GetLocations(); in VisitStringNewStringFromBytes()
2536 new (codegen_->GetScopedAllocator()) IntrinsicSlowPathARM64(invoke); in VisitStringNewStringFromBytes()
2540 codegen_->InvokeRuntime(kQuickAllocStringFromBytes, invoke, invoke->GetDexPc(), slow_path); in VisitStringNewStringFromBytes()
2545 void IntrinsicLocationsBuilderARM64::VisitStringNewStringFromChars(HInvoke* invoke) { in VisitStringNewStringFromChars() argument
2547 new (allocator_) LocationSummary(invoke, LocationSummary::kCallOnMainOnly, kIntrinsified); in VisitStringNewStringFromChars()
2555 void IntrinsicCodeGeneratorARM64::VisitStringNewStringFromChars(HInvoke* invoke) { in VisitStringNewStringFromChars() argument
2562 codegen_->InvokeRuntime(kQuickAllocStringFromChars, invoke, invoke->GetDexPc()); in VisitStringNewStringFromChars()
2566 void IntrinsicLocationsBuilderARM64::VisitStringNewStringFromString(HInvoke* invoke) { in VisitStringNewStringFromString() argument
2568 invoke, LocationSummary::kCallOnMainAndSlowPath, kIntrinsified); in VisitStringNewStringFromString()
2574 void IntrinsicCodeGeneratorARM64::VisitStringNewStringFromString(HInvoke* invoke) { in VisitStringNewStringFromString() argument
2576 LocationSummary* locations = invoke->GetLocations(); in VisitStringNewStringFromString()
2581 new (codegen_->GetScopedAllocator()) IntrinsicSlowPathARM64(invoke); in VisitStringNewStringFromString()
2585 codegen_->InvokeRuntime(kQuickAllocStringFromString, invoke, invoke->GetDexPc(), slow_path); in VisitStringNewStringFromString()
2590 static void CreateFPToFPCallLocations(ArenaAllocator* allocator, HInvoke* invoke) { in CreateFPToFPCallLocations() argument
2591 DCHECK_EQ(invoke->GetNumberOfArguments(), 1U); in CreateFPToFPCallLocations()
2592 DCHECK(DataType::IsFloatingPointType(invoke->InputAt(0)->GetType())); in CreateFPToFPCallLocations()
2593 DCHECK(DataType::IsFloatingPointType(invoke->GetType())); in CreateFPToFPCallLocations()
2596 new (allocator) LocationSummary(invoke, LocationSummary::kCallOnMainOnly, kIntrinsified); in CreateFPToFPCallLocations()
2600 locations->SetOut(calling_convention.GetReturnLocation(invoke->GetType())); in CreateFPToFPCallLocations()
2603 static void CreateFPFPToFPCallLocations(ArenaAllocator* allocator, HInvoke* invoke) { in CreateFPFPToFPCallLocations() argument
2604 DCHECK_EQ(invoke->GetNumberOfArguments(), 2U); in CreateFPFPToFPCallLocations()
2605 DCHECK(DataType::IsFloatingPointType(invoke->InputAt(0)->GetType())); in CreateFPFPToFPCallLocations()
2606 DCHECK(DataType::IsFloatingPointType(invoke->InputAt(1)->GetType())); in CreateFPFPToFPCallLocations()
2607 DCHECK(DataType::IsFloatingPointType(invoke->GetType())); in CreateFPFPToFPCallLocations()
2610 new (allocator) LocationSummary(invoke, LocationSummary::kCallOnMainOnly, kIntrinsified); in CreateFPFPToFPCallLocations()
2615 locations->SetOut(calling_convention.GetReturnLocation(invoke->GetType())); in CreateFPFPToFPCallLocations()
2618 static void CreateFPFPFPToFPLocations(ArenaAllocator* allocator, HInvoke* invoke) { in CreateFPFPFPToFPLocations() argument
2619 DCHECK_EQ(invoke->GetNumberOfArguments(), 3U); in CreateFPFPFPToFPLocations()
2620 DCHECK(DataType::IsFloatingPointType(invoke->InputAt(0)->GetType())); in CreateFPFPFPToFPLocations()
2621 DCHECK(DataType::IsFloatingPointType(invoke->InputAt(1)->GetType())); in CreateFPFPFPToFPLocations()
2622 DCHECK(DataType::IsFloatingPointType(invoke->InputAt(2)->GetType())); in CreateFPFPFPToFPLocations()
2623 DCHECK(DataType::IsFloatingPointType(invoke->GetType())); in CreateFPFPFPToFPLocations()
2626 new (allocator) LocationSummary(invoke, LocationSummary::kNoCall, kIntrinsified); in CreateFPFPFPToFPLocations()
2634 static void GenFPToFPCall(HInvoke* invoke, in GenFPToFPCall() argument
2637 codegen->InvokeRuntime(entry, invoke, invoke->GetDexPc()); in GenFPToFPCall()
2640 void IntrinsicLocationsBuilderARM64::VisitMathCos(HInvoke* invoke) { in VisitMathCos() argument
2641 CreateFPToFPCallLocations(allocator_, invoke); in VisitMathCos()
2644 void IntrinsicCodeGeneratorARM64::VisitMathCos(HInvoke* invoke) { in VisitMathCos() argument
2645 GenFPToFPCall(invoke, codegen_, kQuickCos); in VisitMathCos()
2648 void IntrinsicLocationsBuilderARM64::VisitMathSin(HInvoke* invoke) { in VisitMathSin() argument
2649 CreateFPToFPCallLocations(allocator_, invoke); in VisitMathSin()
2652 void IntrinsicCodeGeneratorARM64::VisitMathSin(HInvoke* invoke) { in VisitMathSin() argument
2653 GenFPToFPCall(invoke, codegen_, kQuickSin); in VisitMathSin()
2656 void IntrinsicLocationsBuilderARM64::VisitMathAcos(HInvoke* invoke) { in VisitMathAcos() argument
2657 CreateFPToFPCallLocations(allocator_, invoke); in VisitMathAcos()
2660 void IntrinsicCodeGeneratorARM64::VisitMathAcos(HInvoke* invoke) { in VisitMathAcos() argument
2661 GenFPToFPCall(invoke, codegen_, kQuickAcos); in VisitMathAcos()
2664 void IntrinsicLocationsBuilderARM64::VisitMathAsin(HInvoke* invoke) { in VisitMathAsin() argument
2665 CreateFPToFPCallLocations(allocator_, invoke); in VisitMathAsin()
2668 void IntrinsicCodeGeneratorARM64::VisitMathAsin(HInvoke* invoke) { in VisitMathAsin() argument
2669 GenFPToFPCall(invoke, codegen_, kQuickAsin); in VisitMathAsin()
2672 void IntrinsicLocationsBuilderARM64::VisitMathAtan(HInvoke* invoke) { in VisitMathAtan() argument
2673 CreateFPToFPCallLocations(allocator_, invoke); in VisitMathAtan()
2676 void IntrinsicCodeGeneratorARM64::VisitMathAtan(HInvoke* invoke) { in VisitMathAtan() argument
2677 GenFPToFPCall(invoke, codegen_, kQuickAtan); in VisitMathAtan()
2680 void IntrinsicLocationsBuilderARM64::VisitMathCbrt(HInvoke* invoke) { in VisitMathCbrt() argument
2681 CreateFPToFPCallLocations(allocator_, invoke); in VisitMathCbrt()
2684 void IntrinsicCodeGeneratorARM64::VisitMathCbrt(HInvoke* invoke) { in VisitMathCbrt() argument
2685 GenFPToFPCall(invoke, codegen_, kQuickCbrt); in VisitMathCbrt()
2688 void IntrinsicLocationsBuilderARM64::VisitMathCosh(HInvoke* invoke) { in VisitMathCosh() argument
2689 CreateFPToFPCallLocations(allocator_, invoke); in VisitMathCosh()
2692 void IntrinsicCodeGeneratorARM64::VisitMathCosh(HInvoke* invoke) { in VisitMathCosh() argument
2693 GenFPToFPCall(invoke, codegen_, kQuickCosh); in VisitMathCosh()
2696 void IntrinsicLocationsBuilderARM64::VisitMathExp(HInvoke* invoke) { in VisitMathExp() argument
2697 CreateFPToFPCallLocations(allocator_, invoke); in VisitMathExp()
2700 void IntrinsicCodeGeneratorARM64::VisitMathExp(HInvoke* invoke) { in VisitMathExp() argument
2701 GenFPToFPCall(invoke, codegen_, kQuickExp); in VisitMathExp()
2704 void IntrinsicLocationsBuilderARM64::VisitMathExpm1(HInvoke* invoke) { in VisitMathExpm1() argument
2705 CreateFPToFPCallLocations(allocator_, invoke); in VisitMathExpm1()
2708 void IntrinsicCodeGeneratorARM64::VisitMathExpm1(HInvoke* invoke) { in VisitMathExpm1() argument
2709 GenFPToFPCall(invoke, codegen_, kQuickExpm1); in VisitMathExpm1()
2712 void IntrinsicLocationsBuilderARM64::VisitMathLog(HInvoke* invoke) { in VisitMathLog() argument
2713 CreateFPToFPCallLocations(allocator_, invoke); in VisitMathLog()
2716 void IntrinsicCodeGeneratorARM64::VisitMathLog(HInvoke* invoke) { in VisitMathLog() argument
2717 GenFPToFPCall(invoke, codegen_, kQuickLog); in VisitMathLog()
2720 void IntrinsicLocationsBuilderARM64::VisitMathLog10(HInvoke* invoke) { in VisitMathLog10() argument
2721 CreateFPToFPCallLocations(allocator_, invoke); in VisitMathLog10()
2724 void IntrinsicCodeGeneratorARM64::VisitMathLog10(HInvoke* invoke) { in VisitMathLog10() argument
2725 GenFPToFPCall(invoke, codegen_, kQuickLog10); in VisitMathLog10()
2728 void IntrinsicLocationsBuilderARM64::VisitMathSinh(HInvoke* invoke) { in VisitMathSinh() argument
2729 CreateFPToFPCallLocations(allocator_, invoke); in VisitMathSinh()
2732 void IntrinsicCodeGeneratorARM64::VisitMathSinh(HInvoke* invoke) { in VisitMathSinh() argument
2733 GenFPToFPCall(invoke, codegen_, kQuickSinh); in VisitMathSinh()
2736 void IntrinsicLocationsBuilderARM64::VisitMathTan(HInvoke* invoke) { in VisitMathTan() argument
2737 CreateFPToFPCallLocations(allocator_, invoke); in VisitMathTan()
2740 void IntrinsicCodeGeneratorARM64::VisitMathTan(HInvoke* invoke) { in VisitMathTan() argument
2741 GenFPToFPCall(invoke, codegen_, kQuickTan); in VisitMathTan()
2744 void IntrinsicLocationsBuilderARM64::VisitMathTanh(HInvoke* invoke) { in VisitMathTanh() argument
2745 CreateFPToFPCallLocations(allocator_, invoke); in VisitMathTanh()
2748 void IntrinsicCodeGeneratorARM64::VisitMathTanh(HInvoke* invoke) { in VisitMathTanh() argument
2749 GenFPToFPCall(invoke, codegen_, kQuickTanh); in VisitMathTanh()
2752 void IntrinsicLocationsBuilderARM64::VisitMathAtan2(HInvoke* invoke) { in VisitMathAtan2() argument
2753 CreateFPFPToFPCallLocations(allocator_, invoke); in VisitMathAtan2()
2756 void IntrinsicCodeGeneratorARM64::VisitMathAtan2(HInvoke* invoke) { in VisitMathAtan2() argument
2757 GenFPToFPCall(invoke, codegen_, kQuickAtan2); in VisitMathAtan2()
2760 void IntrinsicLocationsBuilderARM64::VisitMathPow(HInvoke* invoke) { in VisitMathPow() argument
2761 CreateFPFPToFPCallLocations(allocator_, invoke); in VisitMathPow()
2764 void IntrinsicCodeGeneratorARM64::VisitMathPow(HInvoke* invoke) { in VisitMathPow() argument
2765 GenFPToFPCall(invoke, codegen_, kQuickPow); in VisitMathPow()
2768 void IntrinsicLocationsBuilderARM64::VisitMathHypot(HInvoke* invoke) { in VisitMathHypot() argument
2769 CreateFPFPToFPCallLocations(allocator_, invoke); in VisitMathHypot()
2772 void IntrinsicCodeGeneratorARM64::VisitMathHypot(HInvoke* invoke) { in VisitMathHypot() argument
2773 GenFPToFPCall(invoke, codegen_, kQuickHypot); in VisitMathHypot()
2776 void IntrinsicLocationsBuilderARM64::VisitMathNextAfter(HInvoke* invoke) { in VisitMathNextAfter() argument
2777 CreateFPFPToFPCallLocations(allocator_, invoke); in VisitMathNextAfter()
2780 void IntrinsicCodeGeneratorARM64::VisitMathNextAfter(HInvoke* invoke) { in VisitMathNextAfter() argument
2781 GenFPToFPCall(invoke, codegen_, kQuickNextAfter); in VisitMathNextAfter()
2784 void IntrinsicLocationsBuilderARM64::VisitStringGetCharsNoCheck(HInvoke* invoke) { in VisitStringGetCharsNoCheck() argument
2786 new (allocator_) LocationSummary(invoke, LocationSummary::kNoCall, kIntrinsified); in VisitStringGetCharsNoCheck()
2796 void IntrinsicCodeGeneratorARM64::VisitStringGetCharsNoCheck(HInvoke* invoke) { in VisitStringGetCharsNoCheck() argument
2798 LocationSummary* locations = invoke->GetLocations(); in VisitStringGetCharsNoCheck()
2932 void IntrinsicLocationsBuilderARM64::VisitSystemArrayCopyChar(HInvoke* invoke) { in VisitSystemArrayCopyChar() argument
2935 HIntConstant* src_pos = invoke->InputAt(1)->AsIntConstantOrNull(); in VisitSystemArrayCopyChar()
2936 HIntConstant* dst_pos = invoke->InputAt(3)->AsIntConstantOrNull(); in VisitSystemArrayCopyChar()
2947 HIntConstant* length = invoke->InputAt(4)->AsIntConstantOrNull(); in VisitSystemArrayCopyChar()
2956 ArenaAllocator* allocator = invoke->GetBlock()->GetGraph()->GetAllocator(); in VisitSystemArrayCopyChar()
2958 new (allocator) LocationSummary(invoke, LocationSummary::kCallOnSlowPath, kIntrinsified); in VisitSystemArrayCopyChar()
2961 locations->SetInAt(1, LocationForSystemArrayCopyInput(invoke->InputAt(1))); in VisitSystemArrayCopyChar()
2963 locations->SetInAt(3, LocationForSystemArrayCopyInput(invoke->InputAt(3))); in VisitSystemArrayCopyChar()
2964 locations->SetInAt(4, LocationForSystemArrayCopyInput(invoke->InputAt(4))); in VisitSystemArrayCopyChar()
3064 void IntrinsicCodeGeneratorARM64::VisitSystemArrayCopyChar(HInvoke* invoke) { in VisitSystemArrayCopyChar() argument
3066 LocationSummary* locations = invoke->GetLocations(); in VisitSystemArrayCopyChar()
3074 new (codegen_->GetScopedAllocator()) IntrinsicSlowPathARM64(invoke); in VisitSystemArrayCopyChar()
3238 void IntrinsicLocationsBuilderARM64::VisitSystemArrayCopy(HInvoke* invoke) { in VisitSystemArrayCopy() argument
3247 invoke, kSystemArrayCopyThreshold, kInitialNumTemps); in VisitSystemArrayCopy()
3249 locations->SetInAt(1, LocationForSystemArrayCopyInput(invoke->InputAt(1))); in VisitSystemArrayCopy()
3250 locations->SetInAt(3, LocationForSystemArrayCopyInput(invoke->InputAt(3))); in VisitSystemArrayCopy()
3251 locations->SetInAt(4, LocationForSystemArrayCopyInput(invoke->InputAt(4))); in VisitSystemArrayCopy()
3268 void IntrinsicCodeGeneratorARM64::VisitSystemArrayCopy(HInvoke* invoke) { in VisitSystemArrayCopy() argument
3274 LocationSummary* locations = invoke->GetLocations(); in VisitSystemArrayCopy()
3293 new (codegen_->GetScopedAllocator()) IntrinsicSlowPathARM64(invoke); in VisitSystemArrayCopy()
3297 SystemArrayCopyOptimizations optimizations(invoke); in VisitSystemArrayCopy()
3326 __ Cmp(RegisterFrom(src_pos, invoke->InputAt(1)->GetType()), in VisitSystemArrayCopy()
3327 OperandFrom(dest_pos, invoke->InputAt(3)->GetType())); in VisitSystemArrayCopy()
3397 codegen_->GenerateFieldLoadWithBakerReadBarrier(invoke, in VisitSystemArrayCopy()
3410 codegen_->GenerateFieldLoadWithBakerReadBarrier(invoke, in VisitSystemArrayCopy()
3539 invoke, LocationFrom(tmp)); in VisitSystemArrayCopy()
3617 void IntrinsicLocationsBuilderARM64::VisitFloatIsInfinite(HInvoke* invoke) { in VisitFloatIsInfinite() argument
3618 CreateFPToIntLocations(allocator_, invoke); in VisitFloatIsInfinite()
3621 void IntrinsicCodeGeneratorARM64::VisitFloatIsInfinite(HInvoke* invoke) { in VisitFloatIsInfinite() argument
3622 GenIsInfinite(invoke->GetLocations(), /* is64bit= */ false, GetVIXLAssembler()); in VisitFloatIsInfinite()
3625 void IntrinsicLocationsBuilderARM64::VisitDoubleIsInfinite(HInvoke* invoke) { in VisitDoubleIsInfinite() argument
3626 CreateFPToIntLocations(allocator_, invoke); in VisitDoubleIsInfinite()
3629 void IntrinsicCodeGeneratorARM64::VisitDoubleIsInfinite(HInvoke* invoke) { in VisitDoubleIsInfinite() argument
3630 GenIsInfinite(invoke->GetLocations(), /* is64bit= */ true, GetVIXLAssembler()); in VisitDoubleIsInfinite()
3634 void IntrinsicLocationsBuilderARM64::Visit##name##ValueOf(HInvoke* invoke) { \
3637 invoke, \
3644 void IntrinsicCodeGeneratorARM64::Visit##name##ValueOf(HInvoke* invoke) { \
3646 IntrinsicVisitor::ComputeValueOfInfo(invoke, \
3652 HandleValueOf(invoke, info, type); \
3657 void IntrinsicCodeGeneratorARM64::HandleValueOf(HInvoke* invoke, in BOXED_TYPES()
3660 LocationSummary* locations = invoke->GetLocations(); in BOXED_TYPES()
3668 codegen_->LoadIntrinsicDeclaringClass(out, invoke); in BOXED_TYPES()
3669 codegen_->InvokeRuntime(kQuickAllocObjectInitialized, invoke, invoke->GetDexPc()); in BOXED_TYPES()
3672 if (invoke->InputAt(0)->IsIntConstant()) { in BOXED_TYPES()
3673 int32_t value = invoke->InputAt(0)->AsIntConstant()->GetValue(); in BOXED_TYPES()
3714 void IntrinsicLocationsBuilderARM64::VisitReferenceGetReferent(HInvoke* invoke) { in VisitReferenceGetReferent() argument
3715 IntrinsicVisitor::CreateReferenceGetReferentLocations(invoke, codegen_); in VisitReferenceGetReferent()
3717 if (codegen_->EmitBakerReadBarrier() && invoke->GetLocations() != nullptr) { in VisitReferenceGetReferent()
3718 invoke->GetLocations()->AddTemp(Location::RequiresRegister()); in VisitReferenceGetReferent()
3722 void IntrinsicCodeGeneratorARM64::VisitReferenceGetReferent(HInvoke* invoke) { in VisitReferenceGetReferent() argument
3724 LocationSummary* locations = invoke->GetLocations(); in VisitReferenceGetReferent()
3729 SlowPathCodeARM64* slow_path = new (GetAllocator()) IntrinsicSlowPathARM64(invoke); in VisitReferenceGetReferent()
3746 codegen_->LoadIntrinsicDeclaringClass(temp, invoke); in VisitReferenceGetReferent()
3760 codegen_->GenerateFieldLoadWithBakerReadBarrier(invoke, in VisitReferenceGetReferent()
3770 invoke, DataType::Type::kReference, WRegisterFrom(out), field, /*needs_null_check=*/ true); in VisitReferenceGetReferent()
3771 codegen_->MaybeGenerateReadBarrierSlow(invoke, out, out, obj, referent_offset); in VisitReferenceGetReferent()
3776 void IntrinsicLocationsBuilderARM64::VisitReferenceRefersTo(HInvoke* invoke) { in VisitReferenceRefersTo() argument
3777 IntrinsicVisitor::CreateReferenceRefersToLocations(invoke, codegen_); in VisitReferenceRefersTo()
3780 void IntrinsicCodeGeneratorARM64::VisitReferenceRefersTo(HInvoke* invoke) { in VisitReferenceRefersTo() argument
3781 LocationSummary* locations = invoke->GetLocations(); in VisitReferenceRefersTo()
3794 codegen_->LoadAcquire(invoke, DataType::Type::kReference, tmp, field, /*needs_null_check=*/ true); in VisitReferenceRefersTo()
3832 void IntrinsicLocationsBuilderARM64::VisitThreadInterrupted(HInvoke* invoke) { in VisitThreadInterrupted() argument
3834 new (allocator_) LocationSummary(invoke, LocationSummary::kNoCall, kIntrinsified); in VisitThreadInterrupted()
3838 void IntrinsicCodeGeneratorARM64::VisitThreadInterrupted(HInvoke* invoke) { in VisitThreadInterrupted() argument
3840 Register out = RegisterFrom(invoke->GetLocations()->Out(), DataType::Type::kInt32); in VisitThreadInterrupted()
3853 void IntrinsicLocationsBuilderARM64::VisitReachabilityFence(HInvoke* invoke) { in VisitReachabilityFence() argument
3855 new (allocator_) LocationSummary(invoke, LocationSummary::kNoCall, kIntrinsified); in VisitReachabilityFence()
3859 void IntrinsicCodeGeneratorARM64::VisitReachabilityFence([[maybe_unused]] HInvoke* invoke) {} in VisitReachabilityFence() argument
3861 void IntrinsicLocationsBuilderARM64::VisitCRC32Update(HInvoke* invoke) { in VisitCRC32Update() argument
3866 LocationSummary* locations = new (allocator_) LocationSummary(invoke, in VisitCRC32Update()
3875 // Lower the invoke of CRC32.update(int crc, int b).
3876 void IntrinsicCodeGeneratorARM64::VisitCRC32Update(HInvoke* invoke) { in VisitCRC32Update() argument
3881 Register crc = InputRegisterAt(invoke, 0); in VisitCRC32Update()
3882 Register val = InputRegisterAt(invoke, 1); in VisitCRC32Update()
3883 Register out = OutputRegister(invoke); in VisitCRC32Update()
4007 void IntrinsicLocationsBuilderARM64::VisitCRC32UpdateBytes(HInvoke* invoke) { in VisitCRC32UpdateBytes() argument
4013 new (allocator_) LocationSummary(invoke, in VisitCRC32UpdateBytes()
4019 locations->SetInAt(2, Location::RegisterOrConstant(invoke->InputAt(2))); in VisitCRC32UpdateBytes()
4025 // Lower the invoke of CRC32.updateBytes(int crc, byte[] b, int off, int len)
4028 void IntrinsicCodeGeneratorARM64::VisitCRC32UpdateBytes(HInvoke* invoke) { in VisitCRC32UpdateBytes() argument
4032 LocationSummary* locations = invoke->GetLocations(); in VisitCRC32UpdateBytes()
4035 new (codegen_->GetScopedAllocator()) IntrinsicSlowPathARM64(invoke); in VisitCRC32UpdateBytes()
4063 void IntrinsicLocationsBuilderARM64::VisitCRC32UpdateByteBuffer(HInvoke* invoke) { in VisitCRC32UpdateByteBuffer() argument
4069 new (allocator_) LocationSummary(invoke, in VisitCRC32UpdateByteBuffer()
4081 // Lower the invoke of CRC32.updateByteBuffer(int crc, long addr, int off, int len)
4090 void IntrinsicCodeGeneratorARM64::VisitCRC32UpdateByteBuffer(HInvoke* invoke) { in VisitCRC32UpdateByteBuffer() argument
4094 LocationSummary* locations = invoke->GetLocations(); in VisitCRC32UpdateByteBuffer()
4106 void IntrinsicLocationsBuilderARM64::VisitFP16ToFloat(HInvoke* invoke) { in VisitFP16ToFloat() argument
4111 LocationSummary* locations = new (allocator_) LocationSummary(invoke, in VisitFP16ToFloat()
4118 void IntrinsicCodeGeneratorARM64::VisitFP16ToFloat(HInvoke* invoke) { in VisitFP16ToFloat() argument
4122 Register bits = InputRegisterAt(invoke, 0); in VisitFP16ToFloat()
4123 VRegister out = SRegisterFrom(invoke->GetLocations()->Out()); in VisitFP16ToFloat()
4129 void IntrinsicLocationsBuilderARM64::VisitFP16ToHalf(HInvoke* invoke) { in VisitFP16ToHalf() argument
4134 LocationSummary* locations = new (allocator_) LocationSummary(invoke, in VisitFP16ToHalf()
4141 void IntrinsicCodeGeneratorARM64::VisitFP16ToHalf(HInvoke* invoke) { in VisitFP16ToHalf() argument
4145 VRegister in = SRegisterFrom(invoke->GetLocations()->InAt(0)); in VisitFP16ToHalf()
4147 Register out = WRegisterFrom(invoke->GetLocations()->Out()); in VisitFP16ToHalf()
4154 void GenerateFP16Round(HInvoke* invoke, in GenerateFP16Round() argument
4159 LocationSummary* locations = invoke->GetLocations(); in GenerateFP16Round()
4169 void IntrinsicLocationsBuilderARM64::VisitFP16Floor(HInvoke* invoke) { in VisitFP16Floor() argument
4174 CreateIntToIntLocations(allocator_, invoke); in VisitFP16Floor()
4177 void IntrinsicCodeGeneratorARM64::VisitFP16Floor(HInvoke* invoke) { in VisitFP16Floor() argument
4182 GenerateFP16Round(invoke, codegen_, masm, roundOp); in VisitFP16Floor()
4185 void IntrinsicLocationsBuilderARM64::VisitFP16Ceil(HInvoke* invoke) { in VisitFP16Ceil() argument
4190 CreateIntToIntLocations(allocator_, invoke); in VisitFP16Ceil()
4193 void IntrinsicCodeGeneratorARM64::VisitFP16Ceil(HInvoke* invoke) { in VisitFP16Ceil() argument
4198 GenerateFP16Round(invoke, codegen_, masm, roundOp); in VisitFP16Ceil()
4201 void IntrinsicLocationsBuilderARM64::VisitFP16Rint(HInvoke* invoke) { in VisitFP16Rint() argument
4206 CreateIntToIntLocations(allocator_, invoke); in VisitFP16Rint()
4209 void IntrinsicCodeGeneratorARM64::VisitFP16Rint(HInvoke* invoke) { in VisitFP16Rint() argument
4214 GenerateFP16Round(invoke, codegen_, masm, roundOp); in VisitFP16Rint()
4217 void FP16ComparisonLocations(HInvoke* invoke, in FP16ComparisonLocations() argument
4225 CreateIntIntToIntLocations(allocator_, invoke); in FP16ComparisonLocations()
4227 invoke->GetLocations()->AddTemp(Location::RequiresFpuRegister()); in FP16ComparisonLocations()
4232 void GenerateFP16Compare(HInvoke* invoke, in GenerateFP16Compare() argument
4237 LocationSummary* locations = invoke->GetLocations(); in GenerateFP16Compare()
4246 static inline void GenerateFP16Compare(HInvoke* invoke, in GenerateFP16Compare() argument
4254 GenerateFP16Compare(invoke, codegen, masm, compareOp); in GenerateFP16Compare()
4257 void IntrinsicLocationsBuilderARM64::VisitFP16Greater(HInvoke* invoke) { in VisitFP16Greater() argument
4258 FP16ComparisonLocations(invoke, allocator_, codegen_, 2); in VisitFP16Greater()
4261 void IntrinsicCodeGeneratorARM64::VisitFP16Greater(HInvoke* invoke) { in VisitFP16Greater() argument
4263 GenerateFP16Compare(invoke, codegen_, masm, gt); in VisitFP16Greater()
4266 void IntrinsicLocationsBuilderARM64::VisitFP16GreaterEquals(HInvoke* invoke) { in VisitFP16GreaterEquals() argument
4267 FP16ComparisonLocations(invoke, allocator_, codegen_, 2); in VisitFP16GreaterEquals()
4270 void IntrinsicCodeGeneratorARM64::VisitFP16GreaterEquals(HInvoke* invoke) { in VisitFP16GreaterEquals() argument
4272 GenerateFP16Compare(invoke, codegen_, masm, ge); in VisitFP16GreaterEquals()
4275 void IntrinsicLocationsBuilderARM64::VisitFP16Less(HInvoke* invoke) { in VisitFP16Less() argument
4276 FP16ComparisonLocations(invoke, allocator_, codegen_, 2); in VisitFP16Less()
4279 void IntrinsicCodeGeneratorARM64::VisitFP16Less(HInvoke* invoke) { in VisitFP16Less() argument
4281 GenerateFP16Compare(invoke, codegen_, masm, mi); in VisitFP16Less()
4284 void IntrinsicLocationsBuilderARM64::VisitFP16LessEquals(HInvoke* invoke) { in VisitFP16LessEquals() argument
4285 FP16ComparisonLocations(invoke, allocator_, codegen_, 2); in VisitFP16LessEquals()
4288 void IntrinsicCodeGeneratorARM64::VisitFP16LessEquals(HInvoke* invoke) { in VisitFP16LessEquals() argument
4290 GenerateFP16Compare(invoke, codegen_, masm, ls); in VisitFP16LessEquals()
4293 void IntrinsicLocationsBuilderARM64::VisitFP16Compare(HInvoke* invoke) { in VisitFP16Compare() argument
4294 FP16ComparisonLocations(invoke, allocator_, codegen_, 2); in VisitFP16Compare()
4297 void IntrinsicCodeGeneratorARM64::VisitFP16Compare(HInvoke* invoke) { in VisitFP16Compare() argument
4347 GenerateFP16Compare(invoke, codegen_, masm, compareOp); in VisitFP16Compare()
4352 static inline void GenerateFP16MinMax(HInvoke* invoke, in GenerateFP16MinMax() argument
4357 LocationSummary* locations = invoke->GetLocations(); in GenerateFP16MinMax()
4402 void IntrinsicLocationsBuilderARM64::VisitFP16Min(HInvoke* invoke) { in VisitFP16Min() argument
4403 FP16ComparisonLocations(invoke, allocator_, codegen_, 1); in VisitFP16Min()
4406 void IntrinsicCodeGeneratorARM64::VisitFP16Min(HInvoke* invoke) { in VisitFP16Min() argument
4409 GenerateFP16MinMax(invoke, codegen_, masm, mi); in VisitFP16Min()
4412 void IntrinsicLocationsBuilderARM64::VisitFP16Max(HInvoke* invoke) { in VisitFP16Max() argument
4413 FP16ComparisonLocations(invoke, allocator_, codegen_, 1); in VisitFP16Max()
4416 void IntrinsicCodeGeneratorARM64::VisitFP16Max(HInvoke* invoke) { in VisitFP16Max() argument
4419 GenerateFP16MinMax(invoke, codegen_, masm, gt); in VisitFP16Max()
4422 static void GenerateDivideUnsigned(HInvoke* invoke, CodeGeneratorARM64* codegen) { in GenerateDivideUnsigned() argument
4423 LocationSummary* locations = invoke->GetLocations(); in GenerateDivideUnsigned()
4425 DataType::Type type = invoke->GetType(); in GenerateDivideUnsigned()
4434 new (codegen->GetScopedAllocator()) IntrinsicSlowPathARM64(invoke); in GenerateDivideUnsigned()
4443 void IntrinsicLocationsBuilderARM64::VisitIntegerDivideUnsigned(HInvoke* invoke) { in VisitIntegerDivideUnsigned() argument
4444 CreateIntIntToIntSlowPathCallLocations(allocator_, invoke); in VisitIntegerDivideUnsigned()
4447 void IntrinsicCodeGeneratorARM64::VisitIntegerDivideUnsigned(HInvoke* invoke) { in VisitIntegerDivideUnsigned() argument
4448 GenerateDivideUnsigned(invoke, codegen_); in VisitIntegerDivideUnsigned()
4451 void IntrinsicLocationsBuilderARM64::VisitLongDivideUnsigned(HInvoke* invoke) { in VisitLongDivideUnsigned() argument
4452 CreateIntIntToIntSlowPathCallLocations(allocator_, invoke); in VisitLongDivideUnsigned()
4455 void IntrinsicCodeGeneratorARM64::VisitLongDivideUnsigned(HInvoke* invoke) { in VisitLongDivideUnsigned() argument
4456 GenerateDivideUnsigned(invoke, codegen_); in VisitLongDivideUnsigned()
4459 void IntrinsicLocationsBuilderARM64::VisitMathMultiplyHigh(HInvoke* invoke) { in VisitMathMultiplyHigh() argument
4460 CreateIntIntToIntLocations(allocator_, invoke); in VisitMathMultiplyHigh()
4463 void IntrinsicCodeGeneratorARM64::VisitMathMultiplyHigh(HInvoke* invoke) { in VisitMathMultiplyHigh() argument
4464 LocationSummary* locations = invoke->GetLocations(); in VisitMathMultiplyHigh()
4466 DataType::Type type = invoke->GetType(); in VisitMathMultiplyHigh()
4476 static void GenerateMathFma(HInvoke* invoke, CodeGeneratorARM64* codegen) { in GenerateMathFma() argument
4479 VRegister n = helpers::InputFPRegisterAt(invoke, 0); in GenerateMathFma()
4480 VRegister m = helpers::InputFPRegisterAt(invoke, 1); in GenerateMathFma()
4481 VRegister a = helpers::InputFPRegisterAt(invoke, 2); in GenerateMathFma()
4482 VRegister out = helpers::OutputFPRegister(invoke); in GenerateMathFma()
4487 void IntrinsicLocationsBuilderARM64::VisitMathFmaDouble(HInvoke* invoke) { in VisitMathFmaDouble() argument
4488 CreateFPFPFPToFPLocations(allocator_, invoke); in VisitMathFmaDouble()
4491 void IntrinsicCodeGeneratorARM64::VisitMathFmaDouble(HInvoke* invoke) { in VisitMathFmaDouble() argument
4492 GenerateMathFma(invoke, codegen_); in VisitMathFmaDouble()
4495 void IntrinsicLocationsBuilderARM64::VisitMathFmaFloat(HInvoke* invoke) { in VisitMathFmaFloat() argument
4496 CreateFPFPFPToFPLocations(allocator_, invoke); in VisitMathFmaFloat()
4499 void IntrinsicCodeGeneratorARM64::VisitMathFmaFloat(HInvoke* invoke) { in VisitMathFmaFloat() argument
4500 GenerateMathFma(invoke, codegen_); in VisitMathFmaFloat()
4505 VarHandleSlowPathARM64(HInvoke* invoke, std::memory_order order) in VarHandleSlowPathARM64() argument
4506 : IntrinsicSlowPathARM64(invoke), in VarHandleSlowPathARM64()
4600 static void GenerateVarHandleAccessModeAndVarTypeChecks(HInvoke* invoke, in GenerateVarHandleAccessModeAndVarTypeChecks() argument
4605 mirror::VarHandle::GetAccessModeByIntrinsic(invoke->GetIntrinsic()); in GenerateVarHandleAccessModeAndVarTypeChecks()
4609 Register varhandle = InputRegisterAt(invoke, 0); in GenerateVarHandleAccessModeAndVarTypeChecks()
4641 size_t expected_coordinates_count = GetExpectedVarHandleCoordinatesCount(invoke); in GenerateVarHandleAccessModeAndVarTypeChecks()
4643 uint32_t number_of_arguments = invoke->GetNumberOfArguments(); in GenerateVarHandleAccessModeAndVarTypeChecks()
4645 HInstruction* arg = invoke->InputAt(arg_index); in GenerateVarHandleAccessModeAndVarTypeChecks()
4648 Register arg_reg = WRegisterFrom(invoke->GetLocations()->InAt(arg_index)); in GenerateVarHandleAccessModeAndVarTypeChecks()
4655 static void GenerateVarHandleStaticFieldCheck(HInvoke* invoke, in GenerateVarHandleStaticFieldCheck() argument
4659 Register varhandle = InputRegisterAt(invoke, 0); in GenerateVarHandleStaticFieldCheck()
4672 static void GenerateVarHandleInstanceFieldChecks(HInvoke* invoke, in GenerateVarHandleInstanceFieldChecks() argument
4675 VarHandleOptimizations optimizations(invoke); in GenerateVarHandleInstanceFieldChecks()
4677 Register varhandle = InputRegisterAt(invoke, 0); in GenerateVarHandleInstanceFieldChecks()
4678 Register object = InputRegisterAt(invoke, 1); in GenerateVarHandleInstanceFieldChecks()
4710 static void GenerateVarHandleArrayChecks(HInvoke* invoke, in GenerateVarHandleArrayChecks() argument
4713 VarHandleOptimizations optimizations(invoke); in GenerateVarHandleArrayChecks()
4715 Register varhandle = InputRegisterAt(invoke, 0); in GenerateVarHandleArrayChecks()
4716 Register object = InputRegisterAt(invoke, 1); in GenerateVarHandleArrayChecks()
4717 Register index = InputRegisterAt(invoke, 2); in GenerateVarHandleArrayChecks()
4719 GetVarHandleExpectedValueType(invoke, /*expected_coordinates_count=*/ 2u); in GenerateVarHandleArrayChecks()
4795 static void GenerateVarHandleCoordinateChecks(HInvoke* invoke, in GenerateVarHandleCoordinateChecks() argument
4798 size_t expected_coordinates_count = GetExpectedVarHandleCoordinatesCount(invoke); in GenerateVarHandleCoordinateChecks()
4800 GenerateVarHandleStaticFieldCheck(invoke, codegen, slow_path); in GenerateVarHandleCoordinateChecks()
4802 GenerateVarHandleInstanceFieldChecks(invoke, codegen, slow_path); in GenerateVarHandleCoordinateChecks()
4805 GenerateVarHandleArrayChecks(invoke, codegen, slow_path); in GenerateVarHandleCoordinateChecks()
4809 static VarHandleSlowPathARM64* GenerateVarHandleChecks(HInvoke* invoke, in GenerateVarHandleChecks() argument
4813 size_t expected_coordinates_count = GetExpectedVarHandleCoordinatesCount(invoke); in GenerateVarHandleChecks()
4814 VarHandleOptimizations optimizations(invoke); in GenerateVarHandleChecks()
4823 new (codegen->GetScopedAllocator()) VarHandleSlowPathARM64(invoke, order); in GenerateVarHandleChecks()
4827 GenerateVarHandleAccessModeAndVarTypeChecks(invoke, codegen, slow_path, type); in GenerateVarHandleChecks()
4829 GenerateVarHandleCoordinateChecks(invoke, codegen, slow_path); in GenerateVarHandleChecks()
4839 static VarHandleTarget GetVarHandleTarget(HInvoke* invoke) { in GetVarHandleTarget() argument
4840 size_t expected_coordinates_count = GetExpectedVarHandleCoordinatesCount(invoke); in GetVarHandleTarget()
4841 LocationSummary* locations = invoke->GetLocations(); in GetVarHandleTarget()
4849 : InputRegisterAt(invoke, 1); in GetVarHandleTarget()
4853 static void GenerateVarHandleTarget(HInvoke* invoke, in GenerateVarHandleTarget() argument
4857 Register varhandle = InputRegisterAt(invoke, 0); in GenerateVarHandleTarget()
4858 size_t expected_coordinates_count = GetExpectedVarHandleCoordinatesCount(invoke); in GenerateVarHandleTarget()
4861 if (VarHandleOptimizations(invoke).GetUseKnownImageVarHandle()) { in GenerateVarHandleTarget()
4863 ArtField* target_field = GetBootImageVarHandleField(invoke); in GenerateVarHandleTarget()
4890 codegen->GenerateGcRootFieldLoad(invoke, in GenerateVarHandleTarget()
4901 GetVarHandleExpectedValueType(invoke, /*expected_coordinates_count=*/ 2u); in GenerateVarHandleTarget()
4905 Register index = InputRegisterAt(invoke, 2); in GenerateVarHandleTarget()
4915 static LocationSummary* CreateVarHandleCommonLocations(HInvoke* invoke, in CreateVarHandleCommonLocations() argument
4917 size_t expected_coordinates_count = GetExpectedVarHandleCoordinatesCount(invoke); in CreateVarHandleCommonLocations()
4918 DataType::Type return_type = invoke->GetType(); in CreateVarHandleCommonLocations()
4920 ArenaAllocator* allocator = invoke->GetBlock()->GetGraph()->GetAllocator(); in CreateVarHandleCommonLocations()
4922 new (allocator) LocationSummary(invoke, LocationSummary::kCallOnSlowPath, kIntrinsified); in CreateVarHandleCommonLocations()
4937 uint32_t number_of_arguments = invoke->GetNumberOfArguments(); in CreateVarHandleCommonLocations()
4939 HInstruction* arg = invoke->InputAt(arg_index); in CreateVarHandleCommonLocations()
4951 GetExpectedVarHandleCoordinatesCount(invoke) == 0u) { // For static fields. in CreateVarHandleCommonLocations()
4967 static void CreateVarHandleGetLocations(HInvoke* invoke, CodeGeneratorARM64* codegen) { in CreateVarHandleGetLocations() argument
4968 VarHandleOptimizations optimizations(invoke); in CreateVarHandleGetLocations()
4974 invoke->GetType() == DataType::Type::kReference && in CreateVarHandleGetLocations()
4975 invoke->GetIntrinsic() != Intrinsics::kVarHandleGet && in CreateVarHandleGetLocations()
4976 invoke->GetIntrinsic() != Intrinsics::kVarHandleGetOpaque) { in CreateVarHandleGetLocations()
4983 CreateVarHandleCommonLocations(invoke, codegen); in CreateVarHandleGetLocations()
4986 static void GenerateVarHandleGet(HInvoke* invoke, in GenerateVarHandleGet() argument
4990 DataType::Type type = invoke->GetType(); in GenerateVarHandleGet()
4993 LocationSummary* locations = invoke->GetLocations(); in GenerateVarHandleGet()
4995 CPURegister out = helpers::OutputCPURegister(invoke); in GenerateVarHandleGet()
4997 VarHandleTarget target = GetVarHandleTarget(invoke); in GenerateVarHandleGet()
5000 slow_path = GenerateVarHandleChecks(invoke, codegen, order, type); in GenerateVarHandleGet()
5001 GenerateVarHandleTarget(invoke, target, codegen); in GenerateVarHandleGet()
5018 codegen->GenerateFieldLoadWithBakerReadBarrier(invoke, in GenerateVarHandleGet()
5043 codegen->LoadAcquire(invoke, load_type, load_reg, address, /*needs_null_check=*/ false); in GenerateVarHandleGet()
5053 codegen->MaybeGenerateReadBarrierSlow(invoke, out_loc, out_loc, object_loc, 0u, offset_loc); in GenerateVarHandleGet()
5065 void IntrinsicLocationsBuilderARM64::VisitVarHandleGet(HInvoke* invoke) { in VisitVarHandleGet() argument
5066 CreateVarHandleGetLocations(invoke, codegen_); in VisitVarHandleGet()
5069 void IntrinsicCodeGeneratorARM64::VisitVarHandleGet(HInvoke* invoke) { in VisitVarHandleGet() argument
5070 GenerateVarHandleGet(invoke, codegen_, std::memory_order_relaxed); in VisitVarHandleGet()
5073 void IntrinsicLocationsBuilderARM64::VisitVarHandleGetOpaque(HInvoke* invoke) { in VisitVarHandleGetOpaque() argument
5074 CreateVarHandleGetLocations(invoke, codegen_); in VisitVarHandleGetOpaque()
5077 void IntrinsicCodeGeneratorARM64::VisitVarHandleGetOpaque(HInvoke* invoke) { in VisitVarHandleGetOpaque() argument
5078 GenerateVarHandleGet(invoke, codegen_, std::memory_order_relaxed); in VisitVarHandleGetOpaque()
5081 void IntrinsicLocationsBuilderARM64::VisitVarHandleGetAcquire(HInvoke* invoke) { in VisitVarHandleGetAcquire() argument
5082 CreateVarHandleGetLocations(invoke, codegen_); in VisitVarHandleGetAcquire()
5085 void IntrinsicCodeGeneratorARM64::VisitVarHandleGetAcquire(HInvoke* invoke) { in VisitVarHandleGetAcquire() argument
5086 GenerateVarHandleGet(invoke, codegen_, std::memory_order_acquire); in VisitVarHandleGetAcquire()
5089 void IntrinsicLocationsBuilderARM64::VisitVarHandleGetVolatile(HInvoke* invoke) { in VisitVarHandleGetVolatile() argument
5090 CreateVarHandleGetLocations(invoke, codegen_); in VisitVarHandleGetVolatile()
5093 void IntrinsicCodeGeneratorARM64::VisitVarHandleGetVolatile(HInvoke* invoke) { in VisitVarHandleGetVolatile() argument
5094 GenerateVarHandleGet(invoke, codegen_, std::memory_order_seq_cst); in VisitVarHandleGetVolatile()
5097 static void CreateVarHandleSetLocations(HInvoke* invoke, CodeGeneratorARM64* codegen) { in CreateVarHandleSetLocations() argument
5098 VarHandleOptimizations optimizations(invoke); in CreateVarHandleSetLocations()
5103 CreateVarHandleCommonLocations(invoke, codegen); in CreateVarHandleSetLocations()
5106 static void GenerateVarHandleSet(HInvoke* invoke, in GenerateVarHandleSet() argument
5110 uint32_t value_index = invoke->GetNumberOfArguments() - 1; in GenerateVarHandleSet()
5111 DataType::Type value_type = GetDataTypeFromShorty(invoke, value_index); in GenerateVarHandleSet()
5114 CPURegister value = InputCPURegisterOrZeroRegAt(invoke, value_index); in GenerateVarHandleSet()
5116 VarHandleTarget target = GetVarHandleTarget(invoke); in GenerateVarHandleSet()
5119 slow_path = GenerateVarHandleChecks(invoke, codegen, order, value_type); in GenerateVarHandleSet()
5120 GenerateVarHandleTarget(invoke, target, codegen); in GenerateVarHandleSet()
5158 codegen->StoreRelease(invoke, value_type, source, address, /*needs_null_check=*/ false); in GenerateVarHandleSet()
5164 if (CodeGenerator::StoreNeedsWriteBarrier(value_type, invoke->InputAt(value_index))) { in GenerateVarHandleSet()
5174 void IntrinsicLocationsBuilderARM64::VisitVarHandleSet(HInvoke* invoke) { in VisitVarHandleSet() argument
5175 CreateVarHandleSetLocations(invoke, codegen_); in VisitVarHandleSet()
5178 void IntrinsicCodeGeneratorARM64::VisitVarHandleSet(HInvoke* invoke) { in VisitVarHandleSet() argument
5179 GenerateVarHandleSet(invoke, codegen_, std::memory_order_relaxed); in VisitVarHandleSet()
5182 void IntrinsicLocationsBuilderARM64::VisitVarHandleSetOpaque(HInvoke* invoke) { in VisitVarHandleSetOpaque() argument
5183 CreateVarHandleSetLocations(invoke, codegen_); in VisitVarHandleSetOpaque()
5186 void IntrinsicCodeGeneratorARM64::VisitVarHandleSetOpaque(HInvoke* invoke) { in VisitVarHandleSetOpaque() argument
5187 GenerateVarHandleSet(invoke, codegen_, std::memory_order_relaxed); in VisitVarHandleSetOpaque()
5190 void IntrinsicLocationsBuilderARM64::VisitVarHandleSetRelease(HInvoke* invoke) { in VisitVarHandleSetRelease() argument
5191 CreateVarHandleSetLocations(invoke, codegen_); in VisitVarHandleSetRelease()
5194 void IntrinsicCodeGeneratorARM64::VisitVarHandleSetRelease(HInvoke* invoke) { in VisitVarHandleSetRelease() argument
5195 GenerateVarHandleSet(invoke, codegen_, std::memory_order_release); in VisitVarHandleSetRelease()
5198 void IntrinsicLocationsBuilderARM64::VisitVarHandleSetVolatile(HInvoke* invoke) { in VisitVarHandleSetVolatile() argument
5199 CreateVarHandleSetLocations(invoke, codegen_); in VisitVarHandleSetVolatile()
5202 void IntrinsicCodeGeneratorARM64::VisitVarHandleSetVolatile(HInvoke* invoke) { in VisitVarHandleSetVolatile() argument
5203 GenerateVarHandleSet(invoke, codegen_, std::memory_order_seq_cst); in VisitVarHandleSetVolatile()
5206 static void CreateVarHandleCompareAndSetOrExchangeLocations(HInvoke* invoke, in CreateVarHandleCompareAndSetOrExchangeLocations() argument
5209 VarHandleOptimizations optimizations(invoke); in CreateVarHandleCompareAndSetOrExchangeLocations()
5214 uint32_t number_of_arguments = invoke->GetNumberOfArguments(); in CreateVarHandleCompareAndSetOrExchangeLocations()
5215 DataType::Type value_type = GetDataTypeFromShorty(invoke, number_of_arguments - 1u); in CreateVarHandleCompareAndSetOrExchangeLocations()
5227 LocationSummary* locations = CreateVarHandleCommonLocations(invoke, codegen); in CreateVarHandleCompareAndSetOrExchangeLocations()
5235 if (GetExpectedVarHandleCoordinatesCount(invoke) == 0u) { // For static fields. in CreateVarHandleCompareAndSetOrExchangeLocations()
5247 DCHECK_EQ(old_temp_count, (GetExpectedVarHandleCoordinatesCount(invoke) == 0) ? 2u : 1u); in CreateVarHandleCompareAndSetOrExchangeLocations()
5253 (IsZeroBitPattern(invoke->InputAt(number_of_arguments - 1u)) ? 1u : 0u) + in CreateVarHandleCompareAndSetOrExchangeLocations()
5254 (IsZeroBitPattern(invoke->InputAt(number_of_arguments - 2u)) ? 1u : 0u); in CreateVarHandleCompareAndSetOrExchangeLocations()
5261 !IsZeroBitPattern(invoke->InputAt(number_of_arguments - 2u)) && in CreateVarHandleCompareAndSetOrExchangeLocations()
5262 !IsZeroBitPattern(invoke->InputAt(number_of_arguments - 1u)) && in CreateVarHandleCompareAndSetOrExchangeLocations()
5263 GetExpectedVarHandleCoordinatesCount(invoke) == 2u) { in CreateVarHandleCompareAndSetOrExchangeLocations()
5295 static void GenerateVarHandleCompareAndSetOrExchange(HInvoke* invoke, in GenerateVarHandleCompareAndSetOrExchange() argument
5303 uint32_t expected_index = invoke->GetNumberOfArguments() - 2; in GenerateVarHandleCompareAndSetOrExchange()
5304 uint32_t new_value_index = invoke->GetNumberOfArguments() - 1; in GenerateVarHandleCompareAndSetOrExchange()
5305 DataType::Type value_type = GetDataTypeFromShorty(invoke, new_value_index); in GenerateVarHandleCompareAndSetOrExchange()
5306 DCHECK_EQ(value_type, GetDataTypeFromShorty(invoke, expected_index)); in GenerateVarHandleCompareAndSetOrExchange()
5309 LocationSummary* locations = invoke->GetLocations(); in GenerateVarHandleCompareAndSetOrExchange()
5310 CPURegister expected = InputCPURegisterOrZeroRegAt(invoke, expected_index); in GenerateVarHandleCompareAndSetOrExchange()
5311 CPURegister new_value = InputCPURegisterOrZeroRegAt(invoke, new_value_index); in GenerateVarHandleCompareAndSetOrExchange()
5312 CPURegister out = helpers::OutputCPURegister(invoke); in GenerateVarHandleCompareAndSetOrExchange()
5314 VarHandleTarget target = GetVarHandleTarget(invoke); in GenerateVarHandleCompareAndSetOrExchange()
5317 slow_path = GenerateVarHandleChecks(invoke, codegen, order, value_type); in GenerateVarHandleCompareAndSetOrExchange()
5318 GenerateVarHandleTarget(invoke, target, codegen); in GenerateVarHandleCompareAndSetOrExchange()
5326 if (CodeGenerator::StoreNeedsWriteBarrier(value_type, invoke->InputAt(new_value_index))) { in GenerateVarHandleCompareAndSetOrExchange()
5422 size_t expected_coordinates_count = GetExpectedVarHandleCoordinatesCount(invoke); in GenerateVarHandleCompareAndSetOrExchange()
5430 invoke, in GenerateVarHandleCompareAndSetOrExchange()
5485 void IntrinsicLocationsBuilderARM64::VisitVarHandleCompareAndExchange(HInvoke* invoke) { in VisitVarHandleCompareAndExchange() argument
5486 CreateVarHandleCompareAndSetOrExchangeLocations(invoke, codegen_, /*return_success=*/ false); in VisitVarHandleCompareAndExchange()
5489 void IntrinsicCodeGeneratorARM64::VisitVarHandleCompareAndExchange(HInvoke* invoke) { in VisitVarHandleCompareAndExchange() argument
5491 invoke, codegen_, std::memory_order_seq_cst, /*return_success=*/ false, /*strong=*/ true); in VisitVarHandleCompareAndExchange()
5494 void IntrinsicLocationsBuilderARM64::VisitVarHandleCompareAndExchangeAcquire(HInvoke* invoke) { in VisitVarHandleCompareAndExchangeAcquire() argument
5495 CreateVarHandleCompareAndSetOrExchangeLocations(invoke, codegen_, /*return_success=*/ false); in VisitVarHandleCompareAndExchangeAcquire()
5498 void IntrinsicCodeGeneratorARM64::VisitVarHandleCompareAndExchangeAcquire(HInvoke* invoke) { in VisitVarHandleCompareAndExchangeAcquire() argument
5500 invoke, codegen_, std::memory_order_acquire, /*return_success=*/ false, /*strong=*/ true); in VisitVarHandleCompareAndExchangeAcquire()
5503 void IntrinsicLocationsBuilderARM64::VisitVarHandleCompareAndExchangeRelease(HInvoke* invoke) { in VisitVarHandleCompareAndExchangeRelease() argument
5504 CreateVarHandleCompareAndSetOrExchangeLocations(invoke, codegen_, /*return_success=*/ false); in VisitVarHandleCompareAndExchangeRelease()
5507 void IntrinsicCodeGeneratorARM64::VisitVarHandleCompareAndExchangeRelease(HInvoke* invoke) { in VisitVarHandleCompareAndExchangeRelease() argument
5509 invoke, codegen_, std::memory_order_release, /*return_success=*/ false, /*strong=*/ true); in VisitVarHandleCompareAndExchangeRelease()
5512 void IntrinsicLocationsBuilderARM64::VisitVarHandleCompareAndSet(HInvoke* invoke) { in VisitVarHandleCompareAndSet() argument
5513 CreateVarHandleCompareAndSetOrExchangeLocations(invoke, codegen_, /*return_success=*/ true); in VisitVarHandleCompareAndSet()
5516 void IntrinsicCodeGeneratorARM64::VisitVarHandleCompareAndSet(HInvoke* invoke) { in VisitVarHandleCompareAndSet() argument
5518 invoke, codegen_, std::memory_order_seq_cst, /*return_success=*/ true, /*strong=*/ true); in VisitVarHandleCompareAndSet()
5521 void IntrinsicLocationsBuilderARM64::VisitVarHandleWeakCompareAndSet(HInvoke* invoke) { in VisitVarHandleWeakCompareAndSet() argument
5522 CreateVarHandleCompareAndSetOrExchangeLocations(invoke, codegen_, /*return_success=*/ true); in VisitVarHandleWeakCompareAndSet()
5525 void IntrinsicCodeGeneratorARM64::VisitVarHandleWeakCompareAndSet(HInvoke* invoke) { in VisitVarHandleWeakCompareAndSet() argument
5527 invoke, codegen_, std::memory_order_seq_cst, /*return_success=*/ true, /*strong=*/ false); in VisitVarHandleWeakCompareAndSet()
5530 void IntrinsicLocationsBuilderARM64::VisitVarHandleWeakCompareAndSetAcquire(HInvoke* invoke) { in VisitVarHandleWeakCompareAndSetAcquire() argument
5531 CreateVarHandleCompareAndSetOrExchangeLocations(invoke, codegen_, /*return_success=*/ true); in VisitVarHandleWeakCompareAndSetAcquire()
5534 void IntrinsicCodeGeneratorARM64::VisitVarHandleWeakCompareAndSetAcquire(HInvoke* invoke) { in VisitVarHandleWeakCompareAndSetAcquire() argument
5536 invoke, codegen_, std::memory_order_acquire, /*return_success=*/ true, /*strong=*/ false); in VisitVarHandleWeakCompareAndSetAcquire()
5539 void IntrinsicLocationsBuilderARM64::VisitVarHandleWeakCompareAndSetPlain(HInvoke* invoke) { in VisitVarHandleWeakCompareAndSetPlain() argument
5540 CreateVarHandleCompareAndSetOrExchangeLocations(invoke, codegen_, /*return_success=*/ true); in VisitVarHandleWeakCompareAndSetPlain()
5543 void IntrinsicCodeGeneratorARM64::VisitVarHandleWeakCompareAndSetPlain(HInvoke* invoke) { in VisitVarHandleWeakCompareAndSetPlain() argument
5545 invoke, codegen_, std::memory_order_relaxed, /*return_success=*/ true, /*strong=*/ false); in VisitVarHandleWeakCompareAndSetPlain()
5548 void IntrinsicLocationsBuilderARM64::VisitVarHandleWeakCompareAndSetRelease(HInvoke* invoke) { in VisitVarHandleWeakCompareAndSetRelease() argument
5549 CreateVarHandleCompareAndSetOrExchangeLocations(invoke, codegen_, /*return_success=*/ true); in VisitVarHandleWeakCompareAndSetRelease()
5552 void IntrinsicCodeGeneratorARM64::VisitVarHandleWeakCompareAndSetRelease(HInvoke* invoke) { in VisitVarHandleWeakCompareAndSetRelease() argument
5554 invoke, codegen_, std::memory_order_release, /*return_success=*/ true, /*strong=*/ false); in VisitVarHandleWeakCompareAndSetRelease()
5557 static void CreateVarHandleGetAndUpdateLocations(HInvoke* invoke, in CreateVarHandleGetAndUpdateLocations() argument
5560 VarHandleOptimizations optimizations(invoke); in CreateVarHandleGetAndUpdateLocations()
5566 uint32_t arg_index = invoke->GetNumberOfArguments() - 1; in CreateVarHandleGetAndUpdateLocations()
5567 DataType::Type value_type = GetDataTypeFromShorty(invoke, arg_index); in CreateVarHandleGetAndUpdateLocations()
5575 LocationSummary* locations = CreateVarHandleCommonLocations(invoke, codegen); in CreateVarHandleGetAndUpdateLocations()
5578 DCHECK_EQ(old_temp_count, (GetExpectedVarHandleCoordinatesCount(invoke) == 0) ? 2u : 1u); in CreateVarHandleGetAndUpdateLocations()
5582 locations->SetInAt(invoke->GetNumberOfArguments() - 1u, Location::RequiresFpuRegister()); in CreateVarHandleGetAndUpdateLocations()
5587 !IsZeroBitPattern(invoke->InputAt(invoke->GetNumberOfArguments() - 1u))) { in CreateVarHandleGetAndUpdateLocations()
5597 GetExpectedVarHandleCoordinatesCount(invoke) == 2u && in CreateVarHandleGetAndUpdateLocations()
5598 !IsZeroBitPattern(invoke->InputAt(invoke->GetNumberOfArguments() - 1u))) { in CreateVarHandleGetAndUpdateLocations()
5606 DataType::Type return_type = invoke->GetType(); in CreateVarHandleGetAndUpdateLocations()
5618 static void GenerateVarHandleGetAndUpdate(HInvoke* invoke, in GenerateVarHandleGetAndUpdate() argument
5624 uint32_t arg_index = invoke->GetNumberOfArguments() - 1; in GenerateVarHandleGetAndUpdate()
5625 DataType::Type value_type = GetDataTypeFromShorty(invoke, arg_index); in GenerateVarHandleGetAndUpdate()
5629 LocationSummary* locations = invoke->GetLocations(); in GenerateVarHandleGetAndUpdate()
5631 ? InputCPURegisterAt(invoke, arg_index) in GenerateVarHandleGetAndUpdate()
5632 : InputCPURegisterOrZeroRegAt(invoke, arg_index); in GenerateVarHandleGetAndUpdate()
5633 DataType::Type return_type = invoke->GetType(); in GenerateVarHandleGetAndUpdate()
5639 helpers::OutputCPURegister(invoke); in GenerateVarHandleGetAndUpdate()
5641 VarHandleTarget target = GetVarHandleTarget(invoke); in GenerateVarHandleGetAndUpdate()
5644 slow_path = GenerateVarHandleChecks(invoke, codegen, order, value_type); in GenerateVarHandleGetAndUpdate()
5645 GenerateVarHandleTarget(invoke, target, codegen); in GenerateVarHandleGetAndUpdate()
5653 if (CodeGenerator::StoreNeedsWriteBarrier(value_type, invoke->InputAt(arg_index))) { in GenerateVarHandleGetAndUpdate()
5747 invoke, in GenerateVarHandleGetAndUpdate()
5763 void IntrinsicLocationsBuilderARM64::VisitVarHandleGetAndSet(HInvoke* invoke) { in VisitVarHandleGetAndSet() argument
5764 CreateVarHandleGetAndUpdateLocations(invoke, codegen_, GetAndUpdateOp::kSet); in VisitVarHandleGetAndSet()
5767 void IntrinsicCodeGeneratorARM64::VisitVarHandleGetAndSet(HInvoke* invoke) { in VisitVarHandleGetAndSet() argument
5768 GenerateVarHandleGetAndUpdate(invoke, codegen_, GetAndUpdateOp::kSet, std::memory_order_seq_cst); in VisitVarHandleGetAndSet()
5771 void IntrinsicLocationsBuilderARM64::VisitVarHandleGetAndSetAcquire(HInvoke* invoke) { in VisitVarHandleGetAndSetAcquire() argument
5772 CreateVarHandleGetAndUpdateLocations(invoke, codegen_, GetAndUpdateOp::kSet); in VisitVarHandleGetAndSetAcquire()
5775 void IntrinsicCodeGeneratorARM64::VisitVarHandleGetAndSetAcquire(HInvoke* invoke) { in VisitVarHandleGetAndSetAcquire() argument
5776 GenerateVarHandleGetAndUpdate(invoke, codegen_, GetAndUpdateOp::kSet, std::memory_order_acquire); in VisitVarHandleGetAndSetAcquire()
5779 void IntrinsicLocationsBuilderARM64::VisitVarHandleGetAndSetRelease(HInvoke* invoke) { in VisitVarHandleGetAndSetRelease() argument
5780 CreateVarHandleGetAndUpdateLocations(invoke, codegen_, GetAndUpdateOp::kSet); in VisitVarHandleGetAndSetRelease()
5783 void IntrinsicCodeGeneratorARM64::VisitVarHandleGetAndSetRelease(HInvoke* invoke) { in VisitVarHandleGetAndSetRelease() argument
5784 GenerateVarHandleGetAndUpdate(invoke, codegen_, GetAndUpdateOp::kSet, std::memory_order_release); in VisitVarHandleGetAndSetRelease()
5787 void IntrinsicLocationsBuilderARM64::VisitVarHandleGetAndAdd(HInvoke* invoke) { in VisitVarHandleGetAndAdd() argument
5788 CreateVarHandleGetAndUpdateLocations(invoke, codegen_, GetAndUpdateOp::kAdd); in VisitVarHandleGetAndAdd()
5791 void IntrinsicCodeGeneratorARM64::VisitVarHandleGetAndAdd(HInvoke* invoke) { in VisitVarHandleGetAndAdd() argument
5792 GenerateVarHandleGetAndUpdate(invoke, codegen_, GetAndUpdateOp::kAdd, std::memory_order_seq_cst); in VisitVarHandleGetAndAdd()
5795 void IntrinsicLocationsBuilderARM64::VisitVarHandleGetAndAddAcquire(HInvoke* invoke) { in VisitVarHandleGetAndAddAcquire() argument
5796 CreateVarHandleGetAndUpdateLocations(invoke, codegen_, GetAndUpdateOp::kAdd); in VisitVarHandleGetAndAddAcquire()
5799 void IntrinsicCodeGeneratorARM64::VisitVarHandleGetAndAddAcquire(HInvoke* invoke) { in VisitVarHandleGetAndAddAcquire() argument
5800 GenerateVarHandleGetAndUpdate(invoke, codegen_, GetAndUpdateOp::kAdd, std::memory_order_acquire); in VisitVarHandleGetAndAddAcquire()
5803 void IntrinsicLocationsBuilderARM64::VisitVarHandleGetAndAddRelease(HInvoke* invoke) { in VisitVarHandleGetAndAddRelease() argument
5804 CreateVarHandleGetAndUpdateLocations(invoke, codegen_, GetAndUpdateOp::kAdd); in VisitVarHandleGetAndAddRelease()
5807 void IntrinsicCodeGeneratorARM64::VisitVarHandleGetAndAddRelease(HInvoke* invoke) { in VisitVarHandleGetAndAddRelease() argument
5808 GenerateVarHandleGetAndUpdate(invoke, codegen_, GetAndUpdateOp::kAdd, std::memory_order_release); in VisitVarHandleGetAndAddRelease()
5811 void IntrinsicLocationsBuilderARM64::VisitVarHandleGetAndBitwiseAnd(HInvoke* invoke) { in VisitVarHandleGetAndBitwiseAnd() argument
5812 CreateVarHandleGetAndUpdateLocations(invoke, codegen_, GetAndUpdateOp::kAnd); in VisitVarHandleGetAndBitwiseAnd()
5815 void IntrinsicCodeGeneratorARM64::VisitVarHandleGetAndBitwiseAnd(HInvoke* invoke) { in VisitVarHandleGetAndBitwiseAnd() argument
5816 GenerateVarHandleGetAndUpdate(invoke, codegen_, GetAndUpdateOp::kAnd, std::memory_order_seq_cst); in VisitVarHandleGetAndBitwiseAnd()
5819 void IntrinsicLocationsBuilderARM64::VisitVarHandleGetAndBitwiseAndAcquire(HInvoke* invoke) { in VisitVarHandleGetAndBitwiseAndAcquire() argument
5820 CreateVarHandleGetAndUpdateLocations(invoke, codegen_, GetAndUpdateOp::kAnd); in VisitVarHandleGetAndBitwiseAndAcquire()
5823 void IntrinsicCodeGeneratorARM64::VisitVarHandleGetAndBitwiseAndAcquire(HInvoke* invoke) { in VisitVarHandleGetAndBitwiseAndAcquire() argument
5824 GenerateVarHandleGetAndUpdate(invoke, codegen_, GetAndUpdateOp::kAnd, std::memory_order_acquire); in VisitVarHandleGetAndBitwiseAndAcquire()
5827 void IntrinsicLocationsBuilderARM64::VisitVarHandleGetAndBitwiseAndRelease(HInvoke* invoke) { in VisitVarHandleGetAndBitwiseAndRelease() argument
5828 CreateVarHandleGetAndUpdateLocations(invoke, codegen_, GetAndUpdateOp::kAnd); in VisitVarHandleGetAndBitwiseAndRelease()
5831 void IntrinsicCodeGeneratorARM64::VisitVarHandleGetAndBitwiseAndRelease(HInvoke* invoke) { in VisitVarHandleGetAndBitwiseAndRelease() argument
5832 GenerateVarHandleGetAndUpdate(invoke, codegen_, GetAndUpdateOp::kAnd, std::memory_order_release); in VisitVarHandleGetAndBitwiseAndRelease()
5835 void IntrinsicLocationsBuilderARM64::VisitVarHandleGetAndBitwiseOr(HInvoke* invoke) { in VisitVarHandleGetAndBitwiseOr() argument
5836 CreateVarHandleGetAndUpdateLocations(invoke, codegen_, GetAndUpdateOp::kOr); in VisitVarHandleGetAndBitwiseOr()
5839 void IntrinsicCodeGeneratorARM64::VisitVarHandleGetAndBitwiseOr(HInvoke* invoke) { in VisitVarHandleGetAndBitwiseOr() argument
5840 GenerateVarHandleGetAndUpdate(invoke, codegen_, GetAndUpdateOp::kOr, std::memory_order_seq_cst); in VisitVarHandleGetAndBitwiseOr()
5843 void IntrinsicLocationsBuilderARM64::VisitVarHandleGetAndBitwiseOrAcquire(HInvoke* invoke) { in VisitVarHandleGetAndBitwiseOrAcquire() argument
5844 CreateVarHandleGetAndUpdateLocations(invoke, codegen_, GetAndUpdateOp::kOr); in VisitVarHandleGetAndBitwiseOrAcquire()
5847 void IntrinsicCodeGeneratorARM64::VisitVarHandleGetAndBitwiseOrAcquire(HInvoke* invoke) { in VisitVarHandleGetAndBitwiseOrAcquire() argument
5848 GenerateVarHandleGetAndUpdate(invoke, codegen_, GetAndUpdateOp::kOr, std::memory_order_acquire); in VisitVarHandleGetAndBitwiseOrAcquire()
5851 void IntrinsicLocationsBuilderARM64::VisitVarHandleGetAndBitwiseOrRelease(HInvoke* invoke) { in VisitVarHandleGetAndBitwiseOrRelease() argument
5852 CreateVarHandleGetAndUpdateLocations(invoke, codegen_, GetAndUpdateOp::kOr); in VisitVarHandleGetAndBitwiseOrRelease()
5855 void IntrinsicCodeGeneratorARM64::VisitVarHandleGetAndBitwiseOrRelease(HInvoke* invoke) { in VisitVarHandleGetAndBitwiseOrRelease() argument
5856 GenerateVarHandleGetAndUpdate(invoke, codegen_, GetAndUpdateOp::kOr, std::memory_order_release); in VisitVarHandleGetAndBitwiseOrRelease()
5859 void IntrinsicLocationsBuilderARM64::VisitVarHandleGetAndBitwiseXor(HInvoke* invoke) { in VisitVarHandleGetAndBitwiseXor() argument
5860 CreateVarHandleGetAndUpdateLocations(invoke, codegen_, GetAndUpdateOp::kXor); in VisitVarHandleGetAndBitwiseXor()
5863 void IntrinsicCodeGeneratorARM64::VisitVarHandleGetAndBitwiseXor(HInvoke* invoke) { in VisitVarHandleGetAndBitwiseXor() argument
5864 GenerateVarHandleGetAndUpdate(invoke, codegen_, GetAndUpdateOp::kXor, std::memory_order_seq_cst); in VisitVarHandleGetAndBitwiseXor()
5867 void IntrinsicLocationsBuilderARM64::VisitVarHandleGetAndBitwiseXorAcquire(HInvoke* invoke) { in VisitVarHandleGetAndBitwiseXorAcquire() argument
5868 CreateVarHandleGetAndUpdateLocations(invoke, codegen_, GetAndUpdateOp::kXor); in VisitVarHandleGetAndBitwiseXorAcquire()
5871 void IntrinsicCodeGeneratorARM64::VisitVarHandleGetAndBitwiseXorAcquire(HInvoke* invoke) { in VisitVarHandleGetAndBitwiseXorAcquire() argument
5872 GenerateVarHandleGetAndUpdate(invoke, codegen_, GetAndUpdateOp::kXor, std::memory_order_acquire); in VisitVarHandleGetAndBitwiseXorAcquire()
5875 void IntrinsicLocationsBuilderARM64::VisitVarHandleGetAndBitwiseXorRelease(HInvoke* invoke) { in VisitVarHandleGetAndBitwiseXorRelease() argument
5876 CreateVarHandleGetAndUpdateLocations(invoke, codegen_, GetAndUpdateOp::kXor); in VisitVarHandleGetAndBitwiseXorRelease()
5879 void IntrinsicCodeGeneratorARM64::VisitVarHandleGetAndBitwiseXorRelease(HInvoke* invoke) { in VisitVarHandleGetAndBitwiseXorRelease() argument
5880 GenerateVarHandleGetAndUpdate(invoke, codegen_, GetAndUpdateOp::kXor, std::memory_order_release); in VisitVarHandleGetAndBitwiseXorRelease()
5887 HInvoke* invoke = GetInvoke(); in EmitByteArrayViewCode() local
5890 GetVarHandleExpectedValueType(invoke, /*expected_coordinates_count=*/ 2u); in EmitByteArrayViewCode()
5894 Register varhandle = InputRegisterAt(invoke, 0); in EmitByteArrayViewCode()
5895 Register object = InputRegisterAt(invoke, 1); in EmitByteArrayViewCode()
5896 Register index = InputRegisterAt(invoke, 2); in EmitByteArrayViewCode()
5905 VarHandleTarget target = GetVarHandleTarget(invoke); in EmitByteArrayViewCode()
5940 IsZeroBitPattern(invoke->InputAt(invoke->GetNumberOfArguments() - 1u))) { in EmitByteArrayViewCode()
5952 GenerateVarHandleGet(invoke, codegen, order_, /*byte_swap=*/ true); in EmitByteArrayViewCode()
5955 GenerateVarHandleSet(invoke, codegen, order_, /*byte_swap=*/ true); in EmitByteArrayViewCode()
5960 invoke, codegen, order_, return_success_, strong_, /*byte_swap=*/ true); in EmitByteArrayViewCode()
5964 invoke, codegen, get_and_update_op_, order_, /*byte_swap=*/ true); in EmitByteArrayViewCode()
5970 void IntrinsicLocationsBuilderARM64::VisitMethodHandleInvokeExact(HInvoke* invoke) { in VisitMethodHandleInvokeExact() argument
5971 ArenaAllocator* allocator = invoke->GetBlock()->GetGraph()->GetAllocator(); in VisitMethodHandleInvokeExact()
5973 LocationSummary(invoke, LocationSummary::kCallOnMainAndSlowPath, kIntrinsified); in VisitMethodHandleInvokeExact()
5976 locations->SetOut(calling_convention.GetReturnLocation(invoke->GetType())); in VisitMethodHandleInvokeExact()
5980 // Accomodating LocationSummary for underlying invoke-* call. in VisitMethodHandleInvokeExact()
5981 uint32_t number_of_args = invoke->GetNumberOfArguments(); in VisitMethodHandleInvokeExact()
5983 locations->SetInAt(i, calling_convention.GetNextLocation(invoke->InputAt(i)->GetType())); in VisitMethodHandleInvokeExact()
5993 void IntrinsicCodeGeneratorARM64::VisitMethodHandleInvokeExact(HInvoke* invoke) { in VisitMethodHandleInvokeExact() argument
5994 LocationSummary* locations = invoke->GetLocations(); in VisitMethodHandleInvokeExact()
5996 Register method_handle = InputRegisterAt(invoke, 0); in VisitMethodHandleInvokeExact()
5999 new (codegen_->GetScopedAllocator()) InvokePolymorphicSlowPathARM64(invoke, method_handle); in VisitMethodHandleInvokeExact()
6003 Register call_site_type = InputRegisterAt(invoke, invoke->GetNumberOfArguments()); in VisitMethodHandleInvokeExact()
6021 codegen_->RecordPcInfo(invoke, invoke->GetDexPc(), slow_path); in VisitMethodHandleInvokeExact()