xref: /aosp_15_r20/external/swiftshader/src/Pipeline/SpirvShaderControlFlow.cpp (revision 03ce13f70fcc45d86ee91b7ee4cab1936a95046e)
1*03ce13f7SAndroid Build Coastguard Worker // Copyright 2019 The SwiftShader Authors. All Rights Reserved.
2*03ce13f7SAndroid Build Coastguard Worker //
3*03ce13f7SAndroid Build Coastguard Worker // Licensed under the Apache License, Version 2.0 (the "License");
4*03ce13f7SAndroid Build Coastguard Worker // you may not use this file except in compliance with the License.
5*03ce13f7SAndroid Build Coastguard Worker // You may obtain a copy of the License at
6*03ce13f7SAndroid Build Coastguard Worker //
7*03ce13f7SAndroid Build Coastguard Worker //    http://www.apache.org/licenses/LICENSE-2.0
8*03ce13f7SAndroid Build Coastguard Worker //
9*03ce13f7SAndroid Build Coastguard Worker // Unless required by applicable law or agreed to in writing, software
10*03ce13f7SAndroid Build Coastguard Worker // distributed under the License is distributed on an "AS IS" BASIS,
11*03ce13f7SAndroid Build Coastguard Worker // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12*03ce13f7SAndroid Build Coastguard Worker // See the License for the specific language governing permissions and
13*03ce13f7SAndroid Build Coastguard Worker // limitations under the License.
14*03ce13f7SAndroid Build Coastguard Worker 
15*03ce13f7SAndroid Build Coastguard Worker #include "SpirvShader.hpp"
16*03ce13f7SAndroid Build Coastguard Worker #include "SpirvShaderDebug.hpp"
17*03ce13f7SAndroid Build Coastguard Worker 
18*03ce13f7SAndroid Build Coastguard Worker #include "Reactor/Coroutine.hpp"  // rr::Yield
19*03ce13f7SAndroid Build Coastguard Worker 
20*03ce13f7SAndroid Build Coastguard Worker #include "ShaderCore.hpp"
21*03ce13f7SAndroid Build Coastguard Worker 
22*03ce13f7SAndroid Build Coastguard Worker #include <spirv/unified1/spirv.hpp>
23*03ce13f7SAndroid Build Coastguard Worker 
24*03ce13f7SAndroid Build Coastguard Worker #include <queue>
25*03ce13f7SAndroid Build Coastguard Worker 
26*03ce13f7SAndroid Build Coastguard Worker #include <fstream>
27*03ce13f7SAndroid Build Coastguard Worker #include <iostream>
28*03ce13f7SAndroid Build Coastguard Worker 
29*03ce13f7SAndroid Build Coastguard Worker namespace sw {
30*03ce13f7SAndroid Build Coastguard Worker 
Block(InsnIterator begin,InsnIterator end)31*03ce13f7SAndroid Build Coastguard Worker Spirv::Block::Block(InsnIterator begin, InsnIterator end)
32*03ce13f7SAndroid Build Coastguard Worker     : begin_(begin)
33*03ce13f7SAndroid Build Coastguard Worker     , end_(end)
34*03ce13f7SAndroid Build Coastguard Worker {
35*03ce13f7SAndroid Build Coastguard Worker 	// Default to a Simple, this may change later.
36*03ce13f7SAndroid Build Coastguard Worker 	kind = Block::Simple;
37*03ce13f7SAndroid Build Coastguard Worker 
38*03ce13f7SAndroid Build Coastguard Worker 	// Walk the instructions to find the last two of the block.
39*03ce13f7SAndroid Build Coastguard Worker 	InsnIterator insns[2];
40*03ce13f7SAndroid Build Coastguard Worker 	for(auto insn : *this)
41*03ce13f7SAndroid Build Coastguard Worker 	{
42*03ce13f7SAndroid Build Coastguard Worker 		insns[0] = insns[1];
43*03ce13f7SAndroid Build Coastguard Worker 		insns[1] = insn;
44*03ce13f7SAndroid Build Coastguard Worker 	}
45*03ce13f7SAndroid Build Coastguard Worker 
46*03ce13f7SAndroid Build Coastguard Worker 	switch(insns[1].opcode())
47*03ce13f7SAndroid Build Coastguard Worker 	{
48*03ce13f7SAndroid Build Coastguard Worker 	case spv::OpBranch:
49*03ce13f7SAndroid Build Coastguard Worker 		branchInstruction = insns[1];
50*03ce13f7SAndroid Build Coastguard Worker 		outs.emplace(Block::ID(branchInstruction.word(1)));
51*03ce13f7SAndroid Build Coastguard Worker 
52*03ce13f7SAndroid Build Coastguard Worker 		switch(insns[0].opcode())
53*03ce13f7SAndroid Build Coastguard Worker 		{
54*03ce13f7SAndroid Build Coastguard Worker 		case spv::OpLoopMerge:
55*03ce13f7SAndroid Build Coastguard Worker 			kind = Loop;
56*03ce13f7SAndroid Build Coastguard Worker 			mergeInstruction = insns[0];
57*03ce13f7SAndroid Build Coastguard Worker 			mergeBlock = Block::ID(mergeInstruction.word(1));
58*03ce13f7SAndroid Build Coastguard Worker 			continueTarget = Block::ID(mergeInstruction.word(2));
59*03ce13f7SAndroid Build Coastguard Worker 			break;
60*03ce13f7SAndroid Build Coastguard Worker 
61*03ce13f7SAndroid Build Coastguard Worker 		default:
62*03ce13f7SAndroid Build Coastguard Worker 			kind = Block::Simple;
63*03ce13f7SAndroid Build Coastguard Worker 			break;
64*03ce13f7SAndroid Build Coastguard Worker 		}
65*03ce13f7SAndroid Build Coastguard Worker 		break;
66*03ce13f7SAndroid Build Coastguard Worker 
67*03ce13f7SAndroid Build Coastguard Worker 	case spv::OpBranchConditional:
68*03ce13f7SAndroid Build Coastguard Worker 		branchInstruction = insns[1];
69*03ce13f7SAndroid Build Coastguard Worker 		outs.emplace(Block::ID(branchInstruction.word(2)));
70*03ce13f7SAndroid Build Coastguard Worker 		outs.emplace(Block::ID(branchInstruction.word(3)));
71*03ce13f7SAndroid Build Coastguard Worker 
72*03ce13f7SAndroid Build Coastguard Worker 		switch(insns[0].opcode())
73*03ce13f7SAndroid Build Coastguard Worker 		{
74*03ce13f7SAndroid Build Coastguard Worker 		case spv::OpSelectionMerge:
75*03ce13f7SAndroid Build Coastguard Worker 			kind = StructuredBranchConditional;
76*03ce13f7SAndroid Build Coastguard Worker 			mergeInstruction = insns[0];
77*03ce13f7SAndroid Build Coastguard Worker 			mergeBlock = Block::ID(mergeInstruction.word(1));
78*03ce13f7SAndroid Build Coastguard Worker 			break;
79*03ce13f7SAndroid Build Coastguard Worker 
80*03ce13f7SAndroid Build Coastguard Worker 		case spv::OpLoopMerge:
81*03ce13f7SAndroid Build Coastguard Worker 			kind = Loop;
82*03ce13f7SAndroid Build Coastguard Worker 			mergeInstruction = insns[0];
83*03ce13f7SAndroid Build Coastguard Worker 			mergeBlock = Block::ID(mergeInstruction.word(1));
84*03ce13f7SAndroid Build Coastguard Worker 			continueTarget = Block::ID(mergeInstruction.word(2));
85*03ce13f7SAndroid Build Coastguard Worker 			break;
86*03ce13f7SAndroid Build Coastguard Worker 
87*03ce13f7SAndroid Build Coastguard Worker 		default:
88*03ce13f7SAndroid Build Coastguard Worker 			kind = UnstructuredBranchConditional;
89*03ce13f7SAndroid Build Coastguard Worker 			break;
90*03ce13f7SAndroid Build Coastguard Worker 		}
91*03ce13f7SAndroid Build Coastguard Worker 		break;
92*03ce13f7SAndroid Build Coastguard Worker 
93*03ce13f7SAndroid Build Coastguard Worker 	case spv::OpSwitch:
94*03ce13f7SAndroid Build Coastguard Worker 		branchInstruction = insns[1];
95*03ce13f7SAndroid Build Coastguard Worker 		outs.emplace(Block::ID(branchInstruction.word(2)));
96*03ce13f7SAndroid Build Coastguard Worker 		for(uint32_t w = 4; w < branchInstruction.wordCount(); w += 2)
97*03ce13f7SAndroid Build Coastguard Worker 		{
98*03ce13f7SAndroid Build Coastguard Worker 			outs.emplace(Block::ID(branchInstruction.word(w)));
99*03ce13f7SAndroid Build Coastguard Worker 		}
100*03ce13f7SAndroid Build Coastguard Worker 
101*03ce13f7SAndroid Build Coastguard Worker 		switch(insns[0].opcode())
102*03ce13f7SAndroid Build Coastguard Worker 		{
103*03ce13f7SAndroid Build Coastguard Worker 		case spv::OpSelectionMerge:
104*03ce13f7SAndroid Build Coastguard Worker 			kind = StructuredSwitch;
105*03ce13f7SAndroid Build Coastguard Worker 			mergeInstruction = insns[0];
106*03ce13f7SAndroid Build Coastguard Worker 			mergeBlock = Block::ID(mergeInstruction.word(1));
107*03ce13f7SAndroid Build Coastguard Worker 			break;
108*03ce13f7SAndroid Build Coastguard Worker 
109*03ce13f7SAndroid Build Coastguard Worker 		default:
110*03ce13f7SAndroid Build Coastguard Worker 			kind = UnstructuredSwitch;
111*03ce13f7SAndroid Build Coastguard Worker 			break;
112*03ce13f7SAndroid Build Coastguard Worker 		}
113*03ce13f7SAndroid Build Coastguard Worker 		break;
114*03ce13f7SAndroid Build Coastguard Worker 
115*03ce13f7SAndroid Build Coastguard Worker 	default:
116*03ce13f7SAndroid Build Coastguard Worker 		break;
117*03ce13f7SAndroid Build Coastguard Worker 	}
118*03ce13f7SAndroid Build Coastguard Worker }
119*03ce13f7SAndroid Build Coastguard Worker 
TraverseReachableBlocks(Block::ID id,Block::Set & reachable) const120*03ce13f7SAndroid Build Coastguard Worker void Spirv::Function::TraverseReachableBlocks(Block::ID id, Block::Set &reachable) const
121*03ce13f7SAndroid Build Coastguard Worker {
122*03ce13f7SAndroid Build Coastguard Worker 	if(reachable.count(id) == 0)
123*03ce13f7SAndroid Build Coastguard Worker 	{
124*03ce13f7SAndroid Build Coastguard Worker 		reachable.emplace(id);
125*03ce13f7SAndroid Build Coastguard Worker 		for(auto out : getBlock(id).outs)
126*03ce13f7SAndroid Build Coastguard Worker 		{
127*03ce13f7SAndroid Build Coastguard Worker 			TraverseReachableBlocks(out, reachable);
128*03ce13f7SAndroid Build Coastguard Worker 		}
129*03ce13f7SAndroid Build Coastguard Worker 	}
130*03ce13f7SAndroid Build Coastguard Worker }
131*03ce13f7SAndroid Build Coastguard Worker 
AssignBlockFields()132*03ce13f7SAndroid Build Coastguard Worker void Spirv::Function::AssignBlockFields()
133*03ce13f7SAndroid Build Coastguard Worker {
134*03ce13f7SAndroid Build Coastguard Worker 	Block::Set reachable;
135*03ce13f7SAndroid Build Coastguard Worker 	TraverseReachableBlocks(entry, reachable);
136*03ce13f7SAndroid Build Coastguard Worker 
137*03ce13f7SAndroid Build Coastguard Worker 	for(auto &it : blocks)
138*03ce13f7SAndroid Build Coastguard Worker 	{
139*03ce13f7SAndroid Build Coastguard Worker 		auto &blockId = it.first;
140*03ce13f7SAndroid Build Coastguard Worker 		auto &block = it.second;
141*03ce13f7SAndroid Build Coastguard Worker 		if(reachable.count(blockId) > 0)
142*03ce13f7SAndroid Build Coastguard Worker 		{
143*03ce13f7SAndroid Build Coastguard Worker 			for(auto &outId : it.second.outs)
144*03ce13f7SAndroid Build Coastguard Worker 			{
145*03ce13f7SAndroid Build Coastguard Worker 				auto outIt = blocks.find(outId);
146*03ce13f7SAndroid Build Coastguard Worker 				ASSERT_MSG(outIt != blocks.end(), "Block %d has a non-existent out %d", blockId.value(), outId.value());
147*03ce13f7SAndroid Build Coastguard Worker 				auto &out = outIt->second;
148*03ce13f7SAndroid Build Coastguard Worker 				out.ins.emplace(blockId);
149*03ce13f7SAndroid Build Coastguard Worker 			}
150*03ce13f7SAndroid Build Coastguard Worker 			if(block.kind == Block::Loop)
151*03ce13f7SAndroid Build Coastguard Worker 			{
152*03ce13f7SAndroid Build Coastguard Worker 				auto mergeIt = blocks.find(block.mergeBlock);
153*03ce13f7SAndroid Build Coastguard Worker 				ASSERT_MSG(mergeIt != blocks.end(), "Loop block %d has a non-existent merge block %d", blockId.value(), block.mergeBlock.value());
154*03ce13f7SAndroid Build Coastguard Worker 				mergeIt->second.isLoopMerge = true;
155*03ce13f7SAndroid Build Coastguard Worker 			}
156*03ce13f7SAndroid Build Coastguard Worker 		}
157*03ce13f7SAndroid Build Coastguard Worker 	}
158*03ce13f7SAndroid Build Coastguard Worker }
159*03ce13f7SAndroid Build Coastguard Worker 
ForeachBlockDependency(Block::ID blockId,std::function<void (Block::ID)> f) const160*03ce13f7SAndroid Build Coastguard Worker void Spirv::Function::ForeachBlockDependency(Block::ID blockId, std::function<void(Block::ID)> f) const
161*03ce13f7SAndroid Build Coastguard Worker {
162*03ce13f7SAndroid Build Coastguard Worker 	auto block = getBlock(blockId);
163*03ce13f7SAndroid Build Coastguard Worker 	for(auto dep : block.ins)
164*03ce13f7SAndroid Build Coastguard Worker 	{
165*03ce13f7SAndroid Build Coastguard Worker 		if(block.kind != Block::Loop ||                  // if not a loop...
166*03ce13f7SAndroid Build Coastguard Worker 		   !ExistsPath(blockId, dep, block.mergeBlock))  // or a loop and not a loop back edge
167*03ce13f7SAndroid Build Coastguard Worker 		{
168*03ce13f7SAndroid Build Coastguard Worker 			f(dep);
169*03ce13f7SAndroid Build Coastguard Worker 		}
170*03ce13f7SAndroid Build Coastguard Worker 	}
171*03ce13f7SAndroid Build Coastguard Worker }
172*03ce13f7SAndroid Build Coastguard Worker 
ExistsPath(Block::ID from,Block::ID to,Block::ID notPassingThrough) const173*03ce13f7SAndroid Build Coastguard Worker bool Spirv::Function::ExistsPath(Block::ID from, Block::ID to, Block::ID notPassingThrough) const
174*03ce13f7SAndroid Build Coastguard Worker {
175*03ce13f7SAndroid Build Coastguard Worker 	// TODO: Optimize: This can be cached on the block.
176*03ce13f7SAndroid Build Coastguard Worker 	Block::Set seen;
177*03ce13f7SAndroid Build Coastguard Worker 	seen.emplace(notPassingThrough);
178*03ce13f7SAndroid Build Coastguard Worker 
179*03ce13f7SAndroid Build Coastguard Worker 	std::queue<Block::ID> pending;
180*03ce13f7SAndroid Build Coastguard Worker 	pending.emplace(from);
181*03ce13f7SAndroid Build Coastguard Worker 
182*03ce13f7SAndroid Build Coastguard Worker 	while(pending.size() > 0)
183*03ce13f7SAndroid Build Coastguard Worker 	{
184*03ce13f7SAndroid Build Coastguard Worker 		auto id = pending.front();
185*03ce13f7SAndroid Build Coastguard Worker 		pending.pop();
186*03ce13f7SAndroid Build Coastguard Worker 		for(auto out : getBlock(id).outs)
187*03ce13f7SAndroid Build Coastguard Worker 		{
188*03ce13f7SAndroid Build Coastguard Worker 			if(seen.count(out) != 0) { continue; }
189*03ce13f7SAndroid Build Coastguard Worker 			if(out == to) { return true; }
190*03ce13f7SAndroid Build Coastguard Worker 			pending.emplace(out);
191*03ce13f7SAndroid Build Coastguard Worker 		}
192*03ce13f7SAndroid Build Coastguard Worker 		seen.emplace(id);
193*03ce13f7SAndroid Build Coastguard Worker 	}
194*03ce13f7SAndroid Build Coastguard Worker 
195*03ce13f7SAndroid Build Coastguard Worker 	return false;
196*03ce13f7SAndroid Build Coastguard Worker }
197*03ce13f7SAndroid Build Coastguard Worker 
addOutputActiveLaneMaskEdge(Block::ID to,RValue<SIMD::Int> mask)198*03ce13f7SAndroid Build Coastguard Worker void SpirvEmitter::addOutputActiveLaneMaskEdge(Block::ID to, RValue<SIMD::Int> mask)
199*03ce13f7SAndroid Build Coastguard Worker {
200*03ce13f7SAndroid Build Coastguard Worker 	addActiveLaneMaskEdge(block, to, mask & activeLaneMask());
201*03ce13f7SAndroid Build Coastguard Worker }
202*03ce13f7SAndroid Build Coastguard Worker 
addActiveLaneMaskEdge(Block::ID from,Block::ID to,RValue<SIMD::Int> mask)203*03ce13f7SAndroid Build Coastguard Worker void SpirvEmitter::addActiveLaneMaskEdge(Block::ID from, Block::ID to, RValue<SIMD::Int> mask)
204*03ce13f7SAndroid Build Coastguard Worker {
205*03ce13f7SAndroid Build Coastguard Worker 	auto edge = Block::Edge{ from, to };
206*03ce13f7SAndroid Build Coastguard Worker 	auto it = edgeActiveLaneMasks.find(edge);
207*03ce13f7SAndroid Build Coastguard Worker 	if(it == edgeActiveLaneMasks.end())
208*03ce13f7SAndroid Build Coastguard Worker 	{
209*03ce13f7SAndroid Build Coastguard Worker 		edgeActiveLaneMasks.emplace(edge, mask);
210*03ce13f7SAndroid Build Coastguard Worker 	}
211*03ce13f7SAndroid Build Coastguard Worker 	else
212*03ce13f7SAndroid Build Coastguard Worker 	{
213*03ce13f7SAndroid Build Coastguard Worker 		auto combined = it->second | mask;
214*03ce13f7SAndroid Build Coastguard Worker 		edgeActiveLaneMasks.erase(edge);
215*03ce13f7SAndroid Build Coastguard Worker 		edgeActiveLaneMasks.emplace(edge, combined);
216*03ce13f7SAndroid Build Coastguard Worker 	}
217*03ce13f7SAndroid Build Coastguard Worker }
218*03ce13f7SAndroid Build Coastguard Worker 
GetActiveLaneMaskEdge(Block::ID from,Block::ID to) const219*03ce13f7SAndroid Build Coastguard Worker RValue<SIMD::Int> SpirvEmitter::GetActiveLaneMaskEdge(Block::ID from, Block::ID to) const
220*03ce13f7SAndroid Build Coastguard Worker {
221*03ce13f7SAndroid Build Coastguard Worker 	auto edge = Block::Edge{ from, to };
222*03ce13f7SAndroid Build Coastguard Worker 	auto it = edgeActiveLaneMasks.find(edge);
223*03ce13f7SAndroid Build Coastguard Worker 	ASSERT_MSG(it != edgeActiveLaneMasks.end(), "Could not find edge %d -> %d", from.value(), to.value());
224*03ce13f7SAndroid Build Coastguard Worker 	return it->second;
225*03ce13f7SAndroid Build Coastguard Worker }
226*03ce13f7SAndroid Build Coastguard Worker 
EmitBlocks(Block::ID id,Block::ID ignore)227*03ce13f7SAndroid Build Coastguard Worker void SpirvEmitter::EmitBlocks(Block::ID id, Block::ID ignore /* = 0 */)
228*03ce13f7SAndroid Build Coastguard Worker {
229*03ce13f7SAndroid Build Coastguard Worker 	auto oldPending = this->pending;
230*03ce13f7SAndroid Build Coastguard Worker 	auto &function = shader.getFunction(this->function);
231*03ce13f7SAndroid Build Coastguard Worker 
232*03ce13f7SAndroid Build Coastguard Worker 	std::deque<Block::ID> pending;
233*03ce13f7SAndroid Build Coastguard Worker 	this->pending = &pending;
234*03ce13f7SAndroid Build Coastguard Worker 	pending.push_front(id);
235*03ce13f7SAndroid Build Coastguard Worker 	while(pending.size() > 0)
236*03ce13f7SAndroid Build Coastguard Worker 	{
237*03ce13f7SAndroid Build Coastguard Worker 		auto id = pending.front();
238*03ce13f7SAndroid Build Coastguard Worker 
239*03ce13f7SAndroid Build Coastguard Worker 		const auto &block = function.getBlock(id);
240*03ce13f7SAndroid Build Coastguard Worker 		if(id == ignore)
241*03ce13f7SAndroid Build Coastguard Worker 		{
242*03ce13f7SAndroid Build Coastguard Worker 			pending.pop_front();
243*03ce13f7SAndroid Build Coastguard Worker 			continue;
244*03ce13f7SAndroid Build Coastguard Worker 		}
245*03ce13f7SAndroid Build Coastguard Worker 
246*03ce13f7SAndroid Build Coastguard Worker 		// Ensure all dependency blocks have been generated.
247*03ce13f7SAndroid Build Coastguard Worker 		auto depsDone = true;
248*03ce13f7SAndroid Build Coastguard Worker 		function.ForeachBlockDependency(id, [&](Block::ID dep) {
249*03ce13f7SAndroid Build Coastguard Worker 			if(visited.count(dep) == 0)
250*03ce13f7SAndroid Build Coastguard Worker 			{
251*03ce13f7SAndroid Build Coastguard Worker 				this->pending->push_front(dep);
252*03ce13f7SAndroid Build Coastguard Worker 				depsDone = false;
253*03ce13f7SAndroid Build Coastguard Worker 			}
254*03ce13f7SAndroid Build Coastguard Worker 		});
255*03ce13f7SAndroid Build Coastguard Worker 
256*03ce13f7SAndroid Build Coastguard Worker 		if(!depsDone)
257*03ce13f7SAndroid Build Coastguard Worker 		{
258*03ce13f7SAndroid Build Coastguard Worker 			continue;
259*03ce13f7SAndroid Build Coastguard Worker 		}
260*03ce13f7SAndroid Build Coastguard Worker 
261*03ce13f7SAndroid Build Coastguard Worker 		pending.pop_front();
262*03ce13f7SAndroid Build Coastguard Worker 
263*03ce13f7SAndroid Build Coastguard Worker 		this->block = id;
264*03ce13f7SAndroid Build Coastguard Worker 
265*03ce13f7SAndroid Build Coastguard Worker 		switch(block.kind)
266*03ce13f7SAndroid Build Coastguard Worker 		{
267*03ce13f7SAndroid Build Coastguard Worker 		case Block::Simple:
268*03ce13f7SAndroid Build Coastguard Worker 		case Block::StructuredBranchConditional:
269*03ce13f7SAndroid Build Coastguard Worker 		case Block::UnstructuredBranchConditional:
270*03ce13f7SAndroid Build Coastguard Worker 		case Block::StructuredSwitch:
271*03ce13f7SAndroid Build Coastguard Worker 		case Block::UnstructuredSwitch:
272*03ce13f7SAndroid Build Coastguard Worker 			EmitNonLoop();
273*03ce13f7SAndroid Build Coastguard Worker 			break;
274*03ce13f7SAndroid Build Coastguard Worker 
275*03ce13f7SAndroid Build Coastguard Worker 		case Block::Loop:
276*03ce13f7SAndroid Build Coastguard Worker 			EmitLoop();
277*03ce13f7SAndroid Build Coastguard Worker 			break;
278*03ce13f7SAndroid Build Coastguard Worker 
279*03ce13f7SAndroid Build Coastguard Worker 		default:
280*03ce13f7SAndroid Build Coastguard Worker 			UNREACHABLE("Unexpected Block Kind: %d", int(block.kind));
281*03ce13f7SAndroid Build Coastguard Worker 		}
282*03ce13f7SAndroid Build Coastguard Worker 	}
283*03ce13f7SAndroid Build Coastguard Worker 
284*03ce13f7SAndroid Build Coastguard Worker 	this->pending = oldPending;
285*03ce13f7SAndroid Build Coastguard Worker }
286*03ce13f7SAndroid Build Coastguard Worker 
EmitNonLoop()287*03ce13f7SAndroid Build Coastguard Worker void SpirvEmitter::EmitNonLoop()
288*03ce13f7SAndroid Build Coastguard Worker {
289*03ce13f7SAndroid Build Coastguard Worker 	auto &function = shader.getFunction(this->function);
290*03ce13f7SAndroid Build Coastguard Worker 	auto blockId = block;
291*03ce13f7SAndroid Build Coastguard Worker 	auto block = function.getBlock(blockId);
292*03ce13f7SAndroid Build Coastguard Worker 
293*03ce13f7SAndroid Build Coastguard Worker 	if(!visited.emplace(blockId).second)
294*03ce13f7SAndroid Build Coastguard Worker 	{
295*03ce13f7SAndroid Build Coastguard Worker 		return;  // Already generated this block.
296*03ce13f7SAndroid Build Coastguard Worker 	}
297*03ce13f7SAndroid Build Coastguard Worker 
298*03ce13f7SAndroid Build Coastguard Worker 	if(blockId != function.entry)
299*03ce13f7SAndroid Build Coastguard Worker 	{
300*03ce13f7SAndroid Build Coastguard Worker 		// Set the activeLaneMask.
301*03ce13f7SAndroid Build Coastguard Worker 		SIMD::Int activeLaneMask(0);
302*03ce13f7SAndroid Build Coastguard Worker 		for(auto in : block.ins)
303*03ce13f7SAndroid Build Coastguard Worker 		{
304*03ce13f7SAndroid Build Coastguard Worker 			auto inMask = GetActiveLaneMaskEdge(in, blockId);
305*03ce13f7SAndroid Build Coastguard Worker 			SPIRV_SHADER_DBG("Block {0} -> {1} mask: {2}", in, blockId, inMask);
306*03ce13f7SAndroid Build Coastguard Worker 			activeLaneMask |= inMask;
307*03ce13f7SAndroid Build Coastguard Worker 		}
308*03ce13f7SAndroid Build Coastguard Worker 		SPIRV_SHADER_DBG("Block {0} mask: {1}", blockId, activeLaneMask);
309*03ce13f7SAndroid Build Coastguard Worker 		SetActiveLaneMask(activeLaneMask);
310*03ce13f7SAndroid Build Coastguard Worker 	}
311*03ce13f7SAndroid Build Coastguard Worker 
312*03ce13f7SAndroid Build Coastguard Worker 	EmitInstructions(block.begin(), block.end());
313*03ce13f7SAndroid Build Coastguard Worker 
314*03ce13f7SAndroid Build Coastguard Worker 	for(auto out : block.outs)
315*03ce13f7SAndroid Build Coastguard Worker 	{
316*03ce13f7SAndroid Build Coastguard Worker 		if(visited.count(out) == 0)
317*03ce13f7SAndroid Build Coastguard Worker 		{
318*03ce13f7SAndroid Build Coastguard Worker 			pending->push_back(out);
319*03ce13f7SAndroid Build Coastguard Worker 		}
320*03ce13f7SAndroid Build Coastguard Worker 	}
321*03ce13f7SAndroid Build Coastguard Worker 
322*03ce13f7SAndroid Build Coastguard Worker 	SPIRV_SHADER_DBG("Block {0} done", blockId);
323*03ce13f7SAndroid Build Coastguard Worker }
324*03ce13f7SAndroid Build Coastguard Worker 
EmitLoop()325*03ce13f7SAndroid Build Coastguard Worker void SpirvEmitter::EmitLoop()
326*03ce13f7SAndroid Build Coastguard Worker {
327*03ce13f7SAndroid Build Coastguard Worker 	auto &function = shader.getFunction(this->function);
328*03ce13f7SAndroid Build Coastguard Worker 	auto blockId = block;
329*03ce13f7SAndroid Build Coastguard Worker 	auto &block = function.getBlock(blockId);
330*03ce13f7SAndroid Build Coastguard Worker 	auto mergeBlockId = block.mergeBlock;
331*03ce13f7SAndroid Build Coastguard Worker 	auto &mergeBlock = function.getBlock(mergeBlockId);
332*03ce13f7SAndroid Build Coastguard Worker 
333*03ce13f7SAndroid Build Coastguard Worker 	if(!visited.emplace(blockId).second)
334*03ce13f7SAndroid Build Coastguard Worker 	{
335*03ce13f7SAndroid Build Coastguard Worker 		return;  // Already emitted this loop.
336*03ce13f7SAndroid Build Coastguard Worker 	}
337*03ce13f7SAndroid Build Coastguard Worker 
338*03ce13f7SAndroid Build Coastguard Worker 	SPIRV_SHADER_DBG("*** LOOP HEADER ***");
339*03ce13f7SAndroid Build Coastguard Worker 
340*03ce13f7SAndroid Build Coastguard Worker 	// Gather all the blocks that make up the loop.
341*03ce13f7SAndroid Build Coastguard Worker 	std::unordered_set<Block::ID> loopBlocks;
342*03ce13f7SAndroid Build Coastguard Worker 	loopBlocks.emplace(block.mergeBlock);  // Stop traversal at mergeBlock.
343*03ce13f7SAndroid Build Coastguard Worker 	function.TraverseReachableBlocks(blockId, loopBlocks);
344*03ce13f7SAndroid Build Coastguard Worker 
345*03ce13f7SAndroid Build Coastguard Worker 	// incomingBlocks are block ins that are not back-edges.
346*03ce13f7SAndroid Build Coastguard Worker 	std::unordered_set<Block::ID> incomingBlocks;
347*03ce13f7SAndroid Build Coastguard Worker 	for(auto in : block.ins)
348*03ce13f7SAndroid Build Coastguard Worker 	{
349*03ce13f7SAndroid Build Coastguard Worker 		if(loopBlocks.count(in) == 0)
350*03ce13f7SAndroid Build Coastguard Worker 		{
351*03ce13f7SAndroid Build Coastguard Worker 			incomingBlocks.emplace(in);
352*03ce13f7SAndroid Build Coastguard Worker 		}
353*03ce13f7SAndroid Build Coastguard Worker 	}
354*03ce13f7SAndroid Build Coastguard Worker 
355*03ce13f7SAndroid Build Coastguard Worker 	// Emit the loop phi instructions, and initialize them with a value from
356*03ce13f7SAndroid Build Coastguard Worker 	// the incoming blocks.
357*03ce13f7SAndroid Build Coastguard Worker 	for(auto insn = block.begin(); insn != block.mergeInstruction; insn++)
358*03ce13f7SAndroid Build Coastguard Worker 	{
359*03ce13f7SAndroid Build Coastguard Worker 		if(insn.opcode() == spv::OpPhi)
360*03ce13f7SAndroid Build Coastguard Worker 		{
361*03ce13f7SAndroid Build Coastguard Worker 			StorePhi(blockId, insn, incomingBlocks);
362*03ce13f7SAndroid Build Coastguard Worker 		}
363*03ce13f7SAndroid Build Coastguard Worker 	}
364*03ce13f7SAndroid Build Coastguard Worker 
365*03ce13f7SAndroid Build Coastguard Worker 	// loopActiveLaneMask is the mask of lanes that are continuing to loop.
366*03ce13f7SAndroid Build Coastguard Worker 	// This is initialized with the incoming active lane masks.
367*03ce13f7SAndroid Build Coastguard Worker 	SIMD::Int loopActiveLaneMask = SIMD::Int(0);
368*03ce13f7SAndroid Build Coastguard Worker 	for(auto in : incomingBlocks)
369*03ce13f7SAndroid Build Coastguard Worker 	{
370*03ce13f7SAndroid Build Coastguard Worker 		loopActiveLaneMask |= GetActiveLaneMaskEdge(in, blockId);
371*03ce13f7SAndroid Build Coastguard Worker 	}
372*03ce13f7SAndroid Build Coastguard Worker 
373*03ce13f7SAndroid Build Coastguard Worker 	// mergeActiveLaneMasks contains edge lane masks for the merge block.
374*03ce13f7SAndroid Build Coastguard Worker 	// This is the union of all edge masks across all iterations of the loop.
375*03ce13f7SAndroid Build Coastguard Worker 	std::unordered_map<Block::ID, SIMD::Int> mergeActiveLaneMasks;
376*03ce13f7SAndroid Build Coastguard Worker 	for(auto in : function.getBlock(mergeBlockId).ins)
377*03ce13f7SAndroid Build Coastguard Worker 	{
378*03ce13f7SAndroid Build Coastguard Worker 		mergeActiveLaneMasks.emplace(in, SIMD::Int(0));
379*03ce13f7SAndroid Build Coastguard Worker 	}
380*03ce13f7SAndroid Build Coastguard Worker 
381*03ce13f7SAndroid Build Coastguard Worker 	// Create the loop basic blocks
382*03ce13f7SAndroid Build Coastguard Worker 	auto headerBasicBlock = Nucleus::createBasicBlock();
383*03ce13f7SAndroid Build Coastguard Worker 	auto mergeBasicBlock = Nucleus::createBasicBlock();
384*03ce13f7SAndroid Build Coastguard Worker 
385*03ce13f7SAndroid Build Coastguard Worker 	// Start emitting code inside the loop.
386*03ce13f7SAndroid Build Coastguard Worker 	Nucleus::createBr(headerBasicBlock);
387*03ce13f7SAndroid Build Coastguard Worker 	Nucleus::setInsertBlock(headerBasicBlock);
388*03ce13f7SAndroid Build Coastguard Worker 
389*03ce13f7SAndroid Build Coastguard Worker 	SPIRV_SHADER_DBG("*** LOOP START (mask: {0}) ***", loopActiveLaneMask);
390*03ce13f7SAndroid Build Coastguard Worker 
391*03ce13f7SAndroid Build Coastguard Worker 	// Load the active lane mask.
392*03ce13f7SAndroid Build Coastguard Worker 	SetActiveLaneMask(loopActiveLaneMask);
393*03ce13f7SAndroid Build Coastguard Worker 
394*03ce13f7SAndroid Build Coastguard Worker 	// Emit the non-phi loop header block's instructions.
395*03ce13f7SAndroid Build Coastguard Worker 	for(auto insn = block.begin(); insn != block.end(); insn++)
396*03ce13f7SAndroid Build Coastguard Worker 	{
397*03ce13f7SAndroid Build Coastguard Worker 		if(insn.opcode() == spv::OpPhi)
398*03ce13f7SAndroid Build Coastguard Worker 		{
399*03ce13f7SAndroid Build Coastguard Worker 			LoadPhi(insn);
400*03ce13f7SAndroid Build Coastguard Worker 		}
401*03ce13f7SAndroid Build Coastguard Worker 		else
402*03ce13f7SAndroid Build Coastguard Worker 		{
403*03ce13f7SAndroid Build Coastguard Worker 			EmitInstruction(insn);
404*03ce13f7SAndroid Build Coastguard Worker 		}
405*03ce13f7SAndroid Build Coastguard Worker 	}
406*03ce13f7SAndroid Build Coastguard Worker 
407*03ce13f7SAndroid Build Coastguard Worker 	// Emit all blocks between the loop header and the merge block, but
408*03ce13f7SAndroid Build Coastguard Worker 	// don't emit the merge block yet.
409*03ce13f7SAndroid Build Coastguard Worker 	for(auto out : block.outs)
410*03ce13f7SAndroid Build Coastguard Worker 	{
411*03ce13f7SAndroid Build Coastguard Worker 		EmitBlocks(out, mergeBlockId);
412*03ce13f7SAndroid Build Coastguard Worker 	}
413*03ce13f7SAndroid Build Coastguard Worker 
414*03ce13f7SAndroid Build Coastguard Worker 	// Restore current block id after emitting loop blocks.
415*03ce13f7SAndroid Build Coastguard Worker 	this->block = blockId;
416*03ce13f7SAndroid Build Coastguard Worker 
417*03ce13f7SAndroid Build Coastguard Worker 	// Rebuild the loopActiveLaneMask from the loop back edges.
418*03ce13f7SAndroid Build Coastguard Worker 	loopActiveLaneMask = SIMD::Int(0);
419*03ce13f7SAndroid Build Coastguard Worker 	for(auto in : block.ins)
420*03ce13f7SAndroid Build Coastguard Worker 	{
421*03ce13f7SAndroid Build Coastguard Worker 		if(function.ExistsPath(blockId, in, mergeBlockId))
422*03ce13f7SAndroid Build Coastguard Worker 		{
423*03ce13f7SAndroid Build Coastguard Worker 			loopActiveLaneMask |= GetActiveLaneMaskEdge(in, blockId);
424*03ce13f7SAndroid Build Coastguard Worker 		}
425*03ce13f7SAndroid Build Coastguard Worker 	}
426*03ce13f7SAndroid Build Coastguard Worker 
427*03ce13f7SAndroid Build Coastguard Worker 	// Add active lanes to the merge lane mask.
428*03ce13f7SAndroid Build Coastguard Worker 	for(auto in : function.getBlock(mergeBlockId).ins)
429*03ce13f7SAndroid Build Coastguard Worker 	{
430*03ce13f7SAndroid Build Coastguard Worker 		auto edge = Block::Edge{ in, mergeBlockId };
431*03ce13f7SAndroid Build Coastguard Worker 		auto it = edgeActiveLaneMasks.find(edge);
432*03ce13f7SAndroid Build Coastguard Worker 
433*03ce13f7SAndroid Build Coastguard Worker 		if(it != edgeActiveLaneMasks.end())
434*03ce13f7SAndroid Build Coastguard Worker 		{
435*03ce13f7SAndroid Build Coastguard Worker 			mergeActiveLaneMasks[in] |= it->second;
436*03ce13f7SAndroid Build Coastguard Worker 		}
437*03ce13f7SAndroid Build Coastguard Worker 	}
438*03ce13f7SAndroid Build Coastguard Worker 
439*03ce13f7SAndroid Build Coastguard Worker 	// Update loop phi values.
440*03ce13f7SAndroid Build Coastguard Worker 	for(auto insn = block.begin(); insn != block.mergeInstruction; insn++)
441*03ce13f7SAndroid Build Coastguard Worker 	{
442*03ce13f7SAndroid Build Coastguard Worker 		if(insn.opcode() == spv::OpPhi)
443*03ce13f7SAndroid Build Coastguard Worker 		{
444*03ce13f7SAndroid Build Coastguard Worker 			StorePhi(blockId, insn, loopBlocks);
445*03ce13f7SAndroid Build Coastguard Worker 		}
446*03ce13f7SAndroid Build Coastguard Worker 	}
447*03ce13f7SAndroid Build Coastguard Worker 
448*03ce13f7SAndroid Build Coastguard Worker 	SPIRV_SHADER_DBG("*** LOOP END (mask: {0}) ***", loopActiveLaneMask);
449*03ce13f7SAndroid Build Coastguard Worker 
450*03ce13f7SAndroid Build Coastguard Worker 	// Use the [loop -> merge] active lane masks to update the phi values in
451*03ce13f7SAndroid Build Coastguard Worker 	// the merge block. We need to do this to handle divergent control flow
452*03ce13f7SAndroid Build Coastguard Worker 	// in the loop.
453*03ce13f7SAndroid Build Coastguard Worker 	//
454*03ce13f7SAndroid Build Coastguard Worker 	// Consider the following:
455*03ce13f7SAndroid Build Coastguard Worker 	//
456*03ce13f7SAndroid Build Coastguard Worker 	//     int phi_source = 0;
457*03ce13f7SAndroid Build Coastguard Worker 	//     for(uint i = 0; i < 4; i++)
458*03ce13f7SAndroid Build Coastguard Worker 	//     {
459*03ce13f7SAndroid Build Coastguard Worker 	//         phi_source = 0;
460*03ce13f7SAndroid Build Coastguard Worker 	//         if(gl_GlobalInvocationID.x % 4 == i) // divergent control flow
461*03ce13f7SAndroid Build Coastguard Worker 	//         {
462*03ce13f7SAndroid Build Coastguard Worker 	//             phi_source = 42; // single lane assignment.
463*03ce13f7SAndroid Build Coastguard Worker 	//             break; // activeLaneMask for [loop->merge] is active for a single lane.
464*03ce13f7SAndroid Build Coastguard Worker 	//         }
465*03ce13f7SAndroid Build Coastguard Worker 	//         // -- we are here --
466*03ce13f7SAndroid Build Coastguard Worker 	//     }
467*03ce13f7SAndroid Build Coastguard Worker 	//     // merge block
468*03ce13f7SAndroid Build Coastguard Worker 	//     int phi = phi_source; // OpPhi
469*03ce13f7SAndroid Build Coastguard Worker 	//
470*03ce13f7SAndroid Build Coastguard Worker 	// In this example, with each iteration of the loop, phi_source will
471*03ce13f7SAndroid Build Coastguard Worker 	// only have a single lane assigned. However the 'phi' value in the merge
472*03ce13f7SAndroid Build Coastguard Worker 	// block needs to be assigned the union of all the per-lane assignments
473*03ce13f7SAndroid Build Coastguard Worker 	// of phi_source when that lane exited the loop.
474*03ce13f7SAndroid Build Coastguard Worker 	for(auto insn = mergeBlock.begin(); insn != mergeBlock.end(); insn++)
475*03ce13f7SAndroid Build Coastguard Worker 	{
476*03ce13f7SAndroid Build Coastguard Worker 		if(insn.opcode() == spv::OpPhi)
477*03ce13f7SAndroid Build Coastguard Worker 		{
478*03ce13f7SAndroid Build Coastguard Worker 			StorePhi(mergeBlockId, insn, loopBlocks);
479*03ce13f7SAndroid Build Coastguard Worker 		}
480*03ce13f7SAndroid Build Coastguard Worker 	}
481*03ce13f7SAndroid Build Coastguard Worker 
482*03ce13f7SAndroid Build Coastguard Worker 	// Loop body now done.
483*03ce13f7SAndroid Build Coastguard Worker 	// If any lanes are still active, jump back to the loop header,
484*03ce13f7SAndroid Build Coastguard Worker 	// otherwise jump to the merge block.
485*03ce13f7SAndroid Build Coastguard Worker 	Nucleus::createCondBr(AnyTrue(loopActiveLaneMask).value(), headerBasicBlock, mergeBasicBlock);
486*03ce13f7SAndroid Build Coastguard Worker 
487*03ce13f7SAndroid Build Coastguard Worker 	// Continue emitting from the merge block.
488*03ce13f7SAndroid Build Coastguard Worker 	Nucleus::setInsertBlock(mergeBasicBlock);
489*03ce13f7SAndroid Build Coastguard Worker 	pending->push_back(mergeBlockId);
490*03ce13f7SAndroid Build Coastguard Worker 
491*03ce13f7SAndroid Build Coastguard Worker 	for(const auto &it : mergeActiveLaneMasks)
492*03ce13f7SAndroid Build Coastguard Worker 	{
493*03ce13f7SAndroid Build Coastguard Worker 		addActiveLaneMaskEdge(it.first, mergeBlockId, it.second);
494*03ce13f7SAndroid Build Coastguard Worker 	}
495*03ce13f7SAndroid Build Coastguard Worker }
496*03ce13f7SAndroid Build Coastguard Worker 
EmitBranch(InsnIterator insn)497*03ce13f7SAndroid Build Coastguard Worker void SpirvEmitter::EmitBranch(InsnIterator insn)
498*03ce13f7SAndroid Build Coastguard Worker {
499*03ce13f7SAndroid Build Coastguard Worker 	auto target = Block::ID(insn.word(1));
500*03ce13f7SAndroid Build Coastguard Worker 	addActiveLaneMaskEdge(block, target, activeLaneMask());
501*03ce13f7SAndroid Build Coastguard Worker }
502*03ce13f7SAndroid Build Coastguard Worker 
EmitBranchConditional(InsnIterator insn)503*03ce13f7SAndroid Build Coastguard Worker void SpirvEmitter::EmitBranchConditional(InsnIterator insn)
504*03ce13f7SAndroid Build Coastguard Worker {
505*03ce13f7SAndroid Build Coastguard Worker 	auto &function = shader.getFunction(this->function);
506*03ce13f7SAndroid Build Coastguard Worker 	auto block = function.getBlock(this->block);
507*03ce13f7SAndroid Build Coastguard Worker 	ASSERT(block.branchInstruction == insn);
508*03ce13f7SAndroid Build Coastguard Worker 
509*03ce13f7SAndroid Build Coastguard Worker 	auto condId = Object::ID(block.branchInstruction.word(1));
510*03ce13f7SAndroid Build Coastguard Worker 	auto trueBlockId = Block::ID(block.branchInstruction.word(2));
511*03ce13f7SAndroid Build Coastguard Worker 	auto falseBlockId = Block::ID(block.branchInstruction.word(3));
512*03ce13f7SAndroid Build Coastguard Worker 
513*03ce13f7SAndroid Build Coastguard Worker 	auto cond = Operand(shader, *this, condId);
514*03ce13f7SAndroid Build Coastguard Worker 	ASSERT_MSG(shader.getObjectType(condId).componentCount == 1, "Condition must be a Boolean type scalar");
515*03ce13f7SAndroid Build Coastguard Worker 
516*03ce13f7SAndroid Build Coastguard Worker 	// TODO: Optimize for case where all lanes take same path.
517*03ce13f7SAndroid Build Coastguard Worker 
518*03ce13f7SAndroid Build Coastguard Worker 	addOutputActiveLaneMaskEdge(trueBlockId, cond.Int(0));
519*03ce13f7SAndroid Build Coastguard Worker 	addOutputActiveLaneMaskEdge(falseBlockId, ~cond.Int(0));
520*03ce13f7SAndroid Build Coastguard Worker }
521*03ce13f7SAndroid Build Coastguard Worker 
EmitSwitch(InsnIterator insn)522*03ce13f7SAndroid Build Coastguard Worker void SpirvEmitter::EmitSwitch(InsnIterator insn)
523*03ce13f7SAndroid Build Coastguard Worker {
524*03ce13f7SAndroid Build Coastguard Worker 	auto &function = shader.getFunction(this->function);
525*03ce13f7SAndroid Build Coastguard Worker 	auto block = function.getBlock(this->block);
526*03ce13f7SAndroid Build Coastguard Worker 	ASSERT(block.branchInstruction == insn);
527*03ce13f7SAndroid Build Coastguard Worker 
528*03ce13f7SAndroid Build Coastguard Worker 	auto selId = Object::ID(block.branchInstruction.word(1));
529*03ce13f7SAndroid Build Coastguard Worker 
530*03ce13f7SAndroid Build Coastguard Worker 	auto sel = Operand(shader, *this, selId);
531*03ce13f7SAndroid Build Coastguard Worker 	ASSERT_MSG(sel.componentCount == 1, "Selector must be a scalar");
532*03ce13f7SAndroid Build Coastguard Worker 	SPIRV_SHADER_DBG("switch({0})", sel);
533*03ce13f7SAndroid Build Coastguard Worker 
534*03ce13f7SAndroid Build Coastguard Worker 	auto numCases = (block.branchInstruction.wordCount() - 3) / 2;
535*03ce13f7SAndroid Build Coastguard Worker 
536*03ce13f7SAndroid Build Coastguard Worker 	// TODO: Optimize for case where all lanes take same path.
537*03ce13f7SAndroid Build Coastguard Worker 
538*03ce13f7SAndroid Build Coastguard Worker 	SIMD::Int defaultLaneMask = activeLaneMask();
539*03ce13f7SAndroid Build Coastguard Worker 
540*03ce13f7SAndroid Build Coastguard Worker 	// Gather up the case label matches and calculate defaultLaneMask.
541*03ce13f7SAndroid Build Coastguard Worker 	std::vector<RValue<SIMD::Int>> caseLabelMatches;
542*03ce13f7SAndroid Build Coastguard Worker 	caseLabelMatches.reserve(numCases);
543*03ce13f7SAndroid Build Coastguard Worker 
544*03ce13f7SAndroid Build Coastguard Worker 	for(uint32_t i = 0; i < numCases; i++)
545*03ce13f7SAndroid Build Coastguard Worker 	{
546*03ce13f7SAndroid Build Coastguard Worker 		auto label = block.branchInstruction.word(i * 2 + 3);
547*03ce13f7SAndroid Build Coastguard Worker 		auto caseBlockId = Block::ID(block.branchInstruction.word(i * 2 + 4));
548*03ce13f7SAndroid Build Coastguard Worker 		auto caseLabelMatch = CmpEQ(sel.Int(0), SIMD::Int(label));
549*03ce13f7SAndroid Build Coastguard Worker 		SPIRV_SHADER_DBG("case {0}: {1}", label, caseLabelMatch & activeLaneMask());
550*03ce13f7SAndroid Build Coastguard Worker 		addOutputActiveLaneMaskEdge(caseBlockId, caseLabelMatch);
551*03ce13f7SAndroid Build Coastguard Worker 		defaultLaneMask &= ~caseLabelMatch;
552*03ce13f7SAndroid Build Coastguard Worker 	}
553*03ce13f7SAndroid Build Coastguard Worker 
554*03ce13f7SAndroid Build Coastguard Worker 	auto defaultBlockId = Block::ID(block.branchInstruction.word(2));
555*03ce13f7SAndroid Build Coastguard Worker 	SPIRV_SHADER_DBG("default: {0}", defaultLaneMask);
556*03ce13f7SAndroid Build Coastguard Worker 	addOutputActiveLaneMaskEdge(defaultBlockId, defaultLaneMask);
557*03ce13f7SAndroid Build Coastguard Worker }
558*03ce13f7SAndroid Build Coastguard Worker 
EmitUnreachable(InsnIterator insn)559*03ce13f7SAndroid Build Coastguard Worker void SpirvEmitter::EmitUnreachable(InsnIterator insn)
560*03ce13f7SAndroid Build Coastguard Worker {
561*03ce13f7SAndroid Build Coastguard Worker 	// TODO: Log something in this case?
562*03ce13f7SAndroid Build Coastguard Worker 	SetActiveLaneMask(SIMD::Int(0));
563*03ce13f7SAndroid Build Coastguard Worker }
564*03ce13f7SAndroid Build Coastguard Worker 
EmitReturn(InsnIterator insn)565*03ce13f7SAndroid Build Coastguard Worker void SpirvEmitter::EmitReturn(InsnIterator insn)
566*03ce13f7SAndroid Build Coastguard Worker {
567*03ce13f7SAndroid Build Coastguard Worker 	SetActiveLaneMask(SIMD::Int(0));
568*03ce13f7SAndroid Build Coastguard Worker }
569*03ce13f7SAndroid Build Coastguard Worker 
EmitTerminateInvocation(InsnIterator insn)570*03ce13f7SAndroid Build Coastguard Worker void SpirvEmitter::EmitTerminateInvocation(InsnIterator insn)
571*03ce13f7SAndroid Build Coastguard Worker {
572*03ce13f7SAndroid Build Coastguard Worker 	routine->discardMask |= SignMask(activeLaneMask());
573*03ce13f7SAndroid Build Coastguard Worker 	SetActiveLaneMask(SIMD::Int(0));
574*03ce13f7SAndroid Build Coastguard Worker }
575*03ce13f7SAndroid Build Coastguard Worker 
EmitDemoteToHelperInvocation(InsnIterator insn)576*03ce13f7SAndroid Build Coastguard Worker void SpirvEmitter::EmitDemoteToHelperInvocation(InsnIterator insn)
577*03ce13f7SAndroid Build Coastguard Worker {
578*03ce13f7SAndroid Build Coastguard Worker 	routine->helperInvocation |= activeLaneMask();
579*03ce13f7SAndroid Build Coastguard Worker 	routine->discardMask |= SignMask(activeLaneMask());
580*03ce13f7SAndroid Build Coastguard Worker 	SetStoresAndAtomicsMask(storesAndAtomicsMask() & ~activeLaneMask());
581*03ce13f7SAndroid Build Coastguard Worker }
582*03ce13f7SAndroid Build Coastguard Worker 
EmitIsHelperInvocation(InsnIterator insn)583*03ce13f7SAndroid Build Coastguard Worker void SpirvEmitter::EmitIsHelperInvocation(InsnIterator insn)
584*03ce13f7SAndroid Build Coastguard Worker {
585*03ce13f7SAndroid Build Coastguard Worker 	auto &type = shader.getType(insn.resultTypeId());
586*03ce13f7SAndroid Build Coastguard Worker 	auto &dst = createIntermediate(insn.resultId(), type.componentCount);
587*03ce13f7SAndroid Build Coastguard Worker 	dst.move(0, routine->helperInvocation);
588*03ce13f7SAndroid Build Coastguard Worker }
589*03ce13f7SAndroid Build Coastguard Worker 
EmitFunctionCall(InsnIterator insn)590*03ce13f7SAndroid Build Coastguard Worker void SpirvEmitter::EmitFunctionCall(InsnIterator insn)
591*03ce13f7SAndroid Build Coastguard Worker {
592*03ce13f7SAndroid Build Coastguard Worker 	auto functionId = Spirv::Function::ID(insn.word(3));
593*03ce13f7SAndroid Build Coastguard Worker 	const auto &functionIt = shader.functions.find(functionId);
594*03ce13f7SAndroid Build Coastguard Worker 	ASSERT(functionIt != shader.functions.end());
595*03ce13f7SAndroid Build Coastguard Worker 	auto &function = functionIt->second;
596*03ce13f7SAndroid Build Coastguard Worker 
597*03ce13f7SAndroid Build Coastguard Worker 	// TODO(b/141246700): Add full support for spv::OpFunctionCall
598*03ce13f7SAndroid Build Coastguard Worker 	// The only supported function is a single OpKill wrapped in a
599*03ce13f7SAndroid Build Coastguard Worker 	// function, as a result of the "wrap OpKill" SPIRV-Tools pass
600*03ce13f7SAndroid Build Coastguard Worker 	ASSERT(function.blocks.size() == 1);
601*03ce13f7SAndroid Build Coastguard Worker 	spv::Op wrapOpKill[] = { spv::OpLabel, spv::OpKill };
602*03ce13f7SAndroid Build Coastguard Worker 
603*03ce13f7SAndroid Build Coastguard Worker 	for(const auto &block : function.blocks)
604*03ce13f7SAndroid Build Coastguard Worker 	{
605*03ce13f7SAndroid Build Coastguard Worker 		int insnNumber = 0;
606*03ce13f7SAndroid Build Coastguard Worker 		for(auto blockInsn : block.second)
607*03ce13f7SAndroid Build Coastguard Worker 		{
608*03ce13f7SAndroid Build Coastguard Worker 			if(insnNumber > 1)
609*03ce13f7SAndroid Build Coastguard Worker 			{
610*03ce13f7SAndroid Build Coastguard Worker 				UNIMPLEMENTED("b/141246700: Function block number of instructions: %d", insnNumber);  // FIXME(b/141246700)
611*03ce13f7SAndroid Build Coastguard Worker 			}
612*03ce13f7SAndroid Build Coastguard Worker 
613*03ce13f7SAndroid Build Coastguard Worker 			if(blockInsn.opcode() != wrapOpKill[insnNumber++])
614*03ce13f7SAndroid Build Coastguard Worker 			{
615*03ce13f7SAndroid Build Coastguard Worker 				UNIMPLEMENTED("b/141246700: Function block instruction %d : %s", insnNumber - 1, shader.OpcodeName(blockInsn.opcode()));  // FIXME(b/141246700)
616*03ce13f7SAndroid Build Coastguard Worker 			}
617*03ce13f7SAndroid Build Coastguard Worker 
618*03ce13f7SAndroid Build Coastguard Worker 			if(blockInsn.opcode() == spv::OpKill)
619*03ce13f7SAndroid Build Coastguard Worker 			{
620*03ce13f7SAndroid Build Coastguard Worker 				EmitInstruction(blockInsn);
621*03ce13f7SAndroid Build Coastguard Worker 			}
622*03ce13f7SAndroid Build Coastguard Worker 		}
623*03ce13f7SAndroid Build Coastguard Worker 	}
624*03ce13f7SAndroid Build Coastguard Worker }
625*03ce13f7SAndroid Build Coastguard Worker 
EmitControlBarrier(InsnIterator insn)626*03ce13f7SAndroid Build Coastguard Worker void SpirvEmitter::EmitControlBarrier(InsnIterator insn)
627*03ce13f7SAndroid Build Coastguard Worker {
628*03ce13f7SAndroid Build Coastguard Worker 	auto executionScope = spv::Scope(shader.GetConstScalarInt(insn.word(1)));
629*03ce13f7SAndroid Build Coastguard Worker 	auto semantics = spv::MemorySemanticsMask(shader.GetConstScalarInt(insn.word(3)));
630*03ce13f7SAndroid Build Coastguard Worker 	// TODO(b/176819536): We probably want to consider the memory scope here.
631*03ce13f7SAndroid Build Coastguard Worker 	// For now, just always emit the full fence.
632*03ce13f7SAndroid Build Coastguard Worker 	Fence(semantics);
633*03ce13f7SAndroid Build Coastguard Worker 
634*03ce13f7SAndroid Build Coastguard Worker 	switch(executionScope)
635*03ce13f7SAndroid Build Coastguard Worker 	{
636*03ce13f7SAndroid Build Coastguard Worker 	case spv::ScopeWorkgroup:
637*03ce13f7SAndroid Build Coastguard Worker 		Yield(YieldResult::ControlBarrier);
638*03ce13f7SAndroid Build Coastguard Worker 		break;
639*03ce13f7SAndroid Build Coastguard Worker 	case spv::ScopeSubgroup:
640*03ce13f7SAndroid Build Coastguard Worker 		break;
641*03ce13f7SAndroid Build Coastguard Worker 	default:
642*03ce13f7SAndroid Build Coastguard Worker 		// See Vulkan 1.1 spec, Appendix A, Validation Rules within a Module.
643*03ce13f7SAndroid Build Coastguard Worker 		UNREACHABLE("Scope for execution must be limited to Workgroup or Subgroup");
644*03ce13f7SAndroid Build Coastguard Worker 		break;
645*03ce13f7SAndroid Build Coastguard Worker 	}
646*03ce13f7SAndroid Build Coastguard Worker }
647*03ce13f7SAndroid Build Coastguard Worker 
EmitPhi(InsnIterator insn)648*03ce13f7SAndroid Build Coastguard Worker void SpirvEmitter::EmitPhi(InsnIterator insn)
649*03ce13f7SAndroid Build Coastguard Worker {
650*03ce13f7SAndroid Build Coastguard Worker 	auto &function = shader.getFunction(this->function);
651*03ce13f7SAndroid Build Coastguard Worker 	auto currentBlock = function.getBlock(block);
652*03ce13f7SAndroid Build Coastguard Worker 
653*03ce13f7SAndroid Build Coastguard Worker 	if(!currentBlock.isLoopMerge)
654*03ce13f7SAndroid Build Coastguard Worker 	{
655*03ce13f7SAndroid Build Coastguard Worker 		// If this is a loop merge block, then don't attempt to update the
656*03ce13f7SAndroid Build Coastguard Worker 		// phi values from the ins. EmitLoop() has had to take special care
657*03ce13f7SAndroid Build Coastguard Worker 		// of this phi in order to correctly deal with divergent lanes.
658*03ce13f7SAndroid Build Coastguard Worker 		StorePhi(block, insn, currentBlock.ins);
659*03ce13f7SAndroid Build Coastguard Worker 	}
660*03ce13f7SAndroid Build Coastguard Worker 
661*03ce13f7SAndroid Build Coastguard Worker 	LoadPhi(insn);
662*03ce13f7SAndroid Build Coastguard Worker }
663*03ce13f7SAndroid Build Coastguard Worker 
LoadPhi(InsnIterator insn)664*03ce13f7SAndroid Build Coastguard Worker void SpirvEmitter::LoadPhi(InsnIterator insn)
665*03ce13f7SAndroid Build Coastguard Worker {
666*03ce13f7SAndroid Build Coastguard Worker 	auto typeId = Type::ID(insn.word(1));
667*03ce13f7SAndroid Build Coastguard Worker 	auto type = shader.getType(typeId);
668*03ce13f7SAndroid Build Coastguard Worker 	auto objectId = Object::ID(insn.word(2));
669*03ce13f7SAndroid Build Coastguard Worker 
670*03ce13f7SAndroid Build Coastguard Worker 	auto storageIt = phis.find(objectId);
671*03ce13f7SAndroid Build Coastguard Worker 	ASSERT(storageIt != phis.end());
672*03ce13f7SAndroid Build Coastguard Worker 	const auto &storage = storageIt->second;
673*03ce13f7SAndroid Build Coastguard Worker 
674*03ce13f7SAndroid Build Coastguard Worker 	auto &dst = createIntermediate(objectId, type.componentCount);
675*03ce13f7SAndroid Build Coastguard Worker 
676*03ce13f7SAndroid Build Coastguard Worker 	for(uint32_t i = 0; i < type.componentCount; i++)
677*03ce13f7SAndroid Build Coastguard Worker 	{
678*03ce13f7SAndroid Build Coastguard Worker 		dst.move(i, storage[i]);
679*03ce13f7SAndroid Build Coastguard Worker 		SPIRV_SHADER_DBG("LoadPhi({0}.{1}): {2}", objectId, i, storage[i]);
680*03ce13f7SAndroid Build Coastguard Worker 	}
681*03ce13f7SAndroid Build Coastguard Worker }
682*03ce13f7SAndroid Build Coastguard Worker 
StorePhi(Block::ID currentBlock,InsnIterator insn,const std::unordered_set<Block::ID> & filter)683*03ce13f7SAndroid Build Coastguard Worker void SpirvEmitter::StorePhi(Block::ID currentBlock, InsnIterator insn, const std::unordered_set<Block::ID> &filter)
684*03ce13f7SAndroid Build Coastguard Worker {
685*03ce13f7SAndroid Build Coastguard Worker 	auto typeId = Type::ID(insn.word(1));
686*03ce13f7SAndroid Build Coastguard Worker 	auto type = shader.getType(typeId);
687*03ce13f7SAndroid Build Coastguard Worker 	auto objectId = Object::ID(insn.word(2));
688*03ce13f7SAndroid Build Coastguard Worker 
689*03ce13f7SAndroid Build Coastguard Worker 	auto storageIt = phis.find(objectId);
690*03ce13f7SAndroid Build Coastguard Worker 	ASSERT(storageIt != phis.end());
691*03ce13f7SAndroid Build Coastguard Worker 	auto &storage = storageIt->second;
692*03ce13f7SAndroid Build Coastguard Worker 
693*03ce13f7SAndroid Build Coastguard Worker 	for(uint32_t w = 3; w < insn.wordCount(); w += 2)
694*03ce13f7SAndroid Build Coastguard Worker 	{
695*03ce13f7SAndroid Build Coastguard Worker 		auto varId = Object::ID(insn.word(w + 0));
696*03ce13f7SAndroid Build Coastguard Worker 		auto blockId = Block::ID(insn.word(w + 1));
697*03ce13f7SAndroid Build Coastguard Worker 
698*03ce13f7SAndroid Build Coastguard Worker 		if(filter.count(blockId) == 0)
699*03ce13f7SAndroid Build Coastguard Worker 		{
700*03ce13f7SAndroid Build Coastguard Worker 			continue;
701*03ce13f7SAndroid Build Coastguard Worker 		}
702*03ce13f7SAndroid Build Coastguard Worker 
703*03ce13f7SAndroid Build Coastguard Worker 		auto mask = GetActiveLaneMaskEdge(blockId, currentBlock);
704*03ce13f7SAndroid Build Coastguard Worker 		auto in = Operand(shader, *this, varId);
705*03ce13f7SAndroid Build Coastguard Worker 
706*03ce13f7SAndroid Build Coastguard Worker 		for(uint32_t i = 0; i < type.componentCount; i++)
707*03ce13f7SAndroid Build Coastguard Worker 		{
708*03ce13f7SAndroid Build Coastguard Worker 			storage[i] = As<SIMD::Float>((As<SIMD::Int>(storage[i]) & ~mask) | (in.Int(i) & mask));
709*03ce13f7SAndroid Build Coastguard Worker 			SPIRV_SHADER_DBG("StorePhi({0}.{1}): [{2} <- {3}] {4}: {5}, mask: {6}",
710*03ce13f7SAndroid Build Coastguard Worker 			                 objectId, i, currentBlock, blockId, varId, in.UInt(i), mask);
711*03ce13f7SAndroid Build Coastguard Worker 		}
712*03ce13f7SAndroid Build Coastguard Worker 	}
713*03ce13f7SAndroid Build Coastguard Worker 
714*03ce13f7SAndroid Build Coastguard Worker 	for(uint32_t i = 0; i < type.componentCount; i++)
715*03ce13f7SAndroid Build Coastguard Worker 	{
716*03ce13f7SAndroid Build Coastguard Worker 		SPIRV_SHADER_DBG("StorePhi({0}.{1}): {2}", objectId, i, As<SIMD::UInt>(storage[i]));
717*03ce13f7SAndroid Build Coastguard Worker 	}
718*03ce13f7SAndroid Build Coastguard Worker }
719*03ce13f7SAndroid Build Coastguard Worker 
Yield(YieldResult res) const720*03ce13f7SAndroid Build Coastguard Worker void SpirvEmitter::Yield(YieldResult res) const
721*03ce13f7SAndroid Build Coastguard Worker {
722*03ce13f7SAndroid Build Coastguard Worker 	rr::Yield(RValue<Int>(int(res)));
723*03ce13f7SAndroid Build Coastguard Worker }
724*03ce13f7SAndroid Build Coastguard Worker 
SetActiveLaneMask(RValue<SIMD::Int> mask)725*03ce13f7SAndroid Build Coastguard Worker void SpirvEmitter::SetActiveLaneMask(RValue<SIMD::Int> mask)
726*03ce13f7SAndroid Build Coastguard Worker {
727*03ce13f7SAndroid Build Coastguard Worker 	activeLaneMaskValue = mask.value();
728*03ce13f7SAndroid Build Coastguard Worker }
729*03ce13f7SAndroid Build Coastguard Worker 
SetStoresAndAtomicsMask(RValue<SIMD::Int> mask)730*03ce13f7SAndroid Build Coastguard Worker void SpirvEmitter::SetStoresAndAtomicsMask(RValue<SIMD::Int> mask)
731*03ce13f7SAndroid Build Coastguard Worker {
732*03ce13f7SAndroid Build Coastguard Worker 	storesAndAtomicsMaskValue = mask.value();
733*03ce13f7SAndroid Build Coastguard Worker }
734*03ce13f7SAndroid Build Coastguard Worker 
WriteCFGGraphVizDotFile(const char * path) const735*03ce13f7SAndroid Build Coastguard Worker void Spirv::WriteCFGGraphVizDotFile(const char *path) const
736*03ce13f7SAndroid Build Coastguard Worker {
737*03ce13f7SAndroid Build Coastguard Worker 	std::ofstream file(path);
738*03ce13f7SAndroid Build Coastguard Worker 	file << "digraph D {" << std::endl;
739*03ce13f7SAndroid Build Coastguard Worker 	for(auto &func : functions)
740*03ce13f7SAndroid Build Coastguard Worker 	{
741*03ce13f7SAndroid Build Coastguard Worker 		file << "  subgraph cluster_function_" << func.first.value() << " {"
742*03ce13f7SAndroid Build Coastguard Worker 		     << std::endl;
743*03ce13f7SAndroid Build Coastguard Worker 
744*03ce13f7SAndroid Build Coastguard Worker 		file << "    label = \"function<" << func.first.value() << ">"
745*03ce13f7SAndroid Build Coastguard Worker 		     << (func.first == entryPoint ? " (entry point)" : "")
746*03ce13f7SAndroid Build Coastguard Worker 		     << "\"" << std::endl;
747*03ce13f7SAndroid Build Coastguard Worker 
748*03ce13f7SAndroid Build Coastguard Worker 		for(auto &block : func.second.blocks)
749*03ce13f7SAndroid Build Coastguard Worker 		{
750*03ce13f7SAndroid Build Coastguard Worker 			file << "    block_" << block.first.value() << " ["
751*03ce13f7SAndroid Build Coastguard Worker 			     << "shape=circle "
752*03ce13f7SAndroid Build Coastguard Worker 			     << "label=\"" << block.first.value() << "\""
753*03ce13f7SAndroid Build Coastguard Worker 			     << "]" << std::endl;
754*03ce13f7SAndroid Build Coastguard Worker 		}
755*03ce13f7SAndroid Build Coastguard Worker 		file << std::endl;
756*03ce13f7SAndroid Build Coastguard Worker 		for(auto &block : func.second.blocks)
757*03ce13f7SAndroid Build Coastguard Worker 		{
758*03ce13f7SAndroid Build Coastguard Worker 			file << "    block_" << block.first.value() << " -> {";
759*03ce13f7SAndroid Build Coastguard Worker 			bool first = true;
760*03ce13f7SAndroid Build Coastguard Worker 			for(auto outs : block.second.outs)
761*03ce13f7SAndroid Build Coastguard Worker 			{
762*03ce13f7SAndroid Build Coastguard Worker 				if(!first) { file << ", "; }
763*03ce13f7SAndroid Build Coastguard Worker 				file << "block_" << outs.value();
764*03ce13f7SAndroid Build Coastguard Worker 				first = false;
765*03ce13f7SAndroid Build Coastguard Worker 			}
766*03ce13f7SAndroid Build Coastguard Worker 			file << "}" << std::endl;
767*03ce13f7SAndroid Build Coastguard Worker 		}
768*03ce13f7SAndroid Build Coastguard Worker 		file << std::endl;
769*03ce13f7SAndroid Build Coastguard Worker 		for(auto &block : func.second.blocks)
770*03ce13f7SAndroid Build Coastguard Worker 		{
771*03ce13f7SAndroid Build Coastguard Worker 			if(block.second.kind == Block::Loop)
772*03ce13f7SAndroid Build Coastguard Worker 			{
773*03ce13f7SAndroid Build Coastguard Worker 				if(block.second.mergeBlock != 0)
774*03ce13f7SAndroid Build Coastguard Worker 				{
775*03ce13f7SAndroid Build Coastguard Worker 					file << "    block_" << block.first.value() << " -> "
776*03ce13f7SAndroid Build Coastguard Worker 					     << "block_" << block.second.mergeBlock.value()
777*03ce13f7SAndroid Build Coastguard Worker 					     << "[label=\"M\" style=dashed color=blue]"
778*03ce13f7SAndroid Build Coastguard Worker 					     << std::endl;
779*03ce13f7SAndroid Build Coastguard Worker 				}
780*03ce13f7SAndroid Build Coastguard Worker 
781*03ce13f7SAndroid Build Coastguard Worker 				if(block.second.continueTarget != 0)
782*03ce13f7SAndroid Build Coastguard Worker 				{
783*03ce13f7SAndroid Build Coastguard Worker 					file << "    block_" << block.first.value() << " -> "
784*03ce13f7SAndroid Build Coastguard Worker 					     << "block_" << block.second.continueTarget.value()
785*03ce13f7SAndroid Build Coastguard Worker 					     << "[label=\"C\" style=dashed color=green]"
786*03ce13f7SAndroid Build Coastguard Worker 					     << std::endl;
787*03ce13f7SAndroid Build Coastguard Worker 				}
788*03ce13f7SAndroid Build Coastguard Worker 			}
789*03ce13f7SAndroid Build Coastguard Worker 		}
790*03ce13f7SAndroid Build Coastguard Worker 
791*03ce13f7SAndroid Build Coastguard Worker 		file << "  }" << std::endl;
792*03ce13f7SAndroid Build Coastguard Worker 	}
793*03ce13f7SAndroid Build Coastguard Worker 
794*03ce13f7SAndroid Build Coastguard Worker 	for(auto &func : functions)
795*03ce13f7SAndroid Build Coastguard Worker 	{
796*03ce13f7SAndroid Build Coastguard Worker 		for(auto &block : func.second.blocks)
797*03ce13f7SAndroid Build Coastguard Worker 		{
798*03ce13f7SAndroid Build Coastguard Worker 			for(auto insn : block.second)
799*03ce13f7SAndroid Build Coastguard Worker 			{
800*03ce13f7SAndroid Build Coastguard Worker 				if(insn.opcode() == spv::OpFunctionCall)
801*03ce13f7SAndroid Build Coastguard Worker 				{
802*03ce13f7SAndroid Build Coastguard Worker 					auto target = getFunction(insn.word(3)).entry;
803*03ce13f7SAndroid Build Coastguard Worker 					file << "    block_" << block.first.value() << " -> "
804*03ce13f7SAndroid Build Coastguard Worker 					     << "block_" << target.value()
805*03ce13f7SAndroid Build Coastguard Worker 					     << "[color=\"#00008050\"]"
806*03ce13f7SAndroid Build Coastguard Worker 					     << std::endl;
807*03ce13f7SAndroid Build Coastguard Worker 				}
808*03ce13f7SAndroid Build Coastguard Worker 			}
809*03ce13f7SAndroid Build Coastguard Worker 		}
810*03ce13f7SAndroid Build Coastguard Worker 	}
811*03ce13f7SAndroid Build Coastguard Worker 
812*03ce13f7SAndroid Build Coastguard Worker 	file << "}" << std::endl;
813*03ce13f7SAndroid Build Coastguard Worker }
814*03ce13f7SAndroid Build Coastguard Worker 
815*03ce13f7SAndroid Build Coastguard Worker }  // namespace sw
816