1*03ce13f7SAndroid Build Coastguard Worker // Copyright 2019 The SwiftShader Authors. All Rights Reserved.
2*03ce13f7SAndroid Build Coastguard Worker //
3*03ce13f7SAndroid Build Coastguard Worker // Licensed under the Apache License, Version 2.0 (the "License");
4*03ce13f7SAndroid Build Coastguard Worker // you may not use this file except in compliance with the License.
5*03ce13f7SAndroid Build Coastguard Worker // You may obtain a copy of the License at
6*03ce13f7SAndroid Build Coastguard Worker //
7*03ce13f7SAndroid Build Coastguard Worker // http://www.apache.org/licenses/LICENSE-2.0
8*03ce13f7SAndroid Build Coastguard Worker //
9*03ce13f7SAndroid Build Coastguard Worker // Unless required by applicable law or agreed to in writing, software
10*03ce13f7SAndroid Build Coastguard Worker // distributed under the License is distributed on an "AS IS" BASIS,
11*03ce13f7SAndroid Build Coastguard Worker // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12*03ce13f7SAndroid Build Coastguard Worker // See the License for the specific language governing permissions and
13*03ce13f7SAndroid Build Coastguard Worker // limitations under the License.
14*03ce13f7SAndroid Build Coastguard Worker
15*03ce13f7SAndroid Build Coastguard Worker #include "SpirvShader.hpp"
16*03ce13f7SAndroid Build Coastguard Worker #include "SpirvShaderDebug.hpp"
17*03ce13f7SAndroid Build Coastguard Worker
18*03ce13f7SAndroid Build Coastguard Worker #include "Reactor/Coroutine.hpp" // rr::Yield
19*03ce13f7SAndroid Build Coastguard Worker
20*03ce13f7SAndroid Build Coastguard Worker #include "ShaderCore.hpp"
21*03ce13f7SAndroid Build Coastguard Worker
22*03ce13f7SAndroid Build Coastguard Worker #include <spirv/unified1/spirv.hpp>
23*03ce13f7SAndroid Build Coastguard Worker
24*03ce13f7SAndroid Build Coastguard Worker #include <queue>
25*03ce13f7SAndroid Build Coastguard Worker
26*03ce13f7SAndroid Build Coastguard Worker #include <fstream>
27*03ce13f7SAndroid Build Coastguard Worker #include <iostream>
28*03ce13f7SAndroid Build Coastguard Worker
29*03ce13f7SAndroid Build Coastguard Worker namespace sw {
30*03ce13f7SAndroid Build Coastguard Worker
Block(InsnIterator begin,InsnIterator end)31*03ce13f7SAndroid Build Coastguard Worker Spirv::Block::Block(InsnIterator begin, InsnIterator end)
32*03ce13f7SAndroid Build Coastguard Worker : begin_(begin)
33*03ce13f7SAndroid Build Coastguard Worker , end_(end)
34*03ce13f7SAndroid Build Coastguard Worker {
35*03ce13f7SAndroid Build Coastguard Worker // Default to a Simple, this may change later.
36*03ce13f7SAndroid Build Coastguard Worker kind = Block::Simple;
37*03ce13f7SAndroid Build Coastguard Worker
38*03ce13f7SAndroid Build Coastguard Worker // Walk the instructions to find the last two of the block.
39*03ce13f7SAndroid Build Coastguard Worker InsnIterator insns[2];
40*03ce13f7SAndroid Build Coastguard Worker for(auto insn : *this)
41*03ce13f7SAndroid Build Coastguard Worker {
42*03ce13f7SAndroid Build Coastguard Worker insns[0] = insns[1];
43*03ce13f7SAndroid Build Coastguard Worker insns[1] = insn;
44*03ce13f7SAndroid Build Coastguard Worker }
45*03ce13f7SAndroid Build Coastguard Worker
46*03ce13f7SAndroid Build Coastguard Worker switch(insns[1].opcode())
47*03ce13f7SAndroid Build Coastguard Worker {
48*03ce13f7SAndroid Build Coastguard Worker case spv::OpBranch:
49*03ce13f7SAndroid Build Coastguard Worker branchInstruction = insns[1];
50*03ce13f7SAndroid Build Coastguard Worker outs.emplace(Block::ID(branchInstruction.word(1)));
51*03ce13f7SAndroid Build Coastguard Worker
52*03ce13f7SAndroid Build Coastguard Worker switch(insns[0].opcode())
53*03ce13f7SAndroid Build Coastguard Worker {
54*03ce13f7SAndroid Build Coastguard Worker case spv::OpLoopMerge:
55*03ce13f7SAndroid Build Coastguard Worker kind = Loop;
56*03ce13f7SAndroid Build Coastguard Worker mergeInstruction = insns[0];
57*03ce13f7SAndroid Build Coastguard Worker mergeBlock = Block::ID(mergeInstruction.word(1));
58*03ce13f7SAndroid Build Coastguard Worker continueTarget = Block::ID(mergeInstruction.word(2));
59*03ce13f7SAndroid Build Coastguard Worker break;
60*03ce13f7SAndroid Build Coastguard Worker
61*03ce13f7SAndroid Build Coastguard Worker default:
62*03ce13f7SAndroid Build Coastguard Worker kind = Block::Simple;
63*03ce13f7SAndroid Build Coastguard Worker break;
64*03ce13f7SAndroid Build Coastguard Worker }
65*03ce13f7SAndroid Build Coastguard Worker break;
66*03ce13f7SAndroid Build Coastguard Worker
67*03ce13f7SAndroid Build Coastguard Worker case spv::OpBranchConditional:
68*03ce13f7SAndroid Build Coastguard Worker branchInstruction = insns[1];
69*03ce13f7SAndroid Build Coastguard Worker outs.emplace(Block::ID(branchInstruction.word(2)));
70*03ce13f7SAndroid Build Coastguard Worker outs.emplace(Block::ID(branchInstruction.word(3)));
71*03ce13f7SAndroid Build Coastguard Worker
72*03ce13f7SAndroid Build Coastguard Worker switch(insns[0].opcode())
73*03ce13f7SAndroid Build Coastguard Worker {
74*03ce13f7SAndroid Build Coastguard Worker case spv::OpSelectionMerge:
75*03ce13f7SAndroid Build Coastguard Worker kind = StructuredBranchConditional;
76*03ce13f7SAndroid Build Coastguard Worker mergeInstruction = insns[0];
77*03ce13f7SAndroid Build Coastguard Worker mergeBlock = Block::ID(mergeInstruction.word(1));
78*03ce13f7SAndroid Build Coastguard Worker break;
79*03ce13f7SAndroid Build Coastguard Worker
80*03ce13f7SAndroid Build Coastguard Worker case spv::OpLoopMerge:
81*03ce13f7SAndroid Build Coastguard Worker kind = Loop;
82*03ce13f7SAndroid Build Coastguard Worker mergeInstruction = insns[0];
83*03ce13f7SAndroid Build Coastguard Worker mergeBlock = Block::ID(mergeInstruction.word(1));
84*03ce13f7SAndroid Build Coastguard Worker continueTarget = Block::ID(mergeInstruction.word(2));
85*03ce13f7SAndroid Build Coastguard Worker break;
86*03ce13f7SAndroid Build Coastguard Worker
87*03ce13f7SAndroid Build Coastguard Worker default:
88*03ce13f7SAndroid Build Coastguard Worker kind = UnstructuredBranchConditional;
89*03ce13f7SAndroid Build Coastguard Worker break;
90*03ce13f7SAndroid Build Coastguard Worker }
91*03ce13f7SAndroid Build Coastguard Worker break;
92*03ce13f7SAndroid Build Coastguard Worker
93*03ce13f7SAndroid Build Coastguard Worker case spv::OpSwitch:
94*03ce13f7SAndroid Build Coastguard Worker branchInstruction = insns[1];
95*03ce13f7SAndroid Build Coastguard Worker outs.emplace(Block::ID(branchInstruction.word(2)));
96*03ce13f7SAndroid Build Coastguard Worker for(uint32_t w = 4; w < branchInstruction.wordCount(); w += 2)
97*03ce13f7SAndroid Build Coastguard Worker {
98*03ce13f7SAndroid Build Coastguard Worker outs.emplace(Block::ID(branchInstruction.word(w)));
99*03ce13f7SAndroid Build Coastguard Worker }
100*03ce13f7SAndroid Build Coastguard Worker
101*03ce13f7SAndroid Build Coastguard Worker switch(insns[0].opcode())
102*03ce13f7SAndroid Build Coastguard Worker {
103*03ce13f7SAndroid Build Coastguard Worker case spv::OpSelectionMerge:
104*03ce13f7SAndroid Build Coastguard Worker kind = StructuredSwitch;
105*03ce13f7SAndroid Build Coastguard Worker mergeInstruction = insns[0];
106*03ce13f7SAndroid Build Coastguard Worker mergeBlock = Block::ID(mergeInstruction.word(1));
107*03ce13f7SAndroid Build Coastguard Worker break;
108*03ce13f7SAndroid Build Coastguard Worker
109*03ce13f7SAndroid Build Coastguard Worker default:
110*03ce13f7SAndroid Build Coastguard Worker kind = UnstructuredSwitch;
111*03ce13f7SAndroid Build Coastguard Worker break;
112*03ce13f7SAndroid Build Coastguard Worker }
113*03ce13f7SAndroid Build Coastguard Worker break;
114*03ce13f7SAndroid Build Coastguard Worker
115*03ce13f7SAndroid Build Coastguard Worker default:
116*03ce13f7SAndroid Build Coastguard Worker break;
117*03ce13f7SAndroid Build Coastguard Worker }
118*03ce13f7SAndroid Build Coastguard Worker }
119*03ce13f7SAndroid Build Coastguard Worker
TraverseReachableBlocks(Block::ID id,Block::Set & reachable) const120*03ce13f7SAndroid Build Coastguard Worker void Spirv::Function::TraverseReachableBlocks(Block::ID id, Block::Set &reachable) const
121*03ce13f7SAndroid Build Coastguard Worker {
122*03ce13f7SAndroid Build Coastguard Worker if(reachable.count(id) == 0)
123*03ce13f7SAndroid Build Coastguard Worker {
124*03ce13f7SAndroid Build Coastguard Worker reachable.emplace(id);
125*03ce13f7SAndroid Build Coastguard Worker for(auto out : getBlock(id).outs)
126*03ce13f7SAndroid Build Coastguard Worker {
127*03ce13f7SAndroid Build Coastguard Worker TraverseReachableBlocks(out, reachable);
128*03ce13f7SAndroid Build Coastguard Worker }
129*03ce13f7SAndroid Build Coastguard Worker }
130*03ce13f7SAndroid Build Coastguard Worker }
131*03ce13f7SAndroid Build Coastguard Worker
AssignBlockFields()132*03ce13f7SAndroid Build Coastguard Worker void Spirv::Function::AssignBlockFields()
133*03ce13f7SAndroid Build Coastguard Worker {
134*03ce13f7SAndroid Build Coastguard Worker Block::Set reachable;
135*03ce13f7SAndroid Build Coastguard Worker TraverseReachableBlocks(entry, reachable);
136*03ce13f7SAndroid Build Coastguard Worker
137*03ce13f7SAndroid Build Coastguard Worker for(auto &it : blocks)
138*03ce13f7SAndroid Build Coastguard Worker {
139*03ce13f7SAndroid Build Coastguard Worker auto &blockId = it.first;
140*03ce13f7SAndroid Build Coastguard Worker auto &block = it.second;
141*03ce13f7SAndroid Build Coastguard Worker if(reachable.count(blockId) > 0)
142*03ce13f7SAndroid Build Coastguard Worker {
143*03ce13f7SAndroid Build Coastguard Worker for(auto &outId : it.second.outs)
144*03ce13f7SAndroid Build Coastguard Worker {
145*03ce13f7SAndroid Build Coastguard Worker auto outIt = blocks.find(outId);
146*03ce13f7SAndroid Build Coastguard Worker ASSERT_MSG(outIt != blocks.end(), "Block %d has a non-existent out %d", blockId.value(), outId.value());
147*03ce13f7SAndroid Build Coastguard Worker auto &out = outIt->second;
148*03ce13f7SAndroid Build Coastguard Worker out.ins.emplace(blockId);
149*03ce13f7SAndroid Build Coastguard Worker }
150*03ce13f7SAndroid Build Coastguard Worker if(block.kind == Block::Loop)
151*03ce13f7SAndroid Build Coastguard Worker {
152*03ce13f7SAndroid Build Coastguard Worker auto mergeIt = blocks.find(block.mergeBlock);
153*03ce13f7SAndroid Build Coastguard Worker ASSERT_MSG(mergeIt != blocks.end(), "Loop block %d has a non-existent merge block %d", blockId.value(), block.mergeBlock.value());
154*03ce13f7SAndroid Build Coastguard Worker mergeIt->second.isLoopMerge = true;
155*03ce13f7SAndroid Build Coastguard Worker }
156*03ce13f7SAndroid Build Coastguard Worker }
157*03ce13f7SAndroid Build Coastguard Worker }
158*03ce13f7SAndroid Build Coastguard Worker }
159*03ce13f7SAndroid Build Coastguard Worker
ForeachBlockDependency(Block::ID blockId,std::function<void (Block::ID)> f) const160*03ce13f7SAndroid Build Coastguard Worker void Spirv::Function::ForeachBlockDependency(Block::ID blockId, std::function<void(Block::ID)> f) const
161*03ce13f7SAndroid Build Coastguard Worker {
162*03ce13f7SAndroid Build Coastguard Worker auto block = getBlock(blockId);
163*03ce13f7SAndroid Build Coastguard Worker for(auto dep : block.ins)
164*03ce13f7SAndroid Build Coastguard Worker {
165*03ce13f7SAndroid Build Coastguard Worker if(block.kind != Block::Loop || // if not a loop...
166*03ce13f7SAndroid Build Coastguard Worker !ExistsPath(blockId, dep, block.mergeBlock)) // or a loop and not a loop back edge
167*03ce13f7SAndroid Build Coastguard Worker {
168*03ce13f7SAndroid Build Coastguard Worker f(dep);
169*03ce13f7SAndroid Build Coastguard Worker }
170*03ce13f7SAndroid Build Coastguard Worker }
171*03ce13f7SAndroid Build Coastguard Worker }
172*03ce13f7SAndroid Build Coastguard Worker
ExistsPath(Block::ID from,Block::ID to,Block::ID notPassingThrough) const173*03ce13f7SAndroid Build Coastguard Worker bool Spirv::Function::ExistsPath(Block::ID from, Block::ID to, Block::ID notPassingThrough) const
174*03ce13f7SAndroid Build Coastguard Worker {
175*03ce13f7SAndroid Build Coastguard Worker // TODO: Optimize: This can be cached on the block.
176*03ce13f7SAndroid Build Coastguard Worker Block::Set seen;
177*03ce13f7SAndroid Build Coastguard Worker seen.emplace(notPassingThrough);
178*03ce13f7SAndroid Build Coastguard Worker
179*03ce13f7SAndroid Build Coastguard Worker std::queue<Block::ID> pending;
180*03ce13f7SAndroid Build Coastguard Worker pending.emplace(from);
181*03ce13f7SAndroid Build Coastguard Worker
182*03ce13f7SAndroid Build Coastguard Worker while(pending.size() > 0)
183*03ce13f7SAndroid Build Coastguard Worker {
184*03ce13f7SAndroid Build Coastguard Worker auto id = pending.front();
185*03ce13f7SAndroid Build Coastguard Worker pending.pop();
186*03ce13f7SAndroid Build Coastguard Worker for(auto out : getBlock(id).outs)
187*03ce13f7SAndroid Build Coastguard Worker {
188*03ce13f7SAndroid Build Coastguard Worker if(seen.count(out) != 0) { continue; }
189*03ce13f7SAndroid Build Coastguard Worker if(out == to) { return true; }
190*03ce13f7SAndroid Build Coastguard Worker pending.emplace(out);
191*03ce13f7SAndroid Build Coastguard Worker }
192*03ce13f7SAndroid Build Coastguard Worker seen.emplace(id);
193*03ce13f7SAndroid Build Coastguard Worker }
194*03ce13f7SAndroid Build Coastguard Worker
195*03ce13f7SAndroid Build Coastguard Worker return false;
196*03ce13f7SAndroid Build Coastguard Worker }
197*03ce13f7SAndroid Build Coastguard Worker
addOutputActiveLaneMaskEdge(Block::ID to,RValue<SIMD::Int> mask)198*03ce13f7SAndroid Build Coastguard Worker void SpirvEmitter::addOutputActiveLaneMaskEdge(Block::ID to, RValue<SIMD::Int> mask)
199*03ce13f7SAndroid Build Coastguard Worker {
200*03ce13f7SAndroid Build Coastguard Worker addActiveLaneMaskEdge(block, to, mask & activeLaneMask());
201*03ce13f7SAndroid Build Coastguard Worker }
202*03ce13f7SAndroid Build Coastguard Worker
addActiveLaneMaskEdge(Block::ID from,Block::ID to,RValue<SIMD::Int> mask)203*03ce13f7SAndroid Build Coastguard Worker void SpirvEmitter::addActiveLaneMaskEdge(Block::ID from, Block::ID to, RValue<SIMD::Int> mask)
204*03ce13f7SAndroid Build Coastguard Worker {
205*03ce13f7SAndroid Build Coastguard Worker auto edge = Block::Edge{ from, to };
206*03ce13f7SAndroid Build Coastguard Worker auto it = edgeActiveLaneMasks.find(edge);
207*03ce13f7SAndroid Build Coastguard Worker if(it == edgeActiveLaneMasks.end())
208*03ce13f7SAndroid Build Coastguard Worker {
209*03ce13f7SAndroid Build Coastguard Worker edgeActiveLaneMasks.emplace(edge, mask);
210*03ce13f7SAndroid Build Coastguard Worker }
211*03ce13f7SAndroid Build Coastguard Worker else
212*03ce13f7SAndroid Build Coastguard Worker {
213*03ce13f7SAndroid Build Coastguard Worker auto combined = it->second | mask;
214*03ce13f7SAndroid Build Coastguard Worker edgeActiveLaneMasks.erase(edge);
215*03ce13f7SAndroid Build Coastguard Worker edgeActiveLaneMasks.emplace(edge, combined);
216*03ce13f7SAndroid Build Coastguard Worker }
217*03ce13f7SAndroid Build Coastguard Worker }
218*03ce13f7SAndroid Build Coastguard Worker
GetActiveLaneMaskEdge(Block::ID from,Block::ID to) const219*03ce13f7SAndroid Build Coastguard Worker RValue<SIMD::Int> SpirvEmitter::GetActiveLaneMaskEdge(Block::ID from, Block::ID to) const
220*03ce13f7SAndroid Build Coastguard Worker {
221*03ce13f7SAndroid Build Coastguard Worker auto edge = Block::Edge{ from, to };
222*03ce13f7SAndroid Build Coastguard Worker auto it = edgeActiveLaneMasks.find(edge);
223*03ce13f7SAndroid Build Coastguard Worker ASSERT_MSG(it != edgeActiveLaneMasks.end(), "Could not find edge %d -> %d", from.value(), to.value());
224*03ce13f7SAndroid Build Coastguard Worker return it->second;
225*03ce13f7SAndroid Build Coastguard Worker }
226*03ce13f7SAndroid Build Coastguard Worker
EmitBlocks(Block::ID id,Block::ID ignore)227*03ce13f7SAndroid Build Coastguard Worker void SpirvEmitter::EmitBlocks(Block::ID id, Block::ID ignore /* = 0 */)
228*03ce13f7SAndroid Build Coastguard Worker {
229*03ce13f7SAndroid Build Coastguard Worker auto oldPending = this->pending;
230*03ce13f7SAndroid Build Coastguard Worker auto &function = shader.getFunction(this->function);
231*03ce13f7SAndroid Build Coastguard Worker
232*03ce13f7SAndroid Build Coastguard Worker std::deque<Block::ID> pending;
233*03ce13f7SAndroid Build Coastguard Worker this->pending = &pending;
234*03ce13f7SAndroid Build Coastguard Worker pending.push_front(id);
235*03ce13f7SAndroid Build Coastguard Worker while(pending.size() > 0)
236*03ce13f7SAndroid Build Coastguard Worker {
237*03ce13f7SAndroid Build Coastguard Worker auto id = pending.front();
238*03ce13f7SAndroid Build Coastguard Worker
239*03ce13f7SAndroid Build Coastguard Worker const auto &block = function.getBlock(id);
240*03ce13f7SAndroid Build Coastguard Worker if(id == ignore)
241*03ce13f7SAndroid Build Coastguard Worker {
242*03ce13f7SAndroid Build Coastguard Worker pending.pop_front();
243*03ce13f7SAndroid Build Coastguard Worker continue;
244*03ce13f7SAndroid Build Coastguard Worker }
245*03ce13f7SAndroid Build Coastguard Worker
246*03ce13f7SAndroid Build Coastguard Worker // Ensure all dependency blocks have been generated.
247*03ce13f7SAndroid Build Coastguard Worker auto depsDone = true;
248*03ce13f7SAndroid Build Coastguard Worker function.ForeachBlockDependency(id, [&](Block::ID dep) {
249*03ce13f7SAndroid Build Coastguard Worker if(visited.count(dep) == 0)
250*03ce13f7SAndroid Build Coastguard Worker {
251*03ce13f7SAndroid Build Coastguard Worker this->pending->push_front(dep);
252*03ce13f7SAndroid Build Coastguard Worker depsDone = false;
253*03ce13f7SAndroid Build Coastguard Worker }
254*03ce13f7SAndroid Build Coastguard Worker });
255*03ce13f7SAndroid Build Coastguard Worker
256*03ce13f7SAndroid Build Coastguard Worker if(!depsDone)
257*03ce13f7SAndroid Build Coastguard Worker {
258*03ce13f7SAndroid Build Coastguard Worker continue;
259*03ce13f7SAndroid Build Coastguard Worker }
260*03ce13f7SAndroid Build Coastguard Worker
261*03ce13f7SAndroid Build Coastguard Worker pending.pop_front();
262*03ce13f7SAndroid Build Coastguard Worker
263*03ce13f7SAndroid Build Coastguard Worker this->block = id;
264*03ce13f7SAndroid Build Coastguard Worker
265*03ce13f7SAndroid Build Coastguard Worker switch(block.kind)
266*03ce13f7SAndroid Build Coastguard Worker {
267*03ce13f7SAndroid Build Coastguard Worker case Block::Simple:
268*03ce13f7SAndroid Build Coastguard Worker case Block::StructuredBranchConditional:
269*03ce13f7SAndroid Build Coastguard Worker case Block::UnstructuredBranchConditional:
270*03ce13f7SAndroid Build Coastguard Worker case Block::StructuredSwitch:
271*03ce13f7SAndroid Build Coastguard Worker case Block::UnstructuredSwitch:
272*03ce13f7SAndroid Build Coastguard Worker EmitNonLoop();
273*03ce13f7SAndroid Build Coastguard Worker break;
274*03ce13f7SAndroid Build Coastguard Worker
275*03ce13f7SAndroid Build Coastguard Worker case Block::Loop:
276*03ce13f7SAndroid Build Coastguard Worker EmitLoop();
277*03ce13f7SAndroid Build Coastguard Worker break;
278*03ce13f7SAndroid Build Coastguard Worker
279*03ce13f7SAndroid Build Coastguard Worker default:
280*03ce13f7SAndroid Build Coastguard Worker UNREACHABLE("Unexpected Block Kind: %d", int(block.kind));
281*03ce13f7SAndroid Build Coastguard Worker }
282*03ce13f7SAndroid Build Coastguard Worker }
283*03ce13f7SAndroid Build Coastguard Worker
284*03ce13f7SAndroid Build Coastguard Worker this->pending = oldPending;
285*03ce13f7SAndroid Build Coastguard Worker }
286*03ce13f7SAndroid Build Coastguard Worker
EmitNonLoop()287*03ce13f7SAndroid Build Coastguard Worker void SpirvEmitter::EmitNonLoop()
288*03ce13f7SAndroid Build Coastguard Worker {
289*03ce13f7SAndroid Build Coastguard Worker auto &function = shader.getFunction(this->function);
290*03ce13f7SAndroid Build Coastguard Worker auto blockId = block;
291*03ce13f7SAndroid Build Coastguard Worker auto block = function.getBlock(blockId);
292*03ce13f7SAndroid Build Coastguard Worker
293*03ce13f7SAndroid Build Coastguard Worker if(!visited.emplace(blockId).second)
294*03ce13f7SAndroid Build Coastguard Worker {
295*03ce13f7SAndroid Build Coastguard Worker return; // Already generated this block.
296*03ce13f7SAndroid Build Coastguard Worker }
297*03ce13f7SAndroid Build Coastguard Worker
298*03ce13f7SAndroid Build Coastguard Worker if(blockId != function.entry)
299*03ce13f7SAndroid Build Coastguard Worker {
300*03ce13f7SAndroid Build Coastguard Worker // Set the activeLaneMask.
301*03ce13f7SAndroid Build Coastguard Worker SIMD::Int activeLaneMask(0);
302*03ce13f7SAndroid Build Coastguard Worker for(auto in : block.ins)
303*03ce13f7SAndroid Build Coastguard Worker {
304*03ce13f7SAndroid Build Coastguard Worker auto inMask = GetActiveLaneMaskEdge(in, blockId);
305*03ce13f7SAndroid Build Coastguard Worker SPIRV_SHADER_DBG("Block {0} -> {1} mask: {2}", in, blockId, inMask);
306*03ce13f7SAndroid Build Coastguard Worker activeLaneMask |= inMask;
307*03ce13f7SAndroid Build Coastguard Worker }
308*03ce13f7SAndroid Build Coastguard Worker SPIRV_SHADER_DBG("Block {0} mask: {1}", blockId, activeLaneMask);
309*03ce13f7SAndroid Build Coastguard Worker SetActiveLaneMask(activeLaneMask);
310*03ce13f7SAndroid Build Coastguard Worker }
311*03ce13f7SAndroid Build Coastguard Worker
312*03ce13f7SAndroid Build Coastguard Worker EmitInstructions(block.begin(), block.end());
313*03ce13f7SAndroid Build Coastguard Worker
314*03ce13f7SAndroid Build Coastguard Worker for(auto out : block.outs)
315*03ce13f7SAndroid Build Coastguard Worker {
316*03ce13f7SAndroid Build Coastguard Worker if(visited.count(out) == 0)
317*03ce13f7SAndroid Build Coastguard Worker {
318*03ce13f7SAndroid Build Coastguard Worker pending->push_back(out);
319*03ce13f7SAndroid Build Coastguard Worker }
320*03ce13f7SAndroid Build Coastguard Worker }
321*03ce13f7SAndroid Build Coastguard Worker
322*03ce13f7SAndroid Build Coastguard Worker SPIRV_SHADER_DBG("Block {0} done", blockId);
323*03ce13f7SAndroid Build Coastguard Worker }
324*03ce13f7SAndroid Build Coastguard Worker
EmitLoop()325*03ce13f7SAndroid Build Coastguard Worker void SpirvEmitter::EmitLoop()
326*03ce13f7SAndroid Build Coastguard Worker {
327*03ce13f7SAndroid Build Coastguard Worker auto &function = shader.getFunction(this->function);
328*03ce13f7SAndroid Build Coastguard Worker auto blockId = block;
329*03ce13f7SAndroid Build Coastguard Worker auto &block = function.getBlock(blockId);
330*03ce13f7SAndroid Build Coastguard Worker auto mergeBlockId = block.mergeBlock;
331*03ce13f7SAndroid Build Coastguard Worker auto &mergeBlock = function.getBlock(mergeBlockId);
332*03ce13f7SAndroid Build Coastguard Worker
333*03ce13f7SAndroid Build Coastguard Worker if(!visited.emplace(blockId).second)
334*03ce13f7SAndroid Build Coastguard Worker {
335*03ce13f7SAndroid Build Coastguard Worker return; // Already emitted this loop.
336*03ce13f7SAndroid Build Coastguard Worker }
337*03ce13f7SAndroid Build Coastguard Worker
338*03ce13f7SAndroid Build Coastguard Worker SPIRV_SHADER_DBG("*** LOOP HEADER ***");
339*03ce13f7SAndroid Build Coastguard Worker
340*03ce13f7SAndroid Build Coastguard Worker // Gather all the blocks that make up the loop.
341*03ce13f7SAndroid Build Coastguard Worker std::unordered_set<Block::ID> loopBlocks;
342*03ce13f7SAndroid Build Coastguard Worker loopBlocks.emplace(block.mergeBlock); // Stop traversal at mergeBlock.
343*03ce13f7SAndroid Build Coastguard Worker function.TraverseReachableBlocks(blockId, loopBlocks);
344*03ce13f7SAndroid Build Coastguard Worker
345*03ce13f7SAndroid Build Coastguard Worker // incomingBlocks are block ins that are not back-edges.
346*03ce13f7SAndroid Build Coastguard Worker std::unordered_set<Block::ID> incomingBlocks;
347*03ce13f7SAndroid Build Coastguard Worker for(auto in : block.ins)
348*03ce13f7SAndroid Build Coastguard Worker {
349*03ce13f7SAndroid Build Coastguard Worker if(loopBlocks.count(in) == 0)
350*03ce13f7SAndroid Build Coastguard Worker {
351*03ce13f7SAndroid Build Coastguard Worker incomingBlocks.emplace(in);
352*03ce13f7SAndroid Build Coastguard Worker }
353*03ce13f7SAndroid Build Coastguard Worker }
354*03ce13f7SAndroid Build Coastguard Worker
355*03ce13f7SAndroid Build Coastguard Worker // Emit the loop phi instructions, and initialize them with a value from
356*03ce13f7SAndroid Build Coastguard Worker // the incoming blocks.
357*03ce13f7SAndroid Build Coastguard Worker for(auto insn = block.begin(); insn != block.mergeInstruction; insn++)
358*03ce13f7SAndroid Build Coastguard Worker {
359*03ce13f7SAndroid Build Coastguard Worker if(insn.opcode() == spv::OpPhi)
360*03ce13f7SAndroid Build Coastguard Worker {
361*03ce13f7SAndroid Build Coastguard Worker StorePhi(blockId, insn, incomingBlocks);
362*03ce13f7SAndroid Build Coastguard Worker }
363*03ce13f7SAndroid Build Coastguard Worker }
364*03ce13f7SAndroid Build Coastguard Worker
365*03ce13f7SAndroid Build Coastguard Worker // loopActiveLaneMask is the mask of lanes that are continuing to loop.
366*03ce13f7SAndroid Build Coastguard Worker // This is initialized with the incoming active lane masks.
367*03ce13f7SAndroid Build Coastguard Worker SIMD::Int loopActiveLaneMask = SIMD::Int(0);
368*03ce13f7SAndroid Build Coastguard Worker for(auto in : incomingBlocks)
369*03ce13f7SAndroid Build Coastguard Worker {
370*03ce13f7SAndroid Build Coastguard Worker loopActiveLaneMask |= GetActiveLaneMaskEdge(in, blockId);
371*03ce13f7SAndroid Build Coastguard Worker }
372*03ce13f7SAndroid Build Coastguard Worker
373*03ce13f7SAndroid Build Coastguard Worker // mergeActiveLaneMasks contains edge lane masks for the merge block.
374*03ce13f7SAndroid Build Coastguard Worker // This is the union of all edge masks across all iterations of the loop.
375*03ce13f7SAndroid Build Coastguard Worker std::unordered_map<Block::ID, SIMD::Int> mergeActiveLaneMasks;
376*03ce13f7SAndroid Build Coastguard Worker for(auto in : function.getBlock(mergeBlockId).ins)
377*03ce13f7SAndroid Build Coastguard Worker {
378*03ce13f7SAndroid Build Coastguard Worker mergeActiveLaneMasks.emplace(in, SIMD::Int(0));
379*03ce13f7SAndroid Build Coastguard Worker }
380*03ce13f7SAndroid Build Coastguard Worker
381*03ce13f7SAndroid Build Coastguard Worker // Create the loop basic blocks
382*03ce13f7SAndroid Build Coastguard Worker auto headerBasicBlock = Nucleus::createBasicBlock();
383*03ce13f7SAndroid Build Coastguard Worker auto mergeBasicBlock = Nucleus::createBasicBlock();
384*03ce13f7SAndroid Build Coastguard Worker
385*03ce13f7SAndroid Build Coastguard Worker // Start emitting code inside the loop.
386*03ce13f7SAndroid Build Coastguard Worker Nucleus::createBr(headerBasicBlock);
387*03ce13f7SAndroid Build Coastguard Worker Nucleus::setInsertBlock(headerBasicBlock);
388*03ce13f7SAndroid Build Coastguard Worker
389*03ce13f7SAndroid Build Coastguard Worker SPIRV_SHADER_DBG("*** LOOP START (mask: {0}) ***", loopActiveLaneMask);
390*03ce13f7SAndroid Build Coastguard Worker
391*03ce13f7SAndroid Build Coastguard Worker // Load the active lane mask.
392*03ce13f7SAndroid Build Coastguard Worker SetActiveLaneMask(loopActiveLaneMask);
393*03ce13f7SAndroid Build Coastguard Worker
394*03ce13f7SAndroid Build Coastguard Worker // Emit the non-phi loop header block's instructions.
395*03ce13f7SAndroid Build Coastguard Worker for(auto insn = block.begin(); insn != block.end(); insn++)
396*03ce13f7SAndroid Build Coastguard Worker {
397*03ce13f7SAndroid Build Coastguard Worker if(insn.opcode() == spv::OpPhi)
398*03ce13f7SAndroid Build Coastguard Worker {
399*03ce13f7SAndroid Build Coastguard Worker LoadPhi(insn);
400*03ce13f7SAndroid Build Coastguard Worker }
401*03ce13f7SAndroid Build Coastguard Worker else
402*03ce13f7SAndroid Build Coastguard Worker {
403*03ce13f7SAndroid Build Coastguard Worker EmitInstruction(insn);
404*03ce13f7SAndroid Build Coastguard Worker }
405*03ce13f7SAndroid Build Coastguard Worker }
406*03ce13f7SAndroid Build Coastguard Worker
407*03ce13f7SAndroid Build Coastguard Worker // Emit all blocks between the loop header and the merge block, but
408*03ce13f7SAndroid Build Coastguard Worker // don't emit the merge block yet.
409*03ce13f7SAndroid Build Coastguard Worker for(auto out : block.outs)
410*03ce13f7SAndroid Build Coastguard Worker {
411*03ce13f7SAndroid Build Coastguard Worker EmitBlocks(out, mergeBlockId);
412*03ce13f7SAndroid Build Coastguard Worker }
413*03ce13f7SAndroid Build Coastguard Worker
414*03ce13f7SAndroid Build Coastguard Worker // Restore current block id after emitting loop blocks.
415*03ce13f7SAndroid Build Coastguard Worker this->block = blockId;
416*03ce13f7SAndroid Build Coastguard Worker
417*03ce13f7SAndroid Build Coastguard Worker // Rebuild the loopActiveLaneMask from the loop back edges.
418*03ce13f7SAndroid Build Coastguard Worker loopActiveLaneMask = SIMD::Int(0);
419*03ce13f7SAndroid Build Coastguard Worker for(auto in : block.ins)
420*03ce13f7SAndroid Build Coastguard Worker {
421*03ce13f7SAndroid Build Coastguard Worker if(function.ExistsPath(blockId, in, mergeBlockId))
422*03ce13f7SAndroid Build Coastguard Worker {
423*03ce13f7SAndroid Build Coastguard Worker loopActiveLaneMask |= GetActiveLaneMaskEdge(in, blockId);
424*03ce13f7SAndroid Build Coastguard Worker }
425*03ce13f7SAndroid Build Coastguard Worker }
426*03ce13f7SAndroid Build Coastguard Worker
427*03ce13f7SAndroid Build Coastguard Worker // Add active lanes to the merge lane mask.
428*03ce13f7SAndroid Build Coastguard Worker for(auto in : function.getBlock(mergeBlockId).ins)
429*03ce13f7SAndroid Build Coastguard Worker {
430*03ce13f7SAndroid Build Coastguard Worker auto edge = Block::Edge{ in, mergeBlockId };
431*03ce13f7SAndroid Build Coastguard Worker auto it = edgeActiveLaneMasks.find(edge);
432*03ce13f7SAndroid Build Coastguard Worker
433*03ce13f7SAndroid Build Coastguard Worker if(it != edgeActiveLaneMasks.end())
434*03ce13f7SAndroid Build Coastguard Worker {
435*03ce13f7SAndroid Build Coastguard Worker mergeActiveLaneMasks[in] |= it->second;
436*03ce13f7SAndroid Build Coastguard Worker }
437*03ce13f7SAndroid Build Coastguard Worker }
438*03ce13f7SAndroid Build Coastguard Worker
439*03ce13f7SAndroid Build Coastguard Worker // Update loop phi values.
440*03ce13f7SAndroid Build Coastguard Worker for(auto insn = block.begin(); insn != block.mergeInstruction; insn++)
441*03ce13f7SAndroid Build Coastguard Worker {
442*03ce13f7SAndroid Build Coastguard Worker if(insn.opcode() == spv::OpPhi)
443*03ce13f7SAndroid Build Coastguard Worker {
444*03ce13f7SAndroid Build Coastguard Worker StorePhi(blockId, insn, loopBlocks);
445*03ce13f7SAndroid Build Coastguard Worker }
446*03ce13f7SAndroid Build Coastguard Worker }
447*03ce13f7SAndroid Build Coastguard Worker
448*03ce13f7SAndroid Build Coastguard Worker SPIRV_SHADER_DBG("*** LOOP END (mask: {0}) ***", loopActiveLaneMask);
449*03ce13f7SAndroid Build Coastguard Worker
450*03ce13f7SAndroid Build Coastguard Worker // Use the [loop -> merge] active lane masks to update the phi values in
451*03ce13f7SAndroid Build Coastguard Worker // the merge block. We need to do this to handle divergent control flow
452*03ce13f7SAndroid Build Coastguard Worker // in the loop.
453*03ce13f7SAndroid Build Coastguard Worker //
454*03ce13f7SAndroid Build Coastguard Worker // Consider the following:
455*03ce13f7SAndroid Build Coastguard Worker //
456*03ce13f7SAndroid Build Coastguard Worker // int phi_source = 0;
457*03ce13f7SAndroid Build Coastguard Worker // for(uint i = 0; i < 4; i++)
458*03ce13f7SAndroid Build Coastguard Worker // {
459*03ce13f7SAndroid Build Coastguard Worker // phi_source = 0;
460*03ce13f7SAndroid Build Coastguard Worker // if(gl_GlobalInvocationID.x % 4 == i) // divergent control flow
461*03ce13f7SAndroid Build Coastguard Worker // {
462*03ce13f7SAndroid Build Coastguard Worker // phi_source = 42; // single lane assignment.
463*03ce13f7SAndroid Build Coastguard Worker // break; // activeLaneMask for [loop->merge] is active for a single lane.
464*03ce13f7SAndroid Build Coastguard Worker // }
465*03ce13f7SAndroid Build Coastguard Worker // // -- we are here --
466*03ce13f7SAndroid Build Coastguard Worker // }
467*03ce13f7SAndroid Build Coastguard Worker // // merge block
468*03ce13f7SAndroid Build Coastguard Worker // int phi = phi_source; // OpPhi
469*03ce13f7SAndroid Build Coastguard Worker //
470*03ce13f7SAndroid Build Coastguard Worker // In this example, with each iteration of the loop, phi_source will
471*03ce13f7SAndroid Build Coastguard Worker // only have a single lane assigned. However the 'phi' value in the merge
472*03ce13f7SAndroid Build Coastguard Worker // block needs to be assigned the union of all the per-lane assignments
473*03ce13f7SAndroid Build Coastguard Worker // of phi_source when that lane exited the loop.
474*03ce13f7SAndroid Build Coastguard Worker for(auto insn = mergeBlock.begin(); insn != mergeBlock.end(); insn++)
475*03ce13f7SAndroid Build Coastguard Worker {
476*03ce13f7SAndroid Build Coastguard Worker if(insn.opcode() == spv::OpPhi)
477*03ce13f7SAndroid Build Coastguard Worker {
478*03ce13f7SAndroid Build Coastguard Worker StorePhi(mergeBlockId, insn, loopBlocks);
479*03ce13f7SAndroid Build Coastguard Worker }
480*03ce13f7SAndroid Build Coastguard Worker }
481*03ce13f7SAndroid Build Coastguard Worker
482*03ce13f7SAndroid Build Coastguard Worker // Loop body now done.
483*03ce13f7SAndroid Build Coastguard Worker // If any lanes are still active, jump back to the loop header,
484*03ce13f7SAndroid Build Coastguard Worker // otherwise jump to the merge block.
485*03ce13f7SAndroid Build Coastguard Worker Nucleus::createCondBr(AnyTrue(loopActiveLaneMask).value(), headerBasicBlock, mergeBasicBlock);
486*03ce13f7SAndroid Build Coastguard Worker
487*03ce13f7SAndroid Build Coastguard Worker // Continue emitting from the merge block.
488*03ce13f7SAndroid Build Coastguard Worker Nucleus::setInsertBlock(mergeBasicBlock);
489*03ce13f7SAndroid Build Coastguard Worker pending->push_back(mergeBlockId);
490*03ce13f7SAndroid Build Coastguard Worker
491*03ce13f7SAndroid Build Coastguard Worker for(const auto &it : mergeActiveLaneMasks)
492*03ce13f7SAndroid Build Coastguard Worker {
493*03ce13f7SAndroid Build Coastguard Worker addActiveLaneMaskEdge(it.first, mergeBlockId, it.second);
494*03ce13f7SAndroid Build Coastguard Worker }
495*03ce13f7SAndroid Build Coastguard Worker }
496*03ce13f7SAndroid Build Coastguard Worker
EmitBranch(InsnIterator insn)497*03ce13f7SAndroid Build Coastguard Worker void SpirvEmitter::EmitBranch(InsnIterator insn)
498*03ce13f7SAndroid Build Coastguard Worker {
499*03ce13f7SAndroid Build Coastguard Worker auto target = Block::ID(insn.word(1));
500*03ce13f7SAndroid Build Coastguard Worker addActiveLaneMaskEdge(block, target, activeLaneMask());
501*03ce13f7SAndroid Build Coastguard Worker }
502*03ce13f7SAndroid Build Coastguard Worker
EmitBranchConditional(InsnIterator insn)503*03ce13f7SAndroid Build Coastguard Worker void SpirvEmitter::EmitBranchConditional(InsnIterator insn)
504*03ce13f7SAndroid Build Coastguard Worker {
505*03ce13f7SAndroid Build Coastguard Worker auto &function = shader.getFunction(this->function);
506*03ce13f7SAndroid Build Coastguard Worker auto block = function.getBlock(this->block);
507*03ce13f7SAndroid Build Coastguard Worker ASSERT(block.branchInstruction == insn);
508*03ce13f7SAndroid Build Coastguard Worker
509*03ce13f7SAndroid Build Coastguard Worker auto condId = Object::ID(block.branchInstruction.word(1));
510*03ce13f7SAndroid Build Coastguard Worker auto trueBlockId = Block::ID(block.branchInstruction.word(2));
511*03ce13f7SAndroid Build Coastguard Worker auto falseBlockId = Block::ID(block.branchInstruction.word(3));
512*03ce13f7SAndroid Build Coastguard Worker
513*03ce13f7SAndroid Build Coastguard Worker auto cond = Operand(shader, *this, condId);
514*03ce13f7SAndroid Build Coastguard Worker ASSERT_MSG(shader.getObjectType(condId).componentCount == 1, "Condition must be a Boolean type scalar");
515*03ce13f7SAndroid Build Coastguard Worker
516*03ce13f7SAndroid Build Coastguard Worker // TODO: Optimize for case where all lanes take same path.
517*03ce13f7SAndroid Build Coastguard Worker
518*03ce13f7SAndroid Build Coastguard Worker addOutputActiveLaneMaskEdge(trueBlockId, cond.Int(0));
519*03ce13f7SAndroid Build Coastguard Worker addOutputActiveLaneMaskEdge(falseBlockId, ~cond.Int(0));
520*03ce13f7SAndroid Build Coastguard Worker }
521*03ce13f7SAndroid Build Coastguard Worker
EmitSwitch(InsnIterator insn)522*03ce13f7SAndroid Build Coastguard Worker void SpirvEmitter::EmitSwitch(InsnIterator insn)
523*03ce13f7SAndroid Build Coastguard Worker {
524*03ce13f7SAndroid Build Coastguard Worker auto &function = shader.getFunction(this->function);
525*03ce13f7SAndroid Build Coastguard Worker auto block = function.getBlock(this->block);
526*03ce13f7SAndroid Build Coastguard Worker ASSERT(block.branchInstruction == insn);
527*03ce13f7SAndroid Build Coastguard Worker
528*03ce13f7SAndroid Build Coastguard Worker auto selId = Object::ID(block.branchInstruction.word(1));
529*03ce13f7SAndroid Build Coastguard Worker
530*03ce13f7SAndroid Build Coastguard Worker auto sel = Operand(shader, *this, selId);
531*03ce13f7SAndroid Build Coastguard Worker ASSERT_MSG(sel.componentCount == 1, "Selector must be a scalar");
532*03ce13f7SAndroid Build Coastguard Worker SPIRV_SHADER_DBG("switch({0})", sel);
533*03ce13f7SAndroid Build Coastguard Worker
534*03ce13f7SAndroid Build Coastguard Worker auto numCases = (block.branchInstruction.wordCount() - 3) / 2;
535*03ce13f7SAndroid Build Coastguard Worker
536*03ce13f7SAndroid Build Coastguard Worker // TODO: Optimize for case where all lanes take same path.
537*03ce13f7SAndroid Build Coastguard Worker
538*03ce13f7SAndroid Build Coastguard Worker SIMD::Int defaultLaneMask = activeLaneMask();
539*03ce13f7SAndroid Build Coastguard Worker
540*03ce13f7SAndroid Build Coastguard Worker // Gather up the case label matches and calculate defaultLaneMask.
541*03ce13f7SAndroid Build Coastguard Worker std::vector<RValue<SIMD::Int>> caseLabelMatches;
542*03ce13f7SAndroid Build Coastguard Worker caseLabelMatches.reserve(numCases);
543*03ce13f7SAndroid Build Coastguard Worker
544*03ce13f7SAndroid Build Coastguard Worker for(uint32_t i = 0; i < numCases; i++)
545*03ce13f7SAndroid Build Coastguard Worker {
546*03ce13f7SAndroid Build Coastguard Worker auto label = block.branchInstruction.word(i * 2 + 3);
547*03ce13f7SAndroid Build Coastguard Worker auto caseBlockId = Block::ID(block.branchInstruction.word(i * 2 + 4));
548*03ce13f7SAndroid Build Coastguard Worker auto caseLabelMatch = CmpEQ(sel.Int(0), SIMD::Int(label));
549*03ce13f7SAndroid Build Coastguard Worker SPIRV_SHADER_DBG("case {0}: {1}", label, caseLabelMatch & activeLaneMask());
550*03ce13f7SAndroid Build Coastguard Worker addOutputActiveLaneMaskEdge(caseBlockId, caseLabelMatch);
551*03ce13f7SAndroid Build Coastguard Worker defaultLaneMask &= ~caseLabelMatch;
552*03ce13f7SAndroid Build Coastguard Worker }
553*03ce13f7SAndroid Build Coastguard Worker
554*03ce13f7SAndroid Build Coastguard Worker auto defaultBlockId = Block::ID(block.branchInstruction.word(2));
555*03ce13f7SAndroid Build Coastguard Worker SPIRV_SHADER_DBG("default: {0}", defaultLaneMask);
556*03ce13f7SAndroid Build Coastguard Worker addOutputActiveLaneMaskEdge(defaultBlockId, defaultLaneMask);
557*03ce13f7SAndroid Build Coastguard Worker }
558*03ce13f7SAndroid Build Coastguard Worker
EmitUnreachable(InsnIterator insn)559*03ce13f7SAndroid Build Coastguard Worker void SpirvEmitter::EmitUnreachable(InsnIterator insn)
560*03ce13f7SAndroid Build Coastguard Worker {
561*03ce13f7SAndroid Build Coastguard Worker // TODO: Log something in this case?
562*03ce13f7SAndroid Build Coastguard Worker SetActiveLaneMask(SIMD::Int(0));
563*03ce13f7SAndroid Build Coastguard Worker }
564*03ce13f7SAndroid Build Coastguard Worker
EmitReturn(InsnIterator insn)565*03ce13f7SAndroid Build Coastguard Worker void SpirvEmitter::EmitReturn(InsnIterator insn)
566*03ce13f7SAndroid Build Coastguard Worker {
567*03ce13f7SAndroid Build Coastguard Worker SetActiveLaneMask(SIMD::Int(0));
568*03ce13f7SAndroid Build Coastguard Worker }
569*03ce13f7SAndroid Build Coastguard Worker
EmitTerminateInvocation(InsnIterator insn)570*03ce13f7SAndroid Build Coastguard Worker void SpirvEmitter::EmitTerminateInvocation(InsnIterator insn)
571*03ce13f7SAndroid Build Coastguard Worker {
572*03ce13f7SAndroid Build Coastguard Worker routine->discardMask |= SignMask(activeLaneMask());
573*03ce13f7SAndroid Build Coastguard Worker SetActiveLaneMask(SIMD::Int(0));
574*03ce13f7SAndroid Build Coastguard Worker }
575*03ce13f7SAndroid Build Coastguard Worker
EmitDemoteToHelperInvocation(InsnIterator insn)576*03ce13f7SAndroid Build Coastguard Worker void SpirvEmitter::EmitDemoteToHelperInvocation(InsnIterator insn)
577*03ce13f7SAndroid Build Coastguard Worker {
578*03ce13f7SAndroid Build Coastguard Worker routine->helperInvocation |= activeLaneMask();
579*03ce13f7SAndroid Build Coastguard Worker routine->discardMask |= SignMask(activeLaneMask());
580*03ce13f7SAndroid Build Coastguard Worker SetStoresAndAtomicsMask(storesAndAtomicsMask() & ~activeLaneMask());
581*03ce13f7SAndroid Build Coastguard Worker }
582*03ce13f7SAndroid Build Coastguard Worker
EmitIsHelperInvocation(InsnIterator insn)583*03ce13f7SAndroid Build Coastguard Worker void SpirvEmitter::EmitIsHelperInvocation(InsnIterator insn)
584*03ce13f7SAndroid Build Coastguard Worker {
585*03ce13f7SAndroid Build Coastguard Worker auto &type = shader.getType(insn.resultTypeId());
586*03ce13f7SAndroid Build Coastguard Worker auto &dst = createIntermediate(insn.resultId(), type.componentCount);
587*03ce13f7SAndroid Build Coastguard Worker dst.move(0, routine->helperInvocation);
588*03ce13f7SAndroid Build Coastguard Worker }
589*03ce13f7SAndroid Build Coastguard Worker
EmitFunctionCall(InsnIterator insn)590*03ce13f7SAndroid Build Coastguard Worker void SpirvEmitter::EmitFunctionCall(InsnIterator insn)
591*03ce13f7SAndroid Build Coastguard Worker {
592*03ce13f7SAndroid Build Coastguard Worker auto functionId = Spirv::Function::ID(insn.word(3));
593*03ce13f7SAndroid Build Coastguard Worker const auto &functionIt = shader.functions.find(functionId);
594*03ce13f7SAndroid Build Coastguard Worker ASSERT(functionIt != shader.functions.end());
595*03ce13f7SAndroid Build Coastguard Worker auto &function = functionIt->second;
596*03ce13f7SAndroid Build Coastguard Worker
597*03ce13f7SAndroid Build Coastguard Worker // TODO(b/141246700): Add full support for spv::OpFunctionCall
598*03ce13f7SAndroid Build Coastguard Worker // The only supported function is a single OpKill wrapped in a
599*03ce13f7SAndroid Build Coastguard Worker // function, as a result of the "wrap OpKill" SPIRV-Tools pass
600*03ce13f7SAndroid Build Coastguard Worker ASSERT(function.blocks.size() == 1);
601*03ce13f7SAndroid Build Coastguard Worker spv::Op wrapOpKill[] = { spv::OpLabel, spv::OpKill };
602*03ce13f7SAndroid Build Coastguard Worker
603*03ce13f7SAndroid Build Coastguard Worker for(const auto &block : function.blocks)
604*03ce13f7SAndroid Build Coastguard Worker {
605*03ce13f7SAndroid Build Coastguard Worker int insnNumber = 0;
606*03ce13f7SAndroid Build Coastguard Worker for(auto blockInsn : block.second)
607*03ce13f7SAndroid Build Coastguard Worker {
608*03ce13f7SAndroid Build Coastguard Worker if(insnNumber > 1)
609*03ce13f7SAndroid Build Coastguard Worker {
610*03ce13f7SAndroid Build Coastguard Worker UNIMPLEMENTED("b/141246700: Function block number of instructions: %d", insnNumber); // FIXME(b/141246700)
611*03ce13f7SAndroid Build Coastguard Worker }
612*03ce13f7SAndroid Build Coastguard Worker
613*03ce13f7SAndroid Build Coastguard Worker if(blockInsn.opcode() != wrapOpKill[insnNumber++])
614*03ce13f7SAndroid Build Coastguard Worker {
615*03ce13f7SAndroid Build Coastguard Worker UNIMPLEMENTED("b/141246700: Function block instruction %d : %s", insnNumber - 1, shader.OpcodeName(blockInsn.opcode())); // FIXME(b/141246700)
616*03ce13f7SAndroid Build Coastguard Worker }
617*03ce13f7SAndroid Build Coastguard Worker
618*03ce13f7SAndroid Build Coastguard Worker if(blockInsn.opcode() == spv::OpKill)
619*03ce13f7SAndroid Build Coastguard Worker {
620*03ce13f7SAndroid Build Coastguard Worker EmitInstruction(blockInsn);
621*03ce13f7SAndroid Build Coastguard Worker }
622*03ce13f7SAndroid Build Coastguard Worker }
623*03ce13f7SAndroid Build Coastguard Worker }
624*03ce13f7SAndroid Build Coastguard Worker }
625*03ce13f7SAndroid Build Coastguard Worker
EmitControlBarrier(InsnIterator insn)626*03ce13f7SAndroid Build Coastguard Worker void SpirvEmitter::EmitControlBarrier(InsnIterator insn)
627*03ce13f7SAndroid Build Coastguard Worker {
628*03ce13f7SAndroid Build Coastguard Worker auto executionScope = spv::Scope(shader.GetConstScalarInt(insn.word(1)));
629*03ce13f7SAndroid Build Coastguard Worker auto semantics = spv::MemorySemanticsMask(shader.GetConstScalarInt(insn.word(3)));
630*03ce13f7SAndroid Build Coastguard Worker // TODO(b/176819536): We probably want to consider the memory scope here.
631*03ce13f7SAndroid Build Coastguard Worker // For now, just always emit the full fence.
632*03ce13f7SAndroid Build Coastguard Worker Fence(semantics);
633*03ce13f7SAndroid Build Coastguard Worker
634*03ce13f7SAndroid Build Coastguard Worker switch(executionScope)
635*03ce13f7SAndroid Build Coastguard Worker {
636*03ce13f7SAndroid Build Coastguard Worker case spv::ScopeWorkgroup:
637*03ce13f7SAndroid Build Coastguard Worker Yield(YieldResult::ControlBarrier);
638*03ce13f7SAndroid Build Coastguard Worker break;
639*03ce13f7SAndroid Build Coastguard Worker case spv::ScopeSubgroup:
640*03ce13f7SAndroid Build Coastguard Worker break;
641*03ce13f7SAndroid Build Coastguard Worker default:
642*03ce13f7SAndroid Build Coastguard Worker // See Vulkan 1.1 spec, Appendix A, Validation Rules within a Module.
643*03ce13f7SAndroid Build Coastguard Worker UNREACHABLE("Scope for execution must be limited to Workgroup or Subgroup");
644*03ce13f7SAndroid Build Coastguard Worker break;
645*03ce13f7SAndroid Build Coastguard Worker }
646*03ce13f7SAndroid Build Coastguard Worker }
647*03ce13f7SAndroid Build Coastguard Worker
EmitPhi(InsnIterator insn)648*03ce13f7SAndroid Build Coastguard Worker void SpirvEmitter::EmitPhi(InsnIterator insn)
649*03ce13f7SAndroid Build Coastguard Worker {
650*03ce13f7SAndroid Build Coastguard Worker auto &function = shader.getFunction(this->function);
651*03ce13f7SAndroid Build Coastguard Worker auto currentBlock = function.getBlock(block);
652*03ce13f7SAndroid Build Coastguard Worker
653*03ce13f7SAndroid Build Coastguard Worker if(!currentBlock.isLoopMerge)
654*03ce13f7SAndroid Build Coastguard Worker {
655*03ce13f7SAndroid Build Coastguard Worker // If this is a loop merge block, then don't attempt to update the
656*03ce13f7SAndroid Build Coastguard Worker // phi values from the ins. EmitLoop() has had to take special care
657*03ce13f7SAndroid Build Coastguard Worker // of this phi in order to correctly deal with divergent lanes.
658*03ce13f7SAndroid Build Coastguard Worker StorePhi(block, insn, currentBlock.ins);
659*03ce13f7SAndroid Build Coastguard Worker }
660*03ce13f7SAndroid Build Coastguard Worker
661*03ce13f7SAndroid Build Coastguard Worker LoadPhi(insn);
662*03ce13f7SAndroid Build Coastguard Worker }
663*03ce13f7SAndroid Build Coastguard Worker
LoadPhi(InsnIterator insn)664*03ce13f7SAndroid Build Coastguard Worker void SpirvEmitter::LoadPhi(InsnIterator insn)
665*03ce13f7SAndroid Build Coastguard Worker {
666*03ce13f7SAndroid Build Coastguard Worker auto typeId = Type::ID(insn.word(1));
667*03ce13f7SAndroid Build Coastguard Worker auto type = shader.getType(typeId);
668*03ce13f7SAndroid Build Coastguard Worker auto objectId = Object::ID(insn.word(2));
669*03ce13f7SAndroid Build Coastguard Worker
670*03ce13f7SAndroid Build Coastguard Worker auto storageIt = phis.find(objectId);
671*03ce13f7SAndroid Build Coastguard Worker ASSERT(storageIt != phis.end());
672*03ce13f7SAndroid Build Coastguard Worker const auto &storage = storageIt->second;
673*03ce13f7SAndroid Build Coastguard Worker
674*03ce13f7SAndroid Build Coastguard Worker auto &dst = createIntermediate(objectId, type.componentCount);
675*03ce13f7SAndroid Build Coastguard Worker
676*03ce13f7SAndroid Build Coastguard Worker for(uint32_t i = 0; i < type.componentCount; i++)
677*03ce13f7SAndroid Build Coastguard Worker {
678*03ce13f7SAndroid Build Coastguard Worker dst.move(i, storage[i]);
679*03ce13f7SAndroid Build Coastguard Worker SPIRV_SHADER_DBG("LoadPhi({0}.{1}): {2}", objectId, i, storage[i]);
680*03ce13f7SAndroid Build Coastguard Worker }
681*03ce13f7SAndroid Build Coastguard Worker }
682*03ce13f7SAndroid Build Coastguard Worker
StorePhi(Block::ID currentBlock,InsnIterator insn,const std::unordered_set<Block::ID> & filter)683*03ce13f7SAndroid Build Coastguard Worker void SpirvEmitter::StorePhi(Block::ID currentBlock, InsnIterator insn, const std::unordered_set<Block::ID> &filter)
684*03ce13f7SAndroid Build Coastguard Worker {
685*03ce13f7SAndroid Build Coastguard Worker auto typeId = Type::ID(insn.word(1));
686*03ce13f7SAndroid Build Coastguard Worker auto type = shader.getType(typeId);
687*03ce13f7SAndroid Build Coastguard Worker auto objectId = Object::ID(insn.word(2));
688*03ce13f7SAndroid Build Coastguard Worker
689*03ce13f7SAndroid Build Coastguard Worker auto storageIt = phis.find(objectId);
690*03ce13f7SAndroid Build Coastguard Worker ASSERT(storageIt != phis.end());
691*03ce13f7SAndroid Build Coastguard Worker auto &storage = storageIt->second;
692*03ce13f7SAndroid Build Coastguard Worker
693*03ce13f7SAndroid Build Coastguard Worker for(uint32_t w = 3; w < insn.wordCount(); w += 2)
694*03ce13f7SAndroid Build Coastguard Worker {
695*03ce13f7SAndroid Build Coastguard Worker auto varId = Object::ID(insn.word(w + 0));
696*03ce13f7SAndroid Build Coastguard Worker auto blockId = Block::ID(insn.word(w + 1));
697*03ce13f7SAndroid Build Coastguard Worker
698*03ce13f7SAndroid Build Coastguard Worker if(filter.count(blockId) == 0)
699*03ce13f7SAndroid Build Coastguard Worker {
700*03ce13f7SAndroid Build Coastguard Worker continue;
701*03ce13f7SAndroid Build Coastguard Worker }
702*03ce13f7SAndroid Build Coastguard Worker
703*03ce13f7SAndroid Build Coastguard Worker auto mask = GetActiveLaneMaskEdge(blockId, currentBlock);
704*03ce13f7SAndroid Build Coastguard Worker auto in = Operand(shader, *this, varId);
705*03ce13f7SAndroid Build Coastguard Worker
706*03ce13f7SAndroid Build Coastguard Worker for(uint32_t i = 0; i < type.componentCount; i++)
707*03ce13f7SAndroid Build Coastguard Worker {
708*03ce13f7SAndroid Build Coastguard Worker storage[i] = As<SIMD::Float>((As<SIMD::Int>(storage[i]) & ~mask) | (in.Int(i) & mask));
709*03ce13f7SAndroid Build Coastguard Worker SPIRV_SHADER_DBG("StorePhi({0}.{1}): [{2} <- {3}] {4}: {5}, mask: {6}",
710*03ce13f7SAndroid Build Coastguard Worker objectId, i, currentBlock, blockId, varId, in.UInt(i), mask);
711*03ce13f7SAndroid Build Coastguard Worker }
712*03ce13f7SAndroid Build Coastguard Worker }
713*03ce13f7SAndroid Build Coastguard Worker
714*03ce13f7SAndroid Build Coastguard Worker for(uint32_t i = 0; i < type.componentCount; i++)
715*03ce13f7SAndroid Build Coastguard Worker {
716*03ce13f7SAndroid Build Coastguard Worker SPIRV_SHADER_DBG("StorePhi({0}.{1}): {2}", objectId, i, As<SIMD::UInt>(storage[i]));
717*03ce13f7SAndroid Build Coastguard Worker }
718*03ce13f7SAndroid Build Coastguard Worker }
719*03ce13f7SAndroid Build Coastguard Worker
Yield(YieldResult res) const720*03ce13f7SAndroid Build Coastguard Worker void SpirvEmitter::Yield(YieldResult res) const
721*03ce13f7SAndroid Build Coastguard Worker {
722*03ce13f7SAndroid Build Coastguard Worker rr::Yield(RValue<Int>(int(res)));
723*03ce13f7SAndroid Build Coastguard Worker }
724*03ce13f7SAndroid Build Coastguard Worker
SetActiveLaneMask(RValue<SIMD::Int> mask)725*03ce13f7SAndroid Build Coastguard Worker void SpirvEmitter::SetActiveLaneMask(RValue<SIMD::Int> mask)
726*03ce13f7SAndroid Build Coastguard Worker {
727*03ce13f7SAndroid Build Coastguard Worker activeLaneMaskValue = mask.value();
728*03ce13f7SAndroid Build Coastguard Worker }
729*03ce13f7SAndroid Build Coastguard Worker
SetStoresAndAtomicsMask(RValue<SIMD::Int> mask)730*03ce13f7SAndroid Build Coastguard Worker void SpirvEmitter::SetStoresAndAtomicsMask(RValue<SIMD::Int> mask)
731*03ce13f7SAndroid Build Coastguard Worker {
732*03ce13f7SAndroid Build Coastguard Worker storesAndAtomicsMaskValue = mask.value();
733*03ce13f7SAndroid Build Coastguard Worker }
734*03ce13f7SAndroid Build Coastguard Worker
WriteCFGGraphVizDotFile(const char * path) const735*03ce13f7SAndroid Build Coastguard Worker void Spirv::WriteCFGGraphVizDotFile(const char *path) const
736*03ce13f7SAndroid Build Coastguard Worker {
737*03ce13f7SAndroid Build Coastguard Worker std::ofstream file(path);
738*03ce13f7SAndroid Build Coastguard Worker file << "digraph D {" << std::endl;
739*03ce13f7SAndroid Build Coastguard Worker for(auto &func : functions)
740*03ce13f7SAndroid Build Coastguard Worker {
741*03ce13f7SAndroid Build Coastguard Worker file << " subgraph cluster_function_" << func.first.value() << " {"
742*03ce13f7SAndroid Build Coastguard Worker << std::endl;
743*03ce13f7SAndroid Build Coastguard Worker
744*03ce13f7SAndroid Build Coastguard Worker file << " label = \"function<" << func.first.value() << ">"
745*03ce13f7SAndroid Build Coastguard Worker << (func.first == entryPoint ? " (entry point)" : "")
746*03ce13f7SAndroid Build Coastguard Worker << "\"" << std::endl;
747*03ce13f7SAndroid Build Coastguard Worker
748*03ce13f7SAndroid Build Coastguard Worker for(auto &block : func.second.blocks)
749*03ce13f7SAndroid Build Coastguard Worker {
750*03ce13f7SAndroid Build Coastguard Worker file << " block_" << block.first.value() << " ["
751*03ce13f7SAndroid Build Coastguard Worker << "shape=circle "
752*03ce13f7SAndroid Build Coastguard Worker << "label=\"" << block.first.value() << "\""
753*03ce13f7SAndroid Build Coastguard Worker << "]" << std::endl;
754*03ce13f7SAndroid Build Coastguard Worker }
755*03ce13f7SAndroid Build Coastguard Worker file << std::endl;
756*03ce13f7SAndroid Build Coastguard Worker for(auto &block : func.second.blocks)
757*03ce13f7SAndroid Build Coastguard Worker {
758*03ce13f7SAndroid Build Coastguard Worker file << " block_" << block.first.value() << " -> {";
759*03ce13f7SAndroid Build Coastguard Worker bool first = true;
760*03ce13f7SAndroid Build Coastguard Worker for(auto outs : block.second.outs)
761*03ce13f7SAndroid Build Coastguard Worker {
762*03ce13f7SAndroid Build Coastguard Worker if(!first) { file << ", "; }
763*03ce13f7SAndroid Build Coastguard Worker file << "block_" << outs.value();
764*03ce13f7SAndroid Build Coastguard Worker first = false;
765*03ce13f7SAndroid Build Coastguard Worker }
766*03ce13f7SAndroid Build Coastguard Worker file << "}" << std::endl;
767*03ce13f7SAndroid Build Coastguard Worker }
768*03ce13f7SAndroid Build Coastguard Worker file << std::endl;
769*03ce13f7SAndroid Build Coastguard Worker for(auto &block : func.second.blocks)
770*03ce13f7SAndroid Build Coastguard Worker {
771*03ce13f7SAndroid Build Coastguard Worker if(block.second.kind == Block::Loop)
772*03ce13f7SAndroid Build Coastguard Worker {
773*03ce13f7SAndroid Build Coastguard Worker if(block.second.mergeBlock != 0)
774*03ce13f7SAndroid Build Coastguard Worker {
775*03ce13f7SAndroid Build Coastguard Worker file << " block_" << block.first.value() << " -> "
776*03ce13f7SAndroid Build Coastguard Worker << "block_" << block.second.mergeBlock.value()
777*03ce13f7SAndroid Build Coastguard Worker << "[label=\"M\" style=dashed color=blue]"
778*03ce13f7SAndroid Build Coastguard Worker << std::endl;
779*03ce13f7SAndroid Build Coastguard Worker }
780*03ce13f7SAndroid Build Coastguard Worker
781*03ce13f7SAndroid Build Coastguard Worker if(block.second.continueTarget != 0)
782*03ce13f7SAndroid Build Coastguard Worker {
783*03ce13f7SAndroid Build Coastguard Worker file << " block_" << block.first.value() << " -> "
784*03ce13f7SAndroid Build Coastguard Worker << "block_" << block.second.continueTarget.value()
785*03ce13f7SAndroid Build Coastguard Worker << "[label=\"C\" style=dashed color=green]"
786*03ce13f7SAndroid Build Coastguard Worker << std::endl;
787*03ce13f7SAndroid Build Coastguard Worker }
788*03ce13f7SAndroid Build Coastguard Worker }
789*03ce13f7SAndroid Build Coastguard Worker }
790*03ce13f7SAndroid Build Coastguard Worker
791*03ce13f7SAndroid Build Coastguard Worker file << " }" << std::endl;
792*03ce13f7SAndroid Build Coastguard Worker }
793*03ce13f7SAndroid Build Coastguard Worker
794*03ce13f7SAndroid Build Coastguard Worker for(auto &func : functions)
795*03ce13f7SAndroid Build Coastguard Worker {
796*03ce13f7SAndroid Build Coastguard Worker for(auto &block : func.second.blocks)
797*03ce13f7SAndroid Build Coastguard Worker {
798*03ce13f7SAndroid Build Coastguard Worker for(auto insn : block.second)
799*03ce13f7SAndroid Build Coastguard Worker {
800*03ce13f7SAndroid Build Coastguard Worker if(insn.opcode() == spv::OpFunctionCall)
801*03ce13f7SAndroid Build Coastguard Worker {
802*03ce13f7SAndroid Build Coastguard Worker auto target = getFunction(insn.word(3)).entry;
803*03ce13f7SAndroid Build Coastguard Worker file << " block_" << block.first.value() << " -> "
804*03ce13f7SAndroid Build Coastguard Worker << "block_" << target.value()
805*03ce13f7SAndroid Build Coastguard Worker << "[color=\"#00008050\"]"
806*03ce13f7SAndroid Build Coastguard Worker << std::endl;
807*03ce13f7SAndroid Build Coastguard Worker }
808*03ce13f7SAndroid Build Coastguard Worker }
809*03ce13f7SAndroid Build Coastguard Worker }
810*03ce13f7SAndroid Build Coastguard Worker }
811*03ce13f7SAndroid Build Coastguard Worker
812*03ce13f7SAndroid Build Coastguard Worker file << "}" << std::endl;
813*03ce13f7SAndroid Build Coastguard Worker }
814*03ce13f7SAndroid Build Coastguard Worker
815*03ce13f7SAndroid Build Coastguard Worker } // namespace sw
816