1 /*
2  * Copyright © Microsoft Corporation
3  *
4  * Permission is hereby granted, free of charge, to any person obtaining a
5  * copy of this software and associated documentation files (the "Software"),
6  * to deal in the Software without restriction, including without limitation
7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8  * and/or sell copies of the Software, and to permit persons to whom the
9  * Software is furnished to do so, subject to the following conditions:
10  *
11  * The above copyright notice and this permission notice (including the next
12  * paragraph) shall be included in all copies or substantial portions of the
13  * Software.
14  *
15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21  * IN THE SOFTWARE.
22  */
23 
24 #include "d3d12_video_encoder_references_manager_hevc.h"
25 #include <algorithm>
26 #include <string>
27 #include "d3d12_screen.h"
28 #include "d3d12_resource.h"
29 #include "d3d12_video_buffer.h"
30 
31 using namespace std;
32 
33 bool
get_current_frame_picture_control_data(D3D12_VIDEO_ENCODER_PICTURE_CONTROL_CODEC_DATA & codecAllocation)34 d3d12_video_encoder_references_manager_hevc::get_current_frame_picture_control_data(
35    D3D12_VIDEO_ENCODER_PICTURE_CONTROL_CODEC_DATA &codecAllocation)
36 {
37    assert((codecAllocation.DataSize == sizeof(D3D12_VIDEO_ENCODER_PICTURE_CONTROL_CODEC_DATA_HEVC)) ||
38           (codecAllocation.DataSize == sizeof(D3D12_VIDEO_ENCODER_PICTURE_CONTROL_CODEC_DATA_HEVC1)));
39    memcpy(codecAllocation.pHEVCPicData1, &m_curFrameState, codecAllocation.DataSize);
40    memset(codecAllocation.pHEVCPicData1 + codecAllocation.DataSize, 0, sizeof(m_curFrameState) - codecAllocation.DataSize);
41    return true;
42 }
43 
44 D3D12_VIDEO_ENCODE_REFERENCE_FRAMES
get_current_reference_frames()45 d3d12_video_encoder_references_manager_hevc::get_current_reference_frames()
46 {
47    D3D12_VIDEO_ENCODE_REFERENCE_FRAMES retVal = { 0,
48                                                   // ppTexture2Ds
49                                                   nullptr,
50                                                   // pSubresources
51                                                   nullptr };
52 
53    // Return nullptr for fully intra frames (eg IDR)
54    // and return references information for inter frames (eg.P/B) and I frame that doesn't flush DPB
55 
56    if (m_curFrameState.FrameType != D3D12_VIDEO_ENCODER_FRAME_TYPE_HEVC_IDR_FRAME) {
57       retVal.NumTexture2Ds = m_CurrentFrameReferencesData.ReferenceTextures.pResources.size();
58       retVal.ppTexture2Ds = m_CurrentFrameReferencesData.ReferenceTextures.pResources.data();
59 
60       // D3D12 Encode expects null subresources for AoT
61       bool isAoT = (std::all_of(m_CurrentFrameReferencesData.ReferenceTextures.pSubresources.begin(),
62                                 m_CurrentFrameReferencesData.ReferenceTextures.pSubresources.end(),
63                                 [](UINT i) { return i == 0; }));
64       retVal.pSubresources = isAoT ? nullptr : m_CurrentFrameReferencesData.ReferenceTextures.pSubresources.data();
65    }
66 
67    return retVal;
68 }
69 
70 static const char *
d3d12_video_encoder_friendly_frame_type_hevc(D3D12_VIDEO_ENCODER_FRAME_TYPE_HEVC picType)71 d3d12_video_encoder_friendly_frame_type_hevc(D3D12_VIDEO_ENCODER_FRAME_TYPE_HEVC picType)
72 {
73    switch (picType) {
74       case D3D12_VIDEO_ENCODER_FRAME_TYPE_HEVC_P_FRAME:
75       {
76          return "HEVC_P_FRAME";
77       } break;
78       case D3D12_VIDEO_ENCODER_FRAME_TYPE_HEVC_B_FRAME:
79       {
80          return "HEVC_B_FRAME";
81       } break;
82       case D3D12_VIDEO_ENCODER_FRAME_TYPE_HEVC_I_FRAME:
83       {
84          return "HEVC_I_FRAME";
85       } break;
86       case D3D12_VIDEO_ENCODER_FRAME_TYPE_HEVC_IDR_FRAME:
87       {
88          return "HEVC_IDR_FRAME";
89       } break;
90       default:
91       {
92          unreachable("Unsupported D3D12_VIDEO_ENCODER_FRAME_TYPE_HEVC");
93       } break;
94    }
95 }
96 
97 void
print_l0_l1_lists()98 d3d12_video_encoder_references_manager_hevc::print_l0_l1_lists()
99 {
100    if ((D3D12_DEBUG_VERBOSE & d3d12_debug) &&
101        ((m_curFrameState.FrameType == D3D12_VIDEO_ENCODER_FRAME_TYPE_HEVC_P_FRAME) ||
102         (m_curFrameState.FrameType == D3D12_VIDEO_ENCODER_FRAME_TYPE_HEVC_B_FRAME))) {
103 
104       debug_printf(
105          "[D3D12 Video Encoder Picture Manager HEVC] L0 (%d entries) and L1 (%d entries) lists for frame with POC "
106          "%d and frame_type %s are:\n",
107          m_curFrameState.List0ReferenceFramesCount,
108          m_curFrameState.List1ReferenceFramesCount,
109          m_curFrameState.PictureOrderCountNumber,
110          d3d12_video_encoder_friendly_frame_type_hevc(m_curFrameState.FrameType));
111 
112       std::string list0ContentsString;
113       for (uint32_t idx = 0; idx < m_curFrameState.List0ReferenceFramesCount; idx++) {
114          uint32_t value = m_curFrameState.pList0ReferenceFrames[idx];
115          list0ContentsString += "{ DPBidx: ";
116          list0ContentsString += std::to_string(value);
117          list0ContentsString += " - POC: ";
118          list0ContentsString += std::to_string(
119             m_CurrentFrameReferencesData.pReferenceFramesReconPictureDescriptors[value].PictureOrderCountNumber);
120          list0ContentsString += " }\n";
121       }
122 
123       debug_printf("[D3D12 Video Encoder Picture Manager HEVC] L0 list (%d entries) for frame with POC %d is: \n%s \n",
124                    m_curFrameState.List0ReferenceFramesCount,
125                    m_curFrameState.PictureOrderCountNumber,
126                    list0ContentsString.c_str());
127 
128       std::string modificationOrderList0ContentsString;
129       for (uint32_t idx = 0; idx < m_curFrameState.List0RefPicModificationsCount; idx++) {
130          modificationOrderList0ContentsString += "{ ";
131          modificationOrderList0ContentsString += std::to_string(m_curFrameState.pList0RefPicModifications[idx]);
132          modificationOrderList0ContentsString += " }\n";
133       }
134       debug_printf("[D3D12 Video Encoder Picture Manager HEVC] L0 modification list (%d entries) for frame with POC %d "
135                    " - temporal_id (%d) is: \n%s \n",
136                    m_curFrameState.List0RefPicModificationsCount,
137                    m_curFrameState.PictureOrderCountNumber,
138                    m_curFrameState.TemporalLayerIndex,
139                    modificationOrderList0ContentsString.c_str());
140 
141       std::string list1ContentsString;
142       for (uint32_t idx = 0; idx < m_curFrameState.List1ReferenceFramesCount; idx++) {
143          uint32_t value = m_curFrameState.pList1ReferenceFrames[idx];
144          list1ContentsString += "{ DPBidx: ";
145          list1ContentsString += std::to_string(value);
146          list1ContentsString += " - POC: ";
147          list1ContentsString += std::to_string(
148             m_CurrentFrameReferencesData.pReferenceFramesReconPictureDescriptors[value].PictureOrderCountNumber);
149          list1ContentsString += " }\n";
150       }
151 
152       debug_printf("[D3D12 Video Encoder Picture Manager HEVC] L1 list (%d entries) for frame with POC %d is: \n%s \n",
153                    m_curFrameState.List1ReferenceFramesCount,
154                    m_curFrameState.PictureOrderCountNumber,
155                    list1ContentsString.c_str());
156 
157       std::string modificationOrderList1ContentsString;
158       for (uint32_t idx = 0; idx < m_curFrameState.List1RefPicModificationsCount; idx++) {
159          modificationOrderList1ContentsString += "{ ";
160          modificationOrderList1ContentsString += std::to_string(m_curFrameState.pList1RefPicModifications[idx]);
161          modificationOrderList1ContentsString += " }\n";
162       }
163 
164       debug_printf("[D3D12 Video Encoder Picture Manager HEVC] L1 modification list (%d entries) for frame with POC %d "
165                    "- temporal_id (%d) is: \n%s \n",
166                    m_curFrameState.List1RefPicModificationsCount,
167                    m_curFrameState.PictureOrderCountNumber,
168                    m_curFrameState.TemporalLayerIndex,
169                    modificationOrderList1ContentsString.c_str());
170    }
171 }
172 
173 void
print_dpb()174 d3d12_video_encoder_references_manager_hevc::print_dpb()
175 {
176    if (D3D12_DEBUG_VERBOSE & d3d12_debug) {
177       std::string dpbContents;
178       for (uint32_t dpbResIdx = 0;
179            dpbResIdx < m_CurrentFrameReferencesData.pReferenceFramesReconPictureDescriptors.size();
180            dpbResIdx++) {
181          auto &dpbDesc = m_CurrentFrameReferencesData.pReferenceFramesReconPictureDescriptors[dpbResIdx];
182 
183          dpbContents += "{ DPBidx: ";
184          dpbContents += std::to_string(dpbResIdx);
185          dpbContents += " - POC: ";
186          dpbContents += std::to_string(dpbDesc.PictureOrderCountNumber);
187          dpbContents += " - IsRefUsedByCurrentPic: ";
188          dpbContents += std::to_string(dpbDesc.IsRefUsedByCurrentPic);
189          dpbContents += " - IsLongTermReference: ";
190          dpbContents += std::to_string(dpbDesc.IsLongTermReference);
191          dpbContents += " - TemporalLayerIndex: ";
192          dpbContents += std::to_string(dpbDesc.TemporalLayerIndex);
193          dpbContents += " - DPBStorageIdx: ";
194          dpbContents += std::to_string(dpbDesc.ReconstructedPictureResourceIndex);
195          dpbContents += " - DPBStorageResourcePtr: ";
196          char strBuf[256];
197          memset(&strBuf, '\0', 256);
198          sprintf(strBuf,
199                  "%p",
200                  m_CurrentFrameReferencesData.ReferenceTextures.pResources[dpbDesc.ReconstructedPictureResourceIndex]);
201          dpbContents += std::string(strBuf);
202          dpbContents += " - DPBStorageSubresource: ";
203          dpbContents += std::to_string(
204             m_CurrentFrameReferencesData.ReferenceTextures.pSubresources[dpbDesc.ReconstructedPictureResourceIndex]);
205 
206          if (dpbDesc.PictureOrderCountNumber == m_curFrameState.PictureOrderCountNumber) {
207             dpbContents += " - CURRENT FRAME RECON PIC ";
208          }
209 
210          dpbContents += "}\n";
211       }
212 
213       debug_printf("[D3D12 Video Encoder Picture Manager HEVC] DPB has %d frames - DPB references for frame with POC "
214                    "%d and frame_type %s are: \n%s \n",
215                    static_cast<UINT>(m_CurrentFrameReferencesData.pReferenceFramesReconPictureDescriptors.size()),
216                    m_curFrameState.PictureOrderCountNumber,
217                    d3d12_video_encoder_friendly_frame_type_hevc(m_curFrameState.FrameType),
218                    dpbContents.c_str());
219    }
220 }
221 
222 static D3D12_VIDEO_ENCODER_FRAME_TYPE_HEVC
d3d12_video_encoder_convert_frame_type_hevc(enum pipe_h2645_enc_picture_type picType)223 d3d12_video_encoder_convert_frame_type_hevc(enum pipe_h2645_enc_picture_type picType)
224 {
225    switch (picType) {
226       case PIPE_H2645_ENC_PICTURE_TYPE_P:
227       {
228          return D3D12_VIDEO_ENCODER_FRAME_TYPE_HEVC_P_FRAME;
229       } break;
230       case PIPE_H2645_ENC_PICTURE_TYPE_B:
231       {
232          return D3D12_VIDEO_ENCODER_FRAME_TYPE_HEVC_B_FRAME;
233       } break;
234       case PIPE_H2645_ENC_PICTURE_TYPE_I:
235       {
236          return D3D12_VIDEO_ENCODER_FRAME_TYPE_HEVC_I_FRAME;
237       } break;
238       case PIPE_H2645_ENC_PICTURE_TYPE_IDR:
239       {
240          return D3D12_VIDEO_ENCODER_FRAME_TYPE_HEVC_IDR_FRAME;
241       } break;
242       default:
243       {
244          unreachable("Unsupported pipe_h2645_enc_picture_type");
245       } break;
246    }
247 }
248 
249 void
begin_frame(D3D12_VIDEO_ENCODER_PICTURE_CONTROL_CODEC_DATA curFrameData,bool bUsedAsReference,struct pipe_picture_desc * picture)250 d3d12_video_encoder_references_manager_hevc::begin_frame(D3D12_VIDEO_ENCODER_PICTURE_CONTROL_CODEC_DATA curFrameData,
251                                                          bool bUsedAsReference,
252                                                          struct pipe_picture_desc *picture)
253 {
254    assert((curFrameData.DataSize == sizeof(D3D12_VIDEO_ENCODER_PICTURE_CONTROL_CODEC_DATA_HEVC)) ||
255           (curFrameData.DataSize == sizeof(D3D12_VIDEO_ENCODER_PICTURE_CONTROL_CODEC_DATA_HEVC1)));
256    memcpy(&m_curFrameState, curFrameData.pHEVCPicData1, curFrameData.DataSize);
257    memset(&m_curFrameState + curFrameData.DataSize, 0, sizeof(m_curFrameState) - curFrameData.DataSize);
258 
259    m_isCurrentFrameUsedAsReference = bUsedAsReference;
260 
261    struct pipe_h265_enc_picture_desc *hevcPic = (struct pipe_h265_enc_picture_desc *) picture;
262 
263    ///
264    /// Copy DPB snapshot from pipe params
265    ///
266 
267    m_curFrameState.ReferenceFramesReconPictureDescriptorsCount =
268       static_cast<uint32_t>(m_CurrentFrameReferencesData.pReferenceFramesReconPictureDescriptors.size());
269    m_CurrentFrameReferencesData.pReferenceFramesReconPictureDescriptors.resize(hevcPic->dpb_size);
270    m_CurrentFrameReferencesData.ReferenceTextures.pResources.resize(hevcPic->dpb_size);
271    m_CurrentFrameReferencesData.ReferenceTextures.pSubresources.resize(hevcPic->dpb_size);
272    m_CurrentFrameReferencesData.pReferenceFramesReconPictureDescriptors.resize(hevcPic->dpb_size);
273    for (uint8_t i = 0; i < hevcPic->dpb_size; i++) {
274       //
275       // Set entry DPB members
276       //
277 
278       m_CurrentFrameReferencesData.pReferenceFramesReconPictureDescriptors[i].IsLongTermReference =
279          hevcPic->dpb[i].is_ltr;
280       m_CurrentFrameReferencesData.pReferenceFramesReconPictureDescriptors[i].PictureOrderCountNumber =
281          hevcPic->dpb[i].pic_order_cnt;
282       // mirror indices between DPB entries and allocation arrays
283       m_CurrentFrameReferencesData.pReferenceFramesReconPictureDescriptors[i].ReconstructedPictureResourceIndex = i;
284       m_CurrentFrameReferencesData.pReferenceFramesReconPictureDescriptors[i].TemporalLayerIndex =
285          0u;   // hevcPic->dpb[i].temporal_id;
286 
287       // Check if this i-th dpb descriptor entry is referenced by any entry in L0 or L1 lists
288       // and set IsRefUsedByCurrentPic accordingly
289       auto endItL0 = hevcPic->ref_list0 + (hevcPic->num_ref_idx_l0_active_minus1 + 1);
290       bool bReferencesFromL0 = std::find(hevcPic->ref_list0, endItL0, i) != endItL0;
291       auto endItL1 = hevcPic->ref_list1 + (hevcPic->num_ref_idx_l1_active_minus1 + 1);
292       bool bReferencesFromL1 = std::find(hevcPic->ref_list1, endItL1, i) != endItL1;
293       m_CurrentFrameReferencesData.pReferenceFramesReconPictureDescriptors[i].IsRefUsedByCurrentPic =
294          bReferencesFromL0 || bReferencesFromL1;
295 
296       //
297       // Set texture allocations
298       //
299 
300       struct d3d12_video_buffer *vidbuf = (struct d3d12_video_buffer *) hevcPic->dpb[i].buffer;
301       m_CurrentFrameReferencesData.ReferenceTextures.pResources[i] = d3d12_resource_resource(vidbuf->texture);
302       m_CurrentFrameReferencesData.ReferenceTextures.pSubresources[i] = vidbuf->idx_texarray_slots;
303 
304       if (hevcPic->dpb[i].pic_order_cnt == hevcPic->pic_order_cnt) {
305          m_CurrentFrameReferencesData.ReconstructedPicTexture.pReconstructedPicture =
306             m_CurrentFrameReferencesData.ReferenceTextures.pResources[i];
307          m_CurrentFrameReferencesData.ReconstructedPicTexture.ReconstructedPictureSubresource =
308             m_CurrentFrameReferencesData.ReferenceTextures.pSubresources[i];
309       }
310    }
311 
312    ///
313    /// Set pic control info
314    ///
315 
316    m_curFrameState.FrameType = d3d12_video_encoder_convert_frame_type_hevc(hevcPic->picture_type);
317    m_curFrameState.PictureOrderCountNumber = hevcPic->pic_order_cnt;
318    m_curFrameState.TemporalLayerIndex = 0u;   // hevcPic->temporal_id;
319 
320    ///
321    /// Set reference pics info
322    ///
323 
324    m_curFrameState.List0ReferenceFramesCount = 0;
325    m_curFrameState.pList0ReferenceFrames = nullptr;
326    m_curFrameState.List0RefPicModificationsCount = 0;
327    m_curFrameState.pList0RefPicModifications = nullptr;
328    m_curFrameState.List1ReferenceFramesCount = 0;
329    m_curFrameState.pList1ReferenceFrames = nullptr;
330    m_curFrameState.List1RefPicModificationsCount = 0;
331    m_curFrameState.pList1RefPicModifications = nullptr;
332    m_curFrameState.ReferenceFramesReconPictureDescriptorsCount = 0u;
333    m_curFrameState.pReferenceFramesReconPictureDescriptors = nullptr;
334 
335    if ((m_curFrameState.FrameType == D3D12_VIDEO_ENCODER_FRAME_TYPE_HEVC_P_FRAME) ||
336        (m_curFrameState.FrameType == D3D12_VIDEO_ENCODER_FRAME_TYPE_HEVC_B_FRAME)) {
337 
338       // Set DPB descriptors
339       m_curFrameState.ReferenceFramesReconPictureDescriptorsCount =
340          m_CurrentFrameReferencesData.pReferenceFramesReconPictureDescriptors.size();
341       m_curFrameState.pReferenceFramesReconPictureDescriptors =
342          m_CurrentFrameReferencesData.pReferenceFramesReconPictureDescriptors.data();
343 
344       // Deep Copy L0 list
345       m_curFrameState.List0ReferenceFramesCount = hevcPic->num_ref_idx_l0_active_minus1 + 1;
346       m_CurrentFrameReferencesData.pList0ReferenceFrames.resize(m_curFrameState.List0ReferenceFramesCount);
347       for (unsigned i = 0; i < m_curFrameState.List0ReferenceFramesCount; i++)
348          m_CurrentFrameReferencesData.pList0ReferenceFrames[i] = hevcPic->ref_list0[i];
349       m_curFrameState.pList0ReferenceFrames = m_CurrentFrameReferencesData.pList0ReferenceFrames.data();
350 
351       // Deep Copy L0 ref modification list
352       if (hevcPic->slice.ref_pic_lists_modification.ref_pic_list_modification_flag_l0) {
353          m_curFrameState.List0RefPicModificationsCount = hevcPic->num_ref_idx_l0_active_minus1 + 1;
354          m_CurrentFrameReferencesData.pList0RefPicModifications.resize(m_curFrameState.List0RefPicModificationsCount);
355          for (unsigned i = 0; i < m_curFrameState.List0RefPicModificationsCount; i++)
356             m_CurrentFrameReferencesData.pList0RefPicModifications[i] =
357                hevcPic->slice.ref_pic_lists_modification.list_entry_l0[i];
358          m_curFrameState.pList0RefPicModifications = m_CurrentFrameReferencesData.pList0RefPicModifications.data();
359       }
360    }
361 
362    if (m_curFrameState.FrameType == D3D12_VIDEO_ENCODER_FRAME_TYPE_HEVC_B_FRAME) {
363 
364       // Deep Copy L1 list
365       m_curFrameState.List1ReferenceFramesCount = hevcPic->num_ref_idx_l1_active_minus1 + 1;
366       m_CurrentFrameReferencesData.pList1ReferenceFrames.resize(m_curFrameState.List1ReferenceFramesCount);
367       for (unsigned i = 0; i < m_curFrameState.List1ReferenceFramesCount; i++)
368          m_CurrentFrameReferencesData.pList1ReferenceFrames[i] = hevcPic->ref_list1[i];
369       m_curFrameState.pList1ReferenceFrames = m_CurrentFrameReferencesData.pList1ReferenceFrames.data();
370 
371       // Deep Copy L1 ref modification list
372       if (hevcPic->slice.ref_pic_lists_modification.ref_pic_list_modification_flag_l1) {
373          m_curFrameState.List1RefPicModificationsCount = hevcPic->num_ref_idx_l1_active_minus1 + 1;
374          m_CurrentFrameReferencesData.pList1RefPicModifications.resize(m_curFrameState.List1RefPicModificationsCount);
375          for (unsigned i = 0; i < m_curFrameState.List1RefPicModificationsCount; i++)
376             m_CurrentFrameReferencesData.pList1RefPicModifications[i] =
377                hevcPic->slice.ref_pic_lists_modification.list_entry_l1[i];
378          m_curFrameState.pList1RefPicModifications = m_CurrentFrameReferencesData.pList1RefPicModifications.data();
379       }
380    }
381 
382    print_dpb();
383    print_l0_l1_lists();
384 }
385