1 /*
2  *  Copyright (c) 2019 The WebRTC project authors. All Rights Reserved.
3  *
4  *  Use of this source code is governed by a BSD-style license
5  *  that can be found in the LICENSE file in the root of the source
6  *  tree. An additional intellectual property rights grant can be found
7  *  in the file PATENTS.  All contributing project authors may
8  *  be found in the AUTHORS file in the root of the source tree.
9  */
10 
11 #include "test/pc/e2e/analyzer/video/single_process_encoded_image_data_injector.h"
12 
13 #include <utility>
14 
15 #include "api/video/encoded_image.h"
16 #include "rtc_base/buffer.h"
17 #include "test/gtest.h"
18 
19 namespace webrtc {
20 namespace webrtc_pc_e2e {
21 namespace {
22 
23 rtc::scoped_refptr<EncodedImageBuffer>
CreateEncodedImageBufferOfSizeNFilledWithValuesFromX(size_t n,uint8_t x)24 CreateEncodedImageBufferOfSizeNFilledWithValuesFromX(size_t n, uint8_t x) {
25   auto buffer = EncodedImageBuffer::Create(n);
26   for (size_t i = 0; i < n; ++i) {
27     buffer->data()[i] = static_cast<uint8_t>(x + i);
28   }
29   return buffer;
30 }
31 
CreateEncodedImageOfSizeNFilledWithValuesFromX(size_t n,uint8_t x)32 EncodedImage CreateEncodedImageOfSizeNFilledWithValuesFromX(size_t n,
33                                                             uint8_t x) {
34   EncodedImage image;
35   image.SetEncodedData(
36       CreateEncodedImageBufferOfSizeNFilledWithValuesFromX(n, x));
37   return image;
38 }
39 
DeepCopyEncodedImage(const EncodedImage & source)40 EncodedImage DeepCopyEncodedImage(const EncodedImage& source) {
41   EncodedImage copy = source;
42   copy.SetEncodedData(EncodedImageBuffer::Create(source.data(), source.size()));
43   return copy;
44 }
45 
TEST(SingleProcessEncodedImageDataInjectorTest,InjectExtractDiscardFalse)46 TEST(SingleProcessEncodedImageDataInjectorTest, InjectExtractDiscardFalse) {
47   SingleProcessEncodedImageDataInjector injector;
48   injector.Start(/*expected_receivers_count=*/1);
49 
50   EncodedImage source =
51       CreateEncodedImageOfSizeNFilledWithValuesFromX(/*n=*/10, /*x=*/1);
52   source.SetTimestamp(123456789);
53 
54   EncodedImageExtractionResult out =
55       injector.ExtractData(injector.InjectData(512, false, source));
56   EXPECT_EQ(out.id, 512);
57   EXPECT_FALSE(out.discard);
58   EXPECT_EQ(out.image.size(), 10ul);
59   EXPECT_EQ(out.image.SpatialLayerFrameSize(0).value_or(0), 0ul);
60   for (int i = 0; i < 10; ++i) {
61     EXPECT_EQ(out.image.data()[i], i + 1);
62   }
63 }
64 
TEST(SingleProcessEncodedImageDataInjectorTest,InjectExtractDiscardTrue)65 TEST(SingleProcessEncodedImageDataInjectorTest, InjectExtractDiscardTrue) {
66   SingleProcessEncodedImageDataInjector injector;
67   injector.Start(/*expected_receivers_count=*/1);
68 
69   EncodedImage source =
70       CreateEncodedImageOfSizeNFilledWithValuesFromX(/*n=*/10, /*x=*/1);
71   source.SetTimestamp(123456789);
72 
73   EncodedImageExtractionResult out =
74       injector.ExtractData(injector.InjectData(512, true, source));
75   EXPECT_EQ(out.id, 512);
76   EXPECT_TRUE(out.discard);
77   EXPECT_EQ(out.image.size(), 0ul);
78   EXPECT_EQ(out.image.SpatialLayerFrameSize(0).value_or(0), 0ul);
79 }
80 
TEST(SingleProcessEncodedImageDataInjectorTest,InjectWithUnsetSpatialLayerSizes)81 TEST(SingleProcessEncodedImageDataInjectorTest,
82      InjectWithUnsetSpatialLayerSizes) {
83   SingleProcessEncodedImageDataInjector injector;
84   injector.Start(/*expected_receivers_count=*/1);
85 
86   EncodedImage source =
87       CreateEncodedImageOfSizeNFilledWithValuesFromX(/*n=*/10, /*x=*/1);
88   source.SetTimestamp(123456789);
89 
90   EncodedImage intermediate = injector.InjectData(512, false, source);
91   intermediate.SetSpatialIndex(2);
92 
93   EncodedImageExtractionResult out = injector.ExtractData(intermediate);
94   EXPECT_EQ(out.id, 512);
95   EXPECT_FALSE(out.discard);
96   EXPECT_EQ(out.image.size(), 10ul);
97   for (int i = 0; i < 10; ++i) {
98     EXPECT_EQ(out.image.data()[i], i + 1);
99   }
100   EXPECT_EQ(out.image.SpatialIndex().value_or(0), 2);
101   for (int i = 0; i < 3; ++i) {
102     EXPECT_EQ(out.image.SpatialLayerFrameSize(i).value_or(0), 0ul);
103   }
104 }
105 
TEST(SingleProcessEncodedImageDataInjectorTest,InjectWithZeroSpatialLayerSizes)106 TEST(SingleProcessEncodedImageDataInjectorTest,
107      InjectWithZeroSpatialLayerSizes) {
108   SingleProcessEncodedImageDataInjector injector;
109   injector.Start(/*expected_receivers_count=*/1);
110 
111   EncodedImage source =
112       CreateEncodedImageOfSizeNFilledWithValuesFromX(/*n=*/10, /*x=*/1);
113   source.SetTimestamp(123456789);
114 
115   EncodedImage intermediate = injector.InjectData(512, false, source);
116   intermediate.SetSpatialIndex(2);
117   intermediate.SetSpatialLayerFrameSize(0, 0);
118   intermediate.SetSpatialLayerFrameSize(1, 0);
119   intermediate.SetSpatialLayerFrameSize(2, 0);
120 
121   EncodedImageExtractionResult out = injector.ExtractData(intermediate);
122   EXPECT_EQ(out.id, 512);
123   EXPECT_FALSE(out.discard);
124   EXPECT_EQ(out.image.size(), 10ul);
125   for (int i = 0; i < 10; ++i) {
126     EXPECT_EQ(out.image.data()[i], i + 1);
127   }
128   EXPECT_EQ(out.image.SpatialIndex().value_or(0), 2);
129   for (int i = 0; i < 3; ++i) {
130     EXPECT_EQ(out.image.SpatialLayerFrameSize(i).value_or(0), 0ul);
131   }
132 }
133 
TEST(SingleProcessEncodedImageDataInjectorTest,Inject3Extract3)134 TEST(SingleProcessEncodedImageDataInjectorTest, Inject3Extract3) {
135   SingleProcessEncodedImageDataInjector injector;
136   injector.Start(/*expected_receivers_count=*/1);
137 
138   // 1st frame
139   EncodedImage source1 =
140       CreateEncodedImageOfSizeNFilledWithValuesFromX(/*n=*/10, /*x=*/1);
141   source1.SetTimestamp(123456710);
142   // 2nd frame 1st spatial layer
143   EncodedImage source2 =
144       CreateEncodedImageOfSizeNFilledWithValuesFromX(/*n=*/10, /*x=*/11);
145   source2.SetTimestamp(123456720);
146   // 2nd frame 2nd spatial layer
147   EncodedImage source3 =
148       CreateEncodedImageOfSizeNFilledWithValuesFromX(/*n=*/10, /*x=*/21);
149   source3.SetTimestamp(123456720);
150 
151   EncodedImage intermediate1 = injector.InjectData(510, false, source1);
152   EncodedImage intermediate2 = injector.InjectData(520, true, source2);
153   EncodedImage intermediate3 = injector.InjectData(520, false, source3);
154 
155   // Extract ids in different order.
156   EncodedImageExtractionResult out3 = injector.ExtractData(intermediate3);
157   EncodedImageExtractionResult out1 = injector.ExtractData(intermediate1);
158   EncodedImageExtractionResult out2 = injector.ExtractData(intermediate2);
159 
160   EXPECT_EQ(out1.id, 510);
161   EXPECT_FALSE(out1.discard);
162   EXPECT_EQ(out1.image.size(), 10ul);
163   EXPECT_EQ(out1.image.SpatialLayerFrameSize(0).value_or(0), 0ul);
164   for (int i = 0; i < 10; ++i) {
165     EXPECT_EQ(out1.image.data()[i], i + 1);
166   }
167   EXPECT_EQ(out2.id, 520);
168   EXPECT_TRUE(out2.discard);
169   EXPECT_EQ(out2.image.size(), 0ul);
170   EXPECT_EQ(out2.image.SpatialLayerFrameSize(0).value_or(0), 0ul);
171   EXPECT_EQ(out3.id, 520);
172   EXPECT_FALSE(out3.discard);
173   EXPECT_EQ(out3.image.size(), 10ul);
174   EXPECT_EQ(out3.image.SpatialLayerFrameSize(0).value_or(0), 0ul);
175   for (int i = 0; i < 10; ++i) {
176     EXPECT_EQ(out3.image.data()[i], i + 21);
177   }
178 }
179 
TEST(SingleProcessEncodedImageDataInjectorTest,InjectExtractFromConcatenated)180 TEST(SingleProcessEncodedImageDataInjectorTest, InjectExtractFromConcatenated) {
181   SingleProcessEncodedImageDataInjector injector;
182   injector.Start(/*expected_receivers_count=*/1);
183 
184   EncodedImage source1 =
185       CreateEncodedImageOfSizeNFilledWithValuesFromX(/*n=*/10, /*x=*/1);
186   source1.SetTimestamp(123456710);
187   EncodedImage source2 =
188       CreateEncodedImageOfSizeNFilledWithValuesFromX(/*n=*/10, /*x=*/11);
189   source2.SetTimestamp(123456710);
190   EncodedImage source3 =
191       CreateEncodedImageOfSizeNFilledWithValuesFromX(/*n=*/10, /*x=*/21);
192   source3.SetTimestamp(123456710);
193 
194   // Inject id into 3 images with same frame id.
195   EncodedImage intermediate1 = injector.InjectData(512, false, source1);
196   EncodedImage intermediate2 = injector.InjectData(512, true, source2);
197   EncodedImage intermediate3 = injector.InjectData(512, false, source3);
198 
199   // Concatenate them into single encoded image, like it can be done in jitter
200   // buffer.
201   size_t concatenated_length =
202       intermediate1.size() + intermediate2.size() + intermediate3.size();
203   rtc::Buffer concatenated_buffer;
204   concatenated_buffer.AppendData(intermediate1.data(), intermediate1.size());
205   concatenated_buffer.AppendData(intermediate2.data(), intermediate2.size());
206   concatenated_buffer.AppendData(intermediate3.data(), intermediate3.size());
207   EncodedImage concatenated;
208   concatenated.SetEncodedData(EncodedImageBuffer::Create(
209       concatenated_buffer.data(), concatenated_length));
210   concatenated.SetSpatialIndex(2);
211   concatenated.SetSpatialLayerFrameSize(0, intermediate1.size());
212   concatenated.SetSpatialLayerFrameSize(1, intermediate2.size());
213   concatenated.SetSpatialLayerFrameSize(2, intermediate3.size());
214 
215   // Extract frame id from concatenated image
216   EncodedImageExtractionResult out = injector.ExtractData(concatenated);
217 
218   EXPECT_EQ(out.id, 512);
219   EXPECT_FALSE(out.discard);
220   EXPECT_EQ(out.image.size(), 2 * 10ul);
221   for (int i = 0; i < 10; ++i) {
222     EXPECT_EQ(out.image.data()[i], i + 1);
223     EXPECT_EQ(out.image.data()[i + 10], i + 21);
224   }
225   EXPECT_EQ(out.image.SpatialIndex().value_or(0), 2);
226   EXPECT_EQ(out.image.SpatialLayerFrameSize(0).value_or(0), 10ul);
227   EXPECT_EQ(out.image.SpatialLayerFrameSize(1).value_or(0), 0ul);
228   EXPECT_EQ(out.image.SpatialLayerFrameSize(2).value_or(0), 10ul);
229 }
230 
TEST(SingleProcessEncodedImageDataInjector,InjectExtractFromConcatenatedAllDiscarded)231 TEST(SingleProcessEncodedImageDataInjector,
232      InjectExtractFromConcatenatedAllDiscarded) {
233   SingleProcessEncodedImageDataInjector injector;
234   injector.Start(/*expected_receivers_count=*/1);
235 
236   EncodedImage source1 =
237       CreateEncodedImageOfSizeNFilledWithValuesFromX(/*n=*/10, /*x=*/1);
238   source1.SetTimestamp(123456710);
239   EncodedImage source2 =
240       CreateEncodedImageOfSizeNFilledWithValuesFromX(/*n=*/10, /*x=*/11);
241   source2.SetTimestamp(123456710);
242   EncodedImage source3 =
243       CreateEncodedImageOfSizeNFilledWithValuesFromX(/*n=*/10, /*x=*/21);
244   source3.SetTimestamp(123456710);
245 
246   // Inject id into 3 images with same frame id.
247   EncodedImage intermediate1 = injector.InjectData(512, true, source1);
248   EncodedImage intermediate2 = injector.InjectData(512, true, source2);
249   EncodedImage intermediate3 = injector.InjectData(512, true, source3);
250 
251   // Concatenate them into single encoded image, like it can be done in jitter
252   // buffer.
253   size_t concatenated_length =
254       intermediate1.size() + intermediate2.size() + intermediate3.size();
255   rtc::Buffer concatenated_buffer;
256   concatenated_buffer.AppendData(intermediate1.data(), intermediate1.size());
257   concatenated_buffer.AppendData(intermediate2.data(), intermediate2.size());
258   concatenated_buffer.AppendData(intermediate3.data(), intermediate3.size());
259   EncodedImage concatenated;
260   concatenated.SetEncodedData(EncodedImageBuffer::Create(
261       concatenated_buffer.data(), concatenated_length));
262   concatenated.SetSpatialIndex(2);
263   concatenated.SetSpatialLayerFrameSize(0, intermediate1.size());
264   concatenated.SetSpatialLayerFrameSize(1, intermediate2.size());
265   concatenated.SetSpatialLayerFrameSize(2, intermediate3.size());
266 
267   // Extract frame id from concatenated image
268   EncodedImageExtractionResult out = injector.ExtractData(concatenated);
269 
270   EXPECT_EQ(out.id, 512);
271   EXPECT_TRUE(out.discard);
272   EXPECT_EQ(out.image.size(), 0ul);
273   EXPECT_EQ(out.image.SpatialIndex().value_or(0), 2);
274   for (int i = 0; i < 3; ++i) {
275     EXPECT_EQ(out.image.SpatialLayerFrameSize(i).value_or(0), 0ul);
276   }
277 }
278 
TEST(SingleProcessEncodedImageDataInjectorTest,InjectOnceExtractTwice)279 TEST(SingleProcessEncodedImageDataInjectorTest, InjectOnceExtractTwice) {
280   SingleProcessEncodedImageDataInjector injector;
281   injector.Start(/*expected_receivers_count=*/2);
282 
283   EncodedImage source =
284       CreateEncodedImageOfSizeNFilledWithValuesFromX(/*n=*/10, /*x=*/1);
285   source.SetTimestamp(123456789);
286 
287   EncodedImageExtractionResult out = injector.ExtractData(
288       injector.InjectData(/*id=*/512, /*discard=*/false, source));
289   EXPECT_EQ(out.id, 512);
290   EXPECT_FALSE(out.discard);
291   EXPECT_EQ(out.image.size(), 10ul);
292   EXPECT_EQ(out.image.SpatialLayerFrameSize(0).value_or(0), 0ul);
293   for (int i = 0; i < 10; ++i) {
294     EXPECT_EQ(out.image.data()[i], i + 1);
295   }
296   out = injector.ExtractData(
297       injector.InjectData(/*id=*/512, /*discard=*/false, source));
298   EXPECT_EQ(out.id, 512);
299   EXPECT_FALSE(out.discard);
300   EXPECT_EQ(out.image.size(), 10ul);
301   EXPECT_EQ(out.image.SpatialLayerFrameSize(0).value_or(0), 0ul);
302   for (int i = 0; i < 10; ++i) {
303     EXPECT_EQ(out.image.data()[i], i + 1);
304   }
305 }
306 
TEST(SingleProcessEncodedImageDataInjectorTest,Add1stReceiverAfterStart)307 TEST(SingleProcessEncodedImageDataInjectorTest, Add1stReceiverAfterStart) {
308   SingleProcessEncodedImageDataInjector injector;
309   injector.Start(/*expected_receivers_count=*/0);
310 
311   EncodedImage source =
312       CreateEncodedImageOfSizeNFilledWithValuesFromX(/*n=*/10, /*x=*/1);
313   source.SetTimestamp(123456789);
314   EncodedImage modified_image = injector.InjectData(
315       /*id=*/512, /*discard=*/false, source);
316 
317   injector.AddParticipantInCall();
318   EncodedImageExtractionResult out = injector.ExtractData(modified_image);
319 
320   EXPECT_EQ(out.id, 512);
321   EXPECT_FALSE(out.discard);
322   EXPECT_EQ(out.image.size(), 10ul);
323   EXPECT_EQ(out.image.SpatialLayerFrameSize(0).value_or(0), 0ul);
324   for (int i = 0; i < 10; ++i) {
325     EXPECT_EQ(out.image.data()[i], i + 1);
326   }
327 }
328 
TEST(SingleProcessEncodedImageDataInjectorTest,Add3rdReceiverAfterStart)329 TEST(SingleProcessEncodedImageDataInjectorTest, Add3rdReceiverAfterStart) {
330   SingleProcessEncodedImageDataInjector injector;
331   injector.Start(/*expected_receivers_count=*/2);
332 
333   EncodedImage source =
334       CreateEncodedImageOfSizeNFilledWithValuesFromX(/*n=*/10, /*x=*/1);
335   source.SetTimestamp(123456789);
336   EncodedImage modified_image = injector.InjectData(
337       /*id=*/512, /*discard=*/false, source);
338   injector.ExtractData(modified_image);
339 
340   injector.AddParticipantInCall();
341   injector.ExtractData(modified_image);
342   EncodedImageExtractionResult out = injector.ExtractData(modified_image);
343 
344   EXPECT_EQ(out.id, 512);
345   EXPECT_FALSE(out.discard);
346   EXPECT_EQ(out.image.size(), 10ul);
347   EXPECT_EQ(out.image.SpatialLayerFrameSize(0).value_or(0), 0ul);
348   for (int i = 0; i < 10; ++i) {
349     EXPECT_EQ(out.image.data()[i], i + 1);
350   }
351 }
352 
TEST(SingleProcessEncodedImageDataInjectorTest,RemoveReceiverRemovesOnlyFullyReceivedFrames)353 TEST(SingleProcessEncodedImageDataInjectorTest,
354      RemoveReceiverRemovesOnlyFullyReceivedFrames) {
355   SingleProcessEncodedImageDataInjector injector;
356   injector.Start(/*expected_receivers_count=*/3);
357 
358   EncodedImage source1 =
359       CreateEncodedImageOfSizeNFilledWithValuesFromX(/*n=*/10, /*x=*/1);
360   source1.SetTimestamp(10);
361   EncodedImage source2 =
362       CreateEncodedImageOfSizeNFilledWithValuesFromX(/*n=*/10, /*x=*/1);
363   source2.SetTimestamp(20);
364 
365   EncodedImage modified_image1 = injector.InjectData(
366       /*id=*/512, /*discard=*/false, source1);
367   EncodedImage modified_image2 = injector.InjectData(
368       /*id=*/513, /*discard=*/false, source2);
369 
370   // Out of 3 receivers 1st image received by 2 and 2nd image by 1
371   injector.ExtractData(DeepCopyEncodedImage(modified_image1));
372   injector.ExtractData(DeepCopyEncodedImage(modified_image1));
373   injector.ExtractData(DeepCopyEncodedImage(modified_image2));
374 
375   // When we removed one receiver, 2nd image should still be available for
376   // extraction.
377   injector.RemoveParticipantInCall();
378 
379   EncodedImageExtractionResult out =
380       injector.ExtractData(DeepCopyEncodedImage(modified_image2));
381 
382   EXPECT_EQ(out.id, 513);
383   EXPECT_FALSE(out.discard);
384   EXPECT_EQ(out.image.size(), 10ul);
385   EXPECT_EQ(out.image.SpatialLayerFrameSize(0).value_or(0), 0ul);
386   for (int i = 0; i < 10; ++i) {
387     EXPECT_EQ(out.image.data()[i], i + 1);
388   }
389 }
390 
391 // Death tests.
392 // Disabled on Android because death tests misbehave on Android, see
393 // base/test/gtest_util.h.
394 #if RTC_DCHECK_IS_ON && GTEST_HAS_DEATH_TEST && !defined(WEBRTC_ANDROID)
TEST(SingleProcessEncodedImageDataInjectorTestDeathTest,InjectOnceExtractMoreThenExpected)395 TEST(SingleProcessEncodedImageDataInjectorTestDeathTest,
396      InjectOnceExtractMoreThenExpected) {
397   SingleProcessEncodedImageDataInjector injector;
398   injector.Start(/*expected_receivers_count=*/2);
399 
400   EncodedImage source =
401       CreateEncodedImageOfSizeNFilledWithValuesFromX(/*n=*/10, /*x=*/1);
402   source.SetTimestamp(123456789);
403 
404   EncodedImage modified =
405       injector.InjectData(/*id=*/512, /*discard=*/false, source);
406 
407   injector.ExtractData(DeepCopyEncodedImage(modified));
408   injector.ExtractData(DeepCopyEncodedImage(modified));
409   EXPECT_DEATH(injector.ExtractData(DeepCopyEncodedImage(modified)),
410                "Unknown sub_id=0 for frame_id=512");
411 }
412 
TEST(SingleProcessEncodedImageDataInjectorTestDeathTest,RemoveReceiverRemovesOnlyFullyReceivedFramesVerifyFrameIsRemoved)413 TEST(SingleProcessEncodedImageDataInjectorTestDeathTest,
414      RemoveReceiverRemovesOnlyFullyReceivedFramesVerifyFrameIsRemoved) {
415   SingleProcessEncodedImageDataInjector injector;
416   injector.Start(/*expected_receivers_count=*/3);
417 
418   EncodedImage source1 =
419       CreateEncodedImageOfSizeNFilledWithValuesFromX(/*n=*/10, /*x=*/1);
420   source1.SetTimestamp(10);
421   EncodedImage source2 =
422       CreateEncodedImageOfSizeNFilledWithValuesFromX(/*n=*/10, /*x=*/1);
423   source2.SetTimestamp(20);
424 
425   EncodedImage modified_image1 = injector.InjectData(
426       /*id=*/512, /*discard=*/false, source1);
427   EncodedImage modified_image2 = injector.InjectData(
428       /*id=*/513, /*discard=*/false, source2);
429 
430   // Out of 3 receivers 1st image received by 2 and 2nd image by 1
431   injector.ExtractData(DeepCopyEncodedImage(modified_image1));
432   injector.ExtractData(DeepCopyEncodedImage(modified_image1));
433   injector.ExtractData(DeepCopyEncodedImage(modified_image2));
434 
435   // When we removed one receiver 1st image should be removed.
436   injector.RemoveParticipantInCall();
437 
438   EXPECT_DEATH(injector.ExtractData(DeepCopyEncodedImage(modified_image1)),
439                "Unknown sub_id=0 for frame_id=512");
440 }
441 #endif  // RTC_DCHECK_IS_ON && GTEST_HAS_DEATH_TEST && !defined(WEBRTC_ANDROID)
442 
443 }  // namespace
444 }  // namespace webrtc_pc_e2e
445 }  // namespace webrtc
446