1 /* 2 * Copyright (C) 2021 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package android.videoencodingquality.cts; 18 19 import static com.android.media.videoquality.bdrate.BdRateMain.verifyBdRate; 20 21 import android.cts.host.utils.DeviceJUnit4ClassRunnerWithParameters; 22 import android.cts.host.utils.DeviceJUnit4Parameterized; 23 import android.platform.test.annotations.AppModeFull; 24 25 import com.android.compatibility.common.util.CddTest; 26 import com.android.ddmlib.IDevice; 27 import com.android.ddmlib.testrunner.RemoteAndroidTestRunner; 28 import com.android.ddmlib.testrunner.TestResult.TestStatus; 29 import com.android.tradefed.config.Option; 30 import com.android.tradefed.config.OptionClass; 31 import com.android.tradefed.device.DeviceNotAvailableException; 32 import com.android.tradefed.device.ITestDevice; 33 import com.android.tradefed.log.LogUtil; 34 import com.android.tradefed.result.CollectingTestListener; 35 import com.android.tradefed.result.TestDescription; 36 import com.android.tradefed.result.TestResult; 37 import com.android.tradefed.result.TestRunResult; 38 import com.android.tradefed.testtype.IDeviceTest; 39 40 import org.json.JSONArray; 41 import org.json.JSONObject; 42 import org.junit.Assert; 43 import org.junit.Assume; 44 import org.junit.Test; 45 import org.junit.runner.RunWith; 46 import org.junit.runners.Parameterized; 47 import org.junit.runners.Parameterized.UseParametersRunnerFactory; 48 49 import java.io.BufferedReader; 50 import java.io.File; 51 import java.io.FileReader; 52 import java.io.FileWriter; 53 import java.io.IOException; 54 import java.io.InputStreamReader; 55 import java.nio.charset.StandardCharsets; 56 import java.nio.file.Files; 57 import java.nio.file.Paths; 58 import java.util.ArrayList; 59 import java.util.Arrays; 60 import java.util.List; 61 import java.util.Map; 62 import java.util.concurrent.TimeUnit; 63 import java.util.concurrent.locks.Condition; 64 import java.util.concurrent.locks.Lock; 65 import java.util.concurrent.locks.ReentrantLock; 66 67 import javax.annotation.Nullable; 68 69 /** 70 * This class constitutes host-part of video encoding quality test (go/pc14-veq). This test is 71 * aimed towards benchmarking encoders on the target device. 72 * <p> 73 * Video encoding quality test quantifies encoders on the test device by encoding a set of clips 74 * at various configurations. The encoded output is analysed for vmaf and compared against 75 * reference. This entire process is not carried on the device. The host side of the test 76 * prepares the test environment by installing a VideoEncodingApp on the device. It also pushes 77 * the test vectors and test configurations on to the device. The VideoEncodingApp transcodes the 78 * input clips basing on the configurations shared. The host side of the test then pulls output 79 * files from the device and analyses for vmaf. These values are compared against reference using 80 * Bjontegaard metric. 81 **/ 82 @AppModeFull(reason = "Instant apps cannot access the SD card") 83 @RunWith(DeviceJUnit4Parameterized.class) 84 @UseParametersRunnerFactory(DeviceJUnit4ClassRunnerWithParameters.RunnerFactory.class) 85 @OptionClass(alias = "pc-veq-test") 86 public class CtsVideoEncodingQualityHostTest implements IDeviceTest { 87 private static final String RES_URL = 88 "https://storage.googleapis.com/android_media/cts/hostsidetests/pc14_veq/veqtests-1_4.tar.gz"; 89 90 // variables related to host-side of the test 91 private static final int MEDIA_PERFORMANCE_CLASS_14 = 34; 92 private static final int MINIMUM_VALID_SDK = 31; 93 // test is not valid before sdk 31, aka Android 12, aka Android S 94 95 private static final Lock sLock = new ReentrantLock(); 96 private static final Condition sCondition = sLock.newCondition(); 97 private static boolean sIsTestSetUpDone = false; 98 // install apk, push necessary resources to device to run the test. lock/condition 99 // pair is to keep setupTestEnv() thread safe 100 private static File sHostWorkDir; 101 private static int sMpc; 102 103 // Variables related to device-side of the test. These need to kept in sync with definitions of 104 // VideoEncodingApp.apk 105 private static final String DEVICE_SIDE_TEST_PACKAGE = "android.videoencoding.app"; 106 private static final String DEVICE_SIDE_TEST_CLASS = 107 "android.videoencoding.app.VideoTranscoderTest"; 108 private static final String RUNNER = "androidx.test.runner.AndroidJUnitRunner"; 109 private static final String TEST_CONFIG_INST_ARGS_KEY = "conf-json"; 110 private static final long DEFAULT_SHELL_TIMEOUT_MILLIS = TimeUnit.MINUTES.toMillis(5); 111 private static final String TEST_TIMEOUT_INST_ARGS_KEY = "timeout_msec"; 112 private static final long DEFAULT_TEST_TIMEOUT_MILLIS = TimeUnit.MINUTES.toMillis(3); 113 114 // local variables related to host-side of the test 115 private final String mJsonName; 116 private ITestDevice mDevice; 117 118 @Option(name = "force-to-run", description = "Force to run the test even if the device is not" 119 + " a right performance class device.") 120 private boolean mForceToRun = false; 121 122 @Option(name = "skip-avc", description = "Skip avc encoder testing") 123 private boolean mSkipAvc = false; 124 125 @Option(name = "skip-hevc", description = "Skip hevc encoder testing") 126 private boolean mSkipHevc = false; 127 128 @Option(name = "skip-p", description = "Skip P only testing") 129 private boolean mSkipP = false; 130 131 @Option(name = "skip-b", description = "Skip B frame testing") 132 private boolean mSkipB = false; 133 134 @Option(name = "reset", description = "Start with a fresh directory.") 135 private boolean mReset = false; 136 137 @Option(name = "quick-check", description = "Run a quick check.") 138 private boolean mQuickCheck = false; 139 CtsVideoEncodingQualityHostTest(String jsonName, @SuppressWarnings("unused") String testLabel)140 public CtsVideoEncodingQualityHostTest(String jsonName, 141 @SuppressWarnings("unused") String testLabel) { 142 mJsonName = jsonName; 143 } 144 145 private static final List<Object[]> AVC_VBR_B0_PARAMS = Arrays.asList(new Object[][]{ 146 {"AVICON-MOBILE-Beach-SO04-CRW02-L-420-8bit-SDR-1080p-30fps_hw_avc_vbr_b0.json", 147 "Beach_SO04_CRW02_L_420_8bit_SDR_1080p_30fps_hw_avc_vbr_b0"}, 148 {"AVICON-MOBILE-BirthdayHalfway-SI17-CRUW03-L-420-8bit-SDR-1080p-30fps_hw_avc_vbr_b0" 149 + ".json", 150 "BirthdayHalfway_SI17_CRUW03_L_420_8bit_SDR_1080p_30fps_hw_avc_vbr_b0"}, 151 {"AVICON-MOBILE-SelfieTeenKitchenSocialMedia-SS01-CF01-P-420-8bit-SDR-1080p" 152 + "-30fps_hw_avc_vbr_b0.json", 153 "SelfieTeenKitchenSocialMedia_SS01_CF01_P_420_8bit_SDR_1080p_30fps_hw_avc_" 154 + "vbr_b0"}, 155 {"AVICON-MOBILE-Waterfall-SO05-CRW01-P-420-8bit-SDR-1080p-30fps_hw_avc_vbr_b0.json", 156 "Waterfall_SO05_CRW01_P_420_8bit_SDR_1080p_30fps_hw_avc_vbr_b0"}, 157 {"AVICON-MOBILE-SelfieFamily-SF14-CF01-L-420-8bit-SDR-1080p-30fps_hw_avc_vbr_b0.json" 158 , "SelfieFamily_SF14_CF01_L_420_8bit_SDR_1080p_30fps_hw_avc_vbr_b0"}, 159 {"AVICON-MOBILE-River-SO03-CRW01-L-420-8bit-SDR-1080p-30fps_hw_avc_vbr_b0.json", 160 "River_SO03_CRW01_L_420_8bit_SDR_1080p_30fps_hw_avc_vbr_b0"}, 161 {"AVICON-MOBILE-SelfieGroupGarden-SF15-CF01-P-420-8bit-SDR-1080p-30fps_hw_avc_vbr_b0" 162 + ".json", 163 "SelfieGroupGarden_SF15_CF01_P_420_8bit_SDR_1080p_30fps_hw_avc_vbr_b0"}, 164 {"AVICON-MOBILE-ConcertNear-SI10-CRW01-L-420-8bit-SDR-1080p-30fps_hw_avc_vbr_b0.json" 165 , "ConcertNear_SI10_CRW01_L_420_8bit_SDR_1080p_30fps_hw_avc_vbr_b0"}, 166 {"AVICON-MOBILE-SelfieCoupleCitySocialMedia-SS02-CF01-P-420-8bit-SDR" 167 + "-1080p-30fps_hw_avc_vbr_b0.json", 168 "SelfieCoupleCitySocialMedia_SS02_CF01_P_420_8bit_SDR_1080p_30fps_hw_avc_" 169 + "vbr_b0"}}); 170 171 private static final List<Object[]> AVC_VBR_B3_PARAMS = Arrays.asList(new Object[][]{ 172 {"AVICON-MOBILE-Beach-SO04-CRW02-L-420-8bit-SDR-1080p-30fps_hw_avc_vbr_b3.json", 173 "Beach_SO04_CRW02_L_420_8bit_SDR_1080p_30fps_hw_avc_vbr_b3"}, 174 {"AVICON-MOBILE-BirthdayHalfway-SI17-CRUW03-L-420-8bit-SDR-1080p-30fps_hw_avc_vbr_b3" 175 + ".json", 176 "BirthdayHalfway_SI17_CRUW03_L_420_8bit_SDR_1080p_30fps_hw_avc_vbr_b3"}, 177 {"AVICON-MOBILE-SelfieTeenKitchenSocialMedia-SS01-CF01-P-420-8bit-SDR-1080p" 178 + "-30fps_hw_avc_vbr_b3.json", 179 "SelfieTeenKitchenSocialMedia_SS01_CF01_P_420_8bit_SDR_1080p_30fps_hw_avc_" 180 + "vbr_b3"}, 181 {"AVICON-MOBILE-Waterfall-SO05-CRW01-P-420-8bit-SDR-1080p-30fps_hw_avc_vbr_b3.json", 182 "Waterfall_SO05_CRW01_P_420_8bit_SDR_1080p_30fps_hw_avc_vbr_b3"}, 183 {"AVICON-MOBILE-SelfieFamily-SF14-CF01-L-420-8bit-SDR-1080p-30fps_hw_avc_vbr_b3.json" 184 , "SelfieFamily_SF14_CF01_L_420_8bit_SDR_1080p_30fps_hw_avc_vbr_b3"}, 185 {"AVICON-MOBILE-River-SO03-CRW01-L-420-8bit-SDR-1080p-30fps_hw_avc_vbr_b3.json", 186 "River_SO03_CRW01_L_420_8bit_SDR_1080p_30fps_hw_avc_vbr_b3"}, 187 {"AVICON-MOBILE-SelfieGroupGarden-SF15-CF01-P-420-8bit-SDR-1080p-30fps_hw_avc_vbr_b3" 188 + ".json", 189 "SelfieGroupGarden_SF15_CF01_P_420_8bit_SDR_1080p_30fps_hw_avc_vbr_b3"}, 190 {"AVICON-MOBILE-ConcertNear-SI10-CRW01-L-420-8bit-SDR-1080p-30fps_hw_avc_vbr_b3.json" 191 , "ConcertNear_SI10_CRW01_L_420_8bit_SDR_1080p_30fps_hw_avc_vbr_b3"}, 192 {"AVICON-MOBILE-SelfieCoupleCitySocialMedia-SS02-CF01-P-420-8bit-SDR" 193 + "-1080p-30fps_hw_avc_vbr_b3.json", 194 "SelfieCoupleCitySocialMedia_SS02_CF01_P_420_8bit_SDR_1080p_30fps_hw_avc_" 195 + "vbr_b3"}}); 196 197 private static final List<Object[]> HEVC_VBR_B0_PARAMS = Arrays.asList(new Object[][]{ 198 {"AVICON-MOBILE-Beach-SO04-CRW02-L-420-8bit-SDR-1080p-30fps_hw_hevc_vbr_b0.json", 199 "Beach_SO04_CRW02_L_420_8bit_SDR_1080p_30fps_hw_hevc_vbr_b0"}, 200 {"AVICON-MOBILE-BirthdayHalfway-SI17-CRUW03-L-420-8bit-SDR-1080p-30fps_hw_hevc_vbr_b0" 201 + ".json", 202 "BirthdayHalfway_SI17_CRUW03_L_420_8bit_SDR_1080p_30fps_hw_hevc_vbr_b0"}, 203 {"AVICON-MOBILE-SelfieTeenKitchenSocialMedia-SS01-CF01-P-420-8bit-SDR-1080p" 204 + "-30fps_hw_hevc_vbr_b0.json", 205 "SelfieTeenKitchenSocialMedia_SS01_CF01_P_420_8bit_SDR_1080p_30fps_hw_hevc_" 206 + "vbr_b0"}, 207 {"AVICON-MOBILE-Waterfall-SO05-CRW01-P-420-8bit-SDR-1080p-30fps_hw_hevc_vbr_b0.json", 208 "Waterfall_SO05_CRW01_P_420_8bit_SDR_1080p_30fps_hw_hevc_vbr_b0"}, 209 {"AVICON-MOBILE-SelfieFamily-SF14-CF01-L-420-8bit-SDR-1080p-30fps_hw_hevc_vbr_b0.json" 210 , "SelfieFamily_SF14_CF01_L_420_8bit_SDR_1080p_30fps_hw_hevc_vbr_b0"}, 211 {"AVICON-MOBILE-River-SO03-CRW01-L-420-8bit-SDR-1080p-30fps_hw_hevc_vbr_b0.json", 212 "River_SO03_CRW01_L_420_8bit_SDR_1080p_30fps_hw_hevc_vbr_b0"}, 213 {"AVICON-MOBILE-SelfieGroupGarden-SF15-CF01-P-420-8bit-SDR-1080p-30fps_hw_hevc_vbr_b0" 214 + ".json", 215 "SelfieGroupGarden_SF15_CF01_P_420_8bit_SDR_1080p_30fps_hw_hevc_vbr_b0"}, 216 {"AVICON-MOBILE-ConcertNear-SI10-CRW01-L-420-8bit-SDR-1080p-30fps_hw_hevc_vbr_b0.json" 217 , "ConcertNear_SI10_CRW01_L_420_8bit_SDR_1080p_30fps_hw_hevc_vbr_b0"}, 218 {"AVICON-MOBILE-SelfieCoupleCitySocialMedia-SS02-CF01-P-420-8bit-SDR" 219 + "-1080p-30fps_hw_hevc_vbr_b0.json", 220 "SelfieCoupleCitySocialMedia_SS02_CF01_P_420_8bit_SDR_1080p_30fps_hw_hevc_" 221 + "vbr_b0"}}); 222 223 private static final List<Object[]> HEVC_VBR_B3_PARAMS = Arrays.asList(new Object[][]{ 224 {"AVICON-MOBILE-Beach-SO04-CRW02-L-420-8bit-SDR-1080p-30fps_hw_hevc_vbr_b3.json", 225 "Beach_SO04_CRW02_L_420_8bit_SDR_1080p_30fps_hw_hevc_vbr_b3"}, 226 {"AVICON-MOBILE-BirthdayHalfway-SI17-CRUW03-L-420-8bit-SDR-1080p-30fps_hw_hevc_vbr_b3" 227 + ".json", 228 "BirthdayHalfway_SI17_CRUW03_L_420_8bit_SDR_1080p_30fps_hw_hevc_vbr_b3"}, 229 {"AVICON-MOBILE-SelfieTeenKitchenSocialMedia-SS01-CF01-P-420-8bit-SDR-1080p" 230 + "-30fps_hw_hevc_vbr_b3.json", 231 "SelfieTeenKitchenSocialMedia_SS01_CF01_P_420_8bit_SDR_1080p_30fps_hw_hevc_" 232 + "vbr_b3"}, 233 {"AVICON-MOBILE-Waterfall-SO05-CRW01-P-420-8bit-SDR-1080p-30fps_hw_hevc_vbr_b3.json", 234 "Waterfall_SO05_CRW01_P_420_8bit_SDR_1080p_30fps_hw_hevc_vbr_b3"}, 235 {"AVICON-MOBILE-SelfieFamily-SF14-CF01-L-420-8bit-SDR-1080p-30fps_hw_hevc_vbr_b3.json" 236 , "SelfieFamily_SF14_CF01_L_420_8bit_SDR_1080p_30fps_hw_hevc_vbr_b3"}, 237 {"AVICON-MOBILE-River-SO03-CRW01-L-420-8bit-SDR-1080p-30fps_hw_hevc_vbr_b3.json", 238 "River_SO03_CRW01_L_420_8bit_SDR_1080p_30fps_hw_hevc_vbr_b3"}, 239 // Abnormal curve, not monotonically increasing. 240 /*{"AVICON-MOBILE-SelfieGroupGarden-SF15-CF01-P-420-8bit-SDR-1080p-30fps_hw_hevc_vbr_b3" 241 + ".json", 242 "SelfieGroupGarden_SF15_CF01_P_420_8bit_SDR_1080p_30fps_hw_hevc_vbr_b3"},*/ 243 {"AVICON-MOBILE-ConcertNear-SI10-CRW01-L-420-8bit-SDR-1080p-30fps_hw_hevc_vbr_b3.json" 244 , "ConcertNear_SI10_CRW01_L_420_8bit_SDR_1080p_30fps_hw_hevc_vbr_b3"}, 245 {"AVICON-MOBILE-SelfieCoupleCitySocialMedia-SS02-CF01-P-420-8bit-SDR" 246 + "-1080p-30fps_hw_hevc_vbr_b3.json", 247 "SelfieCoupleCitySocialMedia_SS02_CF01_P_420_8bit_SDR_1080p_30fps_hw_hevc_" 248 + "vbr_b3"}}); 249 250 private static final List<Object[]> QUICK_RUN_PARAMS = Arrays.asList(new Object[][]{ 251 {"AVICON-MOBILE-SelfieTeenKitchenSocialMedia-SS01-CF01-P-420-8bit-SDR-1080p" 252 + "-30fps_hw_avc_vbr_b0.json", 253 "SelfieTeenKitchenSocialMedia_SS01_CF01_P_420_8bit_SDR_1080p_30fps_hw_avc_" + 254 "vbr_b0"}, 255 {"AVICON-MOBILE-SelfieTeenKitchenSocialMedia-SS01-CF01-P-420-8bit-SDR-1080p" 256 + "-30fps_hw_hevc_vbr_b0.json", 257 "SelfieTeenKitchenSocialMedia_SS01_CF01_P_420_8bit_SDR_1080p_30fps_hw_hevc_" 258 + "vbr_b0"}}); 259 260 @Parameterized.Parameters(name = "{index}_{1}") input()261 public static List<Object[]> input() { 262 final List<Object[]> args = new ArrayList<>(); 263 args.addAll(AVC_VBR_B0_PARAMS); 264 args.addAll(AVC_VBR_B3_PARAMS); 265 args.addAll(HEVC_VBR_B0_PARAMS); 266 args.addAll(HEVC_VBR_B3_PARAMS); 267 return args; 268 } 269 270 @Override setDevice(ITestDevice device)271 public void setDevice(ITestDevice device) { 272 mDevice = device; 273 } 274 275 @Override getDevice()276 public ITestDevice getDevice() { 277 return mDevice; 278 } 279 280 /** 281 * Sets up the necessary environment for the video encoding quality test. 282 */ setupTestEnv()283 public void setupTestEnv() throws Exception { 284 String sdkAsString = getDevice().getProperty("ro.build.version.sdk"); 285 int sdk = Integer.parseInt(sdkAsString); 286 Assume.assumeTrue("Test requires sdk >= " + MINIMUM_VALID_SDK 287 + " test device has sdk = " + sdk, sdk >= MINIMUM_VALID_SDK); 288 289 String pcAsString = getDevice().getProperty("ro.odm.build.media_performance_class"); 290 try { 291 sMpc = Integer.parseInt("0" + pcAsString); 292 } catch (Exception e) { 293 LogUtil.CLog.i("Invalid pcAsString: " + pcAsString + ", exception: " + e); 294 } 295 296 Assume.assumeTrue("Performance class advertised by the test device is less than " 297 + MEDIA_PERFORMANCE_CLASS_14, mForceToRun || sMpc >= MEDIA_PERFORMANCE_CLASS_14 298 || (sMpc == 0 && sdk >= 34 /* Build.VERSION_CODES.UPSIDE_DOWN_CAKE */)); 299 300 Assert.assertTrue("Failed to install package on device : " + DEVICE_SIDE_TEST_PACKAGE, 301 getDevice().isPackageInstalled(DEVICE_SIDE_TEST_PACKAGE)); 302 303 // set up host-side working directory 304 String tmpBase = System.getProperty("java.io.tmpdir"); 305 String dirName = "CtsVideoEncodingQualityHostTest_" + getDevice().getSerialNumber(); 306 String tmpDir = tmpBase + "/" + dirName; 307 LogUtil.CLog.i("tmpBase= " + tmpBase + " tmpDir =" + tmpDir); 308 sHostWorkDir = new File(tmpDir); 309 if (mReset || sHostWorkDir.isFile()) { 310 File cwd = new File("."); 311 runCommand("rm -rf " + tmpDir, cwd); 312 } 313 try { 314 if (!sHostWorkDir.isDirectory()) { 315 Assert.assertTrue("Failed to create directory : " + sHostWorkDir.getAbsolutePath(), 316 sHostWorkDir.mkdirs()); 317 } 318 } catch (SecurityException e) { 319 LogUtil.CLog.e("Unable to establish temp directory " + sHostWorkDir.getPath()); 320 } 321 322 // Clean up output folders before starting the test 323 runCommand("rm -rf " + "output_*", sHostWorkDir); 324 325 // Download the test suite tar file. 326 downloadFile(RES_URL, sHostWorkDir); 327 328 // Unpack the test suite tar file. 329 String fileName = RES_URL.substring(RES_URL.lastIndexOf('/') + 1); 330 int result = runCommand("tar xvzf " + fileName, sHostWorkDir); 331 Assert.assertEquals("Failed to untar " + fileName, 0, result); 332 333 // Push input files to device 334 String deviceInDir = getDevice().getMountPoint(IDevice.MNT_EXTERNAL_STORAGE) 335 + "/veq/input/"; 336 String deviceJsonDir = deviceInDir + "json/"; 337 String deviceSamplesDir = deviceInDir + "samples/"; 338 Assert.assertNotNull("Failed to create directory " + deviceJsonDir + " on device ", 339 getDevice().executeAdbCommand("shell", "mkdir", "-p", deviceJsonDir)); 340 Assert.assertNotNull("Failed to create directory " + deviceSamplesDir + " on device ", 341 getDevice().executeAdbCommand("shell", "mkdir", "-p", deviceSamplesDir)); 342 Assert.assertTrue("Failed to push json files to " + deviceJsonDir + " on device ", 343 getDevice().pushDir(new File(sHostWorkDir.getPath() + "/json/"), deviceJsonDir)); 344 Assert.assertTrue("Failed to push mp4 files to " + deviceSamplesDir + " on device ", 345 getDevice().pushDir(new File(sHostWorkDir.getPath() + "/samples/"), 346 deviceSamplesDir)); 347 348 sIsTestSetUpDone = true; 349 } 350 containsJson(String jsonName, List<Object[]> params)351 public static boolean containsJson(String jsonName, List<Object[]> params) { 352 for (Object[] param : params) { 353 if (param[0].equals(jsonName)) { 354 return true; 355 } 356 } 357 return false; 358 } 359 360 /** 361 * Verify the video encoding quality requirements for the performance class 14 devices. 362 */ 363 @CddTest(requirements = {"2.2.7.1/5.8/H-1-1"}) 364 @Test testEncoding()365 public void testEncoding() throws Exception { 366 Assume.assumeFalse("Skipping due to quick run mode", 367 mQuickCheck && !containsJson(mJsonName, QUICK_RUN_PARAMS)); 368 Assume.assumeFalse("Skipping avc encoder tests", 369 mSkipAvc && (containsJson(mJsonName, AVC_VBR_B0_PARAMS) || containsJson(mJsonName, 370 AVC_VBR_B3_PARAMS))); 371 Assume.assumeFalse("Skipping hevc encoder tests", 372 mSkipHevc && (containsJson(mJsonName, HEVC_VBR_B0_PARAMS) || containsJson(mJsonName, 373 HEVC_VBR_B3_PARAMS))); 374 Assume.assumeFalse("Skipping b-frame tests", 375 mSkipB && (containsJson(mJsonName, AVC_VBR_B3_PARAMS) || containsJson(mJsonName, 376 HEVC_VBR_B3_PARAMS))); 377 Assume.assumeFalse("Skipping non b-frame tests", 378 mSkipP && (containsJson(mJsonName, AVC_VBR_B0_PARAMS) || containsJson(mJsonName, 379 HEVC_VBR_B0_PARAMS))); 380 381 // set up test environment 382 sLock.lock(); 383 try { 384 if (!sIsTestSetUpDone) setupTestEnv(); 385 sCondition.signalAll(); 386 } finally { 387 sLock.unlock(); 388 } 389 390 // transcode input 391 runDeviceTests(DEVICE_SIDE_TEST_PACKAGE, DEVICE_SIDE_TEST_CLASS, "testTranscode"); 392 393 // copy the encoded output from the device to the host. 394 String outDir = "output_" + mJsonName.substring(0, mJsonName.indexOf('.')); 395 File outHostPath = new File(sHostWorkDir, outDir); 396 try { 397 if (!outHostPath.isDirectory()) { 398 Assert.assertTrue("Failed to create directory : " + outHostPath.getAbsolutePath(), 399 outHostPath.mkdirs()); 400 } 401 } catch (SecurityException e) { 402 LogUtil.CLog.e("Unable to establish output host directory : " + outHostPath.getPath()); 403 } 404 String outDevPath = getDevice().getMountPoint(IDevice.MNT_EXTERNAL_STORAGE) + "/veq/output/" 405 + outDir; 406 Assert.assertTrue("Failed to pull mp4 files from " + outDevPath 407 + " to " + outHostPath.getPath(), getDevice().pullDir(outDevPath, outHostPath)); 408 getDevice().deleteFile(outDevPath); 409 410 // Parse json file 411 String jsonPath = sHostWorkDir.getPath() + "/json/" + mJsonName; 412 String jsonString = 413 new String(Files.readAllBytes(Paths.get(jsonPath)), StandardCharsets.UTF_8); 414 JSONArray jsonArray = new JSONArray(jsonString); 415 JSONObject obj = jsonArray.getJSONObject(0); 416 String refFileName = obj.getString("RefFileName"); 417 int fps = obj.getInt("FrameRate"); 418 int frameCount = obj.getInt("FrameCount"); 419 int clipDuration = frameCount / fps; 420 421 // Compute Vmaf 422 try (FileWriter writer = new FileWriter(outHostPath.getPath() + "/" + "all_vmafs.txt")) { 423 JSONArray codecConfigs = obj.getJSONArray("CodecConfigs"); 424 int th = Runtime.getRuntime().availableProcessors() / 2; 425 th = Math.min(Math.max(1, th), 8); 426 String filter = 427 "[0:v]setpts=PTS-STARTPTS[reference];[1:v]setpts=PTS-STARTPTS[distorted];" 428 + "[distorted][reference]libvmaf=feature=name=psnr:model=version" 429 + "=vmaf_v0.6.1:n_threads=" + th; 430 for (int i = 0; i < codecConfigs.length(); i++) { 431 JSONObject codecConfig = codecConfigs.getJSONObject(i); 432 String outputName = codecConfig.getString("EncodedFileName"); 433 outputName = outputName.substring(0, outputName.lastIndexOf(".")); 434 String outputVmafPath = outDir + "/" + outputName + ".txt"; 435 String cmd = "./bin/ffmpeg"; 436 cmd += " -hide_banner"; 437 cmd += " -r " + fps; 438 cmd += " -i " + "samples/" + refFileName + " -an"; // reference video 439 cmd += " -r " + fps; 440 cmd += " -i " + outDir + "/" + outputName + ".mp4" + " -an"; // distorted video 441 cmd += " -filter_complex " + "\"" + filter + "\""; 442 cmd += " -f null -"; 443 cmd += " > " + outputVmafPath + " 2>&1"; 444 LogUtil.CLog.i("ffmpeg command : " + cmd); 445 int result = runCommand(cmd, sHostWorkDir); 446 if (sMpc >= MEDIA_PERFORMANCE_CLASS_14) { 447 Assert.assertEquals("Encountered error during vmaf computation.", 0, result); 448 } else { 449 Assume.assumeTrue("Encountered error during vmaf computation but the " 450 + "test device does not advertise performance class", result == 0); 451 } 452 String vmafLine = ""; 453 try (BufferedReader reader = new BufferedReader( 454 new FileReader(sHostWorkDir.getPath() + "/" + outputVmafPath))) { 455 String token = "VMAF score: "; 456 String line; 457 while ((line = reader.readLine()) != null) { 458 if (line.contains(token)) { 459 line = line.substring(line.indexOf(token)); 460 vmafLine = "VMAF score = " + line.substring(token.length()); 461 LogUtil.CLog.i(vmafLine); 462 break; 463 } 464 } 465 } catch (IOException e) { 466 throw new AssertionError("Unexpected IOException: " + e.getMessage()); 467 } 468 469 writer.write(vmafLine + "\n"); 470 writer.write("Y4M file = " + refFileName + "\n"); 471 writer.write("MP4 file = " + refFileName + "\n"); 472 File file = new File(outHostPath + "/" + outputName + ".mp4"); 473 Assert.assertTrue("output file from device missing", file.exists()); 474 long fileSize = file.length(); 475 writer.write("Filesize = " + fileSize + "\n"); 476 writer.write("FPS = " + fps + "\n"); 477 writer.write("FRAME_COUNT = " + frameCount + "\n"); 478 writer.write("CLIP_DURATION = " + clipDuration + "\n"); 479 long totalBits = fileSize * 8; 480 long totalBits_kbps = totalBits / 1000; 481 long bitrate_kbps = totalBits_kbps / clipDuration; 482 writer.write("Bitrate kbps = " + bitrate_kbps + "\n"); 483 } 484 } catch (IOException e) { 485 throw new AssertionError("Unexpected IOException: " + e.getMessage()); 486 } 487 488 // bd rate verification 489 String refJsonFilePath = sHostWorkDir.getPath() + "/json/" + mJsonName; 490 String testVmafFilePath = sHostWorkDir.getPath() + "/" + outDir + "/" + "all_vmafs.txt"; 491 String resultFilePath = sHostWorkDir.getPath() + "/" + outDir + "/result.txt"; 492 int result = verifyBdRate(refJsonFilePath, testVmafFilePath, resultFilePath); 493 if (sMpc >= MEDIA_PERFORMANCE_CLASS_14) { 494 Assert.assertEquals("bd rate validation failed.", 0, result); 495 } else { 496 Assume.assumeTrue("bd rate validation failed but the test device does not " 497 + "advertise performance class", result == 0); 498 } 499 LogUtil.CLog.i("Finished executing the process."); 500 } 501 runCommand(String command, File dir)502 private int runCommand(String command, File dir) throws IOException, InterruptedException { 503 Process p = new ProcessBuilder("/bin/sh", "-c", command) 504 .directory(dir) 505 .redirectErrorStream(true) 506 .redirectOutput(ProcessBuilder.Redirect.INHERIT) 507 .start(); 508 509 BufferedReader stdInput = new BufferedReader(new InputStreamReader(p.getInputStream())); 510 BufferedReader stdError = new BufferedReader(new InputStreamReader(p.getErrorStream())); 511 String line; 512 while ((line = stdInput.readLine()) != null || (line = stdError.readLine()) != null) { 513 LogUtil.CLog.i(line + "\n"); 514 } 515 return p.waitFor(); 516 } 517 518 // Download the indicated file (within the base_url folder) to our desired destination 519 // simple caching -- if file exists, we do not re-download downloadFile(String url, File destDir)520 private void downloadFile(String url, File destDir) { 521 String fileName = url.substring(RES_URL.lastIndexOf('/') + 1); 522 File destination = new File(destDir, fileName); 523 524 // save bandwidth, also allows a user to manually preload files 525 LogUtil.CLog.i("Do we already have a copy of file " + destination.getPath()); 526 if (destination.isFile()) { 527 LogUtil.CLog.i("Skipping re-download of file " + destination.getPath()); 528 return; 529 } 530 531 String cmd = "wget -O " + destination.getPath() + " " + url; 532 LogUtil.CLog.i("wget_cmd = " + cmd); 533 534 int result = 0; 535 try { 536 result = runCommand(cmd, destDir); 537 } catch (IOException e) { 538 result = -2; 539 } catch (InterruptedException e) { 540 result = -3; 541 } 542 Assert.assertEquals("download file failed.\n", 0, result); 543 } 544 runDeviceTests(String pkgName, @Nullable String testClassName, @Nullable String testMethodName)545 private void runDeviceTests(String pkgName, @Nullable String testClassName, 546 @Nullable String testMethodName) throws DeviceNotAvailableException { 547 RemoteAndroidTestRunner testRunner = getTestRunner(pkgName, testClassName, testMethodName); 548 CollectingTestListener listener = new CollectingTestListener(); 549 Assert.assertTrue(getDevice().runInstrumentationTests(testRunner, listener)); 550 assertTestsPassed(listener.getCurrentRunResults()); 551 } 552 getTestRunner(String pkgName, String testClassName, String testMethodName)553 private RemoteAndroidTestRunner getTestRunner(String pkgName, String testClassName, 554 String testMethodName) { 555 if (testClassName != null && testClassName.startsWith(".")) { 556 testClassName = pkgName + testClassName; 557 } 558 RemoteAndroidTestRunner testRunner = 559 new RemoteAndroidTestRunner(pkgName, RUNNER, getDevice().getIDevice()); 560 testRunner.setMaxTimeToOutputResponse(DEFAULT_SHELL_TIMEOUT_MILLIS, TimeUnit.MILLISECONDS); 561 testRunner.addInstrumentationArg(TEST_TIMEOUT_INST_ARGS_KEY, 562 Long.toString(DEFAULT_TEST_TIMEOUT_MILLIS)); 563 testRunner.addInstrumentationArg(TEST_CONFIG_INST_ARGS_KEY, mJsonName); 564 if (testClassName != null && testMethodName != null) { 565 testRunner.setMethodName(testClassName, testMethodName); 566 } else if (testClassName != null) { 567 testRunner.setClassName(testClassName); 568 } 569 return testRunner; 570 } 571 assertTestsPassed(TestRunResult testRunResult)572 private void assertTestsPassed(TestRunResult testRunResult) { 573 if (testRunResult.isRunFailure()) { 574 throw new AssertionError("Failed to successfully run device tests for " 575 + testRunResult.getName() + ": " + testRunResult.getRunFailureMessage()); 576 } 577 if (testRunResult.getNumTests() != testRunResult.getPassedTests().size()) { 578 for (Map.Entry<TestDescription, TestResult> resultEntry : 579 testRunResult.getTestResults().entrySet()) { 580 if (resultEntry.getValue().getStatus().equals(TestStatus.FAILURE)) { 581 StringBuilder errorBuilder = new StringBuilder("On-device tests failed:\n"); 582 errorBuilder.append(resultEntry.getKey().toString()); 583 errorBuilder.append(":\n"); 584 errorBuilder.append(resultEntry.getValue().getStackTrace()); 585 throw new AssertionError(errorBuilder.toString()); 586 } 587 if (resultEntry.getValue().getStatus().equals(TestStatus.ASSUMPTION_FAILURE)) { 588 StringBuilder errorBuilder = 589 new StringBuilder("On-device tests assumption failed:\n"); 590 errorBuilder.append(resultEntry.getKey().toString()); 591 errorBuilder.append(":\n"); 592 errorBuilder.append(resultEntry.getValue().getStackTrace()); 593 Assume.assumeTrue(errorBuilder.toString(), false); 594 } 595 } 596 } 597 } 598 } 599