1#!/usr/bin/env python3
2#
3#   Copyright 2019 - The Android Open Source Project
4#
5#   Licensed under the Apache License, Version 2.0 (the "License");
6#   you may not use this file except in compliance with the License.
7#   You may obtain a copy of the License at
8#
9#       http://www.apache.org/licenses/LICENSE-2.0
10#
11#   Unless required by applicable law or agreed to in writing, software
12#   distributed under the License is distributed on an "AS IS" BASIS,
13#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14#   See the License for the specific language governing permissions and
15#   limitations under the License.
16'''Python Module for GNSS test log utilities.'''
17
18import re as regex
19import datetime
20import functools as fts
21import numpy as npy
22import pandas as pds
23from acts import logger
24
25# GPS API Log Reading Config
26CONFIG_GPSAPILOG = {
27    'phone_time':
28    r'^(?P<date>\d+\/\d+\/\d+)\s+(?P<time>\d+:\d+:\d+)\s+'
29    r'Read:\s+(?P<logsize>\d+)\s+bytes',
30    'SpaceVehicle':
31    r'^Fix:\s+(?P<Fix>\w+)\s+Type:\s+(?P<Type>\w+)\s+'
32    r'SV:\s+(?P<SV>\d+)\s+C\/No:\s+(?P<CNo>\d+\.\d+)\s+'
33    r'Elevation:\s+(?P<Elevation>\d+\.\d+)\s+'
34    r'Azimuth:\s+(?P<Azimuth>\d+\.\d+)\s+'
35    r'Signal:\s+(?P<Signal>\w+)\s+'
36    r'Frequency:\s+(?P<Frequency>\d+\.\d+)\s+'
37    r'EPH:\s+(?P<EPH>\w+)\s+ALM:\s+(?P<ALM>\w+)',
38    'SpaceVehicle_wBB':
39    r'^Fix:\s+(?P<Fix>\w+)\s+Type:\s+(?P<Type>\w+)\s+'
40    r'SV:\s+(?P<SV>\d+)\s+C\/No:\s+(?P<AntCNo>\d+\.\d+),\s+'
41    r'(?P<BbCNo>\d+\.\d+)\s+'
42    r'Elevation:\s+(?P<Elevation>\d+\.\d+)\s+'
43    r'Azimuth:\s+(?P<Azimuth>\d+\.\d+)\s+'
44    r'Signal:\s+(?P<Signal>\w+)\s+'
45    r'Frequency:\s+(?P<Frequency>\d+\.\d+)\s+'
46    r'EPH:\s+(?P<EPH>\w+)\s+ALM:\s+(?P<ALM>\w+)',
47    'HistoryAvgTop4CNo':
48    r'^History\s+Avg\s+Top4\s+:\s+(?P<HistoryAvgTop4CNo>\d+\.\d+)',
49    'CurrentAvgTop4CNo':
50    r'^Current\s+Avg\s+Top4\s+:\s+(?P<CurrentAvgTop4CNo>\d+\.\d+)',
51    'HistoryAvgCNo':
52    r'^History\s+Avg\s+:\s+(?P<HistoryAvgCNo>\d+\.\d+)',
53    'CurrentAvgCNo':
54    r'^Current\s+Avg\s+:\s+(?P<CurrentAvgCNo>\d+\.\d+)',
55    'AntennaHistoryAvgTop4CNo':
56    r'^Antenna_History\s+Avg\s+Top4\s+:\s+(?P<AntennaHistoryAvgTop4CNo>\d+\.\d+)',
57    'AntennaCurrentAvgTop4CNo':
58    r'^Antenna_Current\s+Avg\s+Top4\s+:\s+(?P<AntennaCurrentAvgTop4CNo>\d+\.\d+)',
59    'AntennaHistoryAvgCNo':
60    r'^Antenna_History\s+Avg\s+:\s+(?P<AntennaHistoryAvgCNo>\d+\.\d+)',
61    'AntennaCurrentAvgCNo':
62    r'^Antenna_Current\s+Avg\s+:\s+(?P<AntennaCurrentAvgCNo>\d+\.\d+)',
63    'BasebandHistoryAvgTop4CNo':
64    r'^Baseband_History\s+Avg\s+Top4\s+:\s+(?P<BasebandHistoryAvgTop4CNo>\d+\.\d+)',
65    'BasebandCurrentAvgTop4CNo':
66    r'^Baseband_Current\s+Avg\s+Top4\s+:\s+(?P<BasebandCurrentAvgTop4CNo>\d+\.\d+)',
67    'BasebandHistoryAvgCNo':
68    r'^Baseband_History\s+Avg\s+:\s+(?P<BasebandHistoryAvgCNo>\d+\.\d+)',
69    'BasebandCurrentAvgCNo':
70    r'^Baseband_Current\s+Avg\s+:\s+(?P<BasebandCurrentAvgCNo>\d+\.\d+)',
71    'L5inFix':
72    r'^L5\s+used\s+in\s+fix:\s+(?P<L5inFix>\w+)',
73    'L5EngagingRate':
74    r'^L5\s+engaging\s+rate:\s+(?P<L5EngagingRate>\d+.\d+)%',
75    'Provider':
76    r'^Provider:\s+(?P<Provider>\w+)',
77    'Latitude':
78    r'^Latitude:\s+(?P<Latitude>-?\d+.\d+)',
79    'Longitude':
80    r'^Longitude:\s+(?P<Longitude>-?\d+.\d+)',
81    'Altitude':
82    r'^Altitude:\s+(?P<Altitude>-?\d+.\d+)',
83    'GNSSTime':
84    r'^Time:\s+(?P<Date>\d+\/\d+\/\d+)\s+'
85    r'(?P<Time>\d+:\d+:\d+)',
86    'Speed':
87    r'^Speed:\s+(?P<Speed>\d+.\d+)',
88    'Bearing':
89    r'^Bearing:\s+(?P<Bearing>\d+.\d+)',
90}
91
92# Space Vehicle Statistics Dataframe List
93# Handle the pre GPSTool 2.12.24 case
94LIST_SVSTAT = [
95    'HistoryAvgTop4CNo', 'CurrentAvgTop4CNo', 'HistoryAvgCNo', 'CurrentAvgCNo',
96    'L5inFix', 'L5EngagingRate'
97]
98# Handle the post GPSTool 2.12.24 case with baseband CNo
99LIST_SVSTAT_WBB = [
100    'AntennaHistoryAvgTop4CNo', 'AntennaCurrentAvgTop4CNo',
101    'AntennaHistoryAvgCNo', 'AntennaCurrentAvgCNo',
102    'BasebandHistoryAvgTop4CNo', 'BasebandCurrentAvgTop4CNo',
103    'BasebandHistoryAvgCNo', 'BasebandCurrentAvgCNo', 'L5inFix',
104    'L5EngagingRate'
105]
106
107# Location Fix Info Dataframe List
108LIST_LOCINFO = [
109    'Provider', 'Latitude', 'Longitude', 'Altitude', 'GNSSTime', 'Speed',
110    'Bearing'
111]
112
113# GPS TTFF Log Reading Config
114CONFIG_GPSTTFFLOG = {
115    'ttff_info':
116    r'Loop:(?P<loop>\d+)\s+'
117    r'(?P<start_datetime>\d+\/\d+\/\d+-\d+:\d+:\d+.\d+)\s+'
118    r'(?P<stop_datetime>\d+\/\d+\/\d+-\d+:\d+:\d+.\d+)\s+'
119    r'(?P<ttff>\d+.\d+)\s+'
120    r'\[Antenna_Avg Top4 : (?P<ant_avg_top4_cn0>\d+.\d+)\]\s'
121    r'\[Antenna_Avg : (?P<ant_avg_cn0>\d+.\d+)\]\s'
122    r'\[Baseband_Avg Top4 : (?P<bb_avg_top4_cn0>\d+.\d+)\]\s'
123    r'\[Baseband_Avg : (?P<bb_avg_cn0>\d+.\d+)\]\s+\[(?P<fix_type>\d+\w+ fix)\]\s+'
124    r'\[Satellites used for fix : (?P<satnum_for_fix>\d+)\]'
125}
126LOGPARSE_UTIL_LOGGER = logger.create_logger()
127
128
129def parse_log_to_df(filename, configs, index_rownum=True):
130    r"""Parse log to a dictionary of Pandas dataframes.
131
132    Args:
133      filename: log file name.
134        Type String.
135      configs: configs dictionary of parsed Pandas dataframes.
136        Type dictionary.
137        dict key, the parsed pattern name, such as 'Speed',
138        dict value, regex of the config pattern,
139          Type Raw String.
140      index_rownum: index row number from raw data.
141        Type Boolean.
142        Default, True.
143
144    Returns:
145      parsed_data: dictionary of parsed data.
146        Type dictionary.
147        dict key, the parsed pattern name, such as 'Speed',
148        dict value, the corresponding parsed dataframe.
149
150    Examples:
151      configs = {
152          'GNSSTime':
153          r'Time:\s+(?P<Date>\d+\/\d+\/\d+)\s+
154          r(?P<Time>\d+:\d+:\d+)')},
155          'Speed': r'Speed:\s+(?P<Speed>\d+.\d+)',
156      }
157    """
158    # Init a local config dictionary to hold compiled regex and match dict.
159    configs_local = {}
160    # Construct parsed data dictionary
161    parsed_data = {}
162
163    # Loop the config dictionary to compile regex and init data list
164    for key, regex_string in configs.items():
165        configs_local[key] = {
166            'cregex': regex.compile(regex_string),
167            'datalist': [],
168        }
169
170    # Open the file, loop and parse
171    with open(filename, 'r') as fid:
172
173        for idx_line, current_line in enumerate(fid):
174            for _, config in configs_local.items():
175                matched_log_object = config['cregex'].search(current_line)
176
177                if matched_log_object:
178                    matched_data = matched_log_object.groupdict()
179                    matched_data['rownumber'] = idx_line + 1
180                    config['datalist'].append(matched_data)
181
182    # Loop to generate parsed data from configs list
183    for key, config in configs_local.items():
184        parsed_data[key] = pds.DataFrame(config['datalist'])
185        if index_rownum and not parsed_data[key].empty:
186            parsed_data[key].set_index('rownumber', inplace=True)
187        elif parsed_data[key].empty:
188            LOGPARSE_UTIL_LOGGER.debug(
189                'The parsed dataframe of "%s" is empty.', key)
190
191    # Return parsed data list
192    return parsed_data
193
194
195def parse_gpstool_ttfflog_to_df(filename):
196    """Parse GPSTool ttff log to Pandas dataframes.
197
198    Args:
199      filename: full log file name.
200        Type, String.
201
202    Returns:
203      ttff_df: TTFF Data Frame.
204        Type, Pandas DataFrame.
205    """
206    # Get parsed dataframe list
207    parsed_data = parse_log_to_df(
208        filename=filename,
209        configs=CONFIG_GPSTTFFLOG,
210    )
211    ttff_df = parsed_data['ttff_info']
212    if not ttff_df.empty:
213        # Data Conversion
214        ttff_df['loop'] = ttff_df['loop'].astype(int)
215        ttff_df['start_datetime'] = pds.to_datetime(ttff_df['start_datetime'])
216        ttff_df['stop_datetime'] = pds.to_datetime(ttff_df['stop_datetime'])
217        ttff_df['ttff_time'] = ttff_df['ttff'].astype(float)
218        ttff_df['ant_avg_top4_cn0'] = ttff_df['ant_avg_top4_cn0'].astype(float)
219        ttff_df['ant_avg_cn0'] = ttff_df['ant_avg_cn0'].astype(float)
220        ttff_df['bb_avg_top4_cn0'] = ttff_df['bb_avg_top4_cn0'].astype(float)
221        ttff_df['bb_avg_cn0'] = ttff_df['bb_avg_cn0'].astype(float)
222        ttff_df['satnum_for_fix'] = ttff_df['satnum_for_fix'].astype(int)
223
224    # return ttff dataframe
225    return ttff_df
226
227
228def parse_gpsapilog_to_df(filename):
229    """Parse GPS API log to Pandas dataframes.
230
231    Args:
232      filename: full log file name.
233        Type, String.
234
235    Returns:
236      timestamp_df: Timestamp Data Frame.
237        Type, Pandas DataFrame.
238      sv_info_df: GNSS SV info Data Frame.
239        Type, Pandas DataFrame.
240      sv_stat_df: GNSS SV statistic Data Frame.
241        Type, Pandas DataFrame
242      loc_info_df: Location Information Data Frame.
243        Type, Pandas DataFrame.
244        include Provider, Latitude, Longitude, Altitude, GNSSTime, Speed, Bearing
245    """
246    def get_phone_time(target_df_row, timestamp_df):
247        """subfunction to get the phone_time."""
248
249        try:
250            row_num = timestamp_df[
251                timestamp_df.index < target_df_row.name].iloc[-1].name
252            phone_time = timestamp_df.loc[row_num]['phone_time']
253        except IndexError:
254            row_num = npy.NaN
255            phone_time = npy.NaN
256
257        return phone_time, row_num
258
259    # Get parsed dataframe list
260    parsed_data = parse_log_to_df(
261        filename=filename,
262        configs=CONFIG_GPSAPILOG,
263    )
264
265    # get DUT Timestamp
266    timestamp_df = parsed_data['phone_time']
267    timestamp_df['phone_time'] = timestamp_df.apply(
268        lambda row: datetime.datetime.strptime(row.date + '-' + row.time,
269                                               '%Y/%m/%d-%H:%M:%S'),
270        axis=1)
271
272    # Add phone_time from timestamp_df dataframe by row number
273    for key in parsed_data:
274        if key != 'phone_time':
275            current_df = parsed_data[key]
276            time_n_row_num = current_df.apply(get_phone_time,
277                                              axis=1,
278                                              timestamp_df=timestamp_df)
279            current_df[['phone_time', 'time_row_num'
280                        ]] = pds.DataFrame(time_n_row_num.apply(pds.Series))
281
282    # Get space vehicle info dataframe
283    sv_info_df = parsed_data['SpaceVehicle']
284
285    # Get space vehicle statistics dataframe
286    # First merge all dataframe from LIST_SVSTAT[1:],
287    # Drop duplicated 'phone_time', based on time_row_num
288    sv_stat_df = fts.reduce(
289        lambda item1, item2: pds.merge(item1, item2, on='time_row_num'), [
290            parsed_data[key].drop(['phone_time'], axis=1)
291            for key in LIST_SVSTAT[1:]
292        ])
293    # Then merge with LIST_SVSTAT[0]
294    sv_stat_df = pds.merge(sv_stat_df,
295                           parsed_data[LIST_SVSTAT[0]],
296                           on='time_row_num')
297
298    # Get location fix information dataframe
299    # First merge all dataframe from LIST_LOCINFO[1:],
300    # Drop duplicated 'phone_time', based on time_row_num
301    loc_info_df = fts.reduce(
302        lambda item1, item2: pds.merge(item1, item2, on='time_row_num'), [
303            parsed_data[key].drop(['phone_time'], axis=1)
304            for key in LIST_LOCINFO[1:]
305        ])
306    # Then merge with LIST_LOCINFO[8]
307    loc_info_df = pds.merge(loc_info_df,
308                            parsed_data[LIST_LOCINFO[0]],
309                            on='time_row_num')
310    # Convert GNSS Time
311    loc_info_df['gnsstime'] = loc_info_df.apply(
312        lambda row: datetime.datetime.strptime(row.Date + '-' + row.Time,
313                                               '%Y/%m/%d-%H:%M:%S'),
314        axis=1)
315
316    return timestamp_df, sv_info_df, sv_stat_df, loc_info_df
317
318
319def parse_gpsapilog_to_df_v2(filename):
320    """Parse GPS API log to Pandas dataframes, by using merge_asof.
321
322    Args:
323      filename: full log file name.
324        Type, String.
325
326    Returns:
327      timestamp_df: Timestamp Data Frame.
328        Type, Pandas DataFrame.
329      sv_info_df: GNSS SV info Data Frame.
330        Type, Pandas DataFrame.
331      sv_stat_df: GNSS SV statistic Data Frame.
332        Type, Pandas DataFrame
333      loc_info_df: Location Information Data Frame.
334        Type, Pandas DataFrame.
335        include Provider, Latitude, Longitude, Altitude, GNSSTime, Speed, Bearing
336    """
337    # Get parsed dataframe list
338    parsed_data = parse_log_to_df(
339        filename=filename,
340        configs=CONFIG_GPSAPILOG,
341    )
342
343    # get DUT Timestamp
344    timestamp_df = parsed_data['phone_time']
345    timestamp_df['phone_time'] = timestamp_df.apply(
346        lambda row: datetime.datetime.strptime(row.date + '-' + row.time,
347                                               '%Y/%m/%d-%H:%M:%S'),
348        axis=1)
349    # drop logsize, date, time
350    parsed_data['phone_time'] = timestamp_df.drop(['logsize', 'date', 'time'],
351                                                  axis=1)
352
353    # Add phone_time from timestamp dataframe by row number
354    for key in parsed_data:
355        if (key != 'phone_time') and (not parsed_data[key].empty):
356            parsed_data[key] = pds.merge_asof(parsed_data[key],
357                                              parsed_data['phone_time'],
358                                              left_index=True,
359                                              right_index=True)
360
361    # Get space vehicle info dataframe
362    # Handle the pre GPSTool 2.12.24 case
363    if not parsed_data['SpaceVehicle'].empty:
364        sv_info_df = parsed_data['SpaceVehicle']
365
366    # Handle the post GPSTool 2.12.24 case with baseband CNo
367    elif not parsed_data['SpaceVehicle_wBB'].empty:
368        sv_info_df = parsed_data['SpaceVehicle_wBB']
369
370    # Get space vehicle statistics dataframe
371    # Handle the pre GPSTool 2.12.24 case
372    if not parsed_data['HistoryAvgTop4CNo'].empty:
373        # First merge all dataframe from LIST_SVSTAT[1:],
374        sv_stat_df = fts.reduce(
375            lambda item1, item2: pds.merge(item1, item2, on='phone_time'),
376            [parsed_data[key] for key in LIST_SVSTAT[1:]])
377        # Then merge with LIST_SVSTAT[0]
378        sv_stat_df = pds.merge(sv_stat_df,
379                               parsed_data[LIST_SVSTAT[0]],
380                               on='phone_time')
381
382    # Handle the post GPSTool 2.12.24 case with baseband CNo
383    elif not parsed_data['AntennaHistoryAvgTop4CNo'].empty:
384        # First merge all dataframe from LIST_SVSTAT[1:],
385        sv_stat_df = fts.reduce(
386            lambda item1, item2: pds.merge(item1, item2, on='phone_time'),
387            [parsed_data[key] for key in LIST_SVSTAT_WBB[1:]])
388        # Then merge with LIST_SVSTAT[0]
389        sv_stat_df = pds.merge(sv_stat_df,
390                               parsed_data[LIST_SVSTAT_WBB[0]],
391                               on='phone_time')
392
393    # Get location fix information dataframe
394    # First merge all dataframe from LIST_LOCINFO[1:],
395    loc_info_df = fts.reduce(
396        lambda item1, item2: pds.merge(item1, item2, on='phone_time'),
397        [parsed_data[key] for key in LIST_LOCINFO[1:]])
398    # Then merge with LIST_LOCINFO[8]
399    loc_info_df = pds.merge(loc_info_df,
400                            parsed_data[LIST_LOCINFO[0]],
401                            on='phone_time')
402    # Convert GNSS Time
403    loc_info_df['gnsstime'] = loc_info_df.apply(
404        lambda row: datetime.datetime.strptime(row.Date + '-' + row.Time,
405                                               '%Y/%m/%d-%H:%M:%S'),
406        axis=1)
407
408    # Data Conversion
409    timestamp_df['logsize'] = timestamp_df['logsize'].astype(int)
410
411    sv_info_df['SV'] = sv_info_df['SV'].astype(int)
412    sv_info_df['Elevation'] = sv_info_df['Elevation'].astype(float)
413    sv_info_df['Azimuth'] = sv_info_df['Azimuth'].astype(float)
414    sv_info_df['Frequency'] = sv_info_df['Frequency'].astype(float)
415
416    if 'CNo' in list(sv_info_df.columns):
417        sv_info_df['CNo'] = sv_info_df['CNo'].astype(float)
418        sv_info_df['AntCNo'] = sv_info_df['CNo']
419    elif 'AntCNo' in list(sv_info_df.columns):
420        sv_info_df['AntCNo'] = sv_info_df['AntCNo'].astype(float)
421        sv_info_df['BbCNo'] = sv_info_df['BbCNo'].astype(float)
422
423    if 'CurrentAvgTop4CNo' in list(sv_stat_df.columns):
424        sv_stat_df['CurrentAvgTop4CNo'] = sv_stat_df[
425            'CurrentAvgTop4CNo'].astype(float)
426        sv_stat_df['CurrentAvgCNo'] = sv_stat_df['CurrentAvgCNo'].astype(float)
427        sv_stat_df['HistoryAvgTop4CNo'] = sv_stat_df[
428            'HistoryAvgTop4CNo'].astype(float)
429        sv_stat_df['HistoryAvgCNo'] = sv_stat_df['HistoryAvgCNo'].astype(float)
430        sv_stat_df['AntennaCurrentAvgTop4CNo'] = sv_stat_df[
431            'CurrentAvgTop4CNo']
432        sv_stat_df['AntennaCurrentAvgCNo'] = sv_stat_df['CurrentAvgCNo']
433        sv_stat_df['AntennaHistoryAvgTop4CNo'] = sv_stat_df[
434            'HistoryAvgTop4CNo']
435        sv_stat_df['AntennaHistoryAvgCNo'] = sv_stat_df['HistoryAvgCNo']
436        sv_stat_df['BasebandCurrentAvgTop4CNo'] = npy.nan
437        sv_stat_df['BasebandCurrentAvgCNo'] = npy.nan
438        sv_stat_df['BasebandHistoryAvgTop4CNo'] = npy.nan
439        sv_stat_df['BasebandHistoryAvgCNo'] = npy.nan
440
441    elif 'AntennaCurrentAvgTop4CNo' in list(sv_stat_df.columns):
442        sv_stat_df['AntennaCurrentAvgTop4CNo'] = sv_stat_df[
443            'AntennaCurrentAvgTop4CNo'].astype(float)
444        sv_stat_df['AntennaCurrentAvgCNo'] = sv_stat_df[
445            'AntennaCurrentAvgCNo'].astype(float)
446        sv_stat_df['AntennaHistoryAvgTop4CNo'] = sv_stat_df[
447            'AntennaHistoryAvgTop4CNo'].astype(float)
448        sv_stat_df['AntennaHistoryAvgCNo'] = sv_stat_df[
449            'AntennaHistoryAvgCNo'].astype(float)
450        sv_stat_df['BasebandCurrentAvgTop4CNo'] = sv_stat_df[
451            'BasebandCurrentAvgTop4CNo'].astype(float)
452        sv_stat_df['BasebandCurrentAvgCNo'] = sv_stat_df[
453            'BasebandCurrentAvgCNo'].astype(float)
454        sv_stat_df['BasebandHistoryAvgTop4CNo'] = sv_stat_df[
455            'BasebandHistoryAvgTop4CNo'].astype(float)
456        sv_stat_df['BasebandHistoryAvgCNo'] = sv_stat_df[
457            'BasebandHistoryAvgCNo'].astype(float)
458
459    sv_stat_df['L5EngagingRate'] = sv_stat_df['L5EngagingRate'].astype(float)
460
461    loc_info_df['Latitude'] = loc_info_df['Latitude'].astype(float)
462    loc_info_df['Longitude'] = loc_info_df['Longitude'].astype(float)
463    loc_info_df['Altitude'] = loc_info_df['Altitude'].astype(float)
464    loc_info_df['Speed'] = loc_info_df['Speed'].astype(float)
465    loc_info_df['Bearing'] = loc_info_df['Bearing'].astype(float)
466
467    return timestamp_df, sv_info_df, sv_stat_df, loc_info_df
468