1d1935f69SMatthias Ringwald#!/usr/bin/env python 2d1935f69SMatthias Ringwald# 3d1935f69SMatthias Ringwald# Scrape SDP UUIDs from Bluetooth SIG page 4d1935f69SMatthias Ringwald# Copyright 2017 BlueKitchen GmbH 5d1935f69SMatthias Ringwald# 6d1935f69SMatthias Ringwald 7d1935f69SMatthias Ringwaldfrom lxml import html 8d1935f69SMatthias Ringwaldimport datetime 9d1935f69SMatthias Ringwaldimport requests 10d1935f69SMatthias Ringwaldimport sys 11d1935f69SMatthias Ringwaldimport os 12d1935f69SMatthias Ringwaldimport codecs 13d1935f69SMatthias Ringwaldimport re 14d1935f69SMatthias Ringwald 15cccaab80SMatthias Ringwaldheaders = {'user-agent': 'curl/7.63.0'} 16cccaab80SMatthias Ringwald 17d1935f69SMatthias Ringwaldprogram_info = ''' 18d1935f69SMatthias RingwaldBTstack SDP UUID Scraper for BTstack 19d1935f69SMatthias RingwaldCopyright 2017, BlueKitchen GmbH 20d1935f69SMatthias Ringwald''' 21d1935f69SMatthias Ringwald 22b816bb66SMatthias Ringwaldheader = '''/** 23b816bb66SMatthias Ringwald * bluetooth_sdp.h generated from Bluetooth SIG website for BTstack by tool/bluetooth_sdp.py 24b816bb66SMatthias Ringwald * {page} 25b436f177SMatthias Ringwald * {datetime} 26d1935f69SMatthias Ringwald */ 27d1935f69SMatthias Ringwald 28*80e33422SMatthias Ringwald#ifndef BLUETOOTH_SDP_H 29*80e33422SMatthias Ringwald#define BLUETOOTH_SDP_H 30d1935f69SMatthias Ringwald 31d1935f69SMatthias Ringwald''' 32d1935f69SMatthias Ringwald 33d1935f69SMatthias Ringwaldtrailer = ''' 34d1935f69SMatthias Ringwald#endif 35d1935f69SMatthias Ringwald''' 36d1935f69SMatthias Ringwald 37b816bb66SMatthias Ringwalddefines = [] 38b816bb66SMatthias Ringwald 39d1935f69SMatthias Ringwald# Convert CamelCase to snake_case from http://stackoverflow.com/a/1176023 40d1935f69SMatthias Ringwalddef camel_to_underscore(name): 41d1935f69SMatthias Ringwald s1 = re.sub('(.)([A-Z][a-z]+)', r'\1_\2', name) 42d1935f69SMatthias Ringwald return re.sub('([a-z0-9])([A-Z])', r'\1_\2', s1).upper() 43d1935f69SMatthias Ringwald 44d1935f69SMatthias Ringwalddef create_pretty_define(name): 45b816bb66SMatthias Ringwald name = name.lstrip() 46b816bb66SMatthias Ringwald to_delete = [ '(FTP v1.2 and later)', '(Deprecated)', '(FTP v1.2 and later)', '(GOEP v2.0 and later)', 47b816bb66SMatthias Ringwald '(BIP v1.1 and later)', '(MAP v1.2 and later)', '(OPP v1.2 and later)', '(Not used in PAN v1.0)', '(PBAP v1.2 and later)'] 48b816bb66SMatthias Ringwald for item in to_delete: 49b816bb66SMatthias Ringwald name = name.replace(item, '') 50b816bb66SMatthias Ringwald name = name.rstrip() 51d1935f69SMatthias Ringwald name = name.replace(' - ', '_') 52d1935f69SMatthias Ringwald name = name.replace(' ', '_') 53d1935f69SMatthias Ringwald name = name.replace('/','') 54d1935f69SMatthias Ringwald name = name.replace('(','_') 55d1935f69SMatthias Ringwald name = name.replace(')','') 56d1935f69SMatthias Ringwald name = name.replace('-','_') 57cccaab80SMatthias Ringwald name = name.replace('.','_') 58d1935f69SMatthias Ringwald name = name.replace('PnP', 'PNP') 59b816bb66SMatthias Ringwald name = name.replace('IPv', 'IPV') 60d38c4adfSMatthias Ringwald name = name.replace('ServiceDiscoveryServerServiceClassID', 'ServiceDiscoveryServer') 61d38c4adfSMatthias Ringwald name = name.replace('BrowseGroupDescriptorServiceClassID', 'BrowseGroupDescriptor') 62b816bb66SMatthias Ringwald name = name.replace('&','and') 63cccaab80SMatthias Ringwald name = name.replace('__','_') 64d1935f69SMatthias Ringwald return camel_to_underscore(name).replace('__','_').replace('3_D','3D').replace('L2_CAP','L2CAP') 65d1935f69SMatthias Ringwald 66b816bb66SMatthias Ringwalddef remove_newlines(remark): 67d1935f69SMatthias Ringwald return " ".join(remark.split()) 68d1935f69SMatthias Ringwald 69cccaab80SMatthias Ringwalddef process_rows(fout, rows, pattern): 70d1935f69SMatthias Ringwald for row in rows: 71d1935f69SMatthias Ringwald columns = row.getchildren() 72d1935f69SMatthias Ringwald name = columns[0].text_content().encode('ascii','ignore') 73d1935f69SMatthias Ringwald value = columns[1].text_content().encode('ascii','ignore') 74b816bb66SMatthias Ringwald remark = '' 75b816bb66SMatthias Ringwald if (len(columns) > 2): 76d1935f69SMatthias Ringwald remark = columns[2].text_content().encode('ascii','ignore') 773860edc8SMatthias Ringwald # skip tbody headers 7846b21291SMatthias Ringwald if name in ["Protocol Name", "Service Class Name", "Attribute Name", "UUID Name", 79b816bb66SMatthias Ringwald "Reserved", 'Reserved for HID Attributes', 'Available for HID Language Strings']: 80d1935f69SMatthias Ringwald continue 813860edc8SMatthias Ringwald # skip tbody footers 82d1935f69SMatthias Ringwald if value.startswith('(Max value '): 83d1935f69SMatthias Ringwald continue 84d1935f69SMatthias Ringwald name = create_pretty_define(name) 85b816bb66SMatthias Ringwald # skip duplicate attributes 86b816bb66SMatthias Ringwald if name in defines: 87b816bb66SMatthias Ringwald continue 88b816bb66SMatthias Ringwald value = remove_newlines(value) 89b816bb66SMatthias Ringwald remark = remove_newlines(remark) 90d1935f69SMatthias Ringwald fout.write(pattern % (name, value, remark)) 91b816bb66SMatthias Ringwald defines.append(name) 92b816bb66SMatthias Ringwald 93d1935f69SMatthias Ringwalddef scrape_page(fout, url): 94d1935f69SMatthias Ringwald print("Parsing %s" % url) 95b436f177SMatthias Ringwald fout.write(header.format(page=url,datetime=str(datetime.datetime.now()))) 96d1935f69SMatthias Ringwald 97d1935f69SMatthias Ringwald # get from web 98cccaab80SMatthias Ringwald r = requests.get(url, headers=headers) 99b816bb66SMatthias Ringwald content = r.text 1003860edc8SMatthias Ringwald 101d1935f69SMatthias Ringwald # test: fetch from local file 'service-discovery.html' 102b816bb66SMatthias Ringwald # f = codecs.open("service-discovery.html", "r", "utf-8") 103b816bb66SMatthias Ringwald # content = f.read(); 104d1935f69SMatthias Ringwald 105d1935f69SMatthias Ringwald tree = html.fromstring(content) 106d1935f69SMatthias Ringwald 107cccaab80SMatthias Ringwald # Protocol Identifiers 108b816bb66SMatthias Ringwald fout.write('/**\n') 109b816bb66SMatthias Ringwald fout.write(' * Protocol Identifiers\n') 110b816bb66SMatthias Ringwald fout.write(' */\n') 111cccaab80SMatthias Ringwald rows = tree.xpath("//table[2]/tbody/tr") 112cccaab80SMatthias Ringwald process_rows(fout, rows, '#define BLUETOOTH_PROTOCOL_%-55s %s // %s\n') 113b816bb66SMatthias Ringwald fout.write('\n') 114d1935f69SMatthias Ringwald 115cccaab80SMatthias Ringwald # Service Classes 116b816bb66SMatthias Ringwald fout.write('/**\n') 117b816bb66SMatthias Ringwald fout.write(' * Service Classes\n') 118b816bb66SMatthias Ringwald fout.write(' */\n') 119cccaab80SMatthias Ringwald rows = tree.xpath("//table[3]/tr") 120cccaab80SMatthias Ringwald process_rows(fout, rows, '#define BLUETOOTH_SERVICE_CLASS_%-50s %s // %s\n') 121b816bb66SMatthias Ringwald fout.write('\n') 122b816bb66SMatthias Ringwald 123b816bb66SMatthias Ringwald # Attributes 124b816bb66SMatthias Ringwald fout.write('/**\n') 125b816bb66SMatthias Ringwald fout.write(' * Attributes\n') 126b816bb66SMatthias Ringwald fout.write(' */\n') 127b816bb66SMatthias Ringwald table_names = [ 128b816bb66SMatthias Ringwald # 'Base Universally Unique Identifier (UUID)', 12946b21291SMatthias Ringwald 'Browse Group Identifiers', 130b816bb66SMatthias Ringwald 'Attribute Identifiers', 131b816bb66SMatthias Ringwald # 'Audio/Video Remote Control Profile (AVRCP)', 132b816bb66SMatthias Ringwald 'Basic Imaging Profile (BIP)', 133b816bb66SMatthias Ringwald 'Basic Printing Profile (BPP)', 134b816bb66SMatthias Ringwald 'Bluetooth Core Specification: Universal Attributes', 135b816bb66SMatthias Ringwald 'Bluetooth Core Specification: Service Discovery Service', 136b816bb66SMatthias Ringwald # 'Bluetooth Core Specification: Browse Group Descriptor Service', 137b816bb66SMatthias Ringwald # 'Cordless Telephony Profile [DEPRECATED]', 138b816bb66SMatthias Ringwald 'Device Identification Profile', 139b816bb66SMatthias Ringwald # 'Fax Profile [DEPRECATED]', 140b816bb66SMatthias Ringwald 'File Transfer Profile', 141b816bb66SMatthias Ringwald 'Generic Object Exchange Profile', 142b816bb66SMatthias Ringwald # 'Global Navigation Satellite System Profile (GNSS)', -- note: SupportedFeatures, but different UUID 143b816bb66SMatthias Ringwald 'Hands-Free Profile', 144b816bb66SMatthias Ringwald 'Hardcopy Replacement Profile ', 145b816bb66SMatthias Ringwald 'Headset Profile', 146b816bb66SMatthias Ringwald 'Health Device Profile', 147b816bb66SMatthias Ringwald 'Human Interface Device Profile', 148b816bb66SMatthias Ringwald # 'Interoperability Requirements for Bluetooth technology as a WAP Bearer [DEPRECATED]', 149b816bb66SMatthias Ringwald 'Message Access Profile', 150b816bb66SMatthias Ringwald 'Object Push Profile', 151b816bb66SMatthias Ringwald 'Personal Area Networking Profile', 152b816bb66SMatthias Ringwald 'Phone Book Access Profile', 153b816bb66SMatthias Ringwald 'Synchronization Profile', 154b816bb66SMatthias Ringwald # 'Attribute ID Offsets for Strings', 155b816bb66SMatthias Ringwald # 'Protocol Parameters', 156b816bb66SMatthias Ringwald 'Multi-Profile', 157b816bb66SMatthias Ringwald 'Calendar Tasks and Notes', 158b816bb66SMatthias Ringwald ] 159b816bb66SMatthias Ringwald for table_name in table_names: 160cccaab80SMatthias Ringwald rows = tree.xpath("//table[preceding-sibling::h3 = '" + table_name +"']/tr") 161cccaab80SMatthias Ringwald process_rows(fout, rows, '#define BLUETOOTH_ATTRIBUTE_%-54s %s // %s\n') 162cccaab80SMatthias Ringwald # scrape_attributes(fout, tree, table_name) 163b816bb66SMatthias Ringwald # see above 164b816bb66SMatthias Ringwald fout.write('#define BLUETOOTH_ATTRIBUTE_GNSS_SUPPORTED_FEATURES 0x0200\n'); 165b816bb66SMatthias Ringwald 166b816bb66SMatthias Ringwald 167d1935f69SMatthias Ringwald 168d1935f69SMatthias Ringwaldbtstack_root = os.path.abspath(os.path.dirname(sys.argv[0]) + '/..') 169d1935f69SMatthias Ringwaldgen_path = btstack_root + '/src/bluetooth_sdp.h' 170d1935f69SMatthias Ringwald 171d1935f69SMatthias Ringwaldprint(program_info) 172d1935f69SMatthias Ringwald 173d1935f69SMatthias Ringwaldwith open(gen_path, 'wt') as fout: 174d1935f69SMatthias Ringwald scrape_page(fout, 'https://www.bluetooth.com/specifications/assigned-numbers/service-discovery') 175d1935f69SMatthias Ringwald fout.write(trailer) 176d1935f69SMatthias Ringwald 177d1935f69SMatthias Ringwaldprint('Scraping successful!\n') 178