xref: /aosp_15_r20/external/noto-fonts/emoji-compat/createfont.py (revision e5825d3be9fd13b272e7df556d285d1f07f3b027)

#!/usr/bin/env python3
#
# Copyright (C) 2017 The Android Open Source Project
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""
Creates the EmojiCompat font with the metadata. Metadata is embedded in FlatBuffers binary format
under a meta tag with name 'Emji'.

In order to create the final font the followings are used as inputs:

- NotoColorEmoji.ttf: Emoji font in the Android framework. Currently at
external/noto-fonts/emoji/NotoColorEmoji.ttf

- Unicode files: Unicode files that are in the framework, and lists information about all the
emojis. These files are emoji-data.txt, emoji-sequences.txt, emoji-zwj-sequences.txt,
and emoji-variation-sequences.txt. Currently at external/unicode/.

- additions/emoji-zwj-sequences.txt: Includes emojis that are not defined in Unicode files, but are
in the Android font. Resides in framework and currently under external/unicode/.

- data/emoji_metadata.txt: The file that includes the id, codepoints, the first
Android OS version that the emoji was added (sdkAdded), and finally the first EmojiCompat font
version that the emoji was added (compatAdded). Updated when the script is executed.

- data/emoji_metadata.fbs: The flatbuffer schema file. See http://google.github.io/flatbuffers/.

After execution the following files are generated if they don't exist otherwise, they are updated:
- font/NotoColorEmojiCompat.ttf
- supported-emojis/emojis.txt
- data/emoji_metadata.txt
- src/java/android/support/text/emoji/flatbuffer/*
"""

import contextlib
import csv
import hashlib
import itertools
import json
import os
import re
import shutil
import subprocess
import sys
import tempfile
from fontTools import ttLib
from fontTools.ttLib.tables import otTables
from nototools import font_data

########### UPDATE OR CHECK WHEN A NEW FONT IS BEING GENERATED ###########
# Last Android SDK Version
SDK_VERSION = 31
# metadata version that will be embedded into font. If there are updates to the font that would
# cause data/emoji_metadata.txt to change, this integer number should be incremented. This number
# defines in which EmojiCompat metadata version the emoji is added to the font.
METADATA_VERSION = 8

####### main directories where output files are created #######
SCRIPT_DIR = os.path.abspath(os.path.dirname(__file__))
FONT_DIR = os.path.join(SCRIPT_DIR, 'font')
DATA_DIR = os.path.join(SCRIPT_DIR, 'data')
SUPPORTED_EMOJIS_DIR = os.path.join(SCRIPT_DIR, 'supported-emojis')
JAVA_SRC_DIR = os.path.join('src', 'java')
####### output files #######
# font file
FONT_PATH = os.path.join(FONT_DIR, 'NotoColorEmojiCompat.ttf')
# emoji metadata json output file
OUTPUT_META_FILE = os.path.join(DATA_DIR, 'emoji_metadata.txt')
# emojis test file
TEST_DATA_PATH = os.path.join(SUPPORTED_EMOJIS_DIR, 'emojis.txt')
####### input files #######
# Unicode file names to read emoji data
EMOJI_DATA_FILE = 'emoji-data.txt'
EMOJI_SEQ_FILE = 'emoji-sequences.txt'
EMOJI_ZWJ_FILE = 'emoji-zwj-sequences.txt'
EMOJI_VARIATION_SEQ_FILE = 'emoji-variation-sequences.txt'
# Android OS emoji file for emojis that are not in Unicode files
ANDROID_EMOJI_ZWJ_SEQ_FILE = os.path.join('additions', 'emoji-zwj-sequences.txt')
ANDROID_EMOJIS_SEQ_FILE = os.path.join('additions', 'emoji-sequences.txt')
# Android OS emoji style override file. Codepoints that are rendered with emoji style by default
# even though not defined so in <code>emoji-data.txt</code>.
EMOJI_STYLE_OVERRIDE_FILE = os.path.join('additions', 'emoji-data.txt')
# emoji metadata file
INPUT_META_FILE = OUTPUT_META_FILE
# default flatbuffer module location (if not specified by caller)
FLATBUFFER_MODULE_DIR = os.path.join(SCRIPT_DIR, '..', 'emoji-compat-flatbuffers')
# flatbuffer schema
FLATBUFFER_SCHEMA = os.path.join(FLATBUFFER_MODULE_DIR, 'data', 'emoji_metadata.fbs')
# file path for java header, it will be prepended to flatbuffer java files
FLATBUFFER_HEADER = os.path.join(FLATBUFFER_MODULE_DIR, 'data', 'flatbuffer_header.txt')
# temporary emoji metadata json output file
OUTPUT_JSON_FILE_NAME = 'emoji_metadata.json'
# temporary binary file generated by flatbuffer
FLATBUFFER_BIN = 'emoji_metadata.bin'
# directory representation for flatbuffer java package
FLATBUFFER_PACKAGE_PATH = os.path.join('androidx', 'text', 'emoji', 'flatbuffer', '')
# temporary directory that contains flatbuffer java files
FLATBUFFER_JAVA_PATH = os.path.join(FLATBUFFER_PACKAGE_PATH)
FLATBUFFER_METADATA_LIST_JAVA = "MetadataList.java"
FLATBUFFER_METADATA_ITEM_JAVA = "MetadataItem.java"
# directory under source where flatbuffer java files will be copied into
FLATBUFFER_JAVA_TARGET = os.path.join(FLATBUFFER_MODULE_DIR, JAVA_SRC_DIR, FLATBUFFER_PACKAGE_PATH)
# meta tag name used in the font to embed the emoji metadata. This value is also used in
# MetadataListReader.java in order to locate the metadata location.
EMOJI_META_TAG_NAME = 'Emji'

EMOJI_STR = 'EMOJI'
EMOJI_PRESENTATION_STR = 'EMOJI_PRESENTATION'
ACCEPTED_EMOJI_PROPERTIES = [EMOJI_PRESENTATION_STR, EMOJI_STR]
STD_VARIANTS_EMOJI_STYLE = 'EMOJI STYLE'

DEFAULT_EMOJI_ID = 0xF0001
EMOJI_STYLE_VS = 0xFE0F

# The reference code point to be used for filling metrics of wartermark glyph
WATERMARK_REF_CODE_POINT = 0x1F600
# The code point and glyph name used for watermark.
WATERMARK_NEW_CODE_POINT = 0x10FF00
WATERMARK_NEW_GLYPH_ID = 'u10FF00'

def to_hex_str(value):
    """Converts given int value to hex without the 0x prefix"""
    return format(value, 'X')

def hex_str_to_int(string):
    """Convert a hex string into int"""
    return int(string, 16)

def codepoint_to_string(codepoints):
    """Converts a list of codepoints into a string separated with space."""
    return ' '.join([to_hex_str(x) for x in codepoints])

def prepend_header_to_file(file_path, header_path):
    """Prepends the header to the file. Used to update flatbuffer java files with header, comments
    and annotations."""
    with open(file_path, "r+") as original_file:
        with open(header_path, "r") as copyright_file:
            original_content = original_file.read()
            original_file.seek(0)
            original_file.write(copyright_file.read() + "\n" + original_content)

def is_ri(codepoint):
  return 0x1F1E6 <= codepoint and codepoint <= 0x1F1FF

def is_flag_seq(codepoints):
  return all(is_ri(x) for x in codepoints)


def update_flatbuffer_java_files(flatbuffer_java_dir, header_dir, target_dir):
    """Prepends headers to flatbuffer java files and copies to the final destination"""
    tmp_metadata_list = flatbuffer_java_dir + FLATBUFFER_METADATA_LIST_JAVA
    tmp_metadata_item = flatbuffer_java_dir + FLATBUFFER_METADATA_ITEM_JAVA
    prepend_header_to_file(tmp_metadata_list, header_dir)
    prepend_header_to_file(tmp_metadata_item, header_dir)

    if not os.path.exists(target_dir):
        os.makedirs(target_dir)

    shutil.copy(tmp_metadata_list, os.path.join(target_dir, FLATBUFFER_METADATA_LIST_JAVA))
    shutil.copy(tmp_metadata_item, os.path.join(target_dir, FLATBUFFER_METADATA_ITEM_JAVA))

def create_test_data(unicode_path):
    """Read all the emojis in the unicode files and update the test file"""
    lines = read_emoji_lines(os.path.join(unicode_path, EMOJI_ZWJ_FILE))
    lines += read_emoji_lines(os.path.join(unicode_path, EMOJI_SEQ_FILE))

    lines += read_emoji_lines(os.path.join(unicode_path, ANDROID_EMOJI_ZWJ_SEQ_FILE), optional=True)
    lines += read_emoji_lines(os.path.join(unicode_path, ANDROID_EMOJIS_SEQ_FILE), optional=True)

    # standardized variants contains a huge list of sequences, only read the ones that are emojis
    # and also the ones with FE0F (emoji style)
    standardized_variants_lines = read_emoji_lines(
        os.path.join(unicode_path, EMOJI_VARIATION_SEQ_FILE))
    for line in standardized_variants_lines:
        if STD_VARIANTS_EMOJI_STYLE in line:
            lines.append(line)

    emojis_set = set()
    for line in lines:
        # In unicode 12.0, "emoji-sequences.txt" contains "Basic_Emoji" session. We ignore them
        # here since we are already checking the emoji presentations with
        # emoji-variation-sequences.txt.
        if "BASIC_EMOJI" in line:
            continue
        codepoints = [hex_str_to_int(x) for x in line.split(';')[0].strip().split(' ')]
        emojis_set.add(codepoint_to_string(codepoints).upper())

    emoji_data_lines = read_emoji_lines(os.path.join(unicode_path, EMOJI_DATA_FILE))
    for line in emoji_data_lines:
        codepoints_range, emoji_property = codepoints_and_emoji_prop(line)
        if not emoji_property in ACCEPTED_EMOJI_PROPERTIES:
            continue
        is_emoji_style = emoji_property == EMOJI_PRESENTATION_STR
        if is_emoji_style:
            codepoints = [to_hex_str(x) for x in
                          codepoints_for_emojirange(codepoints_range)]
            emojis_set.update(codepoints)

    emoji_style_exceptions = get_emoji_style_exceptions(unicode_path)
    #  finally add the android default emoji exceptions
    emojis_set.update([to_hex_str(x) for x in emoji_style_exceptions])

    emojis_list = list(emojis_set)
    emojis_list.sort()
    with open(TEST_DATA_PATH, "w") as test_file:
        for line in emojis_list:
            test_file.write("%s\n" % line)

class _EmojiData(object):
    """Holds the information about a single emoji."""

    def __init__(self, codepoints, is_emoji_style):
        self.codepoints = codepoints
        self.emoji_style = is_emoji_style
        self.emoji_id = 0
        self.width = 0
        self.height = 0
        self.sdk_added = SDK_VERSION
        self.compat_added = METADATA_VERSION

    def update_metrics(self, metrics):
        """Updates width/height instance variables with the values given in metrics dictionary.
        :param metrics: a dictionary object that has width and height values.
        """
        self.width = metrics.width
        self.height = metrics.height

    def __repr__(self):
        return '<EmojiData {0} - {1}>'.format(self.emoji_style,
                                              codepoint_to_string(self.codepoints))

    def create_json_element(self):
        """Creates the json representation of EmojiData."""
        json_element = {}
        json_element['id'] = self.emoji_id
        json_element['emojiStyle'] = self.emoji_style
        json_element['sdkAdded'] = self.sdk_added
        json_element['compatAdded'] = self.compat_added
        json_element['width'] = self.width
        json_element['height'] = self.height
        json_element['codepoints'] = self.codepoints
        return json_element

    def create_txt_row(self):
        """Creates array of values for CSV of EmojiData."""
        row = [to_hex_str(self.emoji_id), self.sdk_added, self.compat_added]
        row += [to_hex_str(x) for x in self.codepoints]
        return row

    def update(self, emoji_id, sdk_added, compat_added):
        """Updates current EmojiData with the values in a json element"""
        self.emoji_id = emoji_id
        self.sdk_added = sdk_added
        self.compat_added = compat_added


def read_emoji_lines(file_path, optional=False):
    """Read all lines in an unicode emoji file into a list of uppercase strings. Ignore the empty
    lines and comments
    :param file_path: unicode emoji file path
    :param optional: if True no exception is raised when the file cannot be read
    :return: list of uppercase strings
    """
    result = []
    try:
        with open(file_path) as file_stream:
            for line in file_stream:
                line = line.strip()
                if line and not line.startswith('#'):
                    result.append(line.upper())
    except IOError:
        if optional:
            pass
        else:
            raise

    return result

def get_emoji_style_exceptions(unicode_path):
    """Read EMOJI_STYLE_OVERRIDE_FILE and return the codepoints as integers"""
    lines = read_emoji_lines(os.path.join(unicode_path, EMOJI_STYLE_OVERRIDE_FILE))
    exceptions = []
    for line in lines:
        codepoint = hex_str_to_int(codepoints_and_emoji_prop(line)[0])
        exceptions.append(codepoint)
    return exceptions

def codepoints_for_emojirange(codepoints_range):
    """ Return codepoints given in emoji files. Expand the codepoints that are given as a range
    such as XYZ ... UVT
    """
    codepoints = []
    if '..' in codepoints_range:
        range_start, range_end = codepoints_range.split('..')
        codepoints_range = range(hex_str_to_int(range_start),
                                 hex_str_to_int(range_end) + 1)
        codepoints.extend(codepoints_range)
    else:
        codepoints.append(hex_str_to_int(codepoints_range))
    return codepoints

def codepoints_and_emoji_prop(line):
    """For a given emoji file line, return codepoints and emoji property in the line.
    1F93C..1F93E ; [Emoji|Emoji_Presentation|Emoji_Modifier_Base|Emoji_Component
    |Extended_Pictographic] # [...]"""
    line = line.strip()
    if '#' in line:
        line = line[:line.index('#')]
    else:
        raise ValueError("Line is expected to have # in it")
    line = line.split(';')
    codepoints_range = line[0].strip()
    emoji_property = line[1].strip()

    return codepoints_range, emoji_property

def read_emoji_intervals(emoji_data_map, file_path, emoji_style_exceptions):
    """Read unicode lines of unicode emoji file in which each line describes a set of codepoint
    intervals. Expands the interval on a line and inserts related EmojiDatas into emoji_data_map.
    A line format that is expected is as follows:
    1F93C..1F93E ; [Emoji|Emoji_Presentation|Emoji_Modifier_Base|Emoji_Component
    |Extended_Pictographic] # [...]"""
    lines = read_emoji_lines(file_path)

    for line in lines:
        codepoints_range, emoji_property = codepoints_and_emoji_prop(line)
        if not emoji_property in ACCEPTED_EMOJI_PROPERTIES:
            continue
        is_emoji_style = emoji_property == EMOJI_PRESENTATION_STR
        codepoints = codepoints_for_emojirange(codepoints_range)

        for codepoint in codepoints:
            key = codepoint_to_string([codepoint])
            codepoint_is_emoji_style = is_emoji_style or codepoint in emoji_style_exceptions
            if key in emoji_data_map:
                # since there are multiple definitions of emojis, only update when emoji style is
                # True
                if codepoint_is_emoji_style:
                    emoji_data_map[key].emoji_style = True
            else:
                emoji_data = _EmojiData([codepoint], codepoint_is_emoji_style)
                emoji_data_map[key] = emoji_data


def read_emoji_sequences(emoji_data_map, file_path, optional=False, filter=None):
    """Reads the content of the file which contains emoji sequences. Creates EmojiData for each
    line and puts into emoji_data_map."""
    lines = read_emoji_lines(file_path, optional)
    # 1F1E6 1F1E8 ; Name ; [...]
    for line in lines:
        # In unicode 12.0, "emoji-sequences.txt" contains "Basic_Emoji" session. We ignore them
        # here since we are already checking the emoji presentations with
        # emoji-variation-sequences.txt.
        if "BASIC_EMOJI" in line:
            continue
        codepoints = [hex_str_to_int(x) for x in line.split(';')[0].strip().split(' ')]
        codepoints = [x for x in codepoints if x != EMOJI_STYLE_VS]
        if filter:
          if filter(codepoints):
            continue
        key = codepoint_to_string(codepoints)
        if not key in emoji_data_map:
            emoji_data = _EmojiData(codepoints, False)
            emoji_data_map[key] = emoji_data


def load_emoji_data_map(unicode_path, without_flags):
    """Reads the emoji data files, constructs a map of space separated codepoints to EmojiData.
    :return: map of space separated codepoints to EmojiData
    """
    if without_flags:
      filter = lambda x: is_flag_seq(x)
    else:
      filter = None
    emoji_data_map = {}
    emoji_style_exceptions = get_emoji_style_exceptions(unicode_path)
    read_emoji_intervals(emoji_data_map, os.path.join(unicode_path, EMOJI_DATA_FILE),
                         emoji_style_exceptions)
    read_emoji_sequences(emoji_data_map, os.path.join(unicode_path, EMOJI_ZWJ_FILE))
    read_emoji_sequences(emoji_data_map, os.path.join(unicode_path, EMOJI_SEQ_FILE), filter=filter)

    # Add the optional ANDROID_EMOJI_ZWJ_SEQ_FILE if it exists.
    read_emoji_sequences(emoji_data_map, os.path.join(unicode_path, ANDROID_EMOJI_ZWJ_SEQ_FILE),
                         optional=True)
    # Add the optional ANDROID_EMOJIS_SEQ_FILE if it exists.
    read_emoji_sequences(emoji_data_map, os.path.join(unicode_path, ANDROID_EMOJIS_SEQ_FILE),
                         optional=True)

    return emoji_data_map


def load_previous_metadata(emoji_data_map):
    """Updates emoji data elements in emoji_data_map using the id, sdk_added and compat_added fields
       in emoji_metadata.txt. Returns the smallest available emoji id to use. i.e. if the largest
       emoji id emoji_metadata.txt is 1, function would return 2. If emoji_metadata.txt does not
       exist, or contains no emojis defined returns DEFAULT_EMOJI_ID"""
    current_emoji_id = DEFAULT_EMOJI_ID
    if os.path.isfile(INPUT_META_FILE):
        with open(INPUT_META_FILE) as csvfile:
            reader = csv.reader(csvfile, delimiter=' ')
            for row in reader:
                if row[0].startswith('#'):
                    continue
                emoji_id = hex_str_to_int(row[0])
                sdk_added = int(row[1])
                compat_added = int(row[2])
                key = codepoint_to_string(hex_str_to_int(x) for x in row[3:])
                if key in emoji_data_map:
                    emoji_data = emoji_data_map[key]
                    emoji_data.update(emoji_id, sdk_added, compat_added)
                    if emoji_data.emoji_id >= current_emoji_id:
                        current_emoji_id = emoji_data.emoji_id + 1

    return current_emoji_id


def update_ttlib_orig_sort():
    """Updates the ttLib tag sort with a closure that makes the meta table first."""
    orig_sort = ttLib.sortedTagList

    def meta_first_table_sort(tag_list, table_order=None):
        """Sorts the tables with the original ttLib sort, then makes the meta table first."""
        tag_list = orig_sort(tag_list, table_order)
        tag_list.remove('meta')
        tag_list.insert(0, 'meta')
        return tag_list

    ttLib.sortedTagList = meta_first_table_sort


def inject_meta_into_font(ttf, flatbuffer_bin_filename):
    """inject metadata binary into font"""
    if not 'meta' in ttf:
        ttf['meta'] = ttLib.getTableClass('meta')()
    meta = ttf['meta']
    with open(flatbuffer_bin_filename, 'rb') as flatbuffer_bin_file:
        meta.data[EMOJI_META_TAG_NAME] = flatbuffer_bin_file.read()

    # sort meta tables for faster access
    update_ttlib_orig_sort()


def validate_input_files(font_path, unicode_path, flatbuffer_path):
    """Validate the existence of font file and the unicode files"""
    if not os.path.isfile(font_path):
        raise ValueError("Font file does not exist: " + font_path)

    if not os.path.isdir(unicode_path):
        raise ValueError(
            "Unicode directory does not exist or is not a directory " + unicode_path)

    emoji_filenames = [os.path.join(unicode_path, EMOJI_DATA_FILE),
                       os.path.join(unicode_path, EMOJI_ZWJ_FILE),
                       os.path.join(unicode_path, EMOJI_SEQ_FILE)]
    for emoji_filename in emoji_filenames:
        if not os.path.isfile(emoji_filename):
            raise ValueError("Unicode emoji data file does not exist: " + emoji_filename)

    if not os.path.isdir(flatbuffer_path):
        raise ValueError(
            "Flatbuffer directory does not exist or is not a directory " + flatbuffer_path)

    flatbuffer_filenames = [os.path.join(flatbuffer_path, FLATBUFFER_SCHEMA),
                            os.path.join(flatbuffer_path, FLATBUFFER_HEADER)]
    for flatbuffer_filename in flatbuffer_filenames:
        if not os.path.isfile(flatbuffer_filename):
            raise ValueError("Flatbuffer file does not exist: " + flatbuffer_filename)


def add_file_to_sha(sha_algo, file_path):
    with open(file_path, 'rb') as input_file:
        for data in iter(lambda: input_file.read(8192), b''):
            sha_algo.update(data)

def create_sha_from_source_files(font_paths):
    """Creates a SHA from the given font files"""
    sha_algo = hashlib.sha256()
    for file_path in font_paths:
        add_file_to_sha(sha_algo, file_path)
    return sha_algo.hexdigest()


class EmojiFontCreator(object):
    """Creates the EmojiCompat font"""

    def __init__(self, font_path, unicode_path, without_flags):
        validate_input_files(font_path, unicode_path, FLATBUFFER_MODULE_DIR)

        self.font_path = font_path
        self.unicode_path = unicode_path
        self.without_flags = without_flags
        self.emoji_data_map = {}
        self.remapped_codepoints = {}
        self.glyph_to_image_metrics_map = {}
        # set default emoji id to start of Supplemental Private Use Area-A
        self.emoji_id = DEFAULT_EMOJI_ID

    def update_emoji_data(self, codepoints, glyph_name):
        """Updates the existing EmojiData identified with codepoints. The fields that are set are:
        - emoji_id (if it does not exist)
        - image width/height"""
        key = codepoint_to_string(codepoints)
        if key in self.emoji_data_map:
            # add emoji to final data
            emoji_data = self.emoji_data_map[key]
            emoji_data.update_metrics(self.glyph_to_image_metrics_map[glyph_name])
            if emoji_data.emoji_id == 0:
                emoji_data.emoji_id = self.emoji_id
                self.emoji_id = self.emoji_id + 1
            self.remapped_codepoints[emoji_data.emoji_id] = glyph_name

    def read_cbdt(self, ttf):
        """Read image size data from CBDT."""
        cbdt = ttf['CBDT']
        for strike_data in cbdt.strikeData:
            for key, data in strike_data.items():
                data.decompile()
                self.glyph_to_image_metrics_map[key] = data.metrics

    def read_cmap12(self, ttf, glyph_to_codepoint_map):
        """Reads single code point emojis that are in cmap12, updates glyph_to_codepoint_map and
        finally clears all elements in CMAP 12"""
        cmap = ttf['cmap']
        for table in cmap.tables:
            if table.format == 12 and table.platformID == 3 and table.platEncID == 10:
                for codepoint, glyph_name in table.cmap.items():
                    glyph_to_codepoint_map[glyph_name] = codepoint
                    self.update_emoji_data([codepoint], glyph_name)
                return table
        raise ValueError("Font doesn't contain cmap with format:12, platformID:3 and platEncID:10")

    def read_gsub(self, ttf, glyph_to_codepoint_map):
        """Reads the emoji sequences defined in GSUB and clear all elements under GSUB"""
        gsub = ttf['GSUB']
        ligature_subtables = []
        context_subtables = []
        # this code is font dependent, implementing all gsub rules is out of scope of EmojiCompat
        # and would be expensive with little value
        for lookup in gsub.table.LookupList.Lookup:
            for subtable in lookup.SubTable:
                if subtable.LookupType == 5:
                    context_subtables.append(subtable)
                elif subtable.LookupType == 4:
                    ligature_subtables.append(subtable)

        for subtable in context_subtables:
            self.add_gsub_context_subtable(subtable, gsub.table.LookupList, glyph_to_codepoint_map)

        for subtable in ligature_subtables:
            self.add_gsub_ligature_subtable(subtable, glyph_to_codepoint_map)

    def add_gsub_context_subtable(self, subtable, lookup_list, glyph_to_codepoint_map):
        """Add substitutions defined as OpenType Context Substitution"""
        for sub_class_set in subtable.SubClassSet:
            if sub_class_set:
                for sub_class_rule in sub_class_set.SubClassRule:
                    # prepare holder for substitution list. each rule will have a list that is added
                    # to the subs_list.
                    subs_list = len(sub_class_rule.SubstLookupRecord) * [None]
                    for record in sub_class_rule.SubstLookupRecord:
                        subs_list[record.SequenceIndex] = self.get_substitutions(lookup_list,
                                                                            record.LookupListIndex)
                    # create combinations or all lists. the combinations will be filtered by
                    # emoji_data_map. the first element that contain as a valid glyph will be used
                    # as the final glyph
                    combinations = list(itertools.product(*subs_list))
                    for seq in combinations:
                        glyph_names = [x["input"] for x in seq]
                        codepoints = [glyph_to_codepoint_map[x] for x in glyph_names]
                        outputs = [x["output"] for x in seq if x["output"]]
                        nonempty_outputs = list(filter(lambda x: x.strip() , outputs))
                        if len(nonempty_outputs) == 0:
                            print("Warning: no output glyph is set for " + str(glyph_names))
                            continue
                        elif len(nonempty_outputs) > 1:
                            print(
                                "Warning: multiple glyph is set for "
                                    + str(glyph_names) + ", will use the first one")

                        glyph = nonempty_outputs[0]
                        self.update_emoji_data(codepoints, glyph)

    def get_substitutions(self, lookup_list, index):
        result = []
        for x in lookup_list.Lookup[index].SubTable:
            for input, output in x.mapping.items():
                result.append({"input": input, "output": output})
        return result

    def add_gsub_ligature_subtable(self, subtable, glyph_to_codepoint_map):
        for name, ligatures in subtable.ligatures.items():
            for ligature in ligatures:
                glyph_names = [name] + ligature.Component
                codepoints = [glyph_to_codepoint_map[x] for x in glyph_names]
                self.update_emoji_data(codepoints, ligature.LigGlyph)

    def write_metadata_json(self, output_json_file_path):
        """Writes the emojis into a json file"""
        output_json = {}
        output_json['version'] = METADATA_VERSION
        output_json['sourceSha'] = create_sha_from_source_files(
            [self.font_path, OUTPUT_META_FILE, FLATBUFFER_SCHEMA])
        output_json['list'] = []

        emoji_data_list = sorted(self.emoji_data_map.values(), key=lambda x: x.emoji_id)

        total_emoji_count = 0
        for emoji_data in emoji_data_list:
            if self.without_flags and is_flag_seq(emoji_data.codepoints):
                continue  # Do not add flags emoji data if this is for subset font.
            element = emoji_data.create_json_element()
            output_json['list'].append(element)
            total_emoji_count = total_emoji_count + 1

        # write the new json file to be processed by FlatBuffers
        with open(output_json_file_path, 'w') as json_file:
            print(json.dumps(output_json, indent=4, sort_keys=True, separators=(',', ':')),
                  file=json_file)

        return total_emoji_count

    def write_metadata_csv(self):
        """Writes emoji metadata into space separated file"""
        with open(OUTPUT_META_FILE, 'w') as csvfile:
            csvwriter = csv.writer(csvfile, delimiter=' ')
            emoji_data_list = sorted(self.emoji_data_map.values(), key=lambda x: x.emoji_id)
            csvwriter.writerow(['#id', 'sdkAdded', 'compatAdded', 'codepoints'])
            for emoji_data in emoji_data_list:
                csvwriter.writerow(emoji_data.create_txt_row())

    def add_watermark(self, ttf):
        cmap = ttf.getBestCmap()
        gsub = ttf['GSUB'].table

        # Obtain Version string
        m = re.search('^Version (\d*)\.(\d*)', font_data.font_version(ttf))
        if not m:
            raise ValueError('The font does not have proper version string.')
        major = m.group(1)
        minor = m.group(2)
        # Replace the dot with space since NotoColorEmoji does not have glyph for dot.
        glyphs = [cmap[ord(x)] for x in '%s %s' % (major, minor)]

        # Update Glyph metrics
        ttf.getGlyphOrder().append(WATERMARK_NEW_GLYPH_ID)
        refGlyphId = cmap[WATERMARK_REF_CODE_POINT]
        ttf['hmtx'].metrics[WATERMARK_NEW_GLYPH_ID] = ttf['hmtx'].metrics[refGlyphId]
        ttf['vmtx'].metrics[WATERMARK_NEW_GLYPH_ID] = ttf['vmtx'].metrics[refGlyphId]

        # Add new Glyph to cmap
        font_data.add_to_cmap(ttf, { WATERMARK_NEW_CODE_POINT : WATERMARK_NEW_GLYPH_ID })

        # Add lookup table for the version string.
        lookups = gsub.LookupList.Lookup
        new_lookup = otTables.Lookup()
        new_lookup.LookupType = 2  # Multiple Substitution Subtable.
        new_lookup.LookupFlag = 0
        new_subtable = otTables.MultipleSubst()
        new_subtable.mapping = { WATERMARK_NEW_GLYPH_ID : tuple(glyphs) }
        new_lookup.SubTable = [ new_subtable ]
        new_lookup_index = len(lookups)
        lookups.append(new_lookup)

        # Add feature
        feature = next(x for x in gsub.FeatureList.FeatureRecord if x.FeatureTag == 'ccmp')
        if not feature:
            raise ValueError("Font doesn't contain ccmp feature.")

        feature.Feature.LookupListIndex.append(new_lookup_index)

    def create_font(self):
        """Creates the EmojiCompat font.
        :param font_path: path to Android NotoColorEmoji font
        :param unicode_path: path to directory that contains unicode files
        """

        tmp_dir = tempfile.mkdtemp()

        # create emoji codepoints to EmojiData map
        self.emoji_data_map = load_emoji_data_map(self.unicode_path, self.without_flags)

        # read previous metadata file to update id, sdkAdded and compatAdded. emoji id that is
        # returned is either default or 1 greater than the largest id in previous data
        self.emoji_id = load_previous_metadata(self.emoji_data_map)

        # recalcTimestamp parameter will keep the modified field same as the original font. Changing
        # the modified field in the font causes the font ttf file to change, which makes it harder
        # to understand if something really changed in the font.
        with contextlib.closing(ttLib.TTFont(self.font_path, recalcTimestamp=False)) as ttf:
            # read image size data
            self.read_cbdt(ttf)

            # glyph name to codepoint map
            glyph_to_codepoint_map = {}

            # read single codepoint emojis under cmap12 and clear the table contents
            cmap12_table = self.read_cmap12(ttf, glyph_to_codepoint_map)

            # read emoji sequences gsub and clear the table contents
            self.read_gsub(ttf, glyph_to_codepoint_map)

            # add all new codepoint to glyph mappings
            cmap12_table.cmap.update(self.remapped_codepoints)

            # final metadata csv will be used to generate the sha, therefore write it before
            # metadata json is written.
            self.write_metadata_csv()

            output_json_file = os.path.join(tmp_dir, OUTPUT_JSON_FILE_NAME)
            flatbuffer_bin_file = os.path.join(tmp_dir, FLATBUFFER_BIN)
            flatbuffer_java_dir = os.path.join(tmp_dir, FLATBUFFER_JAVA_PATH)

            total_emoji_count = self.write_metadata_json(output_json_file)

            # create the flatbuffers binary and java classes
            flatc_command = ['flatc',
                             '-o',
                             tmp_dir,
                             '-b',
                             '-j',
                             FLATBUFFER_SCHEMA,
                             output_json_file]
            subprocess.check_output(flatc_command)

            # inject metadata binary into font
            inject_meta_into_font(ttf, flatbuffer_bin_file)

            # add wartermark glyph for manual verification.
            self.add_watermark(ttf)

            # update CBDT and CBLC versions since older android versions cannot read > 2.0
            ttf['CBDT'].version = 2.0
            ttf['CBLC'].version = 2.0

            # save the new font
            ttf.save(FONT_PATH)

            update_flatbuffer_java_files(flatbuffer_java_dir, #tmp dir
                                         FLATBUFFER_HEADER,
                                         FLATBUFFER_JAVA_TARGET)

            create_test_data(self.unicode_path)

            # clear the tmp output directory
            shutil.rmtree(tmp_dir, ignore_errors=True)

            print(
                "{0} emojis are written to\n{1}".format(total_emoji_count, FONT_DIR))


def print_usage():
    """Prints how to use the script."""
    print("Please specify a path to font and unicode files.\n"
          "usage: createfont.py noto-color-emoji-path unicode-dir-path")

def parse_args(argv):
    # parse manually to avoid any extra dependencies
    if len(argv) == 4:
      without_flags = argv[3] == '--without-flags'
    else:
      without_flags = False

    if len(argv) < 3:
        print_usage()
        sys.exit(1)
    return (sys.argv[1], sys.argv[2], without_flags)

def main():
    font_file, unicode_dir, without_flags = parse_args(sys.argv)
    EmojiFontCreator(font_file, unicode_dir, without_flags).create_font()


if __name__ == '__main__':
    main()