marseymoji/generate_emoji_name_data.py

405 lines
14 KiB
Python
Raw Normal View History

#!/usr/bin/env python
# -*- coding: utf-8 -*-#
#
# Copyright 2015 Google Inc. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Generate name data for emoji resources. Currently in json format."""
import argparse
import collections
import glob
import json
import os
from os import path
import re
import sys
import generate_emoji_html
from nototools import tool_utils
from nototools import unicode_data
def _create_custom_gendered_seq_names():
"""The names have detail that is adequately represented by the image."""
BOY = 0x1f466
GIRL = 0x1f467
MAN = 0x1f468
WOMAN = 0x1f469
HEART = 0x2764 # Heavy Black Heart
KISS_MARK = 0x1f48b
return {
(MAN, HEART, KISS_MARK, MAN): 'Kiss',
(WOMAN, HEART, KISS_MARK, WOMAN): 'Kiss',
(WOMAN, HEART, KISS_MARK, MAN): 'Kiss',
(WOMAN, HEART, MAN): 'Couple with Heart',
(MAN, HEART, MAN): 'Couple with Heart',
(WOMAN, HEART, WOMAN): 'Couple with Heart',
(MAN, GIRL): 'Family',
(MAN, GIRL, GIRL): 'Family',
(MAN, GIRL, BOY): 'Family',
(MAN, BOY): 'Family',
(MAN, BOY, BOY): 'Family',
(MAN, WOMAN, GIRL): 'Family',
(MAN, WOMAN, GIRL, GIRL): 'Family',
(MAN, WOMAN, GIRL, BOY): 'Family',
(MAN, WOMAN, BOY): 'Family',
(MAN, WOMAN, BOY, BOY): 'Family',
(MAN, MAN, GIRL): 'Family',
(MAN, MAN, GIRL, GIRL): 'Family',
(MAN, MAN, GIRL, BOY): 'Family',
(MAN, MAN, BOY): 'Family',
(MAN, MAN, BOY, BOY): 'Family',
(WOMAN, GIRL): 'Family',
(WOMAN, GIRL, GIRL): 'Family',
(WOMAN, GIRL, BOY): 'Family',
(WOMAN, BOY): 'Family',
(WOMAN, BOY, BOY): 'Family',
(WOMAN, WOMAN, GIRL): 'Family',
(WOMAN, WOMAN, GIRL, GIRL): 'Family',
(WOMAN, WOMAN, GIRL, BOY): 'Family',
(WOMAN, WOMAN, BOY): 'Family',
(WOMAN, WOMAN, BOY, BOY): 'Family' }
def _create_custom_seq_names():
"""These have names that often are of the form 'Person xyz-ing' or 'Man Xyz.'
We opt to simplify the former to an activity name or action, and the latter to
drop the gender. This also generally makes the names shorter."""
EYE = 0x1f441
SPEECH = 0x1f5e8
WHITE_FLAG = 0x1f3f3
RAINBOW = 0x1f308
return {
(EYE, SPEECH): 'I Witness',
(WHITE_FLAG, RAINBOW): 'Rainbow Flag',
(0x2695,): 'Health Worker',
(0x2696,): 'Judge',
(0x26f7,): 'Skiing',
(0x26f9,): 'Bouncing a Ball',
(0x2708,): 'Pilot',
(0x1f33e,): 'Farmer',
(0x1f373,): 'Cook',
(0x1f393,): 'Student',
(0x1f3a4,): 'Singer',
(0x1f3a8,): 'Artist',
(0x1f3c2,): 'Snowboarding',
(0x1f3c3,): 'Running',
(0x1f3c4,): 'Surfing',
(0x1f3ca,): 'Swimming',
(0x1f3cb,): 'Weight Lifting',
(0x1f3cc,): 'Golfing',
(0x1f3eb,): 'Teacher',
(0x1f3ed,): 'Factory Worker',
(0x1f46e,): 'Police Officer',
(0x1f46f,): 'Partying',
(0x1f471,): 'Person with Blond Hair',
(0x1f473,): 'Person Wearing Turban',
(0x1f477,): 'Construction Worker',
(0x1f481,): 'Tipping Hand',
(0x1f482,): 'Guard',
(0x1f486,): 'Face Massage',
(0x1f487,): 'Haircut',
(0x1f4bb,): 'Technologist',
(0x1f4bc,): 'Office Worker',
(0x1f527,): 'Mechanic',
(0x1f52c,): 'Scientist',
(0x1f575,): 'Detective',
(0x1f645,): 'No Good Gesture',
(0x1f646,): 'OK Gesture',
(0x1f647,): 'Bowing Deeply',
(0x1f64b,): 'Raising Hand',
(0x1f64d,): 'Frowning',
(0x1f64e,): 'Pouting',
(0x1f680,): 'Astronaut',
(0x1f692,): 'Firefighter',
(0x1f6a3,): 'Rowing',
(0x1f6b4,): 'Bicycling',
(0x1f6b5,): 'Mountain Biking',
(0x1f6b6,): 'Walking',
(0x1f926,): 'Face Palm',
(0x1f937,): 'Shrug',
(0x1f938,): 'Doing a Cartwheel',
(0x1f939,): 'Juggling',
(0x1f93c,): 'Wrestling',
(0x1f93d,): 'Water Polo',
(0x1f93e,): 'Playing Handball',
(0x1f9d6,): 'Person in Steamy Room',
(0x1f9d7,): 'Climbing',
(0x1f9d8,): 'Person in Lotus Position',
(0x1f9d9,): 'Mage',
(0x1f9da,): 'Fairy',
(0x1f9db,): 'Vampire',
(0x1f9dd,): 'Elf',
(0x1f9de,): 'Genie',
(0x1f9df,): 'Zombie',
}
_CUSTOM_GENDERED_SEQ_NAMES = _create_custom_gendered_seq_names()
_CUSTOM_SEQ_NAMES = _create_custom_seq_names()
# Fixes for unusual capitalization or cases we don't care to handle in code.
# Also prevents titlecasing 'S' after apostrophe in posessives. Note we _do_
# want titlecasing after apostrophe in some cases, e.g. O'Clock.
_CUSTOM_CAPS_NAMES = {
(0x26d1,): 'Rescue Worker’s Helmet',
(0x1f170,): 'A Button (blood type)', # a Button (Blood Type)
(0x1f171,): 'B Button (blood type)', # B Button (Blood Type)
(0x1f17e,): 'O Button (blood type)', # O Button (Blood Type)
(0x1f18e,): 'AB Button (blood type)', # Ab Button (Blood Type)
(0x1f191,): 'CL Button', # Cl Button
(0x1f192,): 'COOL Button', # Cool Button
(0x1f193,): 'FREE Button', # Free Button
(0x1f194,): 'ID Button', # Id Button
(0x1f195,): 'NEW Button', # New Button
(0x1f196,): 'NG Button', # Ng Button
(0x1f197,): 'OK Button', # Ok Button
(0x1f198,): 'SOS Button', # Sos Button
(0x1f199,): 'UP! Button', # Up! Button
(0x1f19a,): 'VS Button', # Vs Button
(0x1f3e7,): 'ATM Sign', # Atm Sign
(0x1f44C,): 'OK Hand', # Ok Hand
(0x1f452,): 'Woman’s Hat',
(0x1f45a,): 'Woman’s Clothes',
(0x1f45e,): 'Man’s Shoe',
(0x1f461,): 'Woman’s Sandal',
(0x1f462,): 'Woman’s Boot',
(0x1f519,): 'BACK Arrow', # Back Arrow
(0x1f51a,): 'END Arrow', # End Arrow
(0x1f51b,): 'ON! Arrow', # On! Arrow
(0x1f51c,): 'SOON Arrow', # Soon Arrow
(0x1f51d,): 'TOP Arrow', # Top Arrow
(0x1f6b9,): 'Men’s Room',
(0x1f6ba,): 'Women’s Room',
}
# For the custom sequences we ignore ZWJ, the emoji variation selector
# and skin tone modifiers. We can't always ignore gender because
# the gendered sequences match against them, but we ignore gender in other
# cases so we define a separate set of gendered emoji to remove.
_NON_GENDER_CPS_TO_STRIP = frozenset(
[0xfe0f, 0x200d] +
range(unicode_data._FITZ_START, unicode_data._FITZ_END + 1))
_GENDER_CPS_TO_STRIP = frozenset([0x2640, 0x2642, 0x1f468, 0x1f469])
def _custom_name(seq):
"""Apply three kinds of custom names, based on the sequence."""
seq = tuple([cp for cp in seq if cp not in _NON_GENDER_CPS_TO_STRIP])
name = _CUSTOM_CAPS_NAMES.get(seq)
if name:
return name
# Single characters that participate in sequences (e.g. fire truck in the
# firefighter sequences) should not get converted. Single characters
# are in the custom caps names set but not the other sets.
if len(seq) == 1:
return None
name = _CUSTOM_GENDERED_SEQ_NAMES.get(seq)
if name:
return name
seq = tuple([cp for cp in seq if cp not in _GENDER_CPS_TO_STRIP])
name = _CUSTOM_SEQ_NAMES.get(seq)
return name
def _standard_name(seq):
"""Use the standard emoji name, with some algorithmic modifications.
We want to ignore skin-tone modifiers (but of course if the sequence _is_
the skin-tone modifier itself we keep that). So we strip these so we can
start with the generic name ignoring skin tone.
Non-emoji that are turned into emoji using the emoji VS have '(emoji) '
prepended to them, so strip that.
Regional indicator symbol names are a bit long, so shorten them.
Regional sequences are assumed to be ok as-is in terms of capitalization and
punctuation, so no modifications are applied to them.
After title-casing we make some English articles/prepositions lower-case
again. We also replace '&' with 'and'; Unicode seems rather fond of
ampersand."""
if not unicode_data.is_skintone_modifier(seq[0]):
seq = tuple([cp for cp in seq if not unicode_data.is_skintone_modifier(cp)])
name = unicode_data.get_emoji_sequence_name(seq)
if name.startswith('(emoji) '):
name = name[8:]
if len(seq) == 1 and unicode_data.is_regional_indicator(seq[0]):
return 'Regional Symbol ' + unicode_data.regional_indicator_to_ascii(seq[0])
if (unicode_data.is_regional_indicator_seq(seq) or
unicode_data.is_regional_tag_seq(seq)):
return name
name = name.title()
# Require space delimiting just in case...
name = re.sub(r'\s&\s', ' and ', name)
name = re.sub(
# not \b at start because we retain capital at start of phrase
r'(\s(:?A|And|From|In|Of|With|For))\b', lambda s: s.group(1).lower(),
name)
return name
def _name_data(seq, seq_file):
name = _custom_name(seq) or _standard_name(seq)
# we don't need canonical sequences
sequence = ''.join('&#x%x;' % cp for cp in seq if cp != 0xfe0f)
fname = path.basename(seq_file)
return fname, sequence, name
def generate_names(
src_dir, dst_dir, skip_limit=20, omit_groups=None, pretty_print=False,
verbose=False):
srcdir = tool_utils.resolve_path(src_dir)
if not path.isdir(srcdir):
print >> sys.stderr, '%s is not a directory' % src_dir
return
if omit_groups:
unknown_groups = set(omit_groups) - set(unicode_data.get_emoji_groups())
if unknown_groups:
print >> sys.stderr, 'did not recognize %d group%s: %s' % (
len(unknown_groups), '' if len(unknown_groups) == 1 else 's',
', '.join('"%s"' % g for g in omit_groups if g in unknown_groups))
print >> sys.stderr, 'valid groups are:\n %s' % (
'\n '.join(g for g in unicode_data.get_emoji_groups()))
return
print 'omitting %d group%s: %s' % (
len(omit_groups), '' if len(omit_groups) == 1 else 's',
', '.join('"%s"' % g for g in omit_groups))
else:
# might be None
print 'keeping all groups'
omit_groups = []
# make sure the destination exists
dstdir = tool_utils.ensure_dir_exists(
tool_utils.resolve_path(dst_dir))
# _get_image_data returns canonical cp sequences
print 'src dir:', srcdir
seq_to_file = generate_emoji_html._get_image_data(srcdir, 'png', 'emoji_u')
print 'seq to file has %d sequences' % len(seq_to_file)
# Aliases add non-gendered versions using gendered images for the most part.
# But when we display the images, we don't distinguish genders in the
# naming, we rely on the images-- so these look redundant. So we
# intentionally don't generate images for these.
# However, the alias file also includes the flag aliases, which we do want,
# and it also fails to exclude the unknown flag pua (since it doesn't
# map to anything), so we need to adjust for this.
canonical_aliases = generate_emoji_html._get_canonical_aliases()
aliases = set([
cps for cps in canonical_aliases.keys()
if not unicode_data.is_regional_indicator_seq(cps)])
aliases.add((0xfe82b,)) # unknown flag PUA
excluded = aliases | generate_emoji_html._get_canonical_excluded()
# The flag aliases have distinct names, so we _do_ want to show them
# multiple times.
to_add = {}
for seq in canonical_aliases:
if unicode_data.is_regional_indicator_seq(seq):
replace_seq = canonical_aliases[seq]
if seq in seq_to_file:
print 'warning, alias %s has file %s' % (
unicode_data.regional_indicator_seq_to_string(seq),
seq_to_file[seq])
continue
replace_file = seq_to_file.get(replace_seq)
if replace_file:
to_add[seq] = replace_file
seq_to_file.update(to_add)
data = []
last_skipped_group = None
skipcount = 0
for group in unicode_data.get_emoji_groups():
if group in omit_groups:
continue
name_data = []
for seq in unicode_data.get_emoji_in_group(group):
if seq in excluded:
continue
seq_file = seq_to_file.get(seq, None)
if seq_file is None:
skipcount += 1
if verbose:
if group != last_skipped_group:
print 'group %s' % group
last_skipped_group = group
print ' %s (%s)' % (
unicode_data.seq_to_string(seq),
', '.join(unicode_data.name(cp, 'x') for cp in seq))
if skip_limit >= 0 and skipcount > skip_limit:
raise Exception('skipped too many items')
else:
name_data.append(_name_data(seq, seq_file))
data.append({'category': group, 'emojis': name_data})
outfile = path.join(dstdir, 'data.json')
with open(outfile, 'w') as f:
indent = 2 if pretty_print else None
separators = None if pretty_print else (',', ':')
json.dump(data, f, indent=indent, separators=separators)
print 'wrote %s' % outfile
def main():
DEFAULT_DSTDIR = '[emoji]/emoji'
DEFAULT_IMAGEDIR = '[emoji]/build/compressed_pngs'
parser = argparse.ArgumentParser()
parser.add_argument(
'-s', '--srcdir', help='directory containing images (default %s)' %
DEFAULT_IMAGEDIR, metavar='dir', default=DEFAULT_IMAGEDIR)
parser.add_argument(
'-d', '--dstdir', help='name of destination directory (default %s)' %
DEFAULT_DSTDIR, metavar='fname', default=DEFAULT_DSTDIR)
parser.add_argument(
'-p', '--pretty_print', help='pretty-print json file',
action='store_true')
parser.add_argument(
'-m', '--missing_limit', help='number of missing images before failure '
'(default 20), use -1 for no limit', metavar='n', default=20)
parser.add_argument(
'--omit_groups', help='names of groups to omit (default "Misc")',
metavar='name', default=['Misc'], nargs='*')
parser.add_argument(
'-v', '--verbose', help='print progress information to stdout',
action='store_true')
args = parser.parse_args()
generate_names(
args.srcdir, args.dstdir, args.missing_limit, args.omit_groups,
pretty_print=args.pretty_print, verbose=args.verbose)
if __name__ == "__main__":
main()