406 lines
14 KiB
Python
Executable File
406 lines
14 KiB
Python
Executable File
#!/usr/bin/env python3
|
||
# -*- coding: utf-8 -*-#
|
||
#
|
||
# Copyright 2015 Google Inc. All rights reserved.
|
||
#
|
||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||
# you may not use this file except in compliance with the License.
|
||
# You may obtain a copy of the License at
|
||
#
|
||
# http://www.apache.org/licenses/LICENSE-2.0
|
||
#
|
||
# Unless required by applicable law or agreed to in writing, software
|
||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||
# See the License for the specific language governing permissions and
|
||
# limitations under the License.
|
||
|
||
"""Generate name data for emoji resources. Currently in json format."""
|
||
from __future__ import print_function
|
||
|
||
import argparse
|
||
import collections
|
||
import glob
|
||
import json
|
||
import os
|
||
from os import path
|
||
import re
|
||
import sys
|
||
|
||
import generate_emoji_html
|
||
|
||
from nototools import tool_utils
|
||
from nototools import unicode_data
|
||
|
||
def _create_custom_gendered_seq_names():
|
||
"""The names have detail that is adequately represented by the image."""
|
||
|
||
BOY = 0x1f466
|
||
GIRL = 0x1f467
|
||
MAN = 0x1f468
|
||
WOMAN = 0x1f469
|
||
HEART = 0x2764 # Heavy Black Heart
|
||
KISS_MARK = 0x1f48b
|
||
return {
|
||
(MAN, HEART, KISS_MARK, MAN): 'Kiss',
|
||
(WOMAN, HEART, KISS_MARK, WOMAN): 'Kiss',
|
||
(WOMAN, HEART, KISS_MARK, MAN): 'Kiss',
|
||
(WOMAN, HEART, MAN): 'Couple with Heart',
|
||
(MAN, HEART, MAN): 'Couple with Heart',
|
||
(WOMAN, HEART, WOMAN): 'Couple with Heart',
|
||
(MAN, GIRL): 'Family',
|
||
(MAN, GIRL, GIRL): 'Family',
|
||
(MAN, GIRL, BOY): 'Family',
|
||
(MAN, BOY): 'Family',
|
||
(MAN, BOY, BOY): 'Family',
|
||
(MAN, WOMAN, GIRL): 'Family',
|
||
(MAN, WOMAN, GIRL, GIRL): 'Family',
|
||
(MAN, WOMAN, GIRL, BOY): 'Family',
|
||
(MAN, WOMAN, BOY): 'Family',
|
||
(MAN, WOMAN, BOY, BOY): 'Family',
|
||
(MAN, MAN, GIRL): 'Family',
|
||
(MAN, MAN, GIRL, GIRL): 'Family',
|
||
(MAN, MAN, GIRL, BOY): 'Family',
|
||
(MAN, MAN, BOY): 'Family',
|
||
(MAN, MAN, BOY, BOY): 'Family',
|
||
(WOMAN, GIRL): 'Family',
|
||
(WOMAN, GIRL, GIRL): 'Family',
|
||
(WOMAN, GIRL, BOY): 'Family',
|
||
(WOMAN, BOY): 'Family',
|
||
(WOMAN, BOY, BOY): 'Family',
|
||
(WOMAN, WOMAN, GIRL): 'Family',
|
||
(WOMAN, WOMAN, GIRL, GIRL): 'Family',
|
||
(WOMAN, WOMAN, GIRL, BOY): 'Family',
|
||
(WOMAN, WOMAN, BOY): 'Family',
|
||
(WOMAN, WOMAN, BOY, BOY): 'Family' }
|
||
|
||
def _create_custom_seq_names():
|
||
"""These have names that often are of the form 'Person xyz-ing' or 'Man Xyz.'
|
||
We opt to simplify the former to an activity name or action, and the latter to
|
||
drop the gender. This also generally makes the names shorter."""
|
||
|
||
EYE = 0x1f441
|
||
SPEECH = 0x1f5e8
|
||
WHITE_FLAG = 0x1f3f3
|
||
RAINBOW = 0x1f308
|
||
return {
|
||
(EYE, SPEECH): 'I Witness',
|
||
(WHITE_FLAG, RAINBOW): 'Rainbow Flag',
|
||
(0x2695,): 'Health Worker',
|
||
(0x2696,): 'Judge',
|
||
(0x26f7,): 'Skiing',
|
||
(0x26f9,): 'Bouncing a Ball',
|
||
(0x2708,): 'Pilot',
|
||
(0x1f33e,): 'Farmer',
|
||
(0x1f373,): 'Cook',
|
||
(0x1f393,): 'Student',
|
||
(0x1f3a4,): 'Singer',
|
||
(0x1f3a8,): 'Artist',
|
||
(0x1f3c2,): 'Snowboarding',
|
||
(0x1f3c3,): 'Running',
|
||
(0x1f3c4,): 'Surfing',
|
||
(0x1f3ca,): 'Swimming',
|
||
(0x1f3cb,): 'Weight Lifting',
|
||
(0x1f3cc,): 'Golfing',
|
||
(0x1f3eb,): 'Teacher',
|
||
(0x1f3ed,): 'Factory Worker',
|
||
(0x1f46e,): 'Police Officer',
|
||
(0x1f46f,): 'Partying',
|
||
(0x1f471,): 'Person with Blond Hair',
|
||
(0x1f473,): 'Person Wearing Turban',
|
||
(0x1f477,): 'Construction Worker',
|
||
(0x1f481,): 'Tipping Hand',
|
||
(0x1f482,): 'Guard',
|
||
(0x1f486,): 'Face Massage',
|
||
(0x1f487,): 'Haircut',
|
||
(0x1f4bb,): 'Technologist',
|
||
(0x1f4bc,): 'Office Worker',
|
||
(0x1f527,): 'Mechanic',
|
||
(0x1f52c,): 'Scientist',
|
||
(0x1f575,): 'Detective',
|
||
(0x1f645,): 'No Good Gesture',
|
||
(0x1f646,): 'OK Gesture',
|
||
(0x1f647,): 'Bowing Deeply',
|
||
(0x1f64b,): 'Raising Hand',
|
||
(0x1f64d,): 'Frowning',
|
||
(0x1f64e,): 'Pouting',
|
||
(0x1f680,): 'Astronaut',
|
||
(0x1f692,): 'Firefighter',
|
||
(0x1f6a3,): 'Rowing',
|
||
(0x1f6b4,): 'Bicycling',
|
||
(0x1f6b5,): 'Mountain Biking',
|
||
(0x1f6b6,): 'Walking',
|
||
(0x1f926,): 'Face Palm',
|
||
(0x1f937,): 'Shrug',
|
||
(0x1f938,): 'Doing a Cartwheel',
|
||
(0x1f939,): 'Juggling',
|
||
(0x1f93c,): 'Wrestling',
|
||
(0x1f93d,): 'Water Polo',
|
||
(0x1f93e,): 'Playing Handball',
|
||
(0x1f9d6,): 'Person in Steamy Room',
|
||
(0x1f9d7,): 'Climbing',
|
||
(0x1f9d8,): 'Person in Lotus Position',
|
||
(0x1f9d9,): 'Mage',
|
||
(0x1f9da,): 'Fairy',
|
||
(0x1f9db,): 'Vampire',
|
||
(0x1f9dd,): 'Elf',
|
||
(0x1f9de,): 'Genie',
|
||
(0x1f9df,): 'Zombie',
|
||
}
|
||
|
||
_CUSTOM_GENDERED_SEQ_NAMES = _create_custom_gendered_seq_names()
|
||
_CUSTOM_SEQ_NAMES = _create_custom_seq_names()
|
||
|
||
# Fixes for unusual capitalization or cases we don't care to handle in code.
|
||
# Also prevents titlecasing 'S' after apostrophe in posessives. Note we _do_
|
||
# want titlecasing after apostrophe in some cases, e.g. O'Clock.
|
||
_CUSTOM_CAPS_NAMES = {
|
||
(0x26d1,): 'Rescue Worker’s Helmet',
|
||
(0x1f170,): 'A Button (blood type)', # a Button (Blood Type)
|
||
(0x1f171,): 'B Button (blood type)', # B Button (Blood Type)
|
||
(0x1f17e,): 'O Button (blood type)', # O Button (Blood Type)
|
||
(0x1f18e,): 'AB Button (blood type)', # Ab Button (Blood Type)
|
||
(0x1f191,): 'CL Button', # Cl Button
|
||
(0x1f192,): 'COOL Button', # Cool Button
|
||
(0x1f193,): 'FREE Button', # Free Button
|
||
(0x1f194,): 'ID Button', # Id Button
|
||
(0x1f195,): 'NEW Button', # New Button
|
||
(0x1f196,): 'NG Button', # Ng Button
|
||
(0x1f197,): 'OK Button', # Ok Button
|
||
(0x1f198,): 'SOS Button', # Sos Button
|
||
(0x1f199,): 'UP! Button', # Up! Button
|
||
(0x1f19a,): 'VS Button', # Vs Button
|
||
(0x1f3e7,): 'ATM Sign', # Atm Sign
|
||
(0x1f44C,): 'OK Hand', # Ok Hand
|
||
(0x1f452,): 'Woman’s Hat',
|
||
(0x1f45a,): 'Woman’s Clothes',
|
||
(0x1f45e,): 'Man’s Shoe',
|
||
(0x1f461,): 'Woman’s Sandal',
|
||
(0x1f462,): 'Woman’s Boot',
|
||
(0x1f519,): 'BACK Arrow', # Back Arrow
|
||
(0x1f51a,): 'END Arrow', # End Arrow
|
||
(0x1f51b,): 'ON! Arrow', # On! Arrow
|
||
(0x1f51c,): 'SOON Arrow', # Soon Arrow
|
||
(0x1f51d,): 'TOP Arrow', # Top Arrow
|
||
(0x1f6b9,): 'Men’s Room',
|
||
(0x1f6ba,): 'Women’s Room',
|
||
}
|
||
|
||
# For the custom sequences we ignore ZWJ, the emoji variation selector
|
||
# and skin tone modifiers. We can't always ignore gender because
|
||
# the gendered sequences match against them, but we ignore gender in other
|
||
# cases so we define a separate set of gendered emoji to remove.
|
||
|
||
_NON_GENDER_CPS_TO_STRIP = frozenset(
|
||
[0xfe0f, 0x200d] +
|
||
range(unicode_data._FITZ_START, unicode_data._FITZ_END + 1))
|
||
|
||
_GENDER_CPS_TO_STRIP = frozenset([0x2640, 0x2642, 0x1f468, 0x1f469])
|
||
|
||
def _custom_name(seq):
|
||
"""Apply three kinds of custom names, based on the sequence."""
|
||
|
||
seq = tuple([cp for cp in seq if cp not in _NON_GENDER_CPS_TO_STRIP])
|
||
name = _CUSTOM_CAPS_NAMES.get(seq)
|
||
if name:
|
||
return name
|
||
|
||
# Single characters that participate in sequences (e.g. fire truck in the
|
||
# firefighter sequences) should not get converted. Single characters
|
||
# are in the custom caps names set but not the other sets.
|
||
if len(seq) == 1:
|
||
return None
|
||
|
||
name = _CUSTOM_GENDERED_SEQ_NAMES.get(seq)
|
||
if name:
|
||
return name
|
||
|
||
seq = tuple([cp for cp in seq if cp not in _GENDER_CPS_TO_STRIP])
|
||
name = _CUSTOM_SEQ_NAMES.get(seq)
|
||
|
||
return name
|
||
|
||
|
||
def _standard_name(seq):
|
||
"""Use the standard emoji name, with some algorithmic modifications.
|
||
|
||
We want to ignore skin-tone modifiers (but of course if the sequence _is_
|
||
the skin-tone modifier itself we keep that). So we strip these so we can
|
||
start with the generic name ignoring skin tone.
|
||
|
||
Non-emoji that are turned into emoji using the emoji VS have '(emoji) '
|
||
prepended to them, so strip that.
|
||
|
||
Regional indicator symbol names are a bit long, so shorten them.
|
||
|
||
Regional sequences are assumed to be ok as-is in terms of capitalization and
|
||
punctuation, so no modifications are applied to them.
|
||
|
||
After title-casing we make some English articles/prepositions lower-case
|
||
again. We also replace '&' with 'and'; Unicode seems rather fond of
|
||
ampersand."""
|
||
|
||
if not unicode_data.is_skintone_modifier(seq[0]):
|
||
seq = tuple([cp for cp in seq if not unicode_data.is_skintone_modifier(cp)])
|
||
name = unicode_data.get_emoji_sequence_name(seq)
|
||
|
||
if name.startswith('(emoji) '):
|
||
name = name[8:]
|
||
|
||
if len(seq) == 1 and unicode_data.is_regional_indicator(seq[0]):
|
||
return 'Regional Symbol ' + unicode_data.regional_indicator_to_ascii(seq[0])
|
||
|
||
if (unicode_data.is_regional_indicator_seq(seq) or
|
||
unicode_data.is_regional_tag_seq(seq)):
|
||
return name
|
||
|
||
name = name.title()
|
||
# Require space delimiting just in case...
|
||
name = re.sub(r'\s&\s', ' and ', name)
|
||
name = re.sub(
|
||
# not \b at start because we retain capital at start of phrase
|
||
r'(\s(:?A|And|From|In|Of|With|For))\b', lambda s: s.group(1).lower(),
|
||
name)
|
||
|
||
return name
|
||
|
||
|
||
def _name_data(seq, seq_file):
|
||
name = _custom_name(seq) or _standard_name(seq)
|
||
# we don't need canonical sequences
|
||
sequence = ''.join('&#x%x;' % cp for cp in seq if cp != 0xfe0f)
|
||
fname = path.basename(seq_file)
|
||
return fname, sequence, name
|
||
|
||
|
||
def generate_names(
|
||
src_dir, dst_dir, skip_limit=20, omit_groups=None, pretty_print=False,
|
||
verbose=False):
|
||
srcdir = tool_utils.resolve_path(src_dir)
|
||
if not path.isdir(srcdir):
|
||
print('%s is not a directory' % src_dir, file=sys.stderr)
|
||
return
|
||
|
||
if omit_groups:
|
||
unknown_groups = set(omit_groups) - set(unicode_data.get_emoji_groups())
|
||
if unknown_groups:
|
||
print('did not recognize %d group%s: %s' % (
|
||
len(unknown_groups), '' if len(unknown_groups) == 1 else 's',
|
||
', '.join('"%s"' % g for g in omit_groups if g in unknown_groups)), file=sys.stderr)
|
||
print('valid groups are:\n %s' % (
|
||
'\n '.join(g for g in unicode_data.get_emoji_groups())), file=sys.stderr)
|
||
return
|
||
print('omitting %d group%s: %s' % (
|
||
len(omit_groups), '' if len(omit_groups) == 1 else 's',
|
||
', '.join('"%s"' % g for g in omit_groups)))
|
||
else:
|
||
# might be None
|
||
print('keeping all groups')
|
||
omit_groups = []
|
||
|
||
# make sure the destination exists
|
||
dstdir = tool_utils.ensure_dir_exists(
|
||
tool_utils.resolve_path(dst_dir))
|
||
|
||
# _get_image_data returns canonical cp sequences
|
||
print('src dir:', srcdir)
|
||
seq_to_file = generate_emoji_html._get_image_data(srcdir, 'png', 'emoji_u')
|
||
print('seq to file has %d sequences' % len(seq_to_file))
|
||
|
||
# Aliases add non-gendered versions using gendered images for the most part.
|
||
# But when we display the images, we don't distinguish genders in the
|
||
# naming, we rely on the images-- so these look redundant. So we
|
||
# intentionally don't generate images for these.
|
||
# However, the alias file also includes the flag aliases, which we do want,
|
||
# and it also fails to exclude the unknown flag pua (since it doesn't
|
||
# map to anything), so we need to adjust for this.
|
||
canonical_aliases = generate_emoji_html._get_canonical_aliases()
|
||
|
||
aliases = set([
|
||
cps for cps in canonical_aliases.keys()
|
||
if not unicode_data.is_regional_indicator_seq(cps)])
|
||
aliases.add((0xfe82b,)) # unknown flag PUA
|
||
excluded = aliases | generate_emoji_html._get_canonical_excluded()
|
||
|
||
# The flag aliases have distinct names, so we _do_ want to show them
|
||
# multiple times.
|
||
to_add = {}
|
||
for seq in canonical_aliases:
|
||
if unicode_data.is_regional_indicator_seq(seq):
|
||
replace_seq = canonical_aliases[seq]
|
||
if seq in seq_to_file:
|
||
print('warning, alias %s has file %s' % (
|
||
unicode_data.regional_indicator_seq_to_string(seq),
|
||
seq_to_file[seq]))
|
||
continue
|
||
replace_file = seq_to_file.get(replace_seq)
|
||
if replace_file:
|
||
to_add[seq] = replace_file
|
||
seq_to_file.update(to_add)
|
||
|
||
data = []
|
||
last_skipped_group = None
|
||
skipcount = 0
|
||
for group in unicode_data.get_emoji_groups():
|
||
if group in omit_groups:
|
||
continue
|
||
name_data = []
|
||
for seq in unicode_data.get_emoji_in_group(group):
|
||
if seq in excluded:
|
||
continue
|
||
seq_file = seq_to_file.get(seq, None)
|
||
if seq_file is None:
|
||
skipcount += 1
|
||
if verbose:
|
||
if group != last_skipped_group:
|
||
print('group %s' % group)
|
||
last_skipped_group = group
|
||
print(' %s (%s)' % (
|
||
unicode_data.seq_to_string(seq),
|
||
', '.join(unicode_data.name(cp, 'x') for cp in seq)))
|
||
if skip_limit >= 0 and skipcount > skip_limit:
|
||
raise Exception('skipped too many items')
|
||
else:
|
||
name_data.append(_name_data(seq, seq_file))
|
||
data.append({'category': group, 'emojis': name_data})
|
||
|
||
outfile = path.join(dstdir, 'data.json')
|
||
with open(outfile, 'w') as f:
|
||
indent = 2 if pretty_print else None
|
||
separators = None if pretty_print else (',', ':')
|
||
json.dump(data, f, indent=indent, separators=separators)
|
||
print('wrote %s' % outfile)
|
||
|
||
|
||
def main():
|
||
DEFAULT_DSTDIR = '[emoji]/emoji'
|
||
DEFAULT_IMAGEDIR = '[emoji]/build/compressed_pngs'
|
||
|
||
parser = argparse.ArgumentParser()
|
||
parser.add_argument(
|
||
'-s', '--srcdir', help='directory containing images (default %s)' %
|
||
DEFAULT_IMAGEDIR, metavar='dir', default=DEFAULT_IMAGEDIR)
|
||
parser.add_argument(
|
||
'-d', '--dstdir', help='name of destination directory (default %s)' %
|
||
DEFAULT_DSTDIR, metavar='fname', default=DEFAULT_DSTDIR)
|
||
parser.add_argument(
|
||
'-p', '--pretty_print', help='pretty-print json file',
|
||
action='store_true')
|
||
parser.add_argument(
|
||
'-m', '--missing_limit', help='number of missing images before failure '
|
||
'(default 20), use -1 for no limit', metavar='n', default=20)
|
||
parser.add_argument(
|
||
'--omit_groups', help='names of groups to omit (default "Misc, Flags")',
|
||
metavar='name', default=['Misc', 'Flags'], nargs='*')
|
||
parser.add_argument(
|
||
'-v', '--verbose', help='print progress information to stdout',
|
||
action='store_true')
|
||
args = parser.parse_args()
|
||
generate_names(
|
||
args.srcdir, args.dstdir, args.missing_limit, args.omit_groups,
|
||
pretty_print=args.pretty_print, verbose=args.verbose)
|
||
|
||
|
||
if __name__ == "__main__":
|
||
main()
|