From 0f227e78349a5cdd7122e4d1aeb15912edf9ba6f Mon Sep 17 00:00:00 2001 From: Doug Felt Date: Tue, 3 Nov 2015 18:40:19 -0800 Subject: [PATCH 1/8] wip - update emoji tooling to support tr51 sequences --- Makefile | 3 +- NotoColorEmoji.tmpl.ttx.tmpl | 11 +++ build_emoji_set.py | 73 ++++++++++++++ generate_emoji_placeholders.py | 95 ++++++++++++++++++ third_party/color_emoji/add_glyphs.py | 120 +++++++++++++++++++---- third_party/color_emoji/emoji_builder.py | 32 ++++-- 6 files changed, 306 insertions(+), 28 deletions(-) create mode 100644 build_emoji_set.py create mode 100644 generate_emoji_placeholders.py diff --git a/Makefile b/Makefile index 9217cc4e2..74cd544c2 100644 --- a/Makefile +++ b/Makefile @@ -78,7 +78,8 @@ flag-symlinks: $(WAVED_FLAGS) $(PNG128_FLAGS): flag-symlinks -EMOJI_PNG128 = ./png/128/emoji_u +#EMOJI_PNG128 = ./png/128/emoji_u +EMOJI_PNG128 = /tmp/placeholder_emoji_plus/emoji_u EMOJI_BUILDER = third_party/color_emoji/emoji_builder.py ADD_GLYPHS = third_party/color_emoji/add_glyphs.py diff --git a/NotoColorEmoji.tmpl.ttx.tmpl b/NotoColorEmoji.tmpl.ttx.tmpl index 64461de00..f982d063c 100644 --- a/NotoColorEmoji.tmpl.ttx.tmpl +++ b/NotoColorEmoji.tmpl.ttx.tmpl @@ -4,6 +4,10 @@ + + + + @@ -119,12 +123,19 @@ + + + + + + + diff --git a/build_emoji_set.py b/build_emoji_set.py new file mode 100644 index 000000000..3f0708bfe --- /dev/null +++ b/build_emoji_set.py @@ -0,0 +1,73 @@ +# delete dst, then: +# copy the placeholders to dst +# then copy the noto images to dst +# then copy the draft images to dst, skipping names with parens and +# after fixing the case of the names + +import glob +import os +from os import path +import re +import shutil + +DST = "/tmp/placeholder_emoji_plus" + +SRC_PLACEHOLDER = "/tmp/placeholder_emoji" +SRC_NOTO = "/usr/local/google/users/dougfelt/newnoto/noto-emoji/png/128" +SRC_DRAFT = "/usr/local/google/home/dougfelt/Downloads/PNG_latest_working_draft" + +# First, scan the draft images and select which ones to use. This does +# two things: +# - The download package returns all the images, including previous versions. +# Ensure we use the one with the highest version. +# - The names often mix case. Make sure we have all lower case names. +# +# If something seems amiss, we fail. + +UPDATED_NAMES = {} +FIXED_NAMES = {} +VAR_PAT = re.compile(r'(.*?)\((\d+)\)\.png') +for fname in glob.glob(path.join(SRC_DRAFT, '*.png')): + name = path.basename(fname) + m = VAR_PAT.match(name) + if m: + name = '%s.png' % m.group(1).lower() + version = int(m.group(2)) + if version > UPDATED_NAMES.get(name, (0, None))[0]: + print 'update %s to version %d' % (name, version) + UPDATED_NAMES[name] = (version, fname) + else: + name = name.lower() + FIXED_NAMES[name] = fname + +for name in UPDATED_NAMES: + if name not in FIXED_NAMES: + raise Exception('updated name %s not in names' % name) + fname = UPDATED_NAMES[name][1] + print 'using updated image %s for %s' % (fname, name) + FIXED_NAMES[name] = fname + +# Now, recreate the destination directory and copy the data into it. + +if path.isdir(DST): + shutil.rmtree(DST) +os.makedirs(DST) + +SKIP_PLACEHOLDERS = frozenset([ + 'emoji_u1f468_200d_1f469_200d_1f466.png', + 'emoji_u1f469_200d_2764_fe0f_200d_1f468.png', + 'emoji_u1f469_200d_2764_fe0f_200d_1f48b_200d_1f468.png', +]) + +for fname in glob.glob(path.join(SRC_PLACEHOLDER, '*.png')): + basename = path.basename(fname) + if basename in SKIP_PLACEHOLDERS: + print 'skip %s' % basename + continue + shutil.copy(fname, DST) + +for fname in glob.glob(path.join(SRC_NOTO, '*.png')): + shutil.copy(fname, DST) + +for name, fname in FIXED_NAMES.iteritems(): + shutil.copy(fname, path.join(DST, name)) diff --git a/generate_emoji_placeholders.py b/generate_emoji_placeholders.py new file mode 100644 index 000000000..48b508231 --- /dev/null +++ b/generate_emoji_placeholders.py @@ -0,0 +1,95 @@ +import os +from os import path +import subprocess + +OUTPUT_DIR = '/tmp/placeholder_emoji' + +def generate_image(name, text): + print name, text.replace('\n', '_') + subprocess.check_call( + ['convert', '-size', '100x100', 'label:%s' % text, + '%s/%s' % (OUTPUT_DIR, name)]) + +def is_color_patch(cp): + return cp >= 0x1f3fb and cp <= 0x1f3ff + +def has_color_patch(values): + for v in values: + if is_color_patch(v): + return True + return False + +def regional_to_ascii(cp): + return unichr(ord('A') + cp - 0x1f1e6) + +def is_flag_sequence(values): + if len(values) != 2: + return False + for v in values: + v -= 0x1f1e6 + if v < 0 or v > 25: + return False + return True + +def is_keycap_sequence(values): + return len(values) == 2 and values[1] == 0x20e3 + +def get_keycap_text(values): + return '-%c-' % unichr(values[0]) # convert gags on '[' + +char_map = { + 0x1f468: 'M', + 0x1f469: 'W', + 0x1f466: 'B', + 0x1f467: 'G', + 0x2764: 'H', # heavy black heart, no var sel + 0x1f48b: 'K', # kiss mark + 0x200D: '-', # zwj placeholder + 0xfe0f: '-', # variation selector placeholder + 0x1f441: 'I', # Eye + 0x1f5e8: 'W', # 'witness' (left speech bubble) +} + +def get_combining_text(values): + chars = [] + for v in values: + char = char_map.get(v, None) + if not char: + return None + if char != '-': + chars.append(char) + return ''.join(chars) + + +if not path.isdir(OUTPUT_DIR): + os.makedirs(OUTPUT_DIR) + +with open('sequences.txt', 'r') as f: + for seq in f: + seq = seq.strip() + text = None + values = [int(code, 16) for code in seq.split('_')] + if len(values) == 1: + val = values[0] + text = '%04X' % val # ensure upper case format + elif is_flag_sequence(values): + text = ''.join(regional_to_ascii(cp) for cp in values) + elif has_color_patch(values): + print 'skipping color patch sequence %s' % seq + elif is_keycap_sequence(values): + text = get_keycap_text(values) + else: + text = get_combining_text(values) + if not text: + print 'missing %s' % seq + + if text: + if len(text) > 3: + if len(text) == 4: + hi = text[:2] + lo = text[2:] + else: + hi = text[:-3] + lo = text[-3:] + text = '%s\n%s' % (hi, lo) + generate_image('emoji_u%s.png' % seq, text) diff --git a/third_party/color_emoji/add_glyphs.py b/third_party/color_emoji/add_glyphs.py index a8d986547..bea880064 100644 --- a/third_party/color_emoji/add_glyphs.py +++ b/third_party/color_emoji/add_glyphs.py @@ -1,6 +1,6 @@ #!/usr/bin/env python -import glob, os, sys +import collections, glob, os, sys from fontTools import ttx from fontTools.ttLib.tables import otTables from png import PNG @@ -10,11 +10,31 @@ sys.path.append( import add_emoji_gsub +def is_vs(cp): + return cp >= 0xfe00 and cp <= 0xfe0f + +def codes_to_string(codes): + if "_" in codes: + pieces = codes.split ("_") + string = "".join ([unichr (int (code, 16)) for code in pieces]) + else: + string = unichr (int (codes, 16)) + return string + + +def glyph_sequence(string): + # sequence of names of glyphs that form a ligature + # variation selectors are stripped + return ["u%04X" % ord(char) for char in string if not is_vs(ord(char))] + + def glyph_name(string): + # name of a ligature + # includes variation selectors when present return "_".join (["u%04X" % ord (char) for char in string]) -def add_ligature (font, string): +def add_ligature (font, seq, name): if 'GSUB' not in font: ligature_subst = otTables.LigatureSubst() ligature_subst.ligatures = {} @@ -34,17 +54,27 @@ def add_ligature (font, string): ligatures = lookup.SubTable[0].ligatures lig = otTables.Ligature() - lig.CompCount = len(string) - lig.Component = [glyph_name(ch) for ch in string[1:]] - lig.LigGlyph = glyph_name(string) + lig.CompCount = len(seq) + lig.Component = seq[1:] + lig.LigGlyph = name - first = glyph_name(string[0]) + first = seq[0] try: ligatures[first].append(lig) except KeyError: ligatures[first] = [lig] +# Ligating sequences for emoji that already have a defined codepoint, +# to match the sequences for the related emoji with no codepoint. +# The key is the name of the glyph with the codepoint, the value is the +# name of the sequence in filename form. +EXTRA_SEQUENCES = { + 'u1F46A': '1F468_200D_1F469_200D_1F466', # MWB + 'u1F491': '1F469_200D_2764_FE0F_200D_1F468', # WHM + 'u1F48F': '1F469_200D_2764_FE0F_200D_1F48B_200D_1F468', # WHKM +} + if len (sys.argv) < 4: print >>sys.stderr, """ Usage: @@ -65,23 +95,22 @@ table and the first GSUB lookup (if existing) are modified. in_file = sys.argv[1] out_file = sys.argv[2] -img_prefix = sys.argv[3] +img_prefixen = sys.argv[3:] del sys.argv font = ttx.TTFont() font.importXML (in_file) img_files = {} -glb = "%s*.png" % img_prefix -print "Looking for images matching '%s'." % glb -for img_file in glob.glob (glb): - codes = img_file[len (img_prefix):-4] - if "_" in codes: - pieces = codes.split ("_") - u = "".join ([unichr (int (code, 16)) for code in pieces]) - else: - u = unichr (int (codes, 16)) - img_files[u] = img_file +for img_prefix in img_prefixen: + glb = "%s*.png" % img_prefix + print "Looking for images matching '%s'." % glb + for img_file in glob.glob (glb): + codes = img_file[len (img_prefix):-4] + u = codes_to_string(codes) + if u in img_files: + print 'overwriting %s with %s' % (img_files[u], imag_file) + img_files[u] = img_file if not img_files: raise Exception ("No image files found in '%s'." % glb) @@ -98,20 +127,71 @@ h = font['hmtx'].metrics img_pairs = img_files.items () img_pairs.sort (key=lambda pair: (len (pair[0]), pair[0])) +glyph_names = set() +ligatures = {} + +def add_lig_sequence(ligatures, seq, n): + # Assume sequences with ZWJ are emoji 'ligatures' and rtl order + # is also valid. Internal permutations, though, no. + # We associate a sequence with a filename. We can overwrite the + # sequence with a different filename later. + tseq = tuple(seq) + if tseq in ligatures: + print 'lig sequence %s, replace %s with %s' % ( + tseq, ligatures[tseq], n) + ligatures[tseq] = n + if 'u200D' in seq: + rev_seq = seq[:] + rev_seq.reverse() + trseq = tuple(rev_seq) + if trseq in ligatures: + print 'rev lig sequence %s, replace %s with %s' % ( + trseq, ligatures[trseq], n) + ligatures[trseq] = n + + for (u, filename) in img_pairs: print "Adding glyph for U+%s" % ",".join (["%04X" % ord (char) for char in u]) n = glyph_name (u) + glyph_names.add(n) + g.append (n) for char in u: - if char not in c: + cp = ord(char) + if cp not in c and not is_vs(cp): name = glyph_name (char) - c[ord (char)] = name + c[cp] = name if len (u) > 1: h[name] = [0, 0] (img_width, img_height) = PNG (filename).get_size () advance = int (round ((float (ascent+descent) * img_width / img_height))) h[n] = [advance, 0] if len (u) > 1: - add_ligature (font, u) + seq = glyph_sequence(u) + add_lig_sequence(ligatures, seq, n) + +for n in EXTRA_SEQUENCES: + if n in glyph_names: + seq = glyph_sequence(codes_to_string(EXTRA_SEQUENCES[n])) + add_lig_sequence(ligatures, seq, n) + else: + print 'extras: no glyph for %s' % n + + +keyed_ligatures = collections.defaultdict(list) +for k, v in ligatures.iteritems(): + first = k[0] + keyed_ligatures[first].append((k, v)) + +for base in sorted(keyed_ligatures): + pairs = keyed_ligatures[base] + print 'base %s has %d sequences' % (base, len(pairs)) + # Sort longest first, this ensures longer sequences with common prefixes + # are handled before shorter ones. It would be better to have multiple + # lookups, most likely. + pairs.sort(key = lambda pair: (len(pair[0]), pair[0]), reverse=True) + for seq, name in pairs: + print seq, name + add_ligature(font, seq, name) font.saveXML (out_file) diff --git a/third_party/color_emoji/emoji_builder.py b/third_party/color_emoji/emoji_builder.py index 5a4e646fe..844102ab5 100644 --- a/third_party/color_emoji/emoji_builder.py +++ b/third_party/color_emoji/emoji_builder.py @@ -20,7 +20,8 @@ import sys, struct, StringIO from png import PNG - +import os +from os import path def get_glyph_name_from_gsub (string, font, cmap_dict): ligatures = font['GSUB'].table.LookupList.Lookup[0].SubTable[0].ligatures @@ -83,6 +84,7 @@ class CBDT: write_func = self.image_write_func (image_format) for glyph in glyphs: img_file = glyph_filenames[glyph] + print 'writing data for glyph %s' % path.basename(img_file) offset = self.tell () write_func (PNG (img_file)) self.glyph_maps.append (GlyphMap (glyph, offset, image_format)) @@ -108,6 +110,7 @@ class CBDT: line_ascent = ascent * y_ppem / float (upem) y_bearing = int (round (line_ascent - .5 * (line_height - height))) advance = width + print "small glyph metrics h: %d w: %d a: %d" % (height, width, advance) # smallGlyphMetrics # Type Name # BYTE height @@ -115,10 +118,14 @@ class CBDT: # CHAR BearingX # CHAR BearingY # BYTE Advance - self.write (struct.pack ("BBbbB", + try: + self.write (struct.pack ("BBbbB", height, width, x_bearing, y_bearing, advance)) + except: + raise ValueError("h: %d w: %d a: %d x: %d y: 5d" % ( + height, width, advance, x_braring, y_bearing)) def write_format1 (self, png): @@ -437,8 +444,10 @@ By default they are dropped. eblc.write_header () eblc.start_strikes (len (img_prefixes)) - for img_prefix in img_prefixes: + def is_vs(cp): + return cp >= 0xfe00 and cp <= 0xfe0f + for img_prefix in img_prefixes: print img_files = {} @@ -448,9 +457,14 @@ By default they are dropped. codes = img_file[len (img_prefix):-4] if "_" in codes: pieces = codes.split ("_") - uchars = "".join ([unichr (int (code, 16)) for code in pieces]) + cps = [int(code, 16) for code in pieces] + uchars = "".join ([unichr(cp) for cp in cps if not is_vs(cp)]) else: - uchars = unichr (int (codes, 16)) + cp = int(codes, 16) + if is_vs(cp): + print "ignoring unexpected vs input %04x" % cp + continue + uchars = unichr(cp) img_files[uchars] = img_file if not img_files: raise Exception ("No image files found in '%s'." % glb) @@ -460,7 +474,11 @@ By default they are dropped. advance = width = height = 0 for uchars, img_file in img_files.items (): if len (uchars) == 1: - glyph_name = unicode_cmap.cmap[ord (uchars)] + try: + glyph_name = unicode_cmap.cmap[ord (uchars)] + except: + print "no cmap entry for %x" % ord(uchars) + raise ValueError("%x" % ord(uchars)) else: glyph_name = get_glyph_name_from_gsub (uchars, font, unicode_cmap.cmap) glyph_id = font.getGlyphID (glyph_name) @@ -476,7 +494,7 @@ By default they are dropped. glyphs = sorted (glyph_imgs.keys ()) if not glyphs: - raise Exception ("No common characteres found between font and '%s'." % glb) + raise Exception ("No common characters found between font and '%s'." % glb) print "Embedding images for %d glyphs for this strike." % len (glyphs) advance, width, height = (div (x, len (glyphs)) for x in (advance, width, height)) From d9e320acb53d129e3be5776b57b1a8da057c1107 Mon Sep 17 00:00:00 2001 From: Doug Felt Date: Thu, 5 Nov 2015 11:15:15 -0800 Subject: [PATCH 2/8] updates to support tr51 emoji sequences --- Makefile | 2 + build_emoji_set.py | 47 ++++++++++++++++++++---- third_party/color_emoji/emoji_builder.py | 11 ++++-- 3 files changed, 48 insertions(+), 12 deletions(-) diff --git a/Makefile b/Makefile index 74cd544c2..006dd964f 100644 --- a/Makefile +++ b/Makefile @@ -92,6 +92,8 @@ endif %.ttx: %.ttx.tmpl $(ADD_GLYPHS) $(UNI) $(PNG128_FLAGS) python $(ADD_GLYPHS) "$<" "$@" "$(EMOJI_PNG128)" + + %.ttf: %.ttx @rm -f "$@" ttx "$<" diff --git a/build_emoji_set.py b/build_emoji_set.py index 3f0708bfe..dc3d2ff55 100644 --- a/build_emoji_set.py +++ b/build_emoji_set.py @@ -10,11 +10,11 @@ from os import path import re import shutil -DST = "/tmp/placeholder_emoji_plus" +DST = '/tmp/placeholder_emoji_plus' -SRC_PLACEHOLDER = "/tmp/placeholder_emoji" -SRC_NOTO = "/usr/local/google/users/dougfelt/newnoto/noto-emoji/png/128" -SRC_DRAFT = "/usr/local/google/home/dougfelt/Downloads/PNG_latest_working_draft" +SRC_PLACEHOLDER = '/usr/local/google/users/dougfelt/emoji_images/placeholder' +SRC_NOTO = '/usr/local/google/users/dougfelt/newnoto/noto-emoji/png/128' +SRC_DRAFT = '/usr/local/google/users/dougfelt/emoji_images/png_thurs' # First, scan the draft images and select which ones to use. This does # two things: @@ -29,6 +29,10 @@ FIXED_NAMES = {} VAR_PAT = re.compile(r'(.*?)\((\d+)\)\.png') for fname in glob.glob(path.join(SRC_DRAFT, '*.png')): name = path.basename(fname) + if 'alt' in name: + print 'skip %s' % name + continue + m = VAR_PAT.match(name) if m: name = '%s.png' % m.group(1).lower() @@ -47,22 +51,49 @@ for name in UPDATED_NAMES: print 'using updated image %s for %s' % (fname, name) FIXED_NAMES[name] = fname +EXCLUDE_PAT = re.compile(r'emoji_u1f3f[bcdef].png') +remove = [name for name in FIXED_NAMES if EXCLUDE_PAT.match(name)] +for name in remove: + print 'removing %s' % name + del FIXED_NAMES[name] + + # Now, recreate the destination directory and copy the data into it. if path.isdir(DST): shutil.rmtree(DST) os.makedirs(DST) -SKIP_PLACEHOLDERS = frozenset([ +def flag_emoji_name(flag_ascii): + return 'emoji_u%s.png' % '_'.join( + '%04x' % (ord(cp) - ord('A') + 0x1f1e6) for cp in flag_ascii) + +SKIP_FLAGS = [flag_emoji_name(name) for name in [ + 'AC', 'AQ', 'BL', 'BQ', 'BV', 'CP', 'DG', 'EA', 'EH', 'FK', + 'GF', 'GP', 'GS', 'HM', 'IC', 'MF', 'MQ', 'NC', 'PM', 'RE', + 'SH', 'SJ', 'TA', 'TF', 'UM', 'WF', 'XK', 'YT', + ]] + +def emoji_name(val): + return 'emoji_u%04x.png' % val + +SKIP_SWATCHES = [emoji_name(val) for val in range(0x1f3fb, 0x1f3ff + 1)] + +# these are placeholders for emoji that have a single codepoint yet +# also have a decomposition. +# we want to use the image for the single codepoint. +SKIP_SINGLE_CP_EMOJI = [ 'emoji_u1f468_200d_1f469_200d_1f466.png', 'emoji_u1f469_200d_2764_fe0f_200d_1f468.png', 'emoji_u1f469_200d_2764_fe0f_200d_1f48b_200d_1f468.png', -]) +] -for fname in glob.glob(path.join(SRC_PLACEHOLDER, '*.png')): +SKIP_PLACEHOLDERS = frozenset(SKIP_FLAGS + SKIP_SWATCHES + SKIP_SINGLE_CP_EMOJI) + +for fname in sorted(glob.glob(path.join(SRC_PLACEHOLDER, '*.png'))): basename = path.basename(fname) if basename in SKIP_PLACEHOLDERS: - print 'skip %s' % basename + print 'skip placeholder %s' % basename continue shutil.copy(fname, DST) diff --git a/third_party/color_emoji/emoji_builder.py b/third_party/color_emoji/emoji_builder.py index 844102ab5..0092b0751 100644 --- a/third_party/color_emoji/emoji_builder.py +++ b/third_party/color_emoji/emoji_builder.py @@ -109,8 +109,11 @@ class CBDT: line_height = (ascent + descent) * y_ppem / float (upem) line_ascent = ascent * y_ppem / float (upem) y_bearing = int (round (line_ascent - .5 * (line_height - height))) + # fudge y_bearing if calculations are a bit off + if y_bearing == 128: + y_bearing = 127 advance = width - print "small glyph metrics h: %d w: %d a: %d" % (height, width, advance) + print "small glyph metrics h: %d w: %d" % (height, width) # smallGlyphMetrics # Type Name # BYTE height @@ -123,9 +126,9 @@ class CBDT: height, width, x_bearing, y_bearing, advance)) - except: - raise ValueError("h: %d w: %d a: %d x: %d y: 5d" % ( - height, width, advance, x_braring, y_bearing)) + except Exception as e: + raise ValueError("%s, h: %d w: %d x: %d y: %d %d a:" % ( + e, height, width, x_bearing, y_bearing, advance)) def write_format1 (self, png): From 927b066b3a8fee4e8eea9829bf2c2932aaaeec1b Mon Sep 17 00:00:00 2001 From: Doug Felt Date: Fri, 13 Nov 2015 17:32:41 -0800 Subject: [PATCH 3/8] update template to version 1.23 --- NotoColorEmoji.tmpl.ttx.tmpl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/NotoColorEmoji.tmpl.ttx.tmpl b/NotoColorEmoji.tmpl.ttx.tmpl index f982d063c..9ab095006 100644 --- a/NotoColorEmoji.tmpl.ttx.tmpl +++ b/NotoColorEmoji.tmpl.ttx.tmpl @@ -13,7 +13,7 @@ - + @@ -156,7 +156,7 @@ Noto Color Emoji - Version 1.22 + Version 1.23 NotoColorEmoji From 170bcdad0e8e0e08a564149981aa3c534485acd0 Mon Sep 17 00:00:00 2001 From: Doug Felt Date: Fri, 13 Nov 2015 17:34:15 -0800 Subject: [PATCH 4/8] comment out some print lines used for debugging --- third_party/color_emoji/add_glyphs.py | 14 +++++++++----- third_party/color_emoji/emoji_builder.py | 7 ++++--- 2 files changed, 13 insertions(+), 8 deletions(-) diff --git a/third_party/color_emoji/add_glyphs.py b/third_party/color_emoji/add_glyphs.py index bea880064..14e19682f 100644 --- a/third_party/color_emoji/add_glyphs.py +++ b/third_party/color_emoji/add_glyphs.py @@ -18,7 +18,10 @@ def codes_to_string(codes): pieces = codes.split ("_") string = "".join ([unichr (int (code, 16)) for code in pieces]) else: + try: string = unichr (int (codes, 16)) + except: + raise ValueError("uh-oh, no unichr for '%s'" % codes) return string @@ -145,13 +148,13 @@ def add_lig_sequence(ligatures, seq, n): rev_seq.reverse() trseq = tuple(rev_seq) if trseq in ligatures: - print 'rev lig sequence %s, replace %s with %s' % ( - trseq, ligatures[trseq], n) + # print 'rev lig sequence %s, replace %s with %s' % ( + # trseq, ligatures[trseq], n) ligatures[trseq] = n for (u, filename) in img_pairs: - print "Adding glyph for U+%s" % ",".join (["%04X" % ord (char) for char in u]) + # print "Adding glyph for U+%s" % ",".join (["%04X" % ord (char) for char in u]) n = glyph_name (u) glyph_names.add(n) @@ -185,13 +188,14 @@ for k, v in ligatures.iteritems(): for base in sorted(keyed_ligatures): pairs = keyed_ligatures[base] - print 'base %s has %d sequences' % (base, len(pairs)) + # print 'base %s has %d sequences' % (base, len(pairs)) + # Sort longest first, this ensures longer sequences with common prefixes # are handled before shorter ones. It would be better to have multiple # lookups, most likely. pairs.sort(key = lambda pair: (len(pair[0]), pair[0]), reverse=True) for seq, name in pairs: - print seq, name + # print seq, name add_ligature(font, seq, name) font.saveXML (out_file) diff --git a/third_party/color_emoji/emoji_builder.py b/third_party/color_emoji/emoji_builder.py index 0092b0751..fa9f4115c 100644 --- a/third_party/color_emoji/emoji_builder.py +++ b/third_party/color_emoji/emoji_builder.py @@ -84,7 +84,7 @@ class CBDT: write_func = self.image_write_func (image_format) for glyph in glyphs: img_file = glyph_filenames[glyph] - print 'writing data for glyph %s' % path.basename(img_file) + # print 'writing data for glyph %s' % path.basename(img_file) offset = self.tell () write_func (PNG (img_file)) self.glyph_maps.append (GlyphMap (glyph, offset, image_format)) @@ -113,7 +113,7 @@ class CBDT: if y_bearing == 128: y_bearing = 127 advance = width - print "small glyph metrics h: %d w: %d" % (height, width) + # print "small glyph metrics h: %d w: %d" % (height, width) # smallGlyphMetrics # Type Name # BYTE height @@ -488,7 +488,8 @@ By default they are dropped. glyph_imgs[glyph_id] = img_file if "verbose" in options: uchars_name = ",".join (["%04X" % ord (char) for char in uchars]) - print "Matched U+%s: id=%d name=%s image=%s" % (uchars_name, glyph_id, glyph_name, img_file) + # print "Matched U+%s: id=%d name=%s image=%s" % ( + # uchars_name, glyph_id, glyph_name, img_file) advance += glyph_metrics[glyph_name][0] w, h = PNG (img_file).get_size () From 34fbef4348ded964c203410a21477d11f99b35f6 Mon Sep 17 00:00:00 2001 From: Doug Felt Date: Fri, 13 Nov 2015 17:35:50 -0800 Subject: [PATCH 5/8] Rewrite makefile Change the build process to provide more intermediate steps and maintain intermediate targets. This slows down make's analysis phase, though. Make can be run with -j to parallelize the building, but be careful with multiple targets on the command line (e.g. 'make clean font') since clean's deletion of the output tree can sometimes occur after some of the output directories get built. This is preparatory to replacing the png images with the original internal set. These require a bit more cleanup and processing, which is what these changes do. The overall flow is: - convert all emoji to desired size (136x128). All are smaller than this, most are 128x128 but there are some others. - run waveflag on the flags - convert all flags to desired size(136x128). The flags are 128x128 originally. - 'rename' the flags by creating softlinks with the desired names - run pngquant on all the images and copy them into a single directory, this reduces slight antialiasing differences. - compress all the images, using zopflipng if available, else optipng. zopflipng saves about 10%, but is 10x slower. The remaining steps of building the font are unchanged, for the moment. The intermediate image files are put into a 'build' subdirectory to organize them and keep them out of the way. --- Makefile | 174 ++++++++++++++++++++++++++++++++++++++++++------------- 1 file changed, 134 insertions(+), 40 deletions(-) diff --git a/Makefile b/Makefile index 006dd964f..dceab47a6 100644 --- a/Makefile +++ b/Makefile @@ -12,7 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. - EMOJI = NotoColorEmoji font: $(EMOJI).ttf @@ -20,16 +19,33 @@ CFLAGS = -std=c99 -Wall -Wextra `pkg-config --cflags --libs cairo` LDFLAGS = `pkg-config --libs cairo` PNGQUANTDIR := third_party/pngquant PNGQUANT := $(PNGQUANTDIR)/pngquant -PNGQUANTFLAGS = --speed 1 --skip-if-larger --ext '.png' --force +PNGQUANTFLAGS = --speed 1 --skip-if-larger --force -$(PNGQUANT): - $(MAKE) -C $(PNGQUANTDIR) +# zopflipng is better (about 10%) but much slower. it will be used if +# present. pass ZOPFLIPNG= as an arg to make to use optipng instead. -waveflag: waveflag.c - $(CC) $< -o $@ $(CFLAGS) $(LDFLAGS) +ZOPFLIPNG = zopflipng +OPTIPNG = optipng + +EMOJI_BUILDER = third_party/color_emoji/emoji_builder.py +ADD_GLYPHS = third_party/color_emoji/add_glyphs.py +PUA_ADDER = map_pua_emoji.py +VS_ADDER = add_vs_cmap.py # from nototools + +#EMOJI_SRC_DIR := png/128 +EMOJI_SRC_DIR := /usr/local/google/users/dougfelt/emoji_google/noto-emoji-source/png +FLAGS_SRC_DIR := third_party/region-flags/png + +BUILD_DIR := build +EMOJI_DIR := $(BUILD_DIR)/emoji +FLAGS_DIR := $(BUILD_DIR)/flags +RESIZED_FLAGS_DIR := $(BUILD_DIR)/resized_flags +RENAMED_FLAGS_DIR := $(BUILD_DIR)/renamed_flags +QUANTIZED_DIR := $(BUILD_DIR)/quantized_pngs +COMPRESSED_DIR := $(BUILD_DIR)/compressed_pngs LIMITED_FLAGS = CN DE ES FR GB IT JP KR RU US -FLAGS = AD AE AF AG AI AL AM AO AR AS AT AU AW AX AZ \ +SELECTED_FLAGS = AD AE AF AG AI AL AM AO AR AS AT AU AW AX AZ \ BA BB BD BE BF BG BH BI BJ BM BN BO BR BS BT BW BY BZ \ CA CC CD CF CG CH CI CK CL CM CN CO CR CU CV CW CX CY CZ \ DE DJ DK DM DO DZ \ @@ -54,54 +70,127 @@ FLAGS = AD AE AF AG AI AL AM AO AR AS AT AU AW AX AZ \ WS \ YE \ ZA ZM ZW +ALL_FLAGS = ($basename ($notdir $(wildcard $(FLAGS_SRC_DIR)/*.png))) -FLAGS_SRC_DIR = third_party/region-flags/png -FLAGS_DIR = ./flags +FLAGS = $(SELECTED_FLAGS) -GLYPH_NAMES := $(shell ./flag_glyph_name.py $(FLAGS)) -WAVED_FLAGS := $(foreach flag,$(FLAGS),$(FLAGS_DIR)/$(flag).png) -PNG128_FLAGS := $(foreach glyph_name,$(GLYPH_NAMES),$(addprefix ./png/128/emoji_$(glyph_name),.png)) +FLAG_NAMES = $(FLAGS:%=%.png) +FLAG_FILES = $(addprefix $(FLAGS_DIR)/, $(FLAG_NAMES)) +RESIZED_FLAG_FILES = $(addprefix $(RESIZED_FLAGS_DIR)/, $(FLAG_NAMES)) -$(FLAGS_DIR)/%.png: $(FLAGS_SRC_DIR)/%.png ./waveflag $(PNGQUANT) - mkdir -p $(FLAGS_DIR) - ./waveflag "$<" "$@" - optipng -quiet -o7 "$@" - $(PNGQUANT) $(PNGQUANTFLAGS) "$@" +FLAG_GLYPH_NAMES = $(shell ./flag_glyph_name.py $(FLAGS)) +RENAMED_FLAG_NAMES = $(FLAG_GLYPH_NAMES:%=emoji_%.png) +RENAMED_FLAG_FILES = $(addprefix $(RENAMED_FLAGS_DIR)/, $(RENAMED_FLAG_NAMES)) -flag-symlinks: $(WAVED_FLAGS) - $(subst ^, , \ - $(join \ - $(FLAGS:%=ln^-fs^../../flags/%.png^), \ - $(GLYPH_NAMES:%=./png/128/emoji_%.png;) \ - ) \ - ) +EMOJI_NAMES = $(notdir $(wildcard $(EMOJI_SRC_DIR)/emoji_u*.png)) +EMOJI_FILES= $(addprefix $(EMOJI_DIR)/,$(EMOJI_NAMES))) -$(PNG128_FLAGS): flag-symlinks +ALL_NAMES = $(EMOJI_NAMES) $(RENAMED_FLAG_NAMES) -#EMOJI_PNG128 = ./png/128/emoji_u -EMOJI_PNG128 = /tmp/placeholder_emoji_plus/emoji_u +ALL_QUANTIZED_FILES = $(addprefix $(QUANTIZED_DIR)/, $(ALL_NAMES)) +ALL_COMPRESSED_FILES = $(addprefix $(COMPRESSED_DIR)/, $(ALL_NAMES)) -EMOJI_BUILDER = third_party/color_emoji/emoji_builder.py -ADD_GLYPHS = third_party/color_emoji/add_glyphs.py -PUA_ADDER = map_pua_emoji.py -VS_ADDER = add_vs_cmap.py -ifeq (, $(shell which $(VS_ADDER))) - $(error "$(VS_ADDER) not in path, run setup.py in nototools") +# tool checks +ifeq (,$(shell which $(ZOPFLIPNG))) + ifeq (,$(wildcard $(ZOPFLIPNG))) + MISSING_ZOPFLI = fail + endif endif -%.ttx: %.ttx.tmpl $(ADD_GLYPHS) $(UNI) $(PNG128_FLAGS) - python $(ADD_GLYPHS) "$<" "$@" "$(EMOJI_PNG128)" +ifeq (,$(shell which $(OPTIPNG))) + ifeq (,$(wildcard $(OPTIPNG))) + MISSING_OPTIPNG = fail + endif +endif + +ifeq (, $(shell which $(VS_ADDER))) + MISSING_ADDER = fail +endif +emoji: $(EMOJI_FILES) + +flags: $(FLAG_FILES) + +resized_flags: $(RESIZED_FLAG_FILES) + +renamed_flags: $(RENAMED_FLAG_FILES) + +quantized: $(ALL_QUANTIZED_FILES) + +compressed: $(ALL_COMPRESSED_FILES) + +check_compress_tool: +ifdef MISSING_ZOPFLI + ifdef MISSING_OPTIPNG + $(error "neither $(ZOPFLIPNG) nor $(OPTIPNG) is available") + else + @echo "using $(OPTIPNG)" + endif +else + @echo "using $(ZOPFLIPNG)" +endif + +check_vs_adder: +ifdef MISSING_ADDER + $(error "$(VS_ADDER) not in path, run setup.py in nototools") +endif + + +$(EMOJI_DIR) $(FLAGS_DIR) $(RESIZED_FLAGS_DIR) $(RENAMED_FLAGS_DIR) $(QUANTIZED_DIR) $(COMPRESSED_DIR): + mkdir -p "$@" + +$(PNGQUANT): + $(MAKE) -C $(PNGQUANTDIR) + +waveflag: waveflag.c + $(CC) $< -o $@ $(CFLAGS) $(LDFLAGS) + +$(EMOJI_DIR)/%.png: $(EMOJI_SRC_DIR)/%.png | $(EMOJI_DIR) + echo "emoji $< $@" + @convert -extent 136x128 -gravity center -background none "$<" "$@" + +$(FLAGS_DIR)/%.png: $(FLAGS_SRC_DIR)/%.png ./waveflag $(PNGQUANT) | $(FLAGS_DIR) + @./waveflag "$<" "$@" + +$(RESIZED_FLAGS_DIR)/%.png: $(FLAGS_DIR)/%.png | $(RESIZED_FLAGS_DIR) + @convert -extent 136x128 -gravity center -background none "$<" "$@" + +flag-symlinks: $(RESIZED_FLAG_FILES) | $(RENAMED_FLAGS_DIR) + @$(subst ^, , \ + $(join \ + $(FLAGS:%=ln^-fs^../resized_flags/%.png^), \ + $(RENAMED_FLAG_FILES:%=%; ) \ + ) \ + ) + +$(RENAMED_FLAG_FILES): flag-symlinks + +$(QUANTIZED_DIR)/%.png: $(RENAMED_FLAGS_DIR)/%.png $(PNGQUANT) | $(QUANTIZED_DIR) + $(PNGQUANT) $(PNGQUANTFLAGS) -o "$@" "$<" + +$(QUANTIZED_DIR)/%.png: $(EMOJI_DIR)/%.png $(PNGQUANT) | $(QUANTIZED_DIR) + $(PNGQUANT) $(PNGQUANTFLAGS) -o "$@" "$<" + +$(COMPRESSED_DIR)/%.png: $(QUANTIZED_DIR)/%.png | check_compress_tool $(COMPRESSED_DIR) +ifdef MISSING_ZOPFLI + $(OPTIPNG) -quiet -o7 -force -out "$@" "$<" +else + $(ZOPFLIPNG) -y "$<" "$@" 2> /dev/null +endif + + +%.ttx: %.ttx.tmpl $(ADD_GLYPHS) $(ALL_COMPRESSED_FILES) + @python $(ADD_GLYPHS) "$<" "$@" "$(COMPRESSED_DIR)/emoji_u" %.ttf: %.ttx @rm -f "$@" ttx "$<" $(EMOJI).ttf: $(EMOJI).tmpl.ttf $(EMOJI_BUILDER) $(PUA_ADDER) \ - $(EMOJI_PNG128)*.png $(PNG128_FLAGS) - python $(EMOJI_BUILDER) -V $< "$@" $(EMOJI_PNG128) - python $(PUA_ADDER) "$@" "$@-with-pua" + $(ALL_COMPRESSED_FILES) | check_vs_adder + @python $(EMOJI_BUILDER) -V $< "$@" "$(COMPRESSED_DIR)/emoji_u" + @python $(PUA_ADDER) "$@" "$@-with-pua" $(VS_ADDER) --dstdir '.' -o "$@-with-pua-varsel" "$@-with-pua" mv "$@-with-pua-varsel" "$@" rm "$@-with-pua" @@ -109,5 +198,10 @@ $(EMOJI).ttf: $(EMOJI).tmpl.ttf $(EMOJI_BUILDER) $(PUA_ADDER) \ clean: rm -f $(EMOJI).ttf $(EMOJI).tmpl.ttf $(EMOJI).tmpl.ttx rm -f waveflag - rm -rf $(FLAGS_DIR) - rm -f `find -type l -name "*.png"` + rm -rf $(BUILD_DIR) + +.SECONDARY: $(EMOJI_FILES) $(FLAG_FILES) $(RESIZED_FLAG_FILES) $(RENAMED_FLAG_FILES) \ + $(ALL_QUANTIZED_FILES) $(ALL_COMPRESSED_FILES) + +.PHONY: clean flags emoji renamed_flags quantized compressed check_compress_tool + From 8f39884d6a5cab6f7885ae72c7ccb9d281c3a370 Mon Sep 17 00:00:00 2001 From: Doug Felt Date: Fri, 13 Nov 2015 17:51:16 -0800 Subject: [PATCH 6/8] remove temporary tool build_emoji_set --- build_emoji_set.py | 104 --------------------------------------------- 1 file changed, 104 deletions(-) delete mode 100644 build_emoji_set.py diff --git a/build_emoji_set.py b/build_emoji_set.py deleted file mode 100644 index dc3d2ff55..000000000 --- a/build_emoji_set.py +++ /dev/null @@ -1,104 +0,0 @@ -# delete dst, then: -# copy the placeholders to dst -# then copy the noto images to dst -# then copy the draft images to dst, skipping names with parens and -# after fixing the case of the names - -import glob -import os -from os import path -import re -import shutil - -DST = '/tmp/placeholder_emoji_plus' - -SRC_PLACEHOLDER = '/usr/local/google/users/dougfelt/emoji_images/placeholder' -SRC_NOTO = '/usr/local/google/users/dougfelt/newnoto/noto-emoji/png/128' -SRC_DRAFT = '/usr/local/google/users/dougfelt/emoji_images/png_thurs' - -# First, scan the draft images and select which ones to use. This does -# two things: -# - The download package returns all the images, including previous versions. -# Ensure we use the one with the highest version. -# - The names often mix case. Make sure we have all lower case names. -# -# If something seems amiss, we fail. - -UPDATED_NAMES = {} -FIXED_NAMES = {} -VAR_PAT = re.compile(r'(.*?)\((\d+)\)\.png') -for fname in glob.glob(path.join(SRC_DRAFT, '*.png')): - name = path.basename(fname) - if 'alt' in name: - print 'skip %s' % name - continue - - m = VAR_PAT.match(name) - if m: - name = '%s.png' % m.group(1).lower() - version = int(m.group(2)) - if version > UPDATED_NAMES.get(name, (0, None))[0]: - print 'update %s to version %d' % (name, version) - UPDATED_NAMES[name] = (version, fname) - else: - name = name.lower() - FIXED_NAMES[name] = fname - -for name in UPDATED_NAMES: - if name not in FIXED_NAMES: - raise Exception('updated name %s not in names' % name) - fname = UPDATED_NAMES[name][1] - print 'using updated image %s for %s' % (fname, name) - FIXED_NAMES[name] = fname - -EXCLUDE_PAT = re.compile(r'emoji_u1f3f[bcdef].png') -remove = [name for name in FIXED_NAMES if EXCLUDE_PAT.match(name)] -for name in remove: - print 'removing %s' % name - del FIXED_NAMES[name] - - -# Now, recreate the destination directory and copy the data into it. - -if path.isdir(DST): - shutil.rmtree(DST) -os.makedirs(DST) - -def flag_emoji_name(flag_ascii): - return 'emoji_u%s.png' % '_'.join( - '%04x' % (ord(cp) - ord('A') + 0x1f1e6) for cp in flag_ascii) - -SKIP_FLAGS = [flag_emoji_name(name) for name in [ - 'AC', 'AQ', 'BL', 'BQ', 'BV', 'CP', 'DG', 'EA', 'EH', 'FK', - 'GF', 'GP', 'GS', 'HM', 'IC', 'MF', 'MQ', 'NC', 'PM', 'RE', - 'SH', 'SJ', 'TA', 'TF', 'UM', 'WF', 'XK', 'YT', - ]] - -def emoji_name(val): - return 'emoji_u%04x.png' % val - -SKIP_SWATCHES = [emoji_name(val) for val in range(0x1f3fb, 0x1f3ff + 1)] - -# these are placeholders for emoji that have a single codepoint yet -# also have a decomposition. -# we want to use the image for the single codepoint. -SKIP_SINGLE_CP_EMOJI = [ - 'emoji_u1f468_200d_1f469_200d_1f466.png', - 'emoji_u1f469_200d_2764_fe0f_200d_1f468.png', - 'emoji_u1f469_200d_2764_fe0f_200d_1f48b_200d_1f468.png', -] - -SKIP_PLACEHOLDERS = frozenset(SKIP_FLAGS + SKIP_SWATCHES + SKIP_SINGLE_CP_EMOJI) - -for fname in sorted(glob.glob(path.join(SRC_PLACEHOLDER, '*.png'))): - basename = path.basename(fname) - if basename in SKIP_PLACEHOLDERS: - print 'skip placeholder %s' % basename - continue - shutil.copy(fname, DST) - -for fname in glob.glob(path.join(SRC_NOTO, '*.png')): - shutil.copy(fname, DST) - -for name, fname in FIXED_NAMES.iteritems(): - shutil.copy(fname, path.join(DST, name)) From ed31eb2a9f67aaaf346781c274d2e6170727c11b Mon Sep 17 00:00:00 2001 From: Doug Felt Date: Fri, 13 Nov 2015 17:57:17 -0800 Subject: [PATCH 7/8] fix emoji source path --- Makefile | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Makefile b/Makefile index dceab47a6..596d6e2c0 100644 --- a/Makefile +++ b/Makefile @@ -32,8 +32,7 @@ ADD_GLYPHS = third_party/color_emoji/add_glyphs.py PUA_ADDER = map_pua_emoji.py VS_ADDER = add_vs_cmap.py # from nototools -#EMOJI_SRC_DIR := png/128 -EMOJI_SRC_DIR := /usr/local/google/users/dougfelt/emoji_google/noto-emoji-source/png +EMOJI_SRC_DIR := png/128 FLAGS_SRC_DIR := third_party/region-flags/png BUILD_DIR := build From c67a8c3fa6f46736a68111d0f18891d9abe406a3 Mon Sep 17 00:00:00 2001 From: Doug Felt Date: Fri, 13 Nov 2015 18:15:09 -0800 Subject: [PATCH 8/8] small fixes --- Makefile | 4 ++-- third_party/color_emoji/add_glyphs.py | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Makefile b/Makefile index 596d6e2c0..4773371fe 100644 --- a/Makefile +++ b/Makefile @@ -69,7 +69,7 @@ SELECTED_FLAGS = AD AE AF AG AI AL AM AO AR AS AT AU AW AX AZ \ WS \ YE \ ZA ZM ZW -ALL_FLAGS = ($basename ($notdir $(wildcard $(FLAGS_SRC_DIR)/*.png))) +ALL_FLAGS = $(basename $(notdir $(wildcard $(FLAGS_SRC_DIR)/*.png))) FLAGS = $(SELECTED_FLAGS) @@ -173,7 +173,7 @@ $(QUANTIZED_DIR)/%.png: $(EMOJI_DIR)/%.png $(PNGQUANT) | $(QUANTIZED_DIR) $(COMPRESSED_DIR)/%.png: $(QUANTIZED_DIR)/%.png | check_compress_tool $(COMPRESSED_DIR) ifdef MISSING_ZOPFLI - $(OPTIPNG) -quiet -o7 -force -out "$@" "$<" + $(OPTIPNG) -quiet -o7 -clobber -force -out "$@" "$<" else $(ZOPFLIPNG) -y "$<" "$@" 2> /dev/null endif diff --git a/third_party/color_emoji/add_glyphs.py b/third_party/color_emoji/add_glyphs.py index 14e19682f..26f99b4e9 100644 --- a/third_party/color_emoji/add_glyphs.py +++ b/third_party/color_emoji/add_glyphs.py @@ -147,9 +147,9 @@ def add_lig_sequence(ligatures, seq, n): rev_seq = seq[:] rev_seq.reverse() trseq = tuple(rev_seq) - if trseq in ligatures: - # print 'rev lig sequence %s, replace %s with %s' % ( - # trseq, ligatures[trseq], n) + # if trseq in ligatures: + # print 'rev lig sequence %s, replace %s with %s' % ( + # trseq, ligatures[trseq], n) ligatures[trseq] = n