Support 'ok', 'warning', 'error' annotations and define some.

Formerly the annotations file created a set of sequences that would
cause the name field to display with a special background color.  This
lets you choose one of three colors by defining the 'type' of annotation
in the file.  The file format was enhanced and the code using it takes
the type of annotation into account.

This also adds a sample annotation file with annotations for a number of
situations we currently expect to encounter: missing images that we expect
to be supported by aliases to other images, flags that we expect to not
support, and new unicode 10 emoji that we might not yet have image data
for.

emoji_annotations.txt

@@ -0,0 +1,458 @@
+# annotations
+###
+### aliases
+###
+annotation: ok
+1f3c3 # RUNNER -> man running
+1f3c3 1f3fb # light skin tone
+1f3c3 1f3fc # medium-light skin tone
+1f3c3 1f3fd # medium skin tone
+1f3c3 1f3fe # medium-dark skin tone
+1f3c3 1f3ff # dark skin tone
+1f3c4 # SURFER -> man surfing
+1f3c4 1f3fb # light skin tone
+1f3c4 1f3fc # medium-light skin tone
+1f3c4 1f3fd # medium skin tone
+1f3c4 1f3fe # medium-dark skin tone
+1f3c4 1f3ff # dark skin tone
+1f3ca # SWIMMER -> man swimming
+1f3ca 1f3fb # light skin tone
+1f3ca 1f3fc # medium-light skin tone
+1f3ca 1f3fd # medium skin tone
+1f3ca 1f3fe # medium-dark skin tone
+1f3ca 1f3ff # dark skin tone
+1f3cb # WEIGHT LIFTER -> man lifting weights
+1f3cb 1f3fb # light skin tone
+1f3cb 1f3fc # medium-light skin tone
+1f3cb 1f3fd # medium skin tone
+1f3cb 1f3fe # medium-dark skin tone
+1f3cb 1f3ff # dark skin tone
+1f3cc # GOLFER -> man golfing
+1f3cc 1f3fb # light skin tone
+1f3cc 1f3fc # medium-light skin tone
+1f3cc 1f3fd # medium skin tone
+1f3cc 1f3fe # medium-dark skin tone
+1f3cc 1f3ff # dark skin tone
+1f46a # FAMILY -> family: man, woman, boy
+1f46e # POLICE OFFICER -> man police officer
+1f46e 1f3fb # light skin tone
+1f46e 1f3fc # medium-light skin tone
+1f46e 1f3fd # medium skin tone
+1f46e 1f3fe # medium-dark skin tone
+1f46e 1f3ff # dark skin tone
+1f46f # WOMAN WITH BUNNY EARS -> women with bunny ears partying
+1f471 # PERSON WITH BLOND HAIR -> blond-haired man
+1f471 1f3fb # light skin tone
+1f471 1f3fc # medium-light skin tone
+1f471 1f3fd # medium skin tone
+1f471 1f3fe # medium-dark skin tone
+1f471 1f3ff # dark skin tone
+1f473 # MAN WITH TURBAN -> man wearing turban
+1f473 1f3fb # light skin tone
+1f473 1f3fc # medium-light skin tone
+1f473 1f3fd # medium skin tone
+1f473 1f3fe # medium-dark skin tone
+1f473 1f3ff # dark skin tone
+1f477 # CONSTRUCTION WORKER -> man construction worker
+1f477 1f3fb # light skin tone
+1f477 1f3fc # medium-light skin tone
+1f477 1f3fd # medium skin tone
+1f477 1f3fe # medium-dark skin tone
+1f477 1f3ff # dark skin tone
+1f481 # INFORMATION DESK PERSON -> woman tipping hand
+1f481 1f3fb # light skin tone
+1f481 1f3fc # medium-light skin tone
+1f481 1f3fd # medium skin tone
+1f481 1f3fe # medium-dark skin tone
+1f481 1f3ff # dark skin tone
+1f482 # GUARDSMAN -> man guard
+1f482 1f3fb # light skin tone
+1f482 1f3fc # medium-light skin tone
+1f482 1f3fd # medium skin tone
+1f482 1f3fe # medium-dark skin tone
+1f482 1f3ff # dark skin tone
+1f486 # FACE MASSAGE -> woman getting massage
+1f486 1f3fb # light skin tone
+1f486 1f3fc # medium-light skin tone
+1f486 1f3fd # medium skin tone
+1f486 1f3fe # medium-dark skin tone
+1f486 1f3ff # dark skin tone
+1f487 # HAIRCUT -> woman getting haircut
+1f487 1f3fb # light skin tone
+1f487 1f3fc # medium-light skin tone
+1f487 1f3fd # medium skin tone
+1f487 1f3fe # medium-dark skin tone
+1f487 1f3ff # dark skin tone
+1f48f # KISS -> kiss: woman, man
+1f491 # COUPLE WITH HEART -> couple with heart: woman, man
+1f575 # SLEUTH OR SPY -> man detective
+1f575 1f3fb # light skin tone
+1f575 1f3fc # medium-light skin tone
+1f575 1f3fd # medium skin tone
+1f575 1f3fe # medium-dark skin tone
+1f575 1f3ff # dark skin tone
+1f645 # FACE WITH NO GOOD GESTURE -> woman gesturing NO
+1f645 1f3fb # light skin tone
+1f645 1f3fc # medium-light skin tone
+1f645 1f3fd # medium skin tone
+1f645 1f3fe # medium-dark skin tone
+1f645 1f3ff # dark skin tone
+1f646 # FACE WITH OK GESTURE -> woman gesturing OK
+1f646 1f3fb # light skin tone
+1f646 1f3fc # medium-light skin tone
+1f646 1f3fd # medium skin tone
+1f646 1f3fe # medium-dark skin tone
+1f646 1f3ff # dark skin tone
+1f647 # PERSON BOWING DEEPLY -> man bowing
+1f647 1f3fb # light skin tone
+1f647 1f3fc # medium-light skin tone
+1f647 1f3fd # medium skin tone
+1f647 1f3fe # medium-dark skin tone
+1f647 1f3ff # dark skin tone
+1f64b # HAPPY PERSON RAISING ONE HAND -> woman raising hand
+1f64b 1f3fb # light skin tone
+1f64b 1f3fc # medium-light skin tone
+1f64b 1f3fd # medium skin tone
+1f64b 1f3fe # medium-dark skin tone
+1f64b 1f3ff # dark skin tone
+1f64d # PERSON FROWNING -> woman frowning
+1f64d 1f3fb # light skin tone
+1f64d 1f3fc # medium-light skin tone
+1f64d 1f3fd # medium skin tone
+1f64d 1f3fe # medium-dark skin tone
+1f64d 1f3ff # dark skin tone
+1f64e # PERSON WITH POUTING FACE -> woman pouting
+1f64e 1f3fb # light skin tone
+1f64e 1f3fc # medium-light skin tone
+1f64e 1f3fd # medium skin tone
+1f64e 1f3fe # medium-dark skin tone
+1f64e 1f3ff # dark skin tone
+1f6a3 # ROWBOAT -> man rowing boat
+1f6a3 1f3fb # light skin tone
+1f6a3 1f3fc # medium-light skin tone
+1f6a3 1f3fd # medium skin tone
+1f6a3 1f3fe # medium-dark skin tone
+1f6a3 1f3ff # dark skin tone
+1f6b4 # BICYCLIST -> man biking
+1f6b4 1f3fb # light skin tone
+1f6b4 1f3fc # medium-light skin tone
+1f6b4 1f3fd # medium skin tone
+1f6b4 1f3fe # medium-dark skin tone
+1f6b4 1f3ff # dark skin tone
+1f6b5 # MOUNTAIN BICYCLIST -> man mountain biking
+1f6b5 1f3fb # light skin tone
+1f6b5 1f3fc # medium-light skin tone
+1f6b5 1f3fd # medium skin tone
+1f6b5 1f3fe # medium-dark skin tone
+1f6b5 1f3ff # dark skin tone
+1f6b6 # PEDESTRIAN -> man walking
+1f6b6 1f3fb # light skin tone
+1f6b6 1f3fc # medium-light skin tone
+1f6b6 1f3fd # medium skin tone
+1f6b6 1f3fe # medium-dark skin tone
+1f6b6 1f3ff # dark skin tone
+1f926 # FACE PALM -> woman facepalming
+1f926 1f3fb # light skin tone
+1f926 1f3fc # medium-light skin tone
+1f926 1f3fd # medium skin tone
+1f926 1f3fe # medium-dark skin tone
+1f926 1f3ff # dark skin tone
+1f937 # SHRUG -> woman shrugging
+1f937 1f3fb # light skin tone
+1f937 1f3fc # medium-light skin tone
+1f937 1f3fd # medium skin tone
+1f937 1f3fe # medium-dark skin tone
+1f937 1f3ff # dark skin tone
+1f938 # PERSON DOING CARTWHEEL -> man cartwheeling
+1f938 1f3fb # light skin tone
+1f938 1f3fc # medium-light skin tone
+1f938 1f3fd # medium skin tone
+1f938 1f3fe # medium-dark skin tone
+1f938 1f3ff # dark skin tone
+1f939 # JUGGLING -> man juggling
+1f939 1f3fb # light skin tone
+1f939 1f3fc # medium-light skin tone
+1f939 1f3fd # medium skin tone
+1f939 1f3fe # medium-dark skin tone
+1f939 1f3ff # dark skin tone
+1f93c # WRESTLERS -> men wrestling
+1f93d # WATER POLO -> man playing water polo
+1f93d 1f3fb # light skin tone
+1f93d 1f3fc # medium-light skin tone
+1f93d 1f3fd # medium skin tone
+1f93d 1f3fe # medium-dark skin tone
+1f93d 1f3ff # dark skin tone
+1f93e # HANDBALL -> man playing handball
+1f93e 1f3fb # light skin tone
+1f93e 1f3fc # medium-light skin tone
+1f93e 1f3fd # medium skin tone
+1f93e 1f3fe # medium-dark skin tone
+1f93e 1f3ff # dark skin tone
+26f9 # PERSON WITH BALL -> man bouncing ball
+26f9 1f3fb # light skin tone
+26f9 1f3fc # medium-light skin tone
+26f9 1f3fd # medium skin tone
+26f9 1f3fe # medium-dark skin tone
+26f9 1f3ff # dark skin tone
+fe82b # no name -> no name
+
+# flag aliases
+1f1e7 1f1fb # BV -> NO
+1f1e8 1f1f5 # CP -> FR
+1f1ed 1f1f2 # HM -> AU
+1f1f8 1f1ef # SJ -> NO
+1f1fa 1f1f2 # UM -> US
+
+###
+### unwanted flags
+###
+annotation: error
+1f1e7 1f1f1
+1f1e7 1f1f6
+1f1e9 1f1ec
+1f1ea 1f1e6
+1f1ea 1f1ed
+1f1eb 1f1f0
+1f1ec 1f1eb
+1f1ec 1f1f5
+1f1ec 1f1f8
+1f1f2 1f1eb
+1f1f2 1f1f6
+1f1f3 1f1e8
+1f1f5 1f1f2
+1f1f7 1f1ea
+1f1f9 1f1eb
+1f1fc 1f1eb
+1f1fd 1f1f0
+1f1fe 1f1f9
+
+###
+### new emoji
+###
+annotation: warning
+1f6f7
+1f6f8
+1f91f
+1f91f 1f3fb
+1f91f 1f3fc
+1f91f 1f3fd
+1f91f 1f3fe
+1f91f 1f3ff
+1f928
+1f929
+1f92a
+1f92b
+1f92c
+1f92d
+1f92e
+1f92f
+1f931
+1f931 1f3fb
+1f931 1f3fc
+1f931 1f3fd
+1f931 1f3fe
+1f931 1f3ff
+1f932
+1f932 1f3fb
+1f932 1f3fc
+1f932 1f3fd
+1f932 1f3fe
+1f932 1f3ff
+1f94c
+1f961
+1f962
+1f964
+1f965
+1f966
+1f995
+1f996
+1f997
+1f9d0
+1f9d1
+1f9d1 1f3fb
+1f9d1 1f3fc
+1f9d1 1f3fd
+1f9d1 1f3fe
+1f9d1 1f3ff
+1f9d2
+1f9d2 1f3fb
+1f9d2 1f3fc
+1f9d2 1f3fd
+1f9d2 1f3fe
+1f9d2 1f3ff
+1f9d3
+1f9d3 1f3fb
+1f9d3 1f3fc
+1f9d3 1f3fd
+1f9d3 1f3fe
+1f9d3 1f3ff
+1f9d4
+1f9d4 1f3fb
+1f9d4 1f3fc
+1f9d4 1f3fd
+1f9d4 1f3fe
+1f9d4 1f3ff
+1f9d5
+1f9d5 1f3fb
+1f9d5 1f3fc
+1f9d5 1f3fd
+1f9d5 1f3fe
+1f9d5 1f3ff
+1f9d6
+1f9d6 1f3fb
+1f9d6 1f3fc
+1f9d6 1f3fd
+1f9d6 1f3fe
+1f9d6 1f3ff
+1f9d6 200d 2640
+1f9d6 1f3fb 200d 2640
+1f9d6 1f3fc 200d 2640
+1f9d6 1f3fd 200d 2640
+1f9d6 1f3fe 200d 2640
+1f9d6 1f3ff 200d 2640
+1f9d6 200d 2642
+1f9d6 1f3fb 200d 2642
+1f9d6 1f3fc 200d 2642
+1f9d6 1f3fd 200d 2642
+1f9d6 1f3fe 200d 2642
+1f9d6 1f3ff 200d 2642
+1f9d7
+1f9d7 1f3fb
+1f9d7 1f3fc
+1f9d7 1f3fd
+1f9d7 1f3fe
+1f9d7 1f3ff
+1f9d7 200d 2640
+1f9d7 1f3fb 200d 2640
+1f9d7 1f3fc 200d 2640
+1f9d7 1f3fd 200d 2640
+1f9d7 1f3fe 200d 2640
+1f9d7 1f3ff 200d 2640
+1f9d7 200d 2642
+1f9d7 1f3fb 200d 2642
+1f9d7 1f3fc 200d 2642
+1f9d7 1f3fd 200d 2642
+1f9d7 1f3fe 200d 2642
+1f9d7 1f3ff 200d 2642
+1f9d8
+1f9d8 1f3fb
+1f9d8 1f3fc
+1f9d8 1f3fd
+1f9d8 1f3fe
+1f9d8 1f3ff
+1f9d8 200d 2640
+1f9d8 1f3fb 200d 2640
+1f9d8 1f3fc 200d 2640
+1f9d8 1f3fd 200d 2640
+1f9d8 1f3fe 200d 2640
+1f9d8 1f3ff 200d 2640
+1f9d8 200d 2642
+1f9d8 1f3fb 200d 2642
+1f9d8 1f3fc 200d 2642
+1f9d8 1f3fd 200d 2642
+1f9d8 1f3fe 200d 2642
+1f9d8 1f3ff 200d 2642
+1f9d9
+1f9d9 1f3fb
+1f9d9 1f3fc
+1f9d9 1f3fd
+1f9d9 1f3fe
+1f9d9 1f3ff
+1f9d9 200d 2640
+1f9d9 1f3fb 200d 2640
+1f9d9 1f3fc 200d 2640
+1f9d9 1f3fd 200d 2640
+1f9d9 1f3fe 200d 2640
+1f9d9 1f3ff 200d 2640
+1f9d9 200d 2642
+1f9d9 1f3fb 200d 2642
+1f9d9 1f3fc 200d 2642
+1f9d9 1f3fd 200d 2642
+1f9d9 1f3fe 200d 2642
+1f9d9 1f3ff 200d 2642
+1f9da
+1f9da 1f3fb
+1f9da 1f3fc
+1f9da 1f3fd
+1f9da 1f3fe
+1f9da 1f3ff
+1f9da 200d 2640
+1f9da 1f3fb 200d 2640
+1f9da 1f3fc 200d 2640
+1f9da 1f3fd 200d 2640
+1f9da 1f3fe 200d 2640
+1f9da 1f3ff 200d 2640
+1f9da 200d 2642
+1f9da 1f3fb 200d 2642
+1f9da 1f3fc 200d 2642
+1f9da 1f3fd 200d 2642
+1f9da 1f3fe 200d 2642
+1f9da 1f3ff 200d 2642
+1f9db
+1f9db 1f3fb
+1f9db 1f3fc
+1f9db 1f3fd
+1f9db 1f3fe
+1f9db 1f3ff
+1f9db 200d 2640
+1f9db 1f3fb 200d 2640
+1f9db 1f3fc 200d 2640
+1f9db 1f3fd 200d 2640
+1f9db 1f3fe 200d 2640
+1f9db 1f3ff 200d 2640
+1f9db 200d 2642
+1f9db 1f3fb 200d 2642
+1f9db 1f3fc 200d 2642
+1f9db 1f3fd 200d 2642
+1f9db 1f3fe 200d 2642
+1f9db 1f3ff 200d 2642
+1f9dc
+1f9dc 1f3fb
+1f9dc 1f3fc
+1f9dc 1f3fd
+1f9dc 1f3fe
+1f9dc 1f3ff
+1f9dc 200d 2640
+1f9dc 1f3fb 200d 2640
+1f9dc 1f3fc 200d 2640
+1f9dc 1f3fd 200d 2640
+1f9dc 1f3fe 200d 2640
+1f9dc 1f3ff 200d 2640
+1f9dc 200d 2642
+1f9dc 1f3fb 200d 2642
+1f9dc 1f3fc 200d 2642
+1f9dc 1f3fd 200d 2642
+1f9dc 1f3fe 200d 2642
+1f9dc 1f3ff 200d 2642
+1f9dd
+1f9dd 1f3fb
+1f9dd 1f3fc
+1f9dd 1f3fd
+1f9dd 1f3fe
+1f9dd 1f3ff
+1f9dd 200d 2640
+1f9dd 1f3fb 200d 2640
+1f9dd 1f3fc 200d 2640
+1f9dd 1f3fd 200d 2640
+1f9dd 1f3fe 200d 2640
+1f9dd 1f3ff 200d 2640
+1f9dd 200d 2642
+1f9dd 1f3fb 200d 2642
+1f9dd 1f3fc 200d 2642
+1f9dd 1f3fd 200d 2642
+1f9dd 1f3fe 200d 2642
+1f9dd 1f3ff 200d 2642
+1f9de
+1f9de 200d 2640
+1f9de 200d 2642
+1f9df
+1f9df 200d 2640
+1f9df 200d 2642
+1f9e0
+1f9e1
+1f9e2
+1f9e3
+1f9e4
+1f9e5
+1f9e6
+

generate_emoji_html.py

@@ -118,10 +118,10 @@ def _get_desc(key_tuple, dir_infos, basepaths):
   return CELL_PREFIX + desc
 
 
-def _get_name(key_tuple, annotated_tuples):
+def _get_name(key_tuple, annotations):
+  annotation = None if annotations is None else annotations.get(key_tuple)
   CELL_PREFIX = '<td%s>' % (
-      '' if annotated_tuples is None or key_tuple not in annotated_tuples
+      '' if annotation is None else ' class="%s"' % annotation)
-      else ' class="aname"')
 
   seq_name = unicode_data.get_emoji_sequence_name(key_tuple)
   if seq_name == None:
@@ -164,17 +164,17 @@ def _collect_aux_info(dir_infos, keys):
 
 
 def _generate_content(
-    basedir, font, dir_infos, keys, annotate, standalone, colors):
+    basedir, font, dir_infos, keys, annotations, standalone, colors):
   """Generate an html table for the infos.  Basedir is the parent directory of
   the content, filenames will be made relative to this if underneath it, else
   absolute. If font is not none, generate columns for the text rendered in the
   font before other columns.  Dir_infos is the list of DirInfos in column
   order.  Keys is the list of canonical emoji sequences in row order.  If
-  annotate is not none, highlight sequences that appear in this set.  If
+  annotations is not none, highlight sequences that appear in this map based on
-  standalone is true, the image data and font (if used) will be copied under
+  their map values ('ok', 'error', 'warning').  If standalone is true, the
-  the basedir to make a completely stand-alone page.  Colors is the list of
+  image data and font (if used) will be copied under the basedir to make a
-  background colors, the last DirInfo column will be repeated against each of
+  completely stand-alone page.  Colors is the list of background colors, the
-  these backgrounds.
+  last DirInfo column will be repeated against each of these backgrounds.
   """
 
   basedir = path.abspath(path.expanduser(basedir))
@@ -232,7 +232,7 @@ def _generate_content(
   for key in keys:
     row = _generate_row_cells(key, font, dir_infos, basepaths, colors)
     row.append(_get_desc(key, dir_infos, basepaths))
-    row.append(_get_name(key, annotate))
+    row.append(_get_name(key, annotations))
     lines.append(''.join(row))
 
   return '\n  <tr>'.join(lines) + '\n</table>'
@@ -343,17 +343,42 @@ def _get_keys(dir_infos, limit, all_emoji, emoji_sort):
 
 
 def _parse_annotation_file(afile):
-  annotations = set()
+  """Parse file and return a map from sequences to one of 'ok', 'warning',
-  line_re = re.compile(r'([0-9a-f ]+)')
+  or 'error'.
+
+  The file format consists of two kinds of lines.  One defines the annotation
+  to apply, it consists of the text 'annotation:' followed by one of 'ok',
+  'warning', or 'error'.  The other defines a sequence that should get the most
+  recently defined annotation, this is a series of codepoints expressed in hex
+  separated by spaces.  The initial default annotation is 'error'.  '#' starts
+  a comment to end of line, blank lines are ignored.
+  """
+
+  annotations = {}
+  line_re = re.compile(r'annotation:\s*(ok|warning|error)|([0-9a-f ]+)')
+  annotation = 'error'
   with open(afile, 'r') as f:
     for line in f:
       line = line.strip()
       if not line or line[0] == '#':
         continue
       m = line_re.match(line)
-      if m:
+      if not m:
-        annotations.add(tuple([int(s, 16) for s in m.group(1).split()]))
+        raise Exception('could not parse annotation "%s"' % line)
-  return frozenset(annotations)
+      new_annotation = m.group(1)
+      if new_annotation:
+        annotation = new_annotation
+      else:
+        seq = tuple([int(s, 16) for s in m.group(2).split()])
+        canonical_seq = unicode_data.get_canonical_emoji_sequence(seq)
+        if canonical_seq:
+          seq = canonical_seq
+        if seq in annotations:
+          raise Exception(
+              'duplicate sequence %s in annotations' %
+              unicode_data.seq_to_string(seq))
+        annotations[seq] = annotation
+  return annotations
 
 
 def _instantiate_template(template, arg_dict):
@@ -400,11 +425,13 @@ STYLE = """
          vertical-align: bottom; width: 32px; height: 32px
       }
       td:last-of-type { background-color: white }
-      td.aname { background-color: rgb(250, 65, 75) }
+      td.error { background-color: rgb(250, 65, 75) }
+      td.warning { background-color: rgb(240, 245, 50) }
+      td.ok { background-color: rgb(10, 200, 60) }
 """
 
 def write_html_page(
-    filename, page_title, font, dir_infos, keys, annotate, standalone,
+    filename, page_title, font, dir_infos, keys, annotations, standalone,
     colors):
 
   out_dir = path.dirname(filename)
@@ -430,7 +457,7 @@ def write_html_page(
         font = path.normpath(path.join(common_prefix, rel_font))
 
   content = _generate_content(
-      path.dirname(filename), font, dir_infos, keys, annotate, standalone,
+      path.dirname(filename), font, dir_infos, keys, annotations, standalone,
       colors)
   N_STYLE = STYLE
   if font: