jscoverage/trunk/highlight.c

/*
    highlight.c - JavaScript syntax highlighting
    Copyright (C) 2008 siliconforks.com

    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
    the Free Software Foundation; either version 2 of the License, or
    (at your option) any later version.

    This program is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU General Public License for more details.

    You should have received a copy of the GNU General Public License along
    with this program; if not, write to the Free Software Foundation, Inc.,
    51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
*/

#include <config.h>

#include "highlight.h"

#include <assert.h>
#include <stdlib.h>
#include <string.h>

#include <jslock.h>
#include <jsscan.h>

#include "util.h"

enum Class {
  CLASS_NONE,
  CLASS_COMMENT,
  CLASS_REGEXP,
  CLASS_NUMBER,
  CLASS_STRING,
  CLASS_SPECIALCHAR,
  CLASS_KEYWORD,
  CLASS_TYPE,
  CLASS_SYMBOL,
  CLASS_CBRACKET
};

static const char * get_class_name(enum Class class) {
  switch (class) {
  case CLASS_NONE:
    abort();
    break;
  case CLASS_COMMENT:
    return "c";
    break;
  case CLASS_REGEXP:
    return "s";
    break;
  case CLASS_NUMBER:
    return "s";
    break;
  case CLASS_STRING:
    return "s";
    break;
  case CLASS_SPECIALCHAR:
    return "t";
    break;
  case CLASS_KEYWORD:
    return "k";
    break;
  case CLASS_TYPE:
    return "k";
    break;
  case CLASS_SYMBOL:
    return "k";
    break;
  case CLASS_CBRACKET:
    return "k";
    break;
  default:
    abort();
    break;
  }
}

static const char * g_id;
static const jschar * g_characters;
static size_t g_num_characters;
static Stream * g_output;
static size_t character_offset;
static uint16_t line_num;
static uint16_t column_num;
static enum Class current_class;

static void output_character(jschar c, enum Class class) {
  if (class != current_class) {
    /* output the end tag */
    if (current_class != CLASS_NONE) {
      Stream_write_string(g_output, "</span>");
    }

    current_class = class;

    /* output the start tag */
    if (current_class != CLASS_NONE) {
      Stream_printf(g_output, "<span class=\"%s\">", get_class_name(class));
    }
  }

  switch (c) {
  case '&':
    Stream_write_string(g_output, "&amp;");
    break;
  case '<':
    Stream_write_string(g_output, "&lt;");
    break;
  case '>':
    Stream_write_string(g_output, "&gt;");
    break;
  case '\t':
  case '\n':
    Stream_write_char(g_output, c);
    break;
  default:
    if (32 <= c && c <= 126) {
      Stream_write_char(g_output, c);
    }
    else {
      Stream_printf(g_output, "&#%d;", c);
    }
    break;
  }
}

static void mark_nontoken_chars(uint16_t end_line, uint16_t end_column) {
  enum State {
    STATE_NORMAL,
    STATE_LINE_COMMENT,
    STATE_MULTILINE_COMMENT
  };

  enum State state = STATE_NORMAL;
  while (character_offset < g_num_characters) {
    if (end_line != 0 && line_num > end_line) {
      break;
    }
    else if (line_num == end_line && column_num >= end_column) {
      break;
    }

    jschar c = g_characters[character_offset];
    if (c == '\0') {
      fatal("%s: script contains NULL character", g_id);
    }

    switch (state) {
    case STATE_NORMAL:
      if (c == '/' && character_offset + 1 < g_num_characters && g_characters[character_offset + 1] == '/') {
        state = STATE_LINE_COMMENT;
      }
      else if (c == '/' && character_offset + 1 < g_num_characters && g_characters[character_offset + 1] == '*') {
        state = STATE_MULTILINE_COMMENT;
        output_character('/', CLASS_COMMENT);
        output_character('*', CLASS_COMMENT);
        character_offset += 2;
        if (column_num >= UINT16_MAX - 1) {
          fatal("%s: script contains line with more than 65,535 characters", g_id);
        }
        column_num += 2;
        continue;
      }
      break;
    case STATE_LINE_COMMENT:
      if (c == '\r' || c == '\n' || c == 0x2028 || c == 0x2029) {
        state = STATE_NORMAL;
      }
      break;
    case STATE_MULTILINE_COMMENT:
      if (c == '*' && character_offset + 1 < g_num_characters && g_characters[character_offset + 1] == '/') {
        output_character('*', CLASS_COMMENT);
        output_character('/', CLASS_COMMENT);
        state = STATE_NORMAL;
        character_offset += 2;
        if (column_num >= UINT16_MAX - 1) {
          fatal("%s: script contains line with more than 65,535 characters", g_id);
        }
        column_num += 2;
        continue;
      }
      break;
    }

    character_offset++;
    if (c == '\r' || c == '\n' || c == 0x2028 || c == 0x2029) {
      if (line_num == UINT16_MAX) {
        fatal("%s: script contains more than 65,535 lines", g_id);
      }
      line_num++;
      column_num = 0;
      if (c == '\r' && character_offset < g_num_characters && g_characters[character_offset] == '\n') {
        character_offset++;
      }
      output_character('\n', CLASS_NONE);
    }
    else {
      if (column_num == UINT16_MAX) {
        fatal("%s: script contains line with more than 65,535 characters", g_id);
      }
      column_num++;
      if (state == STATE_NORMAL) {
        output_character(c, CLASS_NONE);
      }
      else {
        output_character(c, CLASS_COMMENT);
      }
    }
  }
}

void jscoverage_highlight_js(JSContext * context, const char * id, const jschar * characters, size_t num_characters, Stream * output) {
  g_id = id;
  g_characters = characters;
  g_num_characters = num_characters;
  g_output = output;

  character_offset = 0;
  line_num = 1;
  column_num = 0;
  current_class = CLASS_NONE;

  /* tokenize the JavaScript */
  JSTokenStream token_stream;
  if (! js_InitTokenStream(context, &token_stream, characters, num_characters, NULL, NULL, 1)) {
    fatal("cannot create token stream from JavaScript file %s", id);
  }

  for (;;) {
    JSTokenType tt = js_GetToken(context, &token_stream);

    if (tt == TOK_ERROR) {
      fatal("JavaScript parse error: %s: line = %d, col = %d\n", id, line_num, column_num);
    }

    if (tt == TOK_EOF) {
      mark_nontoken_chars(0, 0);
      break;
    }

    /* mark the chars before the token */
    JSToken t = CURRENT_TOKEN(&token_stream);
    mark_nontoken_chars(t.pos.begin.lineno, t.pos.begin.index);

    /* mark the token */
    enum Class class;
    switch (tt) {
    case TOK_ERROR:
    case TOK_EOF:
      abort();
    case TOK_EOL:
      class = CLASS_NONE;
      token_stream.flags |= TSF_OPERAND;
      break;
    case TOK_SEMI:
    case TOK_COMMA:
    case TOK_ASSIGN:
    case TOK_HOOK:
    case TOK_COLON:
    case TOK_OR:
    case TOK_AND:
    case TOK_BITOR:
    case TOK_BITXOR:
    case TOK_BITAND:
    case TOK_EQOP:
    case TOK_RELOP:
    case TOK_SHOP:
    case TOK_PLUS:
    case TOK_MINUS:
    case TOK_STAR:
    case TOK_DIVOP:
      class = CLASS_SYMBOL;
      token_stream.flags |= TSF_OPERAND;
      break;
    case TOK_UNARYOP:
      switch (t.t_op) {
      case JSOP_NEG:
      case JSOP_POS:
      case JSOP_NOT:
      case JSOP_BITNOT:
        class = CLASS_SYMBOL;
        token_stream.flags |= TSF_OPERAND;
        break;
      case JSOP_TYPEOF:
        class = CLASS_KEYWORD;
        token_stream.flags |= TSF_OPERAND;
        break;
      case JSOP_VOID:
        class = CLASS_TYPE;
        token_stream.flags |= TSF_OPERAND;
        break;
      default:
        abort();
      }
      break;
    case TOK_INC:
    case TOK_DEC:
      class = CLASS_SYMBOL;
      /* token_stream.flags does not change w.r.t. TSF_OPERAND */
      break;
    case TOK_DOT:
    case TOK_LB:
      class = CLASS_SYMBOL;
      token_stream.flags |= TSF_OPERAND;
      break;
    case TOK_RB:
      class = CLASS_SYMBOL;
      token_stream.flags &= ~TSF_OPERAND;
      break;
    case TOK_LC:
      class = CLASS_CBRACKET;
      token_stream.flags |= TSF_OPERAND;
      break;
    case TOK_RC:
      class = CLASS_CBRACKET;
      token_stream.flags &= ~TSF_OPERAND;
      break;
    case TOK_LP:
      class = CLASS_SYMBOL;
      token_stream.flags |= TSF_OPERAND;
      break;
    case TOK_RP:
      class = CLASS_SYMBOL;
      token_stream.flags &= ~TSF_OPERAND;
      break;
    case TOK_NAME:
      class = CLASS_NONE;
      token_stream.flags &= ~TSF_OPERAND;
      if (js_PeekToken(context, &token_stream) == TOK_LP) {
        /* function */
        class = CLASS_NONE;
      }
      break;
    case TOK_NUMBER:
      class = CLASS_NUMBER;
      token_stream.flags &= ~TSF_OPERAND;
      break;
    case TOK_STRING:
      class = CLASS_STRING;
      token_stream.flags &= ~TSF_OPERAND;
      break;
    case TOK_REGEXP:
      class = CLASS_REGEXP;
      token_stream.flags &= ~TSF_OPERAND;
      break;
    case TOK_PRIMARY:
      switch (t.t_op) {
      case JSOP_TRUE:
      case JSOP_FALSE:
      case JSOP_NULL:
      case JSOP_THIS:
        class = CLASS_KEYWORD;
        token_stream.flags &= ~TSF_OPERAND;
        break;
      default:
        abort();
      }
      break;
    case TOK_FUNCTION:
      class = CLASS_KEYWORD;
      token_stream.flags |= TSF_OPERAND;
      break;
    case TOK_IF:
    case TOK_ELSE:
    case TOK_SWITCH:
    case TOK_CASE:
    case TOK_DEFAULT:
    case TOK_WHILE:
    case TOK_DO:
    case TOK_FOR:
    case TOK_BREAK:
    case TOK_CONTINUE:
    case TOK_IN:
    case TOK_VAR:
    case TOK_WITH:
    case TOK_RETURN:
    case TOK_NEW:
    case TOK_DELETE:
      token_stream.flags |= TSF_OPERAND;
      class = CLASS_KEYWORD;
      break;
    case TOK_DEFSHARP:
    case TOK_USESHARP:
      abort();
      break;
    case TOK_TRY:
    case TOK_CATCH:
    case TOK_FINALLY:
    case TOK_THROW:
    case TOK_INSTANCEOF:
    case TOK_DEBUGGER:
      token_stream.flags |= TSF_OPERAND;
      class = CLASS_KEYWORD;
      break;
    case TOK_XMLSTAGO:
    case TOK_XMLETAGO:
    case TOK_XMLPTAGC:
    case TOK_XMLTAGC:
    case TOK_XMLNAME:
    case TOK_XMLATTR:
    case TOK_XMLSPACE:
    case TOK_XMLTEXT:
    case TOK_XMLCOMMENT:
    case TOK_XMLCDATA:
    case TOK_XMLPI:
    case TOK_AT:
    case TOK_DBLCOLON:
    case TOK_ANYNAME:
    case TOK_DBLDOT:
    case TOK_FILTER:
    case TOK_XMLELEM:
    case TOK_XMLLIST:
      abort();
      break;
    case TOK_YIELD:
      token_stream.flags |= TSF_OPERAND;
      class = CLASS_KEYWORD;
      break;
    case TOK_ARRAYCOMP:
      fatal("token type %d not supported", tt);
      break;
    case TOK_ARRAYPUSH:
      fatal("token type %d not supported", tt);
      break;
    case TOK_LEXICALSCOPE:
      fatal("token type %d not supported", tt);
      break;
    case TOK_LET:
      token_stream.flags |= TSF_OPERAND;
      class = CLASS_KEYWORD;
      break;
    case TOK_BODY:
      fatal("token type %d not supported", tt);
      break;
    case TOK_RESERVED:
    case TOK_LIMIT:
      abort();
      break;
    default:
      abort();
      break;
    }

    if (t.pos.begin.lineno != t.pos.end.lineno) {
      fatal("%s: line %u: token spans multiple lines", id, t.pos.begin.lineno);
    }
    if (t.pos.begin.index > t.pos.end.index) {
      fatal("%s: script contains line with more than 65,535 characters", id);
    }
    for (uint16_t i = t.pos.begin.index; i < t.pos.end.index; i++) {
      assert(character_offset < num_characters);
      jschar c = characters[character_offset];
      if (tt == TOK_STRING && c == '\\') {
        output_character(c, CLASS_SPECIALCHAR);
        character_offset++;
        i++;
        assert(character_offset < num_characters);
        c = characters[character_offset];
        output_character(c, CLASS_SPECIALCHAR);
        character_offset++;
      }
      else {
        output_character(c, class);
        character_offset++;
      }
    }

    line_num = t.pos.end.lineno;
    column_num = t.pos.end.index;
  }

  if (current_class != CLASS_NONE) {
    output_character('\n', CLASS_NONE);
  }

  js_CloseTokenStream(context, &token_stream);
}
1	siliconforks	179	/*
2			highlight.c - JavaScript syntax highlighting
3			Copyright (C) 2008 siliconforks.com
4
5			This program is free software; you can redistribute it and/or modify
6			it under the terms of the GNU General Public License as published by
7			the Free Software Foundation; either version 2 of the License, or
8			(at your option) any later version.
9
10			This program is distributed in the hope that it will be useful,
11			but WITHOUT ANY WARRANTY; without even the implied warranty of
12			MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13			GNU General Public License for more details.
14
15			You should have received a copy of the GNU General Public License along
16			with this program; if not, write to the Free Software Foundation, Inc.,
17			51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
18			*/
19
20			#include <config.h>
21
22			#include "highlight.h"
23
24	siliconforks	213	#include <assert.h>
25	siliconforks	179	#include <stdlib.h>
26			#include <string.h>
27
28			#include <jslock.h>
29			#include <jsscan.h>
30
31			#include "util.h"
32
33			enum Class {
34			CLASS_NONE,
35			CLASS_COMMENT,
36			CLASS_REGEXP,
37			CLASS_NUMBER,
38			CLASS_STRING,
39			CLASS_SPECIALCHAR,
40			CLASS_KEYWORD,
41			CLASS_TYPE,
42			CLASS_SYMBOL,
43			CLASS_CBRACKET
44			};
45
46			static const char * get_class_name(enum Class class) {
47			switch (class) {
48			case CLASS_NONE:
49			abort();
50			break;
51			case CLASS_COMMENT:
52			return "c";
53			break;
54			case CLASS_REGEXP:
55			return "s";
56			break;
57			case CLASS_NUMBER:
58			return "s";
59			break;
60			case CLASS_STRING:
61			return "s";
62			break;
63			case CLASS_SPECIALCHAR:
64			return "t";
65			break;
66			case CLASS_KEYWORD:
67			return "k";
68			break;
69			case CLASS_TYPE:
70			return "k";
71			break;
72			case CLASS_SYMBOL:
73			return "k";
74			break;
75			case CLASS_CBRACKET:
76			return "k";
77			break;
78			default:
79			abort();
80			break;
81			}
82			}
83
84	siliconforks	213	static const char * g_id;
85			static const jschar * g_characters;
86			static size_t g_num_characters;
87			static Stream * g_output;
88			static size_t character_offset;
89			static uint16_t line_num;
90			static uint16_t column_num;
91			static enum Class current_class;
92	siliconforks	179
93	siliconforks	213	static void output_character(jschar c, enum Class class) {
94			if (class != current_class) {
95			/* output the end tag */
96			if (current_class != CLASS_NONE) {
97			Stream_write_string(g_output, "</span>");
98			}
99
100			current_class = class;
101
102			/* output the start tag */
103			if (current_class != CLASS_NONE) {
104			Stream_printf(g_output, "<span class=\"%s\">", get_class_name(class));
105			}
106	siliconforks	179	}
107
108	siliconforks	213	switch (c) {
109			case '&':
110			Stream_write_string(g_output, "&");
111			break;
112			case '<':
113			Stream_write_string(g_output, "<");
114			break;
115			case '>':
116			Stream_write_string(g_output, ">");
117			break;
118	siliconforks	215	case '\t':
119	siliconforks	213	case '\n':
120			Stream_write_char(g_output, c);
121			break;
122			default:
123	siliconforks	215	if (32 <= c && c <= 126) {
124	siliconforks	213	Stream_write_char(g_output, c);
125	siliconforks	179	}
126	siliconforks	213	else {
127			Stream_printf(g_output, "&#%d;", c);
128			}
129			break;
130	siliconforks	179	}
131			}
132
133	siliconforks	213	static void mark_nontoken_chars(uint16_t end_line, uint16_t end_column) {
134	siliconforks	179	enum State {
135			STATE_NORMAL,
136			STATE_LINE_COMMENT,
137			STATE_MULTILINE_COMMENT
138			};
139
140			enum State state = STATE_NORMAL;
141	siliconforks	213	while (character_offset < g_num_characters) {
142			if (end_line != 0 && line_num > end_line) {
143			break;
144	siliconforks	179	}
145	siliconforks	213	else if (line_num == end_line && column_num >= end_column) {
146			break;
147	siliconforks	179	}
148
149	siliconforks	213	jschar c = g_characters[character_offset];
150			if (c == '\0') {
151			fatal("%s: script contains NULL character", g_id);
152	siliconforks	179	}
153	siliconforks	213
154			switch (state) {
155			case STATE_NORMAL:
156			if (c == '/' && character_offset + 1 < g_num_characters && g_characters[character_offset + 1] == '/') {
157			state = STATE_LINE_COMMENT;
158	siliconforks	179	}
159	siliconforks	213	else if (c == '/' && character_offset + 1 < g_num_characters && g_characters[character_offset + 1] == '*') {
160			state = STATE_MULTILINE_COMMENT;
161			output_character('/', CLASS_COMMENT);
162			output_character('*', CLASS_COMMENT);
163			character_offset += 2;
164			if (column_num >= UINT16_MAX - 1) {
165			fatal("%s: script contains line with more than 65,535 characters", g_id);
166			}
167			column_num += 2;
168			continue;
169			}
170			break;
171			case STATE_LINE_COMMENT:
172	siliconforks	179	if (c == '\r' \|\| c == '\n' \|\| c == 0x2028 \|\| c == 0x2029) {
173	siliconforks	213	state = STATE_NORMAL;
174	siliconforks	179	}
175	siliconforks	213	break;
176			case STATE_MULTILINE_COMMENT:
177			if (c == '*' && character_offset + 1 < g_num_characters && g_characters[character_offset + 1] == '/') {
178			output_character('*', CLASS_COMMENT);
179			output_character('/', CLASS_COMMENT);
180			state = STATE_NORMAL;
181			character_offset += 2;
182			if (column_num >= UINT16_MAX - 1) {
183			fatal("%s: script contains line with more than 65,535 characters", g_id);
184			}
185			column_num += 2;
186			continue;
187	siliconforks	179	}
188	siliconforks	213	break;
189	siliconforks	179	}
190
191	siliconforks	213	character_offset++;
192			if (c == '\r' \|\| c == '\n' \|\| c == 0x2028 \|\| c == 0x2029) {
193			if (line_num == UINT16_MAX) {
194			fatal("%s: script contains more than 65,535 lines", g_id);
195	siliconforks	179	}
196	siliconforks	213	line_num++;
197			column_num = 0;
198			if (c == '\r' && character_offset < g_num_characters && g_characters[character_offset] == '\n') {
199			character_offset++;
200			}
201			output_character('\n', CLASS_NONE);
202	siliconforks	179	}
203	siliconforks	213	else {
204			if (column_num == UINT16_MAX) {
205			fatal("%s: script contains line with more than 65,535 characters", g_id);
206	siliconforks	179	}
207	siliconforks	213	column_num++;
208			if (state == STATE_NORMAL) {
209			output_character(c, CLASS_NONE);
210			}
211			else {
212			output_character(c, CLASS_COMMENT);
213			}
214	siliconforks	179	}
215			}
216	siliconforks	213	}
217	siliconforks	179
218	siliconforks	213	void jscoverage_highlight_js(JSContext * context, const char * id, const jschar * characters, size_t num_characters, Stream * output) {
219			g_id = id;
220			g_characters = characters;
221			g_num_characters = num_characters;
222			g_output = output;
223
224			character_offset = 0;
225			line_num = 1;
226			column_num = 0;
227			current_class = CLASS_NONE;
228
229	siliconforks	179	/* tokenize the JavaScript */
230	siliconforks	336	JSTokenStream token_stream;
231			if (! js_InitTokenStream(context, &token_stream, characters, num_characters, NULL, NULL, 1)) {
232	siliconforks	179	fatal("cannot create token stream from JavaScript file %s", id);
233			}
234
235			for (;;) {
236	siliconforks	336	JSTokenType tt = js_GetToken(context, &token_stream);
237	siliconforks	179
238			if (tt == TOK_ERROR) {
239			fatal("JavaScript parse error: %s: line = %d, col = %d\n", id, line_num, column_num);
240			}
241
242			if (tt == TOK_EOF) {
243	siliconforks	213	mark_nontoken_chars(0, 0);
244	siliconforks	179	break;
245			}
246
247			/* mark the chars before the token */
248	siliconforks	336	JSToken t = CURRENT_TOKEN(&token_stream);
249	siliconforks	213	mark_nontoken_chars(t.pos.begin.lineno, t.pos.begin.index);
250	siliconforks	179
251			/* mark the token */
252			enum Class class;
253			switch (tt) {
254			case TOK_ERROR:
255			case TOK_EOF:
256			abort();
257			case TOK_EOL:
258			class = CLASS_NONE;
259	siliconforks	336	token_stream.flags \|= TSF_OPERAND;
260	siliconforks	179	break;
261			case TOK_SEMI:
262			case TOK_COMMA:
263			case TOK_ASSIGN:
264			case TOK_HOOK:
265			case TOK_COLON:
266			case TOK_OR:
267			case TOK_AND:
268			case TOK_BITOR:
269			case TOK_BITXOR:
270			case TOK_BITAND:
271			case TOK_EQOP:
272			case TOK_RELOP:
273			case TOK_SHOP:
274			case TOK_PLUS:
275			case TOK_MINUS:
276			case TOK_STAR:
277			case TOK_DIVOP:
278			class = CLASS_SYMBOL;
279	siliconforks	336	token_stream.flags \|= TSF_OPERAND;
280	siliconforks	179	break;
281			case TOK_UNARYOP:
282			switch (t.t_op) {
283			case JSOP_NEG:
284			case JSOP_POS:
285			case JSOP_NOT:
286			case JSOP_BITNOT:
287			class = CLASS_SYMBOL;
288	siliconforks	336	token_stream.flags \|= TSF_OPERAND;
289	siliconforks	179	break;
290			case JSOP_TYPEOF:
291			class = CLASS_KEYWORD;
292	siliconforks	336	token_stream.flags \|= TSF_OPERAND;
293	siliconforks	179	break;
294			case JSOP_VOID:
295			class = CLASS_TYPE;
296	siliconforks	336	token_stream.flags \|= TSF_OPERAND;
297	siliconforks	179	break;
298			default:
299			abort();
300			}
301			break;
302			case TOK_INC:
303			case TOK_DEC:
304	siliconforks	292	class = CLASS_SYMBOL;
305	siliconforks	336	/* token_stream.flags does not change w.r.t. TSF_OPERAND */
306	siliconforks	292	break;
307	siliconforks	179	case TOK_DOT:
308			case TOK_LB:
309			class = CLASS_SYMBOL;
310	siliconforks	336	token_stream.flags \|= TSF_OPERAND;
311	siliconforks	179	break;
312			case TOK_RB:
313			class = CLASS_SYMBOL;
314	siliconforks	336	token_stream.flags &= ~TSF_OPERAND;
315	siliconforks	179	break;
316			case TOK_LC:
317			class = CLASS_CBRACKET;
318	siliconforks	336	token_stream.flags \|= TSF_OPERAND;
319	siliconforks	179	break;
320			case TOK_RC:
321			class = CLASS_CBRACKET;
322	siliconforks	336	token_stream.flags &= ~TSF_OPERAND;
323	siliconforks	179	break;
324			case TOK_LP:
325			class = CLASS_SYMBOL;
326	siliconforks	336	token_stream.flags \|= TSF_OPERAND;
327	siliconforks	179	break;
328			case TOK_RP:
329			class = CLASS_SYMBOL;
330	siliconforks	336	token_stream.flags &= ~TSF_OPERAND;
331	siliconforks	179	break;
332			case TOK_NAME:
333			class = CLASS_NONE;
334	siliconforks	336	token_stream.flags &= ~TSF_OPERAND;
335			if (js_PeekToken(context, &token_stream) == TOK_LP) {
336	siliconforks	179	/* function */
337			class = CLASS_NONE;
338			}
339			break;
340			case TOK_NUMBER:
341			class = CLASS_NUMBER;
342	siliconforks	336	token_stream.flags &= ~TSF_OPERAND;
343	siliconforks	179	break;
344			case TOK_STRING:
345			class = CLASS_STRING;
346	siliconforks	336	token_stream.flags &= ~TSF_OPERAND;
347	siliconforks	179	break;
348	siliconforks	336	case TOK_REGEXP:
349	siliconforks	179	class = CLASS_REGEXP;
350	siliconforks	336	token_stream.flags &= ~TSF_OPERAND;
351	siliconforks	179	break;
352			case TOK_PRIMARY:
353			switch (t.t_op) {
354			case JSOP_TRUE:
355			case JSOP_FALSE:
356			case JSOP_NULL:
357			case JSOP_THIS:
358			class = CLASS_KEYWORD;
359	siliconforks	336	token_stream.flags &= ~TSF_OPERAND;
360	siliconforks	179	break;
361			default:
362			abort();
363			}
364			break;
365			case TOK_FUNCTION:
366			class = CLASS_KEYWORD;
367	siliconforks	336	token_stream.flags \|= TSF_OPERAND;
368	siliconforks	179	break;
369			case TOK_IF:
370			case TOK_ELSE:
371			case TOK_SWITCH:
372			case TOK_CASE:
373			case TOK_DEFAULT:
374			case TOK_WHILE:
375			case TOK_DO:
376			case TOK_FOR:
377			case TOK_BREAK:
378			case TOK_CONTINUE:
379			case TOK_IN:
380			case TOK_VAR:
381			case TOK_WITH:
382			case TOK_RETURN:
383			case TOK_NEW:
384			case TOK_DELETE:
385	siliconforks	336	token_stream.flags \|= TSF_OPERAND;
386	siliconforks	179	class = CLASS_KEYWORD;
387			break;
388			case TOK_DEFSHARP:
389			case TOK_USESHARP:
390			abort();
391			break;
392			case TOK_TRY:
393			case TOK_CATCH:
394			case TOK_FINALLY:
395			case TOK_THROW:
396			case TOK_INSTANCEOF:
397			case TOK_DEBUGGER:
398	siliconforks	336	token_stream.flags \|= TSF_OPERAND;
399	siliconforks	179	class = CLASS_KEYWORD;
400			break;
401			case TOK_XMLSTAGO:
402			case TOK_XMLETAGO:
403			case TOK_XMLPTAGC:
404			case TOK_XMLTAGC:
405			case TOK_XMLNAME:
406			case TOK_XMLATTR:
407			case TOK_XMLSPACE:
408			case TOK_XMLTEXT:
409			case TOK_XMLCOMMENT:
410			case TOK_XMLCDATA:
411			case TOK_XMLPI:
412			case TOK_AT:
413			case TOK_DBLCOLON:
414			case TOK_ANYNAME:
415			case TOK_DBLDOT:
416			case TOK_FILTER:
417			case TOK_XMLELEM:
418			case TOK_XMLLIST:
419	siliconforks	340	abort();
420			break;
421			case TOK_YIELD:
422			token_stream.flags \|= TSF_OPERAND;
423			class = CLASS_KEYWORD;
424			break;
425			case TOK_ARRAYCOMP:
426			fatal("token type %d not supported", tt);
427			break;
428			case TOK_ARRAYPUSH:
429			fatal("token type %d not supported", tt);
430			break;
431			case TOK_LEXICALSCOPE:
432			fatal("token type %d not supported", tt);
433			break;
434			case TOK_LET:
435			token_stream.flags \|= TSF_OPERAND;
436			class = CLASS_KEYWORD;
437			break;
438			case TOK_BODY:
439			fatal("token type %d not supported", tt);
440			break;
441	siliconforks	179	case TOK_RESERVED:
442			case TOK_LIMIT:
443			abort();
444			break;
445			default:
446			abort();
447			break;
448			}
449	siliconforks	213
450	siliconforks	330	if (t.pos.begin.lineno != t.pos.end.lineno) {
451			fatal("%s: line %u: token spans multiple lines", id, t.pos.begin.lineno);
452			}
453	siliconforks	213	if (t.pos.begin.index > t.pos.end.index) {
454			fatal("%s: script contains line with more than 65,535 characters", id);
455			}
456			for (uint16_t i = t.pos.begin.index; i < t.pos.end.index; i++) {
457			assert(character_offset < num_characters);
458			jschar c = characters[character_offset];
459			if (tt == TOK_STRING && c == '\\') {
460			output_character(c, CLASS_SPECIALCHAR);
461			character_offset++;
462			i++;
463			assert(character_offset < num_characters);
464			c = characters[character_offset];
465			output_character(c, CLASS_SPECIALCHAR);
466			character_offset++;
467	siliconforks	179	}
468	siliconforks	213	else {
469			output_character(c, class);
470			character_offset++;
471			}
472	siliconforks	179	}
473
474			line_num = t.pos.end.lineno;
475			column_num = t.pos.end.index;
476			}
477
478	siliconforks	213	if (current_class != CLASS_NONE) {
479			output_character('\n', CLASS_NONE);
480	siliconforks	179	}
481
482	siliconforks	336	js_CloseTokenStream(context, &token_stream);
483	siliconforks	179	}