jscoverage/trunk/highlight.cpp

/*
    highlight.cpp - JavaScript syntax highlighting
    Copyright (C) 2008, 2009, 2010 siliconforks.com

    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
    the Free Software Foundation; either version 2 of the License, or
    (at your option) any later version.

    This program is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU General Public License for more details.

    You should have received a copy of the GNU General Public License along
    with this program; if not, write to the Free Software Foundation, Inc.,
    51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
*/

#include <config.h>

#include "highlight.h"

#include <assert.h>
#include <stdlib.h>
#include <string.h>

#include <jslock.h>
#include <jsscan.h>

#include "util.h"

#ifndef UINT32_MAX
#define UINT32_MAX ((uint32_t) (-1))
#endif

enum Class {
  CLASS_NONE,
  CLASS_COMMENT,
  CLASS_REGEXP,
  CLASS_NUMBER,
  CLASS_STRING,
  CLASS_SPECIALCHAR,
  CLASS_KEYWORD,
  CLASS_TYPE,
  CLASS_SYMBOL,
  CLASS_CBRACKET
};

static const char * get_class_name(enum Class c) {
  switch (c) {
  case CLASS_NONE:
    abort();
    break;
  case CLASS_COMMENT:
    return "c";
    break;
  case CLASS_REGEXP:
    return "s";
    break;
  case CLASS_NUMBER:
    return "s";
    break;
  case CLASS_STRING:
    return "s";
    break;
  case CLASS_SPECIALCHAR:
    return "t";
    break;
  case CLASS_KEYWORD:
    return "k";
    break;
  case CLASS_TYPE:
    return "k";
    break;
  case CLASS_SYMBOL:
    return "k";
    break;
  case CLASS_CBRACKET:
    return "k";
    break;
  default:
    abort();
    break;
  }
}

static const char * g_id;
static const jschar * g_characters;
static size_t g_num_characters;
static Stream * g_output;
static size_t character_offset;
static uint32_t line_num;
static uint32_t column_num;
static enum Class current_class;

static void output_character(jschar c, enum Class class_) {
  if (c == '\r' || c == '\n' || c == 0x2028 || c == 0x2029) {
    class_ = CLASS_NONE;
  }

  if (class_ != current_class) {
    /* output the end tag */
    if (current_class != CLASS_NONE) {
      Stream_write_string(g_output, "</span>");
    }

    current_class = class_;

    /* output the start tag */
    if (current_class != CLASS_NONE) {
      Stream_printf(g_output, "<span class=\"%s\">", get_class_name(class_));
    }
  }

  if (column_num == UINT32_MAX) {
    fatal("%s: script contains a line with more than 65,535 columns", g_id);
  }
  column_num++;
  switch (c) {
  case '&':
    Stream_write_string(g_output, "&amp;");
    break;
  case '<':
    Stream_write_string(g_output, "&lt;");
    break;
  case '>':
    Stream_write_string(g_output, "&gt;");
    break;
  case '\t':
    Stream_write_char(g_output, c);
    break;
  case '\r':
  case '\n':
  case 0x2028:
  case 0x2029:
    if (c == '\r' && character_offset + 1 < g_num_characters && g_characters[character_offset + 1] == '\n') {
      break;
    }
    Stream_write_char(g_output, '\n');
    column_num = 0;
    if (line_num == UINT32_MAX) {
      fatal("%s: script contains more than 65,535 lines", g_id);
    }
    line_num++;
    break;
  default:
    if (32 <= c && c <= 126) {
      Stream_write_char(g_output, c);
    }
    else {
      Stream_printf(g_output, "&#%d;", c);
    }
    break;
  }
  character_offset++;
}

static void mark_nontoken_chars(uint32_t end_line, uint32_t end_column) {
  enum State {
    STATE_NORMAL,
    STATE_LINE_COMMENT,
    STATE_MULTILINE_COMMENT
  };

  enum State state = STATE_NORMAL;
  while (character_offset < g_num_characters) {
    if (end_line != 0 && line_num > end_line) {
      break;
    }
    else if (line_num == end_line && column_num >= end_column) {
      break;
    }

    jschar c = g_characters[character_offset];
    if (c == '\0') {
      fatal("%s: script contains NULL character", g_id);
    }

    switch (state) {
    case STATE_NORMAL:
      if (c == '/' && character_offset + 1 < g_num_characters && g_characters[character_offset + 1] == '/') {
        state = STATE_LINE_COMMENT;
      }
      else if (c == '/' && character_offset + 1 < g_num_characters && g_characters[character_offset + 1] == '*') {
        state = STATE_MULTILINE_COMMENT;
        output_character('/', CLASS_COMMENT);
        output_character('*', CLASS_COMMENT);
        continue;
      }
      break;
    case STATE_LINE_COMMENT:
      if (c == '\r' || c == '\n' || c == 0x2028 || c == 0x2029) {
        state = STATE_NORMAL;
      }
      break;
    case STATE_MULTILINE_COMMENT:
      if (c == '*' && character_offset + 1 < g_num_characters && g_characters[character_offset + 1] == '/') {
        output_character('*', CLASS_COMMENT);
        output_character('/', CLASS_COMMENT);
        state = STATE_NORMAL;
        continue;
      }
      break;
    }

    if (state == STATE_NORMAL) {
      output_character(c, CLASS_NONE);
    }
    else {
      output_character(c, CLASS_COMMENT);
    }
  }
}

void jscoverage_highlight_js(JSContext * context, const char * id, const jschar * characters, size_t num_characters, Stream * output) {
  g_id = id;
  g_characters = characters;
  g_num_characters = num_characters;
  g_output = output;

  character_offset = 0;
  line_num = 1;
  column_num = 0;
  current_class = CLASS_NONE;

  /* tokenize the JavaScript */
  JSTokenStream token_stream(context);
  if (! token_stream.init(context, characters, num_characters, NULL, NULL, 1)) {
    fatal("cannot create token stream from JavaScript file %s", id);
  }

  for (;;) {
    JSTokenType tt = js_GetToken(context, &token_stream);

    if (tt == TOK_ERROR) {
      fatal("JavaScript parse error: %s: line = %d, col = %d\n", id, line_num, column_num);
    }

    if (tt == TOK_EOF) {
      mark_nontoken_chars(0, 0);
      break;
    }

    /* mark the chars before the token */
    JSToken t = CURRENT_TOKEN(&token_stream);
    mark_nontoken_chars(t.pos.begin.lineno, t.pos.begin.index);

    /* mark the token */
    enum Class class_;
    switch (tt) {
    case TOK_ERROR:
    case TOK_EOF:
      abort();
    case TOK_EOL:
      class_ = CLASS_NONE;
      token_stream.flags |= TSF_OPERAND;
      break;
    case TOK_SEMI:
    case TOK_COMMA:
    case TOK_ASSIGN:
    case TOK_HOOK:
    case TOK_COLON:
    case TOK_OR:
    case TOK_AND:
    case TOK_BITOR:
    case TOK_BITXOR:
    case TOK_BITAND:
    case TOK_EQOP:
    case TOK_RELOP:
    case TOK_SHOP:
    case TOK_PLUS:
    case TOK_MINUS:
    case TOK_STAR:
    case TOK_DIVOP:
      class_ = CLASS_SYMBOL;
      token_stream.flags |= TSF_OPERAND;
      break;
    case TOK_UNARYOP:
      switch (t.t_op) {
      case JSOP_NEG:
      case JSOP_POS:
      case JSOP_NOT:
      case JSOP_BITNOT:
        class_ = CLASS_SYMBOL;
        token_stream.flags |= TSF_OPERAND;
        break;
      case JSOP_TYPEOF:
        class_ = CLASS_KEYWORD;
        token_stream.flags |= TSF_OPERAND;
        break;
      case JSOP_VOID:
        class_ = CLASS_TYPE;
        token_stream.flags |= TSF_OPERAND;
        break;
      default:
        fatal_source(id, t.pos.begin.lineno, "unknown TOK_UNARYOP (%d)", t.t_op);
        break;
      }
      break;
    case TOK_INC:
    case TOK_DEC:
      class_ = CLASS_SYMBOL;
      /* token_stream.flags does not change w.r.t. TSF_OPERAND */
      break;
    case TOK_DOT:
    case TOK_LB:
      class_ = CLASS_SYMBOL;
      token_stream.flags |= TSF_OPERAND;
      break;
    case TOK_RB:
      class_ = CLASS_SYMBOL;
      token_stream.flags &= ~TSF_OPERAND;
      break;
    case TOK_LC:
      class_ = CLASS_CBRACKET;
      token_stream.flags |= TSF_OPERAND;
      break;
    case TOK_RC:
      class_ = CLASS_CBRACKET;
      token_stream.flags &= ~TSF_OPERAND;
      break;
    case TOK_LP:
      class_ = CLASS_SYMBOL;
      token_stream.flags |= TSF_OPERAND;
      break;
    case TOK_RP:
      class_ = CLASS_SYMBOL;
      token_stream.flags &= ~TSF_OPERAND;
      break;
    case TOK_NAME:
      class_ = CLASS_NONE;
      token_stream.flags &= ~TSF_OPERAND;
      if (js_PeekToken(context, &token_stream) == TOK_LP) {
        /* function */
        class_ = CLASS_NONE;
      }
      break;
    case TOK_NUMBER:
      class_ = CLASS_NUMBER;
      token_stream.flags &= ~TSF_OPERAND;
      break;
    case TOK_STRING:
      class_ = CLASS_STRING;
      token_stream.flags &= ~TSF_OPERAND;
      break;
    case TOK_REGEXP:
      class_ = CLASS_REGEXP;
      token_stream.flags &= ~TSF_OPERAND;
      break;
    case TOK_PRIMARY:
      switch (t.t_op) {
      case JSOP_TRUE:
      case JSOP_FALSE:
      case JSOP_NULL:
      case JSOP_THIS:
        class_ = CLASS_KEYWORD;
        token_stream.flags &= ~TSF_OPERAND;
        break;
      default:
        fatal_source(id, t.pos.begin.lineno, "unknown TOK_PRIMARY (%d)", t.t_op);
        break;
      }
      break;
    case TOK_FUNCTION:
      class_ = CLASS_KEYWORD;
      token_stream.flags |= TSF_OPERAND;
      break;
    case TOK_IF:
    case TOK_ELSE:
    case TOK_SWITCH:
    case TOK_CASE:
    case TOK_DEFAULT:
    case TOK_WHILE:
    case TOK_DO:
    case TOK_FOR:
    case TOK_BREAK:
    case TOK_CONTINUE:
    case TOK_IN:
    case TOK_VAR:
    case TOK_WITH:
    case TOK_RETURN:
    case TOK_NEW:
    case TOK_DELETE:
      token_stream.flags |= TSF_OPERAND;
      class_ = CLASS_KEYWORD;
      break;
    case TOK_DEFSHARP:
    case TOK_USESHARP:
      fatal_source(id, t.pos.begin.lineno, "unknown token (%d)", tt);
      break;
    case TOK_TRY:
    case TOK_CATCH:
    case TOK_FINALLY:
    case TOK_THROW:
    case TOK_INSTANCEOF:
    case TOK_DEBUGGER:
      token_stream.flags |= TSF_OPERAND;
      class_ = CLASS_KEYWORD;
      break;
    case TOK_XMLSTAGO:
    case TOK_XMLETAGO:
    case TOK_XMLPTAGC:
    case TOK_XMLTAGC:
    case TOK_XMLNAME:
    case TOK_XMLATTR:
    case TOK_XMLSPACE:
    case TOK_XMLTEXT:
    case TOK_XMLCOMMENT:
    case TOK_XMLCDATA:
    case TOK_XMLPI:
    case TOK_AT:
    case TOK_DBLCOLON:
    case TOK_ANYNAME:
    case TOK_DBLDOT:
    case TOK_FILTER:
    case TOK_XMLELEM:
    case TOK_XMLLIST:
      fatal_source(id, t.pos.begin.lineno, "unknown token (%d)", tt);
      break;
    case TOK_YIELD:
      token_stream.flags |= TSF_OPERAND;
      class_ = CLASS_KEYWORD;
      break;
    case TOK_ARRAYCOMP:
    case TOK_ARRAYPUSH:
    case TOK_LEXICALSCOPE:
      fatal_source(id, t.pos.begin.lineno, "unknown token (%d)", tt);
      break;
    case TOK_LET:
      token_stream.flags |= TSF_OPERAND;
      class_ = CLASS_KEYWORD;
      break;
    case TOK_SEQ:
    case TOK_FORHEAD:
    case TOK_RESERVED:
    case TOK_LIMIT:
      fatal_source(id, t.pos.begin.lineno, "unknown token (%d)", tt);
      break;
    default:
      fatal_source(id, t.pos.begin.lineno, "unknown token (%d)", tt);
      break;
    }

    uint32_t start_line = t.pos.begin.lineno;
    uint32_t end_line = t.pos.end.lineno;
    uint32_t start_column = t.pos.begin.index;
    uint32_t end_column = t.pos.end.index;
    assert(line_num == start_line);
    assert(column_num == start_column);
    if (start_line == end_line && start_column >= end_column) {
      fatal("%s: script contains line with more than 65,535 characters", id);
    }
    for (;;) {
      assert(character_offset < num_characters);
      jschar c = characters[character_offset];
      if (tt == TOK_STRING && c == '\\') {
        output_character(c, CLASS_SPECIALCHAR);
        assert(character_offset < num_characters);
        c = characters[character_offset];
        output_character(c, CLASS_SPECIALCHAR);
      }
      else {
        output_character(c, class_);
      }

      if (line_num > end_line) {
        break;
      }
      else if (line_num == end_line && column_num >= end_column) {
        break;
      }
    }

    assert(line_num == end_line);
    assert(column_num = end_column);
  }

  if (current_class != CLASS_NONE) {
    output_character('\n', CLASS_NONE);
  }

  token_stream.close(context);
}
1	siliconforks	179	/*
2	siliconforks	459	highlight.cpp - JavaScript syntax highlighting
3	siliconforks	505	Copyright (C) 2008, 2009, 2010 siliconforks.com
4	siliconforks	179
5			This program is free software; you can redistribute it and/or modify
6			it under the terms of the GNU General Public License as published by
7			the Free Software Foundation; either version 2 of the License, or
8			(at your option) any later version.
9
10			This program is distributed in the hope that it will be useful,
11			but WITHOUT ANY WARRANTY; without even the implied warranty of
12			MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13			GNU General Public License for more details.
14
15			You should have received a copy of the GNU General Public License along
16			with this program; if not, write to the Free Software Foundation, Inc.,
17			51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
18			*/
19
20			#include <config.h>
21
22			#include "highlight.h"
23
24	siliconforks	213	#include <assert.h>
25	siliconforks	179	#include <stdlib.h>
26			#include <string.h>
27
28			#include <jslock.h>
29			#include <jsscan.h>
30
31			#include "util.h"
32
33	siliconforks	473	#ifndef UINT32_MAX
34			#define UINT32_MAX ((uint32_t) (-1))
35	siliconforks	459	#endif
36
37	siliconforks	179	enum Class {
38			CLASS_NONE,
39			CLASS_COMMENT,
40			CLASS_REGEXP,
41			CLASS_NUMBER,
42			CLASS_STRING,
43			CLASS_SPECIALCHAR,
44			CLASS_KEYWORD,
45			CLASS_TYPE,
46			CLASS_SYMBOL,
47			CLASS_CBRACKET
48			};
49
50	siliconforks	458	static const char * get_class_name(enum Class c) {
51			switch (c) {
52	siliconforks	179	case CLASS_NONE:
53			abort();
54			break;
55			case CLASS_COMMENT:
56			return "c";
57			break;
58			case CLASS_REGEXP:
59			return "s";
60			break;
61			case CLASS_NUMBER:
62			return "s";
63			break;
64			case CLASS_STRING:
65			return "s";
66			break;
67			case CLASS_SPECIALCHAR:
68			return "t";
69			break;
70			case CLASS_KEYWORD:
71			return "k";
72			break;
73			case CLASS_TYPE:
74			return "k";
75			break;
76			case CLASS_SYMBOL:
77			return "k";
78			break;
79			case CLASS_CBRACKET:
80			return "k";
81			break;
82			default:
83			abort();
84			break;
85			}
86			}
87
88	siliconforks	213	static const char * g_id;
89			static const jschar * g_characters;
90			static size_t g_num_characters;
91			static Stream * g_output;
92			static size_t character_offset;
93	siliconforks	473	static uint32_t line_num;
94			static uint32_t column_num;
95	siliconforks	213	static enum Class current_class;
96	siliconforks	179
97	siliconforks	458	static void output_character(jschar c, enum Class class_) {
98	siliconforks	347	if (c == '\r' \|\| c == '\n' \|\| c == 0x2028 \|\| c == 0x2029) {
99	siliconforks	458	class_ = CLASS_NONE;
100	siliconforks	347	}
101
102	siliconforks	458	if (class_ != current_class) {
103	siliconforks	213	/* output the end tag */
104			if (current_class != CLASS_NONE) {
105			Stream_write_string(g_output, "</span>");
106			}
107
108	siliconforks	458	current_class = class_;
109	siliconforks	213
110			/* output the start tag */
111			if (current_class != CLASS_NONE) {
112	siliconforks	458	Stream_printf(g_output, "<span class=\"%s\">", get_class_name(class_));
113	siliconforks	213	}
114	siliconforks	179	}
115
116	siliconforks	473	if (column_num == UINT32_MAX) {
117	siliconforks	347	fatal("%s: script contains a line with more than 65,535 columns", g_id);
118			}
119			column_num++;
120	siliconforks	213	switch (c) {
121			case '&':
122			Stream_write_string(g_output, "&");
123			break;
124			case '<':
125			Stream_write_string(g_output, "<");
126			break;
127			case '>':
128			Stream_write_string(g_output, ">");
129			break;
130	siliconforks	215	case '\t':
131	siliconforks	213	Stream_write_char(g_output, c);
132			break;
133	siliconforks	347	case '\r':
134			case '\n':
135			case 0x2028:
136			case 0x2029:
137			if (c == '\r' && character_offset + 1 < g_num_characters && g_characters[character_offset + 1] == '\n') {
138			break;
139			}
140			Stream_write_char(g_output, '\n');
141			column_num = 0;
142	siliconforks	473	if (line_num == UINT32_MAX) {
143	siliconforks	347	fatal("%s: script contains more than 65,535 lines", g_id);
144			}
145			line_num++;
146			break;
147	siliconforks	213	default:
148	siliconforks	215	if (32 <= c && c <= 126) {
149	siliconforks	213	Stream_write_char(g_output, c);
150	siliconforks	179	}
151	siliconforks	213	else {
152			Stream_printf(g_output, "&#%d;", c);
153			}
154			break;
155	siliconforks	179	}
156	siliconforks	347	character_offset++;
157	siliconforks	179	}
158
159	siliconforks	473	static void mark_nontoken_chars(uint32_t end_line, uint32_t end_column) {
160	siliconforks	179	enum State {
161			STATE_NORMAL,
162			STATE_LINE_COMMENT,
163			STATE_MULTILINE_COMMENT
164			};
165
166			enum State state = STATE_NORMAL;
167	siliconforks	213	while (character_offset < g_num_characters) {
168			if (end_line != 0 && line_num > end_line) {
169			break;
170	siliconforks	179	}
171	siliconforks	213	else if (line_num == end_line && column_num >= end_column) {
172			break;
173	siliconforks	179	}
174
175	siliconforks	213	jschar c = g_characters[character_offset];
176			if (c == '\0') {
177			fatal("%s: script contains NULL character", g_id);
178	siliconforks	179	}
179	siliconforks	213
180			switch (state) {
181			case STATE_NORMAL:
182			if (c == '/' && character_offset + 1 < g_num_characters && g_characters[character_offset + 1] == '/') {
183			state = STATE_LINE_COMMENT;
184	siliconforks	179	}
185	siliconforks	213	else if (c == '/' && character_offset + 1 < g_num_characters && g_characters[character_offset + 1] == '*') {
186			state = STATE_MULTILINE_COMMENT;
187			output_character('/', CLASS_COMMENT);
188			output_character('*', CLASS_COMMENT);
189			continue;
190			}
191			break;
192			case STATE_LINE_COMMENT:
193	siliconforks	179	if (c == '\r' \|\| c == '\n' \|\| c == 0x2028 \|\| c == 0x2029) {
194	siliconforks	213	state = STATE_NORMAL;
195	siliconforks	179	}
196	siliconforks	213	break;
197			case STATE_MULTILINE_COMMENT:
198			if (c == '*' && character_offset + 1 < g_num_characters && g_characters[character_offset + 1] == '/') {
199			output_character('*', CLASS_COMMENT);
200			output_character('/', CLASS_COMMENT);
201			state = STATE_NORMAL;
202			continue;
203	siliconforks	179	}
204	siliconforks	213	break;
205	siliconforks	179	}
206
207	siliconforks	347	if (state == STATE_NORMAL) {
208			output_character(c, CLASS_NONE);
209	siliconforks	179	}
210	siliconforks	213	else {
211	siliconforks	347	output_character(c, CLASS_COMMENT);
212	siliconforks	179	}
213			}
214	siliconforks	213	}
215	siliconforks	179
216	siliconforks	213	void jscoverage_highlight_js(JSContext * context, const char * id, const jschar * characters, size_t num_characters, Stream * output) {
217			g_id = id;
218			g_characters = characters;
219			g_num_characters = num_characters;
220			g_output = output;
221
222			character_offset = 0;
223			line_num = 1;
224			column_num = 0;
225			current_class = CLASS_NONE;
226
227	siliconforks	179	/* tokenize the JavaScript */
228	siliconforks	507	JSTokenStream token_stream(context);
229			if (! token_stream.init(context, characters, num_characters, NULL, NULL, 1)) {
230	siliconforks	179	fatal("cannot create token stream from JavaScript file %s", id);
231			}
232
233			for (;;) {
234	siliconforks	336	JSTokenType tt = js_GetToken(context, &token_stream);
235	siliconforks	179
236			if (tt == TOK_ERROR) {
237			fatal("JavaScript parse error: %s: line = %d, col = %d\n", id, line_num, column_num);
238			}
239
240			if (tt == TOK_EOF) {
241	siliconforks	213	mark_nontoken_chars(0, 0);
242	siliconforks	179	break;
243			}
244
245			/* mark the chars before the token */
246	siliconforks	336	JSToken t = CURRENT_TOKEN(&token_stream);
247	siliconforks	213	mark_nontoken_chars(t.pos.begin.lineno, t.pos.begin.index);
248	siliconforks	179
249			/* mark the token */
250	siliconforks	458	enum Class class_;
251	siliconforks	179	switch (tt) {
252			case TOK_ERROR:
253			case TOK_EOF:
254			abort();
255			case TOK_EOL:
256	siliconforks	458	class_ = CLASS_NONE;
257	siliconforks	336	token_stream.flags \|= TSF_OPERAND;
258	siliconforks	179	break;
259			case TOK_SEMI:
260			case TOK_COMMA:
261			case TOK_ASSIGN:
262			case TOK_HOOK:
263			case TOK_COLON:
264			case TOK_OR:
265			case TOK_AND:
266			case TOK_BITOR:
267			case TOK_BITXOR:
268			case TOK_BITAND:
269			case TOK_EQOP:
270			case TOK_RELOP:
271			case TOK_SHOP:
272			case TOK_PLUS:
273			case TOK_MINUS:
274			case TOK_STAR:
275			case TOK_DIVOP:
276	siliconforks	458	class_ = CLASS_SYMBOL;
277	siliconforks	336	token_stream.flags \|= TSF_OPERAND;
278	siliconforks	179	break;
279			case TOK_UNARYOP:
280			switch (t.t_op) {
281			case JSOP_NEG:
282			case JSOP_POS:
283			case JSOP_NOT:
284			case JSOP_BITNOT:
285	siliconforks	458	class_ = CLASS_SYMBOL;
286	siliconforks	336	token_stream.flags \|= TSF_OPERAND;
287	siliconforks	179	break;
288			case JSOP_TYPEOF:
289	siliconforks	458	class_ = CLASS_KEYWORD;
290	siliconforks	336	token_stream.flags \|= TSF_OPERAND;
291	siliconforks	179	break;
292			case JSOP_VOID:
293	siliconforks	458	class_ = CLASS_TYPE;
294	siliconforks	336	token_stream.flags \|= TSF_OPERAND;
295	siliconforks	179	break;
296			default:
297	siliconforks	477	fatal_source(id, t.pos.begin.lineno, "unknown TOK_UNARYOP (%d)", t.t_op);
298			break;
299	siliconforks	179	}
300			break;
301			case TOK_INC:
302			case TOK_DEC:
303	siliconforks	458	class_ = CLASS_SYMBOL;
304	siliconforks	336	/* token_stream.flags does not change w.r.t. TSF_OPERAND */
305	siliconforks	292	break;
306	siliconforks	179	case TOK_DOT:
307			case TOK_LB:
308	siliconforks	458	class_ = CLASS_SYMBOL;
309	siliconforks	336	token_stream.flags \|= TSF_OPERAND;
310	siliconforks	179	break;
311			case TOK_RB:
312	siliconforks	458	class_ = CLASS_SYMBOL;
313	siliconforks	336	token_stream.flags &= ~TSF_OPERAND;
314	siliconforks	179	break;
315			case TOK_LC:
316	siliconforks	458	class_ = CLASS_CBRACKET;
317	siliconforks	336	token_stream.flags \|= TSF_OPERAND;
318	siliconforks	179	break;
319			case TOK_RC:
320	siliconforks	458	class_ = CLASS_CBRACKET;
321	siliconforks	336	token_stream.flags &= ~TSF_OPERAND;
322	siliconforks	179	break;
323			case TOK_LP:
324	siliconforks	458	class_ = CLASS_SYMBOL;
325	siliconforks	336	token_stream.flags \|= TSF_OPERAND;
326	siliconforks	179	break;
327			case TOK_RP:
328	siliconforks	458	class_ = CLASS_SYMBOL;
329	siliconforks	336	token_stream.flags &= ~TSF_OPERAND;
330	siliconforks	179	break;
331			case TOK_NAME:
332	siliconforks	458	class_ = CLASS_NONE;
333	siliconforks	336	token_stream.flags &= ~TSF_OPERAND;
334			if (js_PeekToken(context, &token_stream) == TOK_LP) {
335	siliconforks	179	/* function */
336	siliconforks	458	class_ = CLASS_NONE;
337	siliconforks	179	}
338			break;
339			case TOK_NUMBER:
340	siliconforks	458	class_ = CLASS_NUMBER;
341	siliconforks	336	token_stream.flags &= ~TSF_OPERAND;
342	siliconforks	179	break;
343			case TOK_STRING:
344	siliconforks	458	class_ = CLASS_STRING;
345	siliconforks	336	token_stream.flags &= ~TSF_OPERAND;
346	siliconforks	179	break;
347	siliconforks	336	case TOK_REGEXP:
348	siliconforks	458	class_ = CLASS_REGEXP;
349	siliconforks	336	token_stream.flags &= ~TSF_OPERAND;
350	siliconforks	179	break;
351			case TOK_PRIMARY:
352			switch (t.t_op) {
353			case JSOP_TRUE:
354			case JSOP_FALSE:
355			case JSOP_NULL:
356			case JSOP_THIS:
357	siliconforks	458	class_ = CLASS_KEYWORD;
358	siliconforks	336	token_stream.flags &= ~TSF_OPERAND;
359	siliconforks	179	break;
360			default:
361	siliconforks	477	fatal_source(id, t.pos.begin.lineno, "unknown TOK_PRIMARY (%d)", t.t_op);
362			break;
363	siliconforks	179	}
364			break;
365			case TOK_FUNCTION:
366	siliconforks	458	class_ = CLASS_KEYWORD;
367	siliconforks	336	token_stream.flags \|= TSF_OPERAND;
368	siliconforks	179	break;
369			case TOK_IF:
370			case TOK_ELSE:
371			case TOK_SWITCH:
372			case TOK_CASE:
373			case TOK_DEFAULT:
374			case TOK_WHILE:
375			case TOK_DO:
376			case TOK_FOR:
377			case TOK_BREAK:
378			case TOK_CONTINUE:
379			case TOK_IN:
380			case TOK_VAR:
381			case TOK_WITH:
382			case TOK_RETURN:
383			case TOK_NEW:
384			case TOK_DELETE:
385	siliconforks	336	token_stream.flags \|= TSF_OPERAND;
386	siliconforks	458	class_ = CLASS_KEYWORD;
387	siliconforks	179	break;
388			case TOK_DEFSHARP:
389			case TOK_USESHARP:
390	siliconforks	477	fatal_source(id, t.pos.begin.lineno, "unknown token (%d)", tt);
391	siliconforks	179	break;
392			case TOK_TRY:
393			case TOK_CATCH:
394			case TOK_FINALLY:
395			case TOK_THROW:
396			case TOK_INSTANCEOF:
397			case TOK_DEBUGGER:
398	siliconforks	336	token_stream.flags \|= TSF_OPERAND;
399	siliconforks	458	class_ = CLASS_KEYWORD;
400	siliconforks	179	break;
401			case TOK_XMLSTAGO:
402			case TOK_XMLETAGO:
403			case TOK_XMLPTAGC:
404			case TOK_XMLTAGC:
405			case TOK_XMLNAME:
406			case TOK_XMLATTR:
407			case TOK_XMLSPACE:
408			case TOK_XMLTEXT:
409			case TOK_XMLCOMMENT:
410			case TOK_XMLCDATA:
411			case TOK_XMLPI:
412			case TOK_AT:
413			case TOK_DBLCOLON:
414			case TOK_ANYNAME:
415			case TOK_DBLDOT:
416			case TOK_FILTER:
417			case TOK_XMLELEM:
418			case TOK_XMLLIST:
419	siliconforks	477	fatal_source(id, t.pos.begin.lineno, "unknown token (%d)", tt);
420	siliconforks	340	break;
421			case TOK_YIELD:
422			token_stream.flags \|= TSF_OPERAND;
423	siliconforks	458	class_ = CLASS_KEYWORD;
424	siliconforks	340	break;
425			case TOK_ARRAYCOMP:
426			case TOK_ARRAYPUSH:
427			case TOK_LEXICALSCOPE:
428	siliconforks	477	fatal_source(id, t.pos.begin.lineno, "unknown token (%d)", tt);
429	siliconforks	340	break;
430			case TOK_LET:
431			token_stream.flags \|= TSF_OPERAND;
432	siliconforks	458	class_ = CLASS_KEYWORD;
433	siliconforks	340	break;
434	siliconforks	399	case TOK_SEQ:
435			case TOK_FORHEAD:
436	siliconforks	179	case TOK_RESERVED:
437			case TOK_LIMIT:
438	siliconforks	477	fatal_source(id, t.pos.begin.lineno, "unknown token (%d)", tt);
439	siliconforks	179	break;
440			default:
441	siliconforks	477	fatal_source(id, t.pos.begin.lineno, "unknown token (%d)", tt);
442	siliconforks	179	break;
443			}
444	siliconforks	213
445	siliconforks	473	uint32_t start_line = t.pos.begin.lineno;
446			uint32_t end_line = t.pos.end.lineno;
447			uint32_t start_column = t.pos.begin.index;
448			uint32_t end_column = t.pos.end.index;
449	siliconforks	347	assert(line_num == start_line);
450			assert(column_num == start_column);
451			if (start_line == end_line && start_column >= end_column) {
452	siliconforks	213	fatal("%s: script contains line with more than 65,535 characters", id);
453			}
454	siliconforks	347	for (;;) {
455	siliconforks	213	assert(character_offset < num_characters);
456			jschar c = characters[character_offset];
457			if (tt == TOK_STRING && c == '\\') {
458			output_character(c, CLASS_SPECIALCHAR);
459			assert(character_offset < num_characters);
460			c = characters[character_offset];
461			output_character(c, CLASS_SPECIALCHAR);
462	siliconforks	179	}
463	siliconforks	213	else {
464	siliconforks	458	output_character(c, class_);
465	siliconforks	213	}
466	siliconforks	347
467			if (line_num > end_line) {
468			break;
469			}
470			else if (line_num == end_line && column_num >= end_column) {
471			break;
472			}
473	siliconforks	179	}
474
475	siliconforks	347	assert(line_num == end_line);
476			assert(column_num = end_column);
477	siliconforks	179	}
478
479	siliconforks	213	if (current_class != CLASS_NONE) {
480			output_character('\n', CLASS_NONE);
481	siliconforks	179	}
482
483	siliconforks	507	token_stream.close(context);
484	siliconforks	179	}