/[jscoverage]/trunk/js/jsscan.h
ViewVC logotype

Contents of /trunk/js/jsscan.h

Parent Directory Parent Directory | Revision Log Revision Log


Revision 460 - (show annotations)
Sat Sep 26 23:15:22 2009 UTC (10 years, 1 month ago) by siliconforks
File MIME type: text/plain
File size: 19601 byte(s)
Upgrade to SpiderMonkey from Firefox 3.5.3.

1 /* -*- Mode: C; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 4 -*-
2 *
3 * ***** BEGIN LICENSE BLOCK *****
4 * Version: MPL 1.1/GPL 2.0/LGPL 2.1
5 *
6 * The contents of this file are subject to the Mozilla Public License Version
7 * 1.1 (the "License"); you may not use this file except in compliance with
8 * the License. You may obtain a copy of the License at
9 * http://www.mozilla.org/MPL/
10 *
11 * Software distributed under the License is distributed on an "AS IS" basis,
12 * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
13 * for the specific language governing rights and limitations under the
14 * License.
15 *
16 * The Original Code is Mozilla Communicator client code, released
17 * March 31, 1998.
18 *
19 * The Initial Developer of the Original Code is
20 * Netscape Communications Corporation.
21 * Portions created by the Initial Developer are Copyright (C) 1998
22 * the Initial Developer. All Rights Reserved.
23 *
24 * Contributor(s):
25 *
26 * Alternatively, the contents of this file may be used under the terms of
27 * either of the GNU General Public License Version 2 or later (the "GPL"),
28 * or the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
29 * in which case the provisions of the GPL or the LGPL are applicable instead
30 * of those above. If you wish to allow use of your version of this file only
31 * under the terms of either the GPL or the LGPL, and not to allow others to
32 * use your version of this file under the terms of the MPL, indicate your
33 * decision by deleting the provisions above and replace them with the notice
34 * and other provisions required by the GPL or the LGPL. If you do not delete
35 * the provisions above, a recipient may use your version of this file under
36 * the terms of any one of the MPL, the GPL or the LGPL.
37 *
38 * ***** END LICENSE BLOCK ***** */
39
40 #ifndef jsscan_h___
41 #define jsscan_h___
42 /*
43 * JS lexical scanner interface.
44 */
45 #include <stddef.h>
46 #include <stdio.h>
47 #include "jsversion.h"
48 #include "jsopcode.h"
49 #include "jsprvtd.h"
50 #include "jspubtd.h"
51
52 JS_BEGIN_EXTERN_C
53
54 #define JS_KEYWORD(keyword, type, op, version) \
55 extern const char js_##keyword##_str[];
56 #include "jskeyword.tbl"
57 #undef JS_KEYWORD
58
59 typedef enum JSTokenType {
60 TOK_ERROR = -1, /* well-known as the only code < EOF */
61 TOK_EOF = 0, /* end of file */
62 TOK_EOL = 1, /* end of line */
63 TOK_SEMI = 2, /* semicolon */
64 TOK_COMMA = 3, /* comma operator */
65 TOK_ASSIGN = 4, /* assignment ops (= += -= etc.) */
66 TOK_HOOK = 5, TOK_COLON = 6, /* conditional (?:) */
67 TOK_OR = 7, /* logical or (||) */
68 TOK_AND = 8, /* logical and (&&) */
69 TOK_BITOR = 9, /* bitwise-or (|) */
70 TOK_BITXOR = 10, /* bitwise-xor (^) */
71 TOK_BITAND = 11, /* bitwise-and (&) */
72 TOK_EQOP = 12, /* equality ops (== !=) */
73 TOK_RELOP = 13, /* relational ops (< <= > >=) */
74 TOK_SHOP = 14, /* shift ops (<< >> >>>) */
75 TOK_PLUS = 15, /* plus */
76 TOK_MINUS = 16, /* minus */
77 TOK_STAR = 17, TOK_DIVOP = 18, /* multiply/divide ops (* / %) */
78 TOK_UNARYOP = 19, /* unary prefix operator */
79 TOK_INC = 20, TOK_DEC = 21, /* increment/decrement (++ --) */
80 TOK_DOT = 22, /* member operator (.) */
81 TOK_LB = 23, TOK_RB = 24, /* left and right brackets */
82 TOK_LC = 25, TOK_RC = 26, /* left and right curlies (braces) */
83 TOK_LP = 27, TOK_RP = 28, /* left and right parentheses */
84 TOK_NAME = 29, /* identifier */
85 TOK_NUMBER = 30, /* numeric constant */
86 TOK_STRING = 31, /* string constant */
87 TOK_REGEXP = 32, /* RegExp constant */
88 TOK_PRIMARY = 33, /* true, false, null, this, super */
89 TOK_FUNCTION = 34, /* function keyword */
90 TOK_IF = 35, /* if keyword */
91 TOK_ELSE = 36, /* else keyword */
92 TOK_SWITCH = 37, /* switch keyword */
93 TOK_CASE = 38, /* case keyword */
94 TOK_DEFAULT = 39, /* default keyword */
95 TOK_WHILE = 40, /* while keyword */
96 TOK_DO = 41, /* do keyword */
97 TOK_FOR = 42, /* for keyword */
98 TOK_BREAK = 43, /* break keyword */
99 TOK_CONTINUE = 44, /* continue keyword */
100 TOK_IN = 45, /* in keyword */
101 TOK_VAR = 46, /* var keyword */
102 TOK_WITH = 47, /* with keyword */
103 TOK_RETURN = 48, /* return keyword */
104 TOK_NEW = 49, /* new keyword */
105 TOK_DELETE = 50, /* delete keyword */
106 TOK_DEFSHARP = 51, /* #n= for object/array initializers */
107 TOK_USESHARP = 52, /* #n# for object/array initializers */
108 TOK_TRY = 53, /* try keyword */
109 TOK_CATCH = 54, /* catch keyword */
110 TOK_FINALLY = 55, /* finally keyword */
111 TOK_THROW = 56, /* throw keyword */
112 TOK_INSTANCEOF = 57, /* instanceof keyword */
113 TOK_DEBUGGER = 58, /* debugger keyword */
114 TOK_XMLSTAGO = 59, /* XML start tag open (<) */
115 TOK_XMLETAGO = 60, /* XML end tag open (</) */
116 TOK_XMLPTAGC = 61, /* XML point tag close (/>) */
117 TOK_XMLTAGC = 62, /* XML start or end tag close (>) */
118 TOK_XMLNAME = 63, /* XML start-tag non-final fragment */
119 TOK_XMLATTR = 64, /* XML quoted attribute value */
120 TOK_XMLSPACE = 65, /* XML whitespace */
121 TOK_XMLTEXT = 66, /* XML text */
122 TOK_XMLCOMMENT = 67, /* XML comment */
123 TOK_XMLCDATA = 68, /* XML CDATA section */
124 TOK_XMLPI = 69, /* XML processing instruction */
125 TOK_AT = 70, /* XML attribute op (@) */
126 TOK_DBLCOLON = 71, /* namespace qualified name op (::) */
127 TOK_ANYNAME = 72, /* XML AnyName singleton (*) */
128 TOK_DBLDOT = 73, /* XML descendant op (..) */
129 TOK_FILTER = 74, /* XML filtering predicate op (.()) */
130 TOK_XMLELEM = 75, /* XML element node type (no token) */
131 TOK_XMLLIST = 76, /* XML list node type (no token) */
132 TOK_YIELD = 77, /* yield from generator function */
133 TOK_ARRAYCOMP = 78, /* array comprehension initialiser */
134 TOK_ARRAYPUSH = 79, /* array push within comprehension */
135 TOK_LEXICALSCOPE = 80, /* block scope AST node label */
136 TOK_LET = 81, /* let keyword */
137 TOK_SEQ = 82, /* synthetic sequence of statements,
138 not a block */
139 TOK_FORHEAD = 83, /* head of for(;;)-style loop */
140 TOK_ARGSBODY = 84, /* formal args in list + body at end */
141 TOK_UPVARS = 85, /* lexical dependencies as JSAtomList
142 of definitions paired with a parse
143 tree full of uses of those names */
144 TOK_RESERVED, /* reserved keywords */
145 TOK_LIMIT /* domain size */
146 } JSTokenType;
147
148 #define IS_PRIMARY_TOKEN(tt) \
149 ((uintN)((tt) - TOK_NAME) <= (uintN)(TOK_PRIMARY - TOK_NAME))
150
151 #define TOKEN_TYPE_IS_XML(tt) \
152 ((tt) == TOK_AT || (tt) == TOK_DBLCOLON || (tt) == TOK_ANYNAME)
153
154 #define TREE_TYPE_IS_XML(tt) \
155 ((tt) == TOK_XMLCOMMENT || (tt) == TOK_XMLCDATA || (tt) == TOK_XMLPI || \
156 (tt) == TOK_XMLELEM || (tt) == TOK_XMLLIST)
157
158 #if JS_HAS_BLOCK_SCOPE
159 # define TOKEN_TYPE_IS_DECL(tt) ((tt) == TOK_VAR || (tt) == TOK_LET)
160 #else
161 # define TOKEN_TYPE_IS_DECL(tt) ((tt) == TOK_VAR)
162 #endif
163
164 struct JSStringBuffer {
165 jschar *base;
166 jschar *limit; /* length limit for quick bounds check */
167 jschar *ptr; /* slot for next non-NUL char to store */
168 void *data;
169 JSBool (*grow)(JSStringBuffer *sb, size_t newlength);
170 void (*free)(JSStringBuffer *sb);
171 };
172
173 #define STRING_BUFFER_ERROR_BASE ((jschar *) 1)
174 #define STRING_BUFFER_OK(sb) ((sb)->base != STRING_BUFFER_ERROR_BASE)
175 #define STRING_BUFFER_OFFSET(sb) ((sb)->ptr -(sb)->base)
176
177 extern void
178 js_InitStringBuffer(JSStringBuffer *sb);
179
180 extern void
181 js_FinishStringBuffer(JSStringBuffer *sb);
182
183 static inline void
184 js_RewindStringBuffer(JSStringBuffer *sb)
185 {
186 JS_ASSERT(STRING_BUFFER_OK(sb));
187 sb->ptr = sb->base;
188 }
189
190 #define ENSURE_STRING_BUFFER(sb,n) \
191 ((sb)->ptr + (n) <= (sb)->limit || sb->grow(sb, n))
192
193 /*
194 * NB: callers are obligated to test STRING_BUFFER_OK(sb) after this returns,
195 * before calling it again -- but not necessarily before calling other sb ops
196 * declared in this header file.
197 *
198 * Thus multiple calls, to ops other than this one that check STRING_BUFFER_OK
199 * and suppress updating sb if true, can consolidate the final STRING_BUFFER_OK
200 * test that conditions a JS_ReportOutOfMemory (if necessary -- the grow hook
201 * can report OOM early, obviating the need for the callers to report).
202 *
203 * This style of error checking is not obviously better, and it could be worse
204 * in efficiency, than the propagated failure return code style used elsewhere
205 * in the engine. I view it as a failed experiment. /be
206 */
207 static inline void
208 js_FastAppendChar(JSStringBuffer *sb, jschar c)
209 {
210 JS_ASSERT(STRING_BUFFER_OK(sb));
211 if (!ENSURE_STRING_BUFFER(sb, 1))
212 return;
213 *sb->ptr++ = c;
214 }
215
216 extern void
217 js_AppendChar(JSStringBuffer *sb, jschar c);
218
219 extern void
220 js_RepeatChar(JSStringBuffer *sb, jschar c, uintN count);
221
222 extern void
223 js_AppendCString(JSStringBuffer *sb, const char *asciiz);
224
225 extern void
226 js_AppendUCString(JSStringBuffer *sb, const jschar *buf, uintN len);
227
228 extern void
229 js_AppendJSString(JSStringBuffer *sb, JSString *str);
230
231 struct JSTokenPtr {
232 uint32 index; /* index of char in physical line */
233 uint32 lineno; /* physical line number */
234
235 bool operator <(const JSTokenPtr& bptr) {
236 return lineno < bptr.lineno ||
237 (lineno == bptr.lineno && index < bptr.index);
238 }
239
240 bool operator <=(const JSTokenPtr& bptr) {
241 return lineno < bptr.lineno ||
242 (lineno == bptr.lineno && index <= bptr.index);
243 }
244
245 bool operator >(const JSTokenPtr& bptr) {
246 return !(*this <= bptr);
247 }
248
249 bool operator >=(const JSTokenPtr& bptr) {
250 return !(*this < bptr);
251 }
252 };
253
254 struct JSTokenPos {
255 JSTokenPtr begin; /* first character and line of token */
256 JSTokenPtr end; /* index 1 past last char, last line */
257
258 bool operator <(const JSTokenPos& bpos) {
259 return begin < bpos.begin;
260 }
261
262 bool operator <=(const JSTokenPos& bpos) {
263 return begin <= bpos.begin;
264 }
265
266 bool operator >(const JSTokenPos& bpos) {
267 return !(*this <= bpos);
268 }
269
270 bool operator >=(const JSTokenPos& bpos) {
271 return !(*this < bpos);
272 }
273 };
274
275 struct JSToken {
276 JSTokenType type; /* char value or above enumerator */
277 JSTokenPos pos; /* token position in file */
278 jschar *ptr; /* beginning of token in line buffer */
279 union {
280 struct { /* name or string literal */
281 JSOp op; /* operator, for minimal parser */
282 JSAtom *atom; /* atom table entry */
283 } s;
284 uintN reflags; /* regexp flags, use tokenbuf to access
285 regexp chars */
286 struct { /* atom pair, for XML PIs */
287 JSAtom *atom2; /* auxiliary atom table entry */
288 JSAtom *atom; /* main atom table entry */
289 } p;
290 jsdouble dval; /* floating point number */
291 } u;
292 };
293
294 #define t_op u.s.op
295 #define t_reflags u.reflags
296 #define t_atom u.s.atom
297 #define t_atom2 u.p.atom2
298 #define t_dval u.dval
299
300 typedef struct JSTokenBuf {
301 jschar *base; /* base of line or stream buffer */
302 jschar *limit; /* limit for quick bounds check */
303 jschar *ptr; /* next char to get, or slot to use */
304 } JSTokenBuf;
305
306 #define JS_LINE_LIMIT 256 /* logical line buffer size limit --
307 physical line length is unlimited */
308 #define NTOKENS 4 /* 1 current + 2 lookahead, rounded */
309 #define NTOKENS_MASK (NTOKENS-1) /* to power of 2 to avoid divmod by 3 */
310
311 struct JSTokenStream {
312 JSToken tokens[NTOKENS];/* circular token buffer */
313 uintN cursor; /* index of last parsed token */
314 uintN lookahead; /* count of lookahead tokens */
315 uintN lineno; /* current line number */
316 uintN ungetpos; /* next free char slot in ungetbuf */
317 jschar ungetbuf[6]; /* at most 6, for \uXXXX lookahead */
318 uintN flags; /* flags -- see below */
319 uint32 linelen; /* physical linebuf segment length */
320 uint32 linepos; /* linebuf offset in physical line */
321 JSTokenBuf linebuf; /* line buffer for diagnostics */
322 JSTokenBuf userbuf; /* user input buffer if !file */
323 JSStringBuffer tokenbuf; /* current token string buffer */
324 const char *filename; /* input filename or null */
325 FILE *file; /* stdio stream if reading from file */
326 JSSourceHandler listener; /* callback for source; eg debugger */
327 void *listenerData; /* listener 'this' data */
328 void *listenerTSData;/* listener data for this TokenStream */
329 jschar *saveEOL; /* save next end of line in userbuf, to
330 optimize for very long lines */
331 };
332
333 #define CURRENT_TOKEN(ts) ((ts)->tokens[(ts)->cursor])
334 #define ON_CURRENT_LINE(ts,pos) ((uint16)(ts)->lineno == (pos).end.lineno)
335
336 /* JSTokenStream flags */
337 #define TSF_ERROR 0x01 /* fatal error while compiling */
338 #define TSF_EOF 0x02 /* hit end of file */
339 #define TSF_NEWLINES 0x04 /* tokenize newlines */
340 #define TSF_OPERAND 0x08 /* looking for operand, not operator */
341 #define TSF_NLFLAG 0x20 /* last linebuf ended with \n */
342 #define TSF_CRFLAG 0x40 /* linebuf would have ended with \r */
343 #define TSF_DIRTYLINE 0x80 /* non-whitespace since start of line */
344 #define TSF_OWNFILENAME 0x100 /* ts->filename is malloc'd */
345 #define TSF_XMLTAGMODE 0x200 /* scanning within an XML tag in E4X */
346 #define TSF_XMLTEXTMODE 0x400 /* scanning XMLText terminal from E4X */
347 #define TSF_XMLONLYMODE 0x800 /* don't scan {expr} within text/tag */
348
349 /* Flag indicating unexpected end of input, i.e. TOK_EOF not at top-level. */
350 #define TSF_UNEXPECTED_EOF 0x1000
351
352 /*
353 * To handle the hard case of contiguous HTML comments, we want to clear the
354 * TSF_DIRTYINPUT flag at the end of each such comment. But we'd rather not
355 * scan for --> within every //-style comment unless we have to. So we set
356 * TSF_IN_HTML_COMMENT when a <!-- is scanned as an HTML begin-comment, and
357 * clear it (and TSF_DIRTYINPUT) when we scan --> either on a clean line, or
358 * only if (ts->flags & TSF_IN_HTML_COMMENT), in a //-style comment.
359 *
360 * This still works as before given a malformed comment hiding hack such as:
361 *
362 * <script>
363 * <!-- comment hiding hack #1
364 * code goes here
365 * // --> oops, markup for script-unaware browsers goes here!
366 * </script>
367 *
368 * It does not cope with malformed comment hiding hacks where --> is hidden
369 * by C-style comments, or on a dirty line. Such cases are already broken.
370 */
371 #define TSF_IN_HTML_COMMENT 0x2000
372
373 /* Ignore keywords and return TOK_NAME instead to the parser. */
374 #define TSF_KEYWORD_IS_NAME 0x4000
375
376 /* Parsing a destructuring object or array initialiser pattern. */
377 #define TSF_DESTRUCTURING 0x8000
378
379 /* Unicode separators that are treated as line terminators, in addition to \n, \r */
380 #define LINE_SEPARATOR 0x2028
381 #define PARA_SEPARATOR 0x2029
382
383 /*
384 * Create a new token stream, either from an input buffer or from a file.
385 * Return null on file-open or memory-allocation failure.
386 *
387 * The function uses JSContext.tempPool to allocate internal buffers. The
388 * caller should release them using JS_ARENA_RELEASE after it has finished
389 * with the token stream and has called js_CloseTokenStream.
390 */
391 extern JSBool
392 js_InitTokenStream(JSContext *cx, JSTokenStream *ts,
393 const jschar *base, size_t length,
394 FILE *fp, const char *filename, uintN lineno);
395
396 extern void
397 js_CloseTokenStream(JSContext *cx, JSTokenStream *ts);
398
399 extern JS_FRIEND_API(int)
400 js_fgets(char *buf, int size, FILE *file);
401
402 /*
403 * If the given char array forms JavaScript keyword, return corresponding
404 * token. Otherwise return TOK_EOF.
405 */
406 extern JSTokenType
407 js_CheckKeyword(const jschar *chars, size_t length);
408
409 /*
410 * Friend-exported API entry point to call a mapping function on each reserved
411 * identifier in the scanner's keyword table.
412 */
413 extern JS_FRIEND_API(void)
414 js_MapKeywords(void (*mapfun)(const char *));
415
416 /*
417 * Check that str forms a valid JS identifier name. The function does not
418 * check if str is a JS keyword.
419 */
420 extern JSBool
421 js_IsIdentifier(JSString *str);
422
423 /*
424 * Report a compile-time error by its number. Return true for a warning, false
425 * for an error. When pn is not null, use it to report error's location.
426 * Otherwise use ts, which must not be null.
427 */
428 JSBool
429 js_ReportCompileErrorNumber(JSContext *cx, JSTokenStream *ts, JSParseNode *pn,
430 uintN flags, uintN errorNumber, ...);
431
432 /*
433 * Steal one JSREPORT_* bit (see jsapi.h) to tell that arguments to the error
434 * message have const jschar* type, not const char*.
435 */
436 #define JSREPORT_UC 0x100
437
438 /*
439 * Look ahead one token and return its type.
440 */
441 extern JSTokenType
442 js_PeekToken(JSContext *cx, JSTokenStream *ts);
443
444 extern JSTokenType
445 js_PeekTokenSameLine(JSContext *cx, JSTokenStream *ts);
446
447 /*
448 * Get the next token from ts.
449 */
450 extern JSTokenType
451 js_GetToken(JSContext *cx, JSTokenStream *ts);
452
453 /*
454 * Push back the last scanned token onto ts.
455 */
456 extern void
457 js_UngetToken(JSTokenStream *ts);
458
459 /*
460 * Get the next token from ts if its type is tt.
461 */
462 extern JSBool
463 js_MatchToken(JSContext *cx, JSTokenStream *ts, JSTokenType tt);
464
465 JS_END_EXTERN_C
466
467 #endif /* jsscan_h___ */

  ViewVC Help
Powered by ViewVC 1.1.24