/[jscoverage]/trunk/js/jstracer.cpp
ViewVC logotype

Contents of /trunk/js/jstracer.cpp

Parent Directory Parent Directory | Revision Log Revision Log


Revision 399 - (show annotations)
Tue Dec 9 03:37:47 2008 UTC (11 years ago) by siliconforks
File size: 259753 byte(s)
Use SpiderMonkey from Firefox 3.1b2.

1 /* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 4 -*-
2 * vim: set ts=4 sw=4 et tw=99:
3 *
4 * ***** BEGIN LICENSE BLOCK *****
5 * Version: MPL 1.1/GPL 2.0/LGPL 2.1
6 *
7 * The contents of this file are subject to the Mozilla Public License Version
8 * 1.1 (the "License"); you may not use this file except in compliance with
9 * the License. You may obtain a copy of the License at
10 * http://www.mozilla.org/MPL/
11 *
12 * Software distributed under the License is distributed on an "AS IS" basis,
13 * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
14 * for the specific language governing rights and limitations under the
15 * License.
16 *
17 * The Original Code is Mozilla SpiderMonkey JavaScript 1.9 code, released
18 * May 28, 2008.
19 *
20 * The Initial Developer of the Original Code is
21 * Brendan Eich <brendan@mozilla.org>
22 *
23 * Contributor(s):
24 * Andreas Gal <gal@mozilla.com>
25 * Mike Shaver <shaver@mozilla.org>
26 * David Anderson <danderson@mozilla.com>
27 *
28 * Alternatively, the contents of this file may be used under the terms of
29 * either of the GNU General Public License Version 2 or later (the "GPL"),
30 * or the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
31 * in which case the provisions of the GPL or the LGPL are applicable instead
32 * of those above. If you wish to allow use of your version of this file only
33 * under the terms of either the GPL or the LGPL, and not to allow others to
34 * use your version of this file under the terms of the MPL, indicate your
35 * decision by deleting the provisions above and replace them with the notice
36 * and other provisions required by the GPL or the LGPL. If you do not delete
37 * the provisions above, a recipient may use your version of this file under
38 * the terms of any one of the MPL, the GPL or the LGPL.
39 *
40 * ***** END LICENSE BLOCK ***** */
41
42 #include "jsstddef.h" // always first
43 #include "jsbit.h" // low-level (NSPR-based) headers next
44 #include "jsprf.h"
45 #include <math.h> // standard headers next
46 #ifdef _MSC_VER
47 #include <malloc.h>
48 #define alloca _alloca
49 #endif
50 #ifdef SOLARIS
51 #include <alloca.h>
52 #endif
53
54 #include "nanojit/nanojit.h"
55 #include "jsarray.h" // higher-level library and API headers
56 #include "jsbool.h"
57 #include "jscntxt.h"
58 #include "jsdbgapi.h"
59 #include "jsemit.h"
60 #include "jsfun.h"
61 #include "jsinterp.h"
62 #include "jsiter.h"
63 #include "jsobj.h"
64 #include "jsopcode.h"
65 #include "jsregexp.h"
66 #include "jsscope.h"
67 #include "jsscript.h"
68 #include "jsdate.h"
69 #include "jsstaticcheck.h"
70 #include "jstracer.h"
71
72 #include "jsautooplen.h" // generated headers last
73
74 /* Never use JSVAL_IS_BOOLEAN because it restricts the value (true, false) and
75 the type. What you want to use is JSVAL_TAG(x) == JSVAL_BOOLEAN and then
76 handle the undefined case properly (bug 457363). */
77 #undef JSVAL_IS_BOOLEAN
78 #define JSVAL_IS_BOOLEAN(x) JS_STATIC_ASSERT(0)
79
80 /* Use a fake tag to represent boxed values, borrowing from the integer tag
81 range since we only use JSVAL_INT to indicate integers. */
82 #define JSVAL_BOXED 3
83
84 /* Map to translate a type tag into a printable representation. */
85 static const char typeChar[] = "OIDVS?B?";
86
87 /* Number of iterations of a loop where we start tracing. That is, we don't
88 start tracing until the beginning of the HOTLOOP-th iteration. */
89 #define HOTLOOP 2
90
91 /* Number of times we wait to exit on a side exit before we try to extend the tree. */
92 #define HOTEXIT 1
93
94 /* Max call depths for inlining. */
95 #define MAX_CALLDEPTH 10
96
97 /* Max number of type mismatchs before we trash the tree. */
98 #define MAX_MISMATCH 20
99
100 /* Max blacklist level of inner tree immediate recompiling */
101 #define MAX_INNER_RECORD_BLACKLIST -16
102
103 /* Max native stack size. */
104 #define MAX_NATIVE_STACK_SLOTS 1024
105
106 /* Max call stack size. */
107 #define MAX_CALL_STACK_ENTRIES 64
108
109 /* Max number of branches per tree. */
110 #define MAX_BRANCHES 16
111
112 /* Macros for demote slot lists */
113 #define ALLOCA_UNDEMOTE_SLOTLIST(num) (unsigned*)alloca(((num) + 1) * sizeof(unsigned))
114 #define ADD_UNDEMOTE_SLOT(list, slot) list[++list[0]] = slot
115 #define NUM_UNDEMOTE_SLOTS(list) list[0]
116 #define CLEAR_UNDEMOTE_SLOTLIST(list) list[0] = 0
117
118 #ifdef JS_JIT_SPEW
119 #define ABORT_TRACE(msg) do { debug_only_v(fprintf(stdout, "abort: %d: %s\n", __LINE__, msg);) return false; } while (0)
120 #else
121 #define ABORT_TRACE(msg) return false
122 #endif
123
124 #ifdef JS_JIT_SPEW
125 struct __jitstats {
126 #define JITSTAT(x) uint64 x;
127 #include "jitstats.tbl"
128 #undef JITSTAT
129 } jitstats = { 0LL, };
130
131 JS_STATIC_ASSERT(sizeof(jitstats) % sizeof(uint64) == 0);
132
133 enum jitstat_ids {
134 #define JITSTAT(x) STAT ## x ## ID,
135 #include "jitstats.tbl"
136 #undef JITSTAT
137 STAT_IDS_TOTAL
138 };
139
140 static JSPropertySpec jitstats_props[] = {
141 #define JITSTAT(x) { #x, STAT ## x ## ID, JSPROP_ENUMERATE | JSPROP_READONLY | JSPROP_PERMANENT },
142 #include "jitstats.tbl"
143 #undef JITSTAT
144 { 0 }
145 };
146
147 static JSBool
148 jitstats_getProperty(JSContext *cx, JSObject *obj, jsid id, jsval *vp)
149 {
150 int index = -1;
151
152 if (JSVAL_IS_STRING(id)) {
153 JSString* str = JSVAL_TO_STRING(id);
154 if (strcmp(JS_GetStringBytes(str), "HOTLOOP") == 0) {
155 *vp = INT_TO_JSVAL(HOTLOOP);
156 return JS_TRUE;
157 }
158 }
159
160 if (JSVAL_IS_INT(id))
161 index = JSVAL_TO_INT(id);
162
163 uint64 result = 0;
164 switch (index) {
165 #define JITSTAT(x) case STAT ## x ## ID: result = jitstats.x; break;
166 #include "jitstats.tbl"
167 #undef JITSTAT
168 default:
169 *vp = JSVAL_VOID;
170 return JS_TRUE;
171 }
172
173 if (result < JSVAL_INT_MAX) {
174 *vp = INT_TO_JSVAL(result);
175 return JS_TRUE;
176 }
177 char retstr[64];
178 JS_snprintf(retstr, sizeof retstr, "%llu", result);
179 *vp = STRING_TO_JSVAL(JS_NewStringCopyZ(cx, retstr));
180 return JS_TRUE;
181 }
182
183 JSClass jitstats_class = {
184 "jitstats",
185 JSCLASS_HAS_PRIVATE,
186 JS_PropertyStub, JS_PropertyStub,
187 jitstats_getProperty, JS_PropertyStub,
188 JS_EnumerateStub, JS_ResolveStub,
189 JS_ConvertStub, JS_FinalizeStub,
190 JSCLASS_NO_OPTIONAL_MEMBERS
191 };
192
193 void
194 js_InitJITStatsClass(JSContext *cx, JSObject *glob)
195 {
196 JS_InitClass(cx, glob, NULL, &jitstats_class, NULL, 0, jitstats_props, NULL, NULL, NULL);
197 }
198
199 #define AUDIT(x) (jitstats.x++)
200 #else
201 #define AUDIT(x) ((void)0)
202 #endif /* JS_JIT_SPEW */
203
204 #define INS_CONST(c) addName(lir->insImm(c), #c)
205 #define INS_CONSTPTR(p) addName(lir->insImmPtr((void*) (p)), #p)
206
207 using namespace avmplus;
208 using namespace nanojit;
209
210 static GC gc = GC();
211 static avmplus::AvmCore s_core = avmplus::AvmCore();
212 static avmplus::AvmCore* core = &s_core;
213
214 #ifdef JS_JIT_SPEW
215 void
216 js_DumpPeerStability(Fragmento* frago, const void* ip);
217 #endif
218
219 /* We really need a better way to configure the JIT. Shaver, where is my fancy JIT object? */
220 static bool nesting_enabled = true;
221 #if defined(NANOJIT_IA32)
222 static bool did_we_check_sse2 = false;
223 #endif
224
225 #ifdef JS_JIT_SPEW
226 static bool verbose_debug = getenv("TRACEMONKEY") && strstr(getenv("TRACEMONKEY"), "verbose");
227 #define debug_only_v(x) if (verbose_debug) { x; }
228 #else
229 #define debug_only_v(x)
230 #endif
231
232 /* The entire VM shares one oracle. Collisions and concurrent updates are tolerated and worst
233 case cause performance regressions. */
234 static Oracle oracle;
235
236 /* Blacklists the root peer fragment at a fragment's PC. This is so blacklisting stays at the
237 top of the peer list and not scattered around. */
238 void
239 js_BlacklistPC(Fragmento* frago, Fragment* frag);
240
241 Tracker::Tracker()
242 {
243 pagelist = 0;
244 }
245
246 Tracker::~Tracker()
247 {
248 clear();
249 }
250
251 jsuword
252 Tracker::getPageBase(const void* v) const
253 {
254 return jsuword(v) & ~jsuword(NJ_PAGE_SIZE-1);
255 }
256
257 struct Tracker::Page*
258 Tracker::findPage(const void* v) const
259 {
260 jsuword base = getPageBase(v);
261 struct Tracker::Page* p = pagelist;
262 while (p) {
263 if (p->base == base) {
264 return p;
265 }
266 p = p->next;
267 }
268 return 0;
269 }
270
271 struct Tracker::Page*
272 Tracker::addPage(const void* v) {
273 jsuword base = getPageBase(v);
274 struct Tracker::Page* p = (struct Tracker::Page*)
275 GC::Alloc(sizeof(*p) - sizeof(p->map) + (NJ_PAGE_SIZE >> 2) * sizeof(LIns*));
276 p->base = base;
277 p->next = pagelist;
278 pagelist = p;
279 return p;
280 }
281
282 void
283 Tracker::clear()
284 {
285 while (pagelist) {
286 Page* p = pagelist;
287 pagelist = pagelist->next;
288 GC::Free(p);
289 }
290 }
291
292 bool
293 Tracker::has(const void *v) const
294 {
295 return get(v) != NULL;
296 }
297
298 #if defined NANOJIT_64BIT
299 #define PAGEMASK 0x7ff
300 #else
301 #define PAGEMASK 0xfff
302 #endif
303
304 LIns*
305 Tracker::get(const void* v) const
306 {
307 struct Tracker::Page* p = findPage(v);
308 if (!p)
309 return NULL;
310 return p->map[(jsuword(v) & PAGEMASK) >> 2];
311 }
312
313 void
314 Tracker::set(const void* v, LIns* i)
315 {
316 struct Tracker::Page* p = findPage(v);
317 if (!p)
318 p = addPage(v);
319 p->map[(jsuword(v) & PAGEMASK) >> 2] = i;
320 }
321
322 static inline bool isNumber(jsval v)
323 {
324 return JSVAL_IS_INT(v) || JSVAL_IS_DOUBLE(v);
325 }
326
327 static inline jsdouble asNumber(jsval v)
328 {
329 JS_ASSERT(isNumber(v));
330 if (JSVAL_IS_DOUBLE(v))
331 return *JSVAL_TO_DOUBLE(v);
332 return (jsdouble)JSVAL_TO_INT(v);
333 }
334
335 static inline bool isInt32(jsval v)
336 {
337 if (!isNumber(v))
338 return false;
339 jsdouble d = asNumber(v);
340 jsint i;
341 return JSDOUBLE_IS_INT(d, i);
342 }
343
344 /* Return JSVAL_DOUBLE for all numbers (int and double) and the tag otherwise. */
345 static inline uint8 getPromotedType(jsval v)
346 {
347 return JSVAL_IS_INT(v) ? JSVAL_DOUBLE : uint8(JSVAL_TAG(v));
348 }
349
350 /* Return JSVAL_INT for all whole numbers that fit into signed 32-bit and the tag otherwise. */
351 static inline uint8 getCoercedType(jsval v)
352 {
353 return isInt32(v) ? JSVAL_INT : (uint8) JSVAL_TAG(v);
354 }
355
356 /* Tell the oracle that a certain global variable should not be demoted. */
357 void
358 Oracle::markGlobalSlotUndemotable(JSScript* script, unsigned slot)
359 {
360 _dontDemote.set(&gc, (slot % ORACLE_SIZE));
361 }
362
363 /* Consult with the oracle whether we shouldn't demote a certain global variable. */
364 bool
365 Oracle::isGlobalSlotUndemotable(JSScript* script, unsigned slot) const
366 {
367 return _dontDemote.get(slot % ORACLE_SIZE);
368 }
369
370 /* Tell the oracle that a certain slot at a certain bytecode location should not be demoted. */
371 void
372 Oracle::markStackSlotUndemotable(JSScript* script, jsbytecode* ip, unsigned slot)
373 {
374 uint32 hash = uint32(intptr_t(ip)) + (slot << 5);
375 hash %= ORACLE_SIZE;
376 _dontDemote.set(&gc, hash);
377 }
378
379 /* Consult with the oracle whether we shouldn't demote a certain slot. */
380 bool
381 Oracle::isStackSlotUndemotable(JSScript* script, jsbytecode* ip, unsigned slot) const
382 {
383 uint32 hash = uint32(intptr_t(ip)) + (slot << 5);
384 hash %= ORACLE_SIZE;
385 return _dontDemote.get(hash);
386 }
387
388 /* Clear the oracle. */
389 void
390 Oracle::clear()
391 {
392 _dontDemote.reset();
393 }
394
395 #if defined(NJ_SOFTFLOAT)
396 JS_DEFINE_CALLINFO_1(static, DOUBLE, i2f, INT32, 1, 1)
397 JS_DEFINE_CALLINFO_1(static, DOUBLE, u2f, UINT32, 1, 1)
398 #endif
399
400 static bool isi2f(LInsp i)
401 {
402 if (i->isop(LIR_i2f))
403 return true;
404
405 #if defined(NJ_SOFTFLOAT)
406 if (i->isop(LIR_qjoin) &&
407 i->oprnd1()->isop(LIR_call) &&
408 i->oprnd2()->isop(LIR_callh))
409 {
410 if (i->oprnd1()->callInfo() == &i2f_ci)
411 return true;
412 }
413 #endif
414
415 return false;
416 }
417
418 static bool isu2f(LInsp i)
419 {
420 if (i->isop(LIR_u2f))
421 return true;
422
423 #if defined(NJ_SOFTFLOAT)
424 if (i->isop(LIR_qjoin) &&
425 i->oprnd1()->isop(LIR_call) &&
426 i->oprnd2()->isop(LIR_callh))
427 {
428 if (i->oprnd1()->callInfo() == &u2f_ci)
429 return true;
430 }
431 #endif
432
433 return false;
434 }
435
436 static LInsp iu2fArg(LInsp i)
437 {
438 #if defined(NJ_SOFTFLOAT)
439 if (i->isop(LIR_qjoin))
440 return i->oprnd1()->arg(0);
441 #endif
442
443 return i->oprnd1();
444 }
445
446
447 static LIns* demote(LirWriter *out, LInsp i)
448 {
449 if (i->isCall())
450 return callArgN(i, 0);
451 if (isi2f(i) || isu2f(i))
452 return iu2fArg(i);
453 if (i->isconst())
454 return i;
455 AvmAssert(i->isconstq());
456 double cf = i->constvalf();
457 int32_t ci = cf > 0x7fffffff ? uint32_t(cf) : int32_t(cf);
458 return out->insImm(ci);
459 }
460
461 static bool isPromoteInt(LIns* i)
462 {
463 jsdouble d;
464 return isi2f(i) || i->isconst() ||
465 (i->isconstq() && (d = i->constvalf()) == jsdouble(jsint(d)) && !JSDOUBLE_IS_NEGZERO(d));
466 }
467
468 static bool isPromoteUint(LIns* i)
469 {
470 jsdouble d;
471 return isu2f(i) || i->isconst() ||
472 (i->isconstq() && (d = i->constvalf()) == (jsdouble)(jsuint)d && !JSDOUBLE_IS_NEGZERO(d));
473 }
474
475 static bool isPromote(LIns* i)
476 {
477 return isPromoteInt(i) || isPromoteUint(i);
478 }
479
480 static bool isconst(LIns* i, int32_t c)
481 {
482 return i->isconst() && i->constval() == c;
483 }
484
485 static bool overflowSafe(LIns* i)
486 {
487 LIns* c;
488 return (i->isop(LIR_and) && ((c = i->oprnd2())->isconst()) &&
489 ((c->constval() & 0xc0000000) == 0)) ||
490 (i->isop(LIR_rsh) && ((c = i->oprnd2())->isconst()) &&
491 ((c->constval() > 0)));
492 }
493
494 #if defined(NJ_SOFTFLOAT)
495 /* soft float */
496
497 JS_DEFINE_CALLINFO_1(static, DOUBLE, fneg, DOUBLE, 1, 1)
498 JS_DEFINE_CALLINFO_2(static, INT32, fcmpeq, DOUBLE, DOUBLE, 1, 1)
499 JS_DEFINE_CALLINFO_2(static, INT32, fcmplt, DOUBLE, DOUBLE, 1, 1)
500 JS_DEFINE_CALLINFO_2(static, INT32, fcmple, DOUBLE, DOUBLE, 1, 1)
501 JS_DEFINE_CALLINFO_2(static, INT32, fcmpgt, DOUBLE, DOUBLE, 1, 1)
502 JS_DEFINE_CALLINFO_2(static, INT32, fcmpge, DOUBLE, DOUBLE, 1, 1)
503 JS_DEFINE_CALLINFO_2(static, DOUBLE, fmul, DOUBLE, DOUBLE, 1, 1)
504 JS_DEFINE_CALLINFO_2(static, DOUBLE, fadd, DOUBLE, DOUBLE, 1, 1)
505 JS_DEFINE_CALLINFO_2(static, DOUBLE, fdiv, DOUBLE, DOUBLE, 1, 1)
506 JS_DEFINE_CALLINFO_2(static, DOUBLE, fsub, DOUBLE, DOUBLE, 1, 1)
507
508 jsdouble FASTCALL
509 fneg(jsdouble x)
510 {
511 return -x;
512 }
513
514 jsdouble FASTCALL
515 i2f(int32 i)
516 {
517 return i;
518 }
519
520 jsdouble FASTCALL
521 u2f(jsuint u)
522 {
523 return u;
524 }
525
526 int32 FASTCALL
527 fcmpeq(jsdouble x, jsdouble y)
528 {
529 return x==y;
530 }
531
532 int32 FASTCALL
533 fcmplt(jsdouble x, jsdouble y)
534 {
535 return x < y;
536 }
537
538 int32 FASTCALL
539 fcmple(jsdouble x, jsdouble y)
540 {
541 return x <= y;
542 }
543
544 int32 FASTCALL
545 fcmpgt(jsdouble x, jsdouble y)
546 {
547 return x > y;
548 }
549
550 int32 FASTCALL
551 fcmpge(jsdouble x, jsdouble y)
552 {
553 return x >= y;
554 }
555
556 jsdouble FASTCALL
557 fmul(jsdouble x, jsdouble y)
558 {
559 return x * y;
560 }
561
562 jsdouble FASTCALL
563 fadd(jsdouble x, jsdouble y)
564 {
565 return x + y;
566 }
567
568 jsdouble FASTCALL
569 fdiv(jsdouble x, jsdouble y)
570 {
571 return x / y;
572 }
573
574 jsdouble FASTCALL
575 fsub(jsdouble x, jsdouble y)
576 {
577 return x - y;
578 }
579
580 class SoftFloatFilter: public LirWriter
581 {
582 public:
583 SoftFloatFilter(LirWriter* out):
584 LirWriter(out)
585 {
586 }
587
588 LInsp quadCall(const CallInfo *ci, LInsp args[]) {
589 LInsp qlo, qhi;
590
591 qlo = out->insCall(ci, args);
592 qhi = out->ins1(LIR_callh, qlo);
593 return out->qjoin(qlo, qhi);
594 }
595
596 LInsp ins1(LOpcode v, LInsp s0)
597 {
598 if (v == LIR_fneg)
599 return quadCall(&fneg_ci, &s0);
600
601 if (v == LIR_i2f)
602 return quadCall(&i2f_ci, &s0);
603
604 if (v == LIR_u2f)
605 return quadCall(&u2f_ci, &s0);
606
607 return out->ins1(v, s0);
608 }
609
610 LInsp ins2(LOpcode v, LInsp s0, LInsp s1)
611 {
612 LInsp args[2];
613 LInsp bv;
614
615 // change the numeric value and order of these LIR opcodes and die
616 if (LIR_fadd <= v && v <= LIR_fdiv) {
617 static const CallInfo *fmap[] = { &fadd_ci, &fsub_ci, &fmul_ci, &fdiv_ci };
618
619 args[0] = s1;
620 args[1] = s0;
621
622 return quadCall(fmap[v - LIR_fadd], args);
623 }
624
625 if (LIR_feq <= v && v <= LIR_fge) {
626 static const CallInfo *fmap[] = { &fcmpeq_ci, &fcmplt_ci, &fcmpgt_ci, &fcmple_ci, &fcmpge_ci };
627
628 args[0] = s1;
629 args[1] = s0;
630
631 bv = out->insCall(fmap[v - LIR_feq], args);
632 return out->ins2(LIR_eq, bv, out->insImm(1));
633 }
634
635 return out->ins2(v, s0, s1);
636 }
637
638 LInsp insCall(const CallInfo *ci, LInsp args[])
639 {
640 // if the return type is ARGSIZE_F, we have
641 // to do a quadCall ( qjoin(call,callh) )
642 if ((ci->_argtypes & 3) == ARGSIZE_F)
643 return quadCall(ci, args);
644
645 return out->insCall(ci, args);
646 }
647 };
648
649 #endif // NJ_SOFTFLOAT
650
651 class FuncFilter: public LirWriter
652 {
653 public:
654 FuncFilter(LirWriter* out):
655 LirWriter(out)
656 {
657 }
658
659 LInsp ins2(LOpcode v, LInsp s0, LInsp s1)
660 {
661 if (s0 == s1 && v == LIR_feq) {
662 if (isPromote(s0)) {
663 // double(int) and double(uint) cannot be nan
664 return insImm(1);
665 }
666 if (s0->isop(LIR_fmul) || s0->isop(LIR_fsub) || s0->isop(LIR_fadd)) {
667 LInsp lhs = s0->oprnd1();
668 LInsp rhs = s0->oprnd2();
669 if (isPromote(lhs) && isPromote(rhs)) {
670 // add/sub/mul promoted ints can't be nan
671 return insImm(1);
672 }
673 }
674 } else if (LIR_feq <= v && v <= LIR_fge) {
675 if (isPromoteInt(s0) && isPromoteInt(s1)) {
676 // demote fcmp to cmp
677 v = LOpcode(v + (LIR_eq - LIR_feq));
678 return out->ins2(v, demote(out, s0), demote(out, s1));
679 } else if (isPromoteUint(s0) && isPromoteUint(s1)) {
680 // uint compare
681 v = LOpcode(v + (LIR_eq - LIR_feq));
682 if (v != LIR_eq)
683 v = LOpcode(v + (LIR_ult - LIR_lt)); // cmp -> ucmp
684 return out->ins2(v, demote(out, s0), demote(out, s1));
685 }
686 } else if (v == LIR_or &&
687 s0->isop(LIR_lsh) && isconst(s0->oprnd2(), 16) &&
688 s1->isop(LIR_and) && isconst(s1->oprnd2(), 0xffff)) {
689 LIns* msw = s0->oprnd1();
690 LIns* lsw = s1->oprnd1();
691 LIns* x;
692 LIns* y;
693 if (lsw->isop(LIR_add) &&
694 lsw->oprnd1()->isop(LIR_and) &&
695 lsw->oprnd2()->isop(LIR_and) &&
696 isconst(lsw->oprnd1()->oprnd2(), 0xffff) &&
697 isconst(lsw->oprnd2()->oprnd2(), 0xffff) &&
698 msw->isop(LIR_add) &&
699 msw->oprnd1()->isop(LIR_add) &&
700 msw->oprnd2()->isop(LIR_rsh) &&
701 msw->oprnd1()->oprnd1()->isop(LIR_rsh) &&
702 msw->oprnd1()->oprnd2()->isop(LIR_rsh) &&
703 isconst(msw->oprnd2()->oprnd2(), 16) &&
704 isconst(msw->oprnd1()->oprnd1()->oprnd2(), 16) &&
705 isconst(msw->oprnd1()->oprnd2()->oprnd2(), 16) &&
706 (x = lsw->oprnd1()->oprnd1()) == msw->oprnd1()->oprnd1()->oprnd1() &&
707 (y = lsw->oprnd2()->oprnd1()) == msw->oprnd1()->oprnd2()->oprnd1() &&
708 lsw == msw->oprnd2()->oprnd1()) {
709 return out->ins2(LIR_add, x, y);
710 }
711 }
712 #ifdef NANOJIT_ARM
713 else if (v == LIR_lsh ||
714 v == LIR_rsh ||
715 v == LIR_ush)
716 {
717 // needed on ARM -- arm doesn't mask shifts to 31 like x86 does
718 if (s1->isconst())
719 s1->setimm16(s1->constval() & 31);
720 else
721 s1 = out->ins2(LIR_and, s1, out->insImm(31));
722 return out->ins2(v, s0, s1);
723 }
724 #endif
725
726 return out->ins2(v, s0, s1);
727 }
728
729 LInsp insCall(const CallInfo *ci, LInsp args[])
730 {
731 LInsp s0 = args[0];
732 if (ci == &js_DoubleToUint32_ci) {
733 if (s0->isconstq())
734 return out->insImm(js_DoubleToECMAUint32(s0->constvalf()));
735 if (isi2f(s0) || isu2f(s0))
736 return iu2fArg(s0);
737 } else if (ci == &js_DoubleToInt32_ci) {
738 if (s0->isconstq())
739 return out->insImm(js_DoubleToECMAInt32(s0->constvalf()));
740 if (s0->isop(LIR_fadd) || s0->isop(LIR_fsub)) {
741 LInsp lhs = s0->oprnd1();
742 LInsp rhs = s0->oprnd2();
743 if (isPromote(lhs) && isPromote(rhs)) {
744 LOpcode op = LOpcode(s0->opcode() & ~LIR64);
745 return out->ins2(op, demote(out, lhs), demote(out, rhs));
746 }
747 }
748 if (isi2f(s0) || isu2f(s0))
749 return iu2fArg(s0);
750 // XXX ARM -- check for qjoin(call(UnboxDouble),call(UnboxDouble))
751 if (s0->isCall() && s0->callInfo() == &js_UnboxDouble_ci) {
752 LIns* args2[] = { callArgN(s0, 0) };
753 return out->insCall(&js_UnboxInt32_ci, args2);
754 }
755 if (s0->isCall() && s0->callInfo() == &js_StringToNumber_ci) {
756 // callArgN's ordering is that as seen by the builtin, not as stored in args here.
757 // True story!
758 LIns* args2[] = { callArgN(s0, 1), callArgN(s0, 0) };
759 return out->insCall(&js_StringToInt32_ci, args2);
760 }
761 } else if (ci == &js_BoxDouble_ci) {
762 JS_ASSERT(s0->isQuad());
763 if (s0->isop(LIR_i2f)) {
764 LIns* args2[] = { s0->oprnd1(), args[1] };
765 return out->insCall(&js_BoxInt32_ci, args2);
766 }
767 if (s0->isCall() && s0->callInfo() == &js_UnboxDouble_ci)
768 return callArgN(s0, 0);
769 }
770 return out->insCall(ci, args);
771 }
772 };
773
774 /* In debug mode vpname contains a textual description of the type of the
775 slot during the forall iteration over al slots. */
776 #ifdef JS_JIT_SPEW
777 #define DEF_VPNAME const char* vpname; unsigned vpnum
778 #define SET_VPNAME(name) do { vpname = name; vpnum = 0; } while(0)
779 #define INC_VPNUM() do { ++vpnum; } while(0)
780 #else
781 #define DEF_VPNAME do {} while (0)
782 #define vpname ""
783 #define vpnum 0
784 #define SET_VPNAME(name) ((void)0)
785 #define INC_VPNUM() ((void)0)
786 #endif
787
788 /* Iterate over all interned global variables. */
789 #define FORALL_GLOBAL_SLOTS(cx, ngslots, gslots, code) \
790 JS_BEGIN_MACRO \
791 DEF_VPNAME; \
792 JSObject* globalObj = JS_GetGlobalForObject(cx, cx->fp->scopeChain); \
793 unsigned n; \
794 jsval* vp; \
795 SET_VPNAME("global"); \
796 for (n = 0; n < ngslots; ++n) { \
797 vp = &STOBJ_GET_SLOT(globalObj, gslots[n]); \
798 { code; } \
799 INC_VPNUM(); \
800 } \
801 JS_END_MACRO
802
803 /* Iterate over all slots in the frame, consisting of args, vars, and stack
804 (except for the top-level frame which does not have args or vars. */
805 #define FORALL_FRAME_SLOTS(fp, depth, code) \
806 JS_BEGIN_MACRO \
807 jsval* vp; \
808 jsval* vpstop; \
809 if (fp->callee) { \
810 if (depth == 0) { \
811 SET_VPNAME("callee"); \
812 vp = &fp->argv[-2]; \
813 { code; } \
814 SET_VPNAME("this"); \
815 vp = &fp->argv[-1]; \
816 { code; } \
817 SET_VPNAME("argv"); \
818 vp = &fp->argv[0]; vpstop = &fp->argv[fp->fun->nargs]; \
819 while (vp < vpstop) { code; ++vp; INC_VPNUM(); } \
820 } \
821 SET_VPNAME("vars"); \
822 vp = fp->slots; vpstop = &fp->slots[fp->script->nfixed]; \
823 while (vp < vpstop) { code; ++vp; INC_VPNUM(); } \
824 } \
825 SET_VPNAME("stack"); \
826 vp = StackBase(fp); vpstop = fp->regs->sp; \
827 while (vp < vpstop) { code; ++vp; INC_VPNUM(); } \
828 if (fsp < fspstop - 1) { \
829 JSStackFrame* fp2 = fsp[1]; \
830 int missing = fp2->fun->nargs - fp2->argc; \
831 if (missing > 0) { \
832 SET_VPNAME("missing"); \
833 vp = fp->regs->sp; \
834 vpstop = vp + missing; \
835 while (vp < vpstop) { code; ++vp; INC_VPNUM(); } \
836 } \
837 } \
838 JS_END_MACRO
839
840 /* Iterate over all slots in each pending frame. */
841 #define FORALL_SLOTS_IN_PENDING_FRAMES(cx, callDepth, code) \
842 JS_BEGIN_MACRO \
843 DEF_VPNAME; \
844 unsigned n; \
845 JSStackFrame* currentFrame = cx->fp; \
846 JSStackFrame* entryFrame; \
847 JSStackFrame* fp = currentFrame; \
848 for (n = 0; n < callDepth; ++n) { fp = fp->down; } \
849 entryFrame = fp; \
850 unsigned frames = callDepth+1; \
851 JSStackFrame** fstack = \
852 (JSStackFrame**) alloca(frames * sizeof (JSStackFrame*)); \
853 JSStackFrame** fspstop = &fstack[frames]; \
854 JSStackFrame** fsp = fspstop-1; \
855 fp = currentFrame; \
856 for (;; fp = fp->down) { *fsp-- = fp; if (fp == entryFrame) break; } \
857 unsigned depth; \
858 for (depth = 0, fsp = fstack; fsp < fspstop; ++fsp, ++depth) { \
859 fp = *fsp; \
860 FORALL_FRAME_SLOTS(fp, depth, code); \
861 } \
862 JS_END_MACRO
863
864 #define FORALL_SLOTS(cx, ngslots, gslots, callDepth, code) \
865 JS_BEGIN_MACRO \
866 FORALL_GLOBAL_SLOTS(cx, ngslots, gslots, code); \
867 FORALL_SLOTS_IN_PENDING_FRAMES(cx, callDepth, code); \
868 JS_END_MACRO
869
870 /* Calculate the total number of native frame slots we need from this frame
871 all the way back to the entry frame, including the current stack usage. */
872 unsigned
873 js_NativeStackSlots(JSContext *cx, unsigned callDepth)
874 {
875 JSStackFrame* fp = cx->fp;
876 unsigned slots = 0;
877 #if defined _DEBUG
878 unsigned int origCallDepth = callDepth;
879 #endif
880 for (;;) {
881 unsigned operands = fp->regs->sp - StackBase(fp);
882 slots += operands;
883 if (fp->callee)
884 slots += fp->script->nfixed;
885 if (callDepth-- == 0) {
886 if (fp->callee)
887 slots += 2/*callee,this*/ + fp->fun->nargs;
888 #if defined _DEBUG
889 unsigned int m = 0;
890 FORALL_SLOTS_IN_PENDING_FRAMES(cx, origCallDepth, m++);
891 JS_ASSERT(m == slots);
892 #endif
893 return slots;
894 }
895 JSStackFrame* fp2 = fp;
896 fp = fp->down;
897 int missing = fp2->fun->nargs - fp2->argc;
898 if (missing > 0)
899 slots += missing;
900 }
901 JS_NOT_REACHED("js_NativeStackSlots");
902 }
903
904 /* Capture the type map for the selected slots of the global object. */
905 void
906 TypeMap::captureGlobalTypes(JSContext* cx, SlotList& slots)
907 {
908 unsigned ngslots = slots.length();
909 uint16* gslots = slots.data();
910 setLength(ngslots);
911 uint8* map = data();
912 uint8* m = map;
913 FORALL_GLOBAL_SLOTS(cx, ngslots, gslots,
914 uint8 type = getCoercedType(*vp);
915 if ((type == JSVAL_INT) && oracle.isGlobalSlotUndemotable(cx->fp->script, gslots[n]))
916 type = JSVAL_DOUBLE;
917 JS_ASSERT(type != JSVAL_BOXED);
918 *m++ = type;
919 );
920 }
921
922 /* Capture the type map for the currently pending stack frames. */
923 void
924 TypeMap::captureStackTypes(JSContext* cx, unsigned callDepth)
925 {
926 setLength(js_NativeStackSlots(cx, callDepth));
927 uint8* map = data();
928 uint8* m = map;
929 FORALL_SLOTS_IN_PENDING_FRAMES(cx, callDepth,
930 uint8 type = getCoercedType(*vp);
931 if ((type == JSVAL_INT) &&
932 oracle.isStackSlotUndemotable(cx->fp->script, cx->fp->regs->pc, unsigned(m - map))) {
933 type = JSVAL_DOUBLE;
934 }
935 debug_only_v(printf("capture %s%d: %d\n", vpname, vpnum, type);)
936 *m++ = type;
937 );
938 }
939
940 /* Compare this type map to another one and see whether they match. */
941 bool
942 TypeMap::matches(TypeMap& other) const
943 {
944 if (length() != other.length())
945 return false;
946 return !memcmp(data(), other.data(), length());
947 }
948
949 /* Use the provided storage area to create a new type map that contains the partial type map
950 with the rest of it filled up from the complete type map. */
951 static void
952 mergeTypeMaps(uint8** partial, unsigned* plength, uint8* complete, unsigned clength, uint8* mem)
953 {
954 unsigned l = *plength;
955 JS_ASSERT(l < clength);
956 memcpy(mem, *partial, l * sizeof(uint8));
957 memcpy(mem + l, complete + l, (clength - l) * sizeof(uint8));
958 *partial = mem;
959 *plength = clength;
960 }
961
962 static void
963 js_TrashTree(JSContext* cx, Fragment* f);
964
965 TraceRecorder::TraceRecorder(JSContext* cx, VMSideExit* _anchor, Fragment* _fragment,
966 TreeInfo* ti, unsigned ngslots, uint8* globalTypeMap, uint8* stackTypeMap,
967 VMSideExit* innermostNestedGuard, Fragment* outerToBlacklist)
968 {
969 JS_ASSERT(!_fragment->vmprivate && ti);
970
971 this->cx = cx;
972 this->traceMonitor = &JS_TRACE_MONITOR(cx);
973 this->globalObj = JS_GetGlobalForObject(cx, cx->fp->scopeChain);
974 this->anchor = _anchor;
975 this->fragment = _fragment;
976 this->lirbuf = _fragment->lirbuf;
977 this->treeInfo = ti;
978 this->callDepth = _anchor ? _anchor->calldepth : 0;
979 this->atoms = cx->fp->script->atomMap.vector;
980 this->deepAborted = false;
981 this->applyingArguments = false;
982 this->trashTree = false;
983 this->whichTreeToTrash = _fragment->root;
984 this->global_dslots = this->globalObj->dslots;
985 this->terminate = false;
986 this->outerToBlacklist = outerToBlacklist;
987 this->wasRootFragment = _fragment == _fragment->root;
988
989 debug_only_v(printf("recording starting from %s:%u@%u\n",
990 cx->fp->script->filename,
991 js_FramePCToLineNumber(cx, cx->fp),
992 FramePCOffset(cx->fp));)
993 debug_only_v(printf("globalObj=%p, shape=%d\n", this->globalObj, OBJ_SHAPE(this->globalObj));)
994
995 lir = lir_buf_writer = new (&gc) LirBufWriter(lirbuf);
996 #ifdef DEBUG
997 if (verbose_debug)
998 lir = verbose_filter = new (&gc) VerboseWriter(&gc, lir, lirbuf->names);
999 #endif
1000 #ifdef NJ_SOFTFLOAT
1001 lir = float_filter = new (&gc) SoftFloatFilter(lir);
1002 #endif
1003 lir = cse_filter = new (&gc) CseFilter(lir, &gc);
1004 lir = expr_filter = new (&gc) ExprFilter(lir);
1005 lir = func_filter = new (&gc) FuncFilter(lir);
1006 lir->ins0(LIR_start);
1007
1008 if (!nanojit::AvmCore::config.tree_opt || fragment->root == fragment)
1009 lirbuf->state = addName(lir->insParam(0, 0), "state");
1010
1011 lirbuf->sp = addName(lir->insLoad(LIR_ldp, lirbuf->state, (int)offsetof(InterpState, sp)), "sp");
1012 lirbuf->rp = addName(lir->insLoad(LIR_ldp, lirbuf->state, offsetof(InterpState, rp)), "rp");
1013 cx_ins = addName(lir->insLoad(LIR_ldp, lirbuf->state, offsetof(InterpState, cx)), "cx");
1014 gp_ins = addName(lir->insLoad(LIR_ldp, lirbuf->state, offsetof(InterpState, gp)), "gp");
1015 eos_ins = addName(lir->insLoad(LIR_ldp, lirbuf->state, offsetof(InterpState, eos)), "eos");
1016 eor_ins = addName(lir->insLoad(LIR_ldp, lirbuf->state, offsetof(InterpState, eor)), "eor");
1017
1018 /* read into registers all values on the stack and all globals we know so far */
1019 import(treeInfo, lirbuf->sp, ngslots, callDepth, globalTypeMap, stackTypeMap);
1020
1021 /* If we are attached to a tree call guard, make sure the guard the inner tree exited from
1022 is what we expect it to be. */
1023 if (_anchor && _anchor->exitType == NESTED_EXIT) {
1024 LIns* nested_ins = addName(lir->insLoad(LIR_ldp, lirbuf->state,
1025 offsetof(InterpState, lastTreeExitGuard)),
1026 "lastTreeExitGuard");
1027 guard(true, lir->ins2(LIR_eq, nested_ins, INS_CONSTPTR(innermostNestedGuard)), NESTED_EXIT);
1028 }
1029 }
1030
1031 TreeInfo::~TreeInfo()
1032 {
1033 UnstableExit* temp;
1034
1035 while (unstableExits) {
1036 temp = unstableExits->next;
1037 delete unstableExits;
1038 unstableExits = temp;
1039 }
1040 }
1041
1042 TraceRecorder::~TraceRecorder()
1043 {
1044 JS_ASSERT(nextRecorderToAbort == NULL);
1045 JS_ASSERT(treeInfo && (fragment || wasDeepAborted()));
1046 #ifdef DEBUG
1047 TraceRecorder* tr = JS_TRACE_MONITOR(cx).abortStack;
1048 while (tr != NULL)
1049 {
1050 JS_ASSERT(this != tr);
1051 tr = tr->nextRecorderToAbort;
1052 }
1053 #endif
1054 if (fragment) {
1055 if (wasRootFragment && !fragment->root->code()) {
1056 JS_ASSERT(!fragment->root->vmprivate);
1057 delete treeInfo;
1058 }
1059 if (trashTree)
1060 js_TrashTree(cx, whichTreeToTrash);
1061 } else if (wasRootFragment) {
1062 delete treeInfo;
1063 }
1064 #ifdef DEBUG
1065 delete verbose_filter;
1066 #endif
1067 delete cse_filter;
1068 delete expr_filter;
1069 delete func_filter;
1070 #ifdef NJ_SOFTFLOAT
1071 delete float_filter;
1072 #endif
1073 delete lir_buf_writer;
1074 }
1075
1076 void TraceRecorder::removeFragmentoReferences()
1077 {
1078 fragment = NULL;
1079 }
1080
1081 /* Add debug information to a LIR instruction as we emit it. */
1082 inline LIns*
1083 TraceRecorder::addName(LIns* ins, const char* name)
1084 {
1085 #ifdef DEBUG
1086 lirbuf->names->addName(ins, name);
1087 #endif
1088 return ins;
1089 }
1090
1091 /* Determine the current call depth (starting with the entry frame.) */
1092 unsigned
1093 TraceRecorder::getCallDepth() const
1094 {
1095 return callDepth;
1096 }
1097
1098 /* Determine the offset in the native global frame for a jsval we track */
1099 ptrdiff_t
1100 TraceRecorder::nativeGlobalOffset(jsval* p) const
1101 {
1102 JS_ASSERT(isGlobal(p));
1103 if (size_t(p - globalObj->fslots) < JS_INITIAL_NSLOTS)
1104 return size_t(p - globalObj->fslots) * sizeof(double);
1105 return ((p - globalObj->dslots) + JS_INITIAL_NSLOTS) * sizeof(double);
1106 }
1107
1108 /* Determine whether a value is a global stack slot */
1109 bool
1110 TraceRecorder::isGlobal(jsval* p) const
1111 {
1112 return ((size_t(p - globalObj->fslots) < JS_INITIAL_NSLOTS) ||
1113 (size_t(p - globalObj->dslots) < (STOBJ_NSLOTS(globalObj) - JS_INITIAL_NSLOTS)));
1114 }
1115
1116 /* Determine the offset in the native stack for a jsval we track */
1117 ptrdiff_t
1118 TraceRecorder::nativeStackOffset(jsval* p) const
1119 {
1120 #ifdef DEBUG
1121 size_t slow_offset = 0;
1122 FORALL_SLOTS_IN_PENDING_FRAMES(cx, callDepth,
1123 if (vp == p) goto done;
1124 slow_offset += sizeof(double)
1125 );
1126
1127 /*
1128 * If it's not in a pending frame, it must be on the stack of the current frame above
1129 * sp but below fp->slots + script->nslots.
1130 */
1131 JS_ASSERT(size_t(p - cx->fp->slots) < cx->fp->script->nslots);
1132 slow_offset += size_t(p - cx->fp->regs->sp) * sizeof(double);
1133
1134 done:
1135 #define RETURN(offset) { JS_ASSERT((offset) == slow_offset); return offset; }
1136 #else
1137 #define RETURN(offset) { return offset; }
1138 #endif
1139 size_t offset = 0;
1140 JSStackFrame* currentFrame = cx->fp;
1141 JSStackFrame* entryFrame;
1142 JSStackFrame* fp = currentFrame;
1143 for (unsigned n = 0; n < callDepth; ++n) { fp = fp->down; }
1144 entryFrame = fp;
1145 unsigned frames = callDepth+1;
1146 JSStackFrame** fstack = (JSStackFrame **)alloca(frames * sizeof (JSStackFrame *));
1147 JSStackFrame** fspstop = &fstack[frames];
1148 JSStackFrame** fsp = fspstop-1;
1149 fp = currentFrame;
1150 for (;; fp = fp->down) { *fsp-- = fp; if (fp == entryFrame) break; }
1151 for (fsp = fstack; fsp < fspstop; ++fsp) {
1152 fp = *fsp;
1153 if (fp->callee) {
1154 if (fsp == fstack) {
1155 if (size_t(p - &fp->argv[-2]) < size_t(2/*callee,this*/ + fp->fun->nargs))
1156 RETURN(offset + size_t(p - &fp->argv[-2]) * sizeof(double));
1157 offset += (2/*callee,this*/ + fp->fun->nargs) * sizeof(double);
1158 }
1159 if (size_t(p - &fp->slots[0]) < fp->script->nfixed)
1160 RETURN(offset + size_t(p - &fp->slots[0]) * sizeof(double));
1161 offset += fp->script->nfixed * sizeof(double);
1162 }
1163 jsval* spbase = StackBase(fp);
1164 if (size_t(p - spbase) < size_t(fp->regs->sp - spbase))
1165 RETURN(offset + size_t(p - spbase) * sizeof(double));
1166 offset += size_t(fp->regs->sp - spbase) * sizeof(double);
1167 if (fsp < fspstop - 1) {
1168 JSStackFrame* fp2 = fsp[1];
1169 int missing = fp2->fun->nargs - fp2->argc;
1170 if (missing > 0) {
1171 if (size_t(p - fp->regs->sp) < size_t(missing))
1172 RETURN(offset + size_t(p - fp->regs->sp) * sizeof(double));
1173 offset += size_t(missing) * sizeof(double);
1174 }
1175 }
1176 }
1177
1178 /*
1179 * If it's not in a pending frame, it must be on the stack of the current frame above
1180 * sp but below fp->slots + script->nslots.
1181 */
1182 JS_ASSERT(size_t(p - currentFrame->slots) < currentFrame->script->nslots);
1183 offset += size_t(p - currentFrame->regs->sp) * sizeof(double);
1184 RETURN(offset);
1185 #undef RETURN
1186 }
1187
1188 /* Track the maximum number of native frame slots we need during
1189 execution. */
1190 void
1191 TraceRecorder::trackNativeStackUse(unsigned slots)
1192 {
1193 if (slots > treeInfo->maxNativeStackSlots)
1194 treeInfo->maxNativeStackSlots = slots;
1195 }
1196
1197 /* Unbox a jsval into a slot. Slots are wide enough to hold double values directly (instead of
1198 storing a pointer to them). We now assert instead of type checking, the caller must ensure the
1199 types are compatible. */
1200 static void
1201 ValueToNative(JSContext* cx, jsval v, uint8 type, double* slot)
1202 {
1203 unsigned tag = JSVAL_TAG(v);
1204 switch (type) {
1205 case JSVAL_INT:
1206 jsint i;
1207 if (JSVAL_IS_INT(v))
1208 *(jsint*)slot = JSVAL_TO_INT(v);
1209 else if ((tag == JSVAL_DOUBLE) && JSDOUBLE_IS_INT(*JSVAL_TO_DOUBLE(v), i))
1210 *(jsint*)slot = i;
1211 else
1212 JS_ASSERT(JSVAL_IS_INT(v));
1213 debug_only_v(printf("int<%d> ", *(jsint*)slot);)
1214 return;
1215 case JSVAL_DOUBLE:
1216 jsdouble d;
1217 if (JSVAL_IS_INT(v))
1218 d = JSVAL_TO_INT(v);
1219 else
1220 d = *JSVAL_TO_DOUBLE(v);
1221 JS_ASSERT(JSVAL_IS_INT(v) || JSVAL_IS_DOUBLE(v));
1222 *(jsdouble*)slot = d;
1223 debug_only_v(printf("double<%g> ", d);)
1224 return;
1225 case JSVAL_BOOLEAN:
1226 JS_ASSERT(tag == JSVAL_BOOLEAN);
1227 *(JSBool*)slot = JSVAL_TO_BOOLEAN(v);
1228 debug_only_v(printf("boolean<%d> ", *(JSBool*)slot);)
1229 return;
1230 case JSVAL_STRING:
1231 JS_ASSERT(tag == JSVAL_STRING);
1232 *(JSString**)slot = JSVAL_TO_STRING(v);
1233 debug_only_v(printf("string<%p> ", *(JSString**)slot);)
1234 return;
1235 default:
1236 /* Note: we should never see JSVAL_BOXED in an entry type map. */
1237 JS_ASSERT(type == JSVAL_OBJECT);
1238 JS_ASSERT(tag == JSVAL_OBJECT);
1239 *(JSObject**)slot = JSVAL_TO_OBJECT(v);
1240 debug_only_v(printf("object<%p:%s> ", JSVAL_TO_OBJECT(v),
1241 JSVAL_IS_NULL(v)
1242 ? "null"
1243 : STOBJ_GET_CLASS(JSVAL_TO_OBJECT(v))->name);)
1244 return;
1245 }
1246 }
1247
1248 /* We maintain an emergency recovery pool of doubles so we can recover safely if a trace runs
1249 out of memory (doubles or objects). */
1250 static jsval
1251 AllocateDoubleFromRecoveryPool(JSContext* cx)
1252 {
1253 JSTraceMonitor* tm = &JS_TRACE_MONITOR(cx);
1254 JS_ASSERT(tm->recoveryDoublePoolPtr > tm->recoveryDoublePool);
1255 return *--tm->recoveryDoublePoolPtr;
1256 }
1257
1258 static bool
1259 js_ReplenishRecoveryPool(JSContext* cx, JSTraceMonitor* tm)
1260 {
1261 /* We should not be called with a full pool. */
1262 JS_ASSERT((size_t) (tm->recoveryDoublePoolPtr - tm->recoveryDoublePool) <
1263 MAX_NATIVE_STACK_SLOTS);
1264
1265 /*
1266 * When the GC runs in js_NewDoubleInRootedValue, it resets
1267 * tm->recoveryDoublePoolPtr back to tm->recoveryDoublePool.
1268 */
1269 JSRuntime* rt = cx->runtime;
1270 uintN gcNumber = rt->gcNumber;
1271 jsval* ptr = tm->recoveryDoublePoolPtr;
1272 while (ptr < tm->recoveryDoublePool + MAX_NATIVE_STACK_SLOTS) {
1273 if (!js_NewDoubleInRootedValue(cx, 0.0, ptr))
1274 goto oom;
1275 if (rt->gcNumber != gcNumber) {
1276 JS_ASSERT(tm->recoveryDoublePoolPtr == tm->recoveryDoublePool);
1277 ptr = tm->recoveryDoublePool;
1278 if (uintN(rt->gcNumber - gcNumber) > uintN(1))
1279 goto oom;
1280 continue;
1281 }
1282 ++ptr;
1283 }
1284 tm->recoveryDoublePoolPtr = ptr;
1285 return true;
1286
1287 oom:
1288 /*
1289 * Already massive GC pressure, no need to hold doubles back.
1290 * We won't run any native code anyway.
1291 */
1292 tm->recoveryDoublePoolPtr = tm->recoveryDoublePool;
1293 return false;
1294 }
1295
1296 /* Box a value from the native stack back into the jsval format. Integers
1297 that are too large to fit into a jsval are automatically boxed into
1298 heap-allocated doubles. */
1299 static bool
1300 NativeToValue(JSContext* cx, jsval& v, uint8 type, double* slot)
1301 {
1302 jsint i;
1303 jsdouble d;
1304 switch (type) {
1305 case JSVAL_BOOLEAN:
1306 v = BOOLEAN_TO_JSVAL(*(JSBool*)slot);
1307 debug_only_v(printf("boolean<%d> ", *(JSBool*)slot);)
1308 break;
1309 case JSVAL_INT:
1310 i = *(jsint*)slot;
1311 debug_only_v(printf("int<%d> ", i);)
1312 store_int:
1313 if (INT_FITS_IN_JSVAL(i)) {
1314 v = INT_TO_JSVAL(i);
1315 break;
1316 }
1317 d = (jsdouble)i;
1318 goto store_double;
1319 case JSVAL_DOUBLE:
1320 d = *slot;
1321 debug_only_v(printf("double<%g> ", d);)
1322 if (JSDOUBLE_IS_INT(d, i))
1323 goto store_int;
1324 store_double: {
1325 /* Its not safe to trigger the GC here, so use an emergency heap if we are out of
1326 double boxes. */
1327 if (cx->doubleFreeList) {
1328 #ifdef DEBUG
1329 bool ok =
1330 #endif
1331 js_NewDoubleInRootedValue(cx, d, &v);
1332 JS_ASSERT(ok);
1333 return true;
1334 }
1335 v = AllocateDoubleFromRecoveryPool(cx);
1336 JS_ASSERT(JSVAL_IS_DOUBLE(v) && *JSVAL_TO_DOUBLE(v) == 0.0);
1337 *JSVAL_TO_DOUBLE(v) = d;
1338 return true;
1339 }
1340 case JSVAL_STRING:
1341 v = STRING_TO_JSVAL(*(JSString**)slot);
1342 JS_ASSERT(JSVAL_TAG(v) == JSVAL_STRING); /* if this fails the pointer was not aligned */
1343 debug_only_v(printf("string<%p> ", *(JSString**)slot);)
1344 break;
1345 case JSVAL_BOXED:
1346 v = *(jsval*)slot;
1347 debug_only_v(printf("box<%lx> ", v));
1348 break;
1349 default:
1350 JS_ASSERT(type == JSVAL_OBJECT);
1351 v = OBJECT_TO_JSVAL(*(JSObject**)slot);
1352 JS_ASSERT(JSVAL_TAG(v) == JSVAL_OBJECT); /* if this fails the pointer was not aligned */
1353 debug_only_v(printf("object<%p:%s> ", JSVAL_TO_OBJECT(v),
1354 JSVAL_IS_NULL(v)
1355 ? "null"
1356 : STOBJ_GET_CLASS(JSVAL_TO_OBJECT(v))->name);)
1357 break;
1358 }
1359 return true;
1360 }
1361
1362 /* Attempt to unbox the given list of interned globals onto the native global frame. */
1363 static void
1364 BuildNativeGlobalFrame(JSContext* cx, unsigned ngslots, uint16* gslots, uint8* mp, double* np)
1365 {
1366 debug_only_v(printf("global: ");)
1367 FORALL_GLOBAL_SLOTS(cx, ngslots, gslots,
1368 ValueToNative(cx, *vp, *mp, np + gslots[n]);
1369 ++mp;
1370 );
1371 debug_only_v(printf("\n");)
1372 }
1373
1374 /* Attempt to unbox the given JS frame onto a native frame. */
1375 static void
1376 BuildNativeStackFrame(JSContext* cx, unsigned callDepth, uint8* mp, double* np)
1377 {
1378 debug_only_v(printf("stack: ");)
1379 FORALL_SLOTS_IN_PENDING_FRAMES(cx, callDepth,
1380 debug_only_v(printf("%s%u=", vpname, vpnum);)
1381 ValueToNative(cx, *vp, *mp, np);
1382 ++mp; ++np;
1383 );
1384 debug_only_v(printf("\n");)
1385 }
1386
1387 /* Box the given native frame into a JS frame. This only fails due to a hard error
1388 (out of memory for example). */
1389 static int
1390 FlushNativeGlobalFrame(JSContext* cx, unsigned ngslots, uint16* gslots, uint8* mp, double* np)
1391 {
1392 uint8* mp_base = mp;
1393 FORALL_GLOBAL_SLOTS(cx, ngslots, gslots,
1394 if (!NativeToValue(cx, *vp, *mp, np + gslots[n]))
1395 return -1;
1396 ++mp;
1397 );
1398 debug_only_v(printf("\n");)
1399 return mp - mp_base;
1400 }
1401
1402 /**
1403 * Box the given native stack frame into the virtual machine stack. This fails
1404 * only due to a hard error (out of memory for example).
1405 *
1406 * @param callDepth the distance between the entry frame into our trace and
1407 * cx->fp when we make this call. If this is not called as a
1408 * result of a nested exit, callDepth is 0.
1409 * @param mp pointer to an array of type tags (JSVAL_INT, etc.) that indicate
1410 * what the types of the things on the stack are.
1411 * @param np pointer to the native stack. We want to copy values from here to
1412 * the JS stack as needed.
1413 * @param stopFrame if non-null, this frame and everything above it should not
1414 * be restored.
1415 * @return the number of things we popped off of np.
1416 */
1417 static int
1418 FlushNativeStackFrame(JSContext* cx, unsigned callDepth, uint8* mp, double* np,
1419 JSStackFrame* stopFrame)
1420 {
1421 jsval* stopAt = stopFrame ? &stopFrame->argv[-2] : NULL;
1422 uint8* mp_base = mp;
1423 /* Root all string and object references first (we don't need to call the GC for this). */
1424 FORALL_SLOTS_IN_PENDING_FRAMES(cx, callDepth,
1425 if (vp == stopAt) goto skip;
1426 debug_only_v(printf("%s%u=", vpname, vpnum);)
1427 if (!NativeToValue(cx, *vp, *mp, np))
1428 return -1;
1429 ++mp; ++np
1430 );
1431 skip:
1432 // Restore thisp from the now-restored argv[-1] in each pending frame.
1433 // Keep in mind that we didn't restore frames at stopFrame and above!
1434 // Scope to keep |fp| from leaking into the macros we're using.
1435 {
1436 unsigned n = callDepth+1; // +1 to make sure we restore the entry frame
1437 JSStackFrame* fp = cx->fp;
1438 if (stopFrame) {
1439 for (; fp != stopFrame; fp = fp->down) {
1440 JS_ASSERT(n != 0);
1441 --n;
1442 }
1443 // Skip over stopFrame itself.
1444 JS_ASSERT(n != 0);
1445 --n;
1446 fp = fp->down;
1447 }
1448 for (; n != 0; fp = fp->down) {
1449 --n;
1450 if (fp->callee) { // might not have it if the entry frame is global
1451 JS_ASSERT(JSVAL_IS_OBJECT(fp->argv[-1]));
1452 fp->thisp = JSVAL_TO_OBJECT(fp->argv[-1]);
1453 }
1454 }
1455 }
1456 debug_only_v(printf("\n");)
1457 return mp - mp_base;
1458 }
1459
1460 /* Emit load instructions onto the trace that read the initial stack state. */
1461 void
1462 TraceRecorder::import(LIns* base, ptrdiff_t offset, jsval* p, uint8& t,
1463 const char *prefix, uintN index, JSStackFrame *fp)
1464 {
1465 LIns* ins;
1466 if (t == JSVAL_INT) { /* demoted */
1467 JS_ASSERT(isInt32(*p));
1468 /* Ok, we have a valid demotion attempt pending, so insert an integer
1469 read and promote it to double since all arithmetic operations expect
1470 to see doubles on entry. The first op to use this slot will emit a
1471 f2i cast which will cancel out the i2f we insert here. */
1472 ins = lir->insLoadi(base, offset);
1473 ins = lir->ins1(LIR_i2f, ins);
1474 } else {
1475 JS_ASSERT(t == JSVAL_BOXED || isNumber(*p) == (t == JSVAL_DOUBLE));
1476 if (t == JSVAL_DOUBLE) {
1477 ins = lir->insLoad(LIR_ldq, base, offset);
1478 } else if (t == JSVAL_BOOLEAN) {
1479 ins = lir->insLoad(LIR_ld, base, offset);
1480 } else {
1481 ins = lir->insLoad(LIR_ldp, base, offset);
1482 }
1483 }
1484 tracker.set(p, ins);
1485 #ifdef DEBUG
1486 char name[64];
1487 JS_ASSERT(strlen(prefix) < 10);
1488 void* mark = NULL;
1489 jsuword* localNames = NULL;
1490 const char* funName = NULL;
1491 if (*prefix == 'a' || *prefix == 'v') {
1492 mark = JS_ARENA_MARK(&cx->tempPool);
1493 if (JS_GET_LOCAL_NAME_COUNT(fp->fun) != 0)
1494 localNames = js_GetLocalNameArray(cx, fp->fun, &cx->tempPool);
1495 funName = fp->fun->atom ? js_AtomToPrintableString(cx, fp->fun->atom) : "<anonymous>";
1496 }
1497 if (!strcmp(prefix, "argv")) {
1498 if (index < fp->fun->nargs) {
1499 JSAtom *atom = JS_LOCAL_NAME_TO_ATOM(localNames[index]);
1500 JS_snprintf(name, sizeof name, "$%s.%s", funName, js_AtomToPrintableString(cx, atom));
1501 } else {
1502 JS_snprintf(name, sizeof name, "$%s.<arg%d>", funName, index);
1503 }
1504 } else if (!strcmp(prefix, "vars")) {
1505 JSAtom *atom = JS_LOCAL_NAME_TO_ATOM(localNames[fp->fun->nargs + index]);
1506 JS_snprintf(name, sizeof name, "$%s.%s", funName, js_AtomToPrintableString(cx, atom));
1507 } else {
1508 JS_snprintf(name, sizeof name, "$%s%d", prefix, index);
1509 }
1510
1511 if (mark)
1512 JS_ARENA_RELEASE(&cx->tempPool, mark);
1513 addName(ins, name);
1514
1515 static const char* typestr[] = {
1516 "object", "int", "double", "3", "string", "5", "boolean", "any"
1517 };
1518 debug_only_v(printf("import vp=%p name=%s type=%s flags=%d\n",
1519 p, name, typestr[t & 7], t >> 3);)
1520 #endif
1521 }
1522
1523 void
1524 TraceRecorder::import(TreeInfo* treeInfo, LIns* sp, unsigned ngslots, unsigned callDepth,
1525 uint8* globalTypeMap, uint8* stackTypeMap)
1526 {
1527 /* If we get a partial list that doesn't have all the types (i.e. recording from a side
1528 exit that was recorded but we added more global slots later), merge the missing types
1529 from the entry type map. This is safe because at the loop edge we verify that we
1530 have compatible types for all globals (entry type and loop edge type match). While
1531 a different trace of the tree might have had a guard with a different type map for
1532 these slots we just filled in here (the guard we continue from didn't know about them),
1533 since we didn't take that particular guard the only way we could have ended up here
1534 is if that other trace had at its end a compatible type distribution with the entry
1535 map. Since thats exactly what we used to fill in the types our current side exit
1536 didn't provide, this is always safe to do. */
1537 unsigned length;
1538 if (ngslots < (length = traceMonitor->globalTypeMap->length()))
1539 mergeTypeMaps(&globalTypeMap, &ngslots,
1540 traceMonitor->globalTypeMap->data(), length,
1541 (uint8*)alloca(sizeof(uint8) * length));
1542 JS_ASSERT(ngslots == traceMonitor->globalTypeMap->length());
1543
1544 /* the first time we compile a tree this will be empty as we add entries lazily */
1545 uint16* gslots = traceMonitor->globalSlots->data();
1546 uint8* m = globalTypeMap;
1547 FORALL_GLOBAL_SLOTS(cx, ngslots, gslots,
1548 import(gp_ins, nativeGlobalOffset(vp), vp, *m, vpname, vpnum, NULL);
1549 m++;
1550 );
1551 ptrdiff_t offset = -treeInfo->nativeStackBase;
1552 m = stackTypeMap;
1553 FORALL_SLOTS_IN_PENDING_FRAMES(cx, callDepth,
1554 import(sp, offset, vp, *m, vpname, vpnum, fp);
1555 m++; offset += sizeof(double);
1556 );
1557 }
1558
1559 /* Lazily import a global slot if we don't already have it in the tracker. */
1560 bool
1561 TraceRecorder::lazilyImportGlobalSlot(unsigned slot)
1562 {
1563 if (slot != uint16(slot)) /* we use a table of 16-bit ints, bail out if that's not enough */
1564 return false;
1565 jsval* vp = &STOBJ_GET_SLOT(globalObj, slot);
1566 if (tracker.has(vp))
1567 return true; /* we already have it */
1568 unsigned index = traceMonitor->globalSlots->length();
1569 /* If this the first global we are adding, remember the shape of the global object. */
1570 if (index == 0)
1571 traceMonitor->globalShape = OBJ_SHAPE(JS_GetGlobalForObject(cx, cx->fp->scopeChain));
1572 /* Add the slot to the list of interned global slots. */
1573 traceMonitor->globalSlots->add(slot);
1574 uint8 type = getCoercedType(*vp);
1575 if ((type == JSVAL_INT) && oracle.isGlobalSlotUndemotable(cx->fp->script, slot))
1576 type = JSVAL_DOUBLE;
1577 traceMonitor->globalTypeMap->add(type);
1578 import(gp_ins, slot*sizeof(double), vp, type, "global", index, NULL);
1579 return true;
1580 }
1581
1582 /* Write back a value onto the stack or global frames. */
1583 LIns*
1584 TraceRecorder::writeBack(LIns* i, LIns* base, ptrdiff_t offset)
1585 {
1586 /* Sink all type casts targeting the stack into the side exit by simply storing the original
1587 (uncasted) value. Each guard generates the side exit map based on the types of the
1588 last stores to every stack location, so its safe to not perform them on-trace. */
1589 if (isPromoteInt(i))
1590 i = ::demote(lir, i);
1591 return lir->insStorei(i, base, offset);
1592 }
1593
1594 /* Update the tracker, then issue a write back store. */
1595 void
1596 TraceRecorder::set(jsval* p, LIns* i, bool initializing)
1597 {
1598 JS_ASSERT(initializing || tracker.has(p));
1599 tracker.set(p, i);
1600 /* If we are writing to this location for the first time, calculate the offset into the
1601 native frame manually, otherwise just look up the last load or store associated with
1602 the same source address (p) and use the same offset/base. */
1603 LIns* x = nativeFrameTracker.get(p);
1604 if (!x) {
1605 if (isGlobal(p))
1606 x = writeBack(i, gp_ins, nativeGlobalOffset(p));
1607 else
1608 x = writeBack(i, lirbuf->sp, -treeInfo->nativeStackBase + nativeStackOffset(p));
1609 nativeFrameTracker.set(p, x);
1610 } else {
1611 #define ASSERT_VALID_CACHE_HIT(base, offset) \
1612 JS_ASSERT(base == lirbuf->sp || base == gp_ins); \
1613 JS_ASSERT(offset == ((base == lirbuf->sp) \
1614 ? -treeInfo->nativeStackBase + nativeStackOffset(p) \
1615 : nativeGlobalOffset(p))); \
1616
1617 if (x->isop(LIR_st) || x->isop(LIR_stq)) {
1618 ASSERT_VALID_CACHE_HIT(x->oprnd2(), x->oprnd3()->constval());
1619 writeBack(i, x->oprnd2(), x->oprnd3()->constval());
1620 } else {
1621 JS_ASSERT(x->isop(LIR_sti) || x->isop(LIR_stqi));
1622 ASSERT_VALID_CACHE_HIT(x->oprnd2(), x->immdisp());
1623 writeBack(i, x->oprnd2(), x->immdisp());
1624 }
1625 }
1626 #undef ASSERT_VALID_CACHE_HIT
1627 }
1628
1629 LIns*
1630 TraceRecorder::get(jsval* p) const
1631 {
1632 return tracker.get(p);
1633 }
1634
1635 /* Determine whether the current branch instruction terminates the loop. */
1636 static bool
1637 js_IsLoopExit(jsbytecode* pc, jsbytecode* header)
1638 {
1639 switch (*pc) {
1640 case JSOP_LT:
1641 case JSOP_GT:
1642 case JSOP_LE:
1643 case JSOP_GE:
1644 case JSOP_NE:
1645 case JSOP_EQ:
1646 /* These ops try to dispatch a JSOP_IFEQ or JSOP_IFNE that follows. */
1647 JS_ASSERT(js_CodeSpec[*pc].length == 1);
1648 pc++;
1649 break;
1650
1651 default:
1652 for (;;) {
1653 if (*pc == JSOP_AND || *pc == JSOP_OR)
1654 pc += GET_JUMP_OFFSET(pc);
1655 else if (*pc == JSOP_ANDX || *pc == JSOP_ORX)
1656 pc += GET_JUMPX_OFFSET(pc);
1657 else
1658 break;
1659 }
1660 }
1661
1662 switch (*pc) {
1663 case JSOP_IFEQ:
1664 case JSOP_IFNE:
1665 /*
1666 * Forward jumps are usually intra-branch, but for-in loops jump to the
1667 * trailing enditer to clean up, so check for that case here.
1668 */
1669 if (pc[GET_JUMP_OFFSET(pc)] == JSOP_ENDITER)
1670 return true;
1671 return pc + GET_JUMP_OFFSET(pc) == header;
1672
1673 case JSOP_IFEQX:
1674 case JSOP_IFNEX:
1675 if (pc[GET_JUMPX_OFFSET(pc)] == JSOP_ENDITER)
1676 return true;
1677 return pc + GET_JUMPX_OFFSET(pc) == header;
1678
1679 default:;
1680 }
1681 return false;
1682 }
1683
1684 /* Determine whether the current branch is a loop edge (taken or not taken). */
1685 static bool
1686 js_IsLoopEdge(jsbytecode* pc, jsbytecode* header)
1687 {
1688 switch (*pc) {
1689 case JSOP_IFEQ:
1690 case JSOP_IFNE:
1691 return ((pc + GET_JUMP_OFFSET(pc)) == header);
1692 case JSOP_IFEQX:
1693 case JSOP_IFNEX:
1694 return ((pc + GET_JUMPX_OFFSET(pc)) == header);
1695 default:
1696 JS_ASSERT((*pc == JSOP_AND) || (*pc == JSOP_ANDX) ||
1697 (*pc == JSOP_OR) || (*pc == JSOP_ORX));
1698 }
1699 return false;
1700 }
1701
1702 /* Promote slots if necessary to match the called tree' type map and report error if thats
1703 impossible. */
1704 bool
1705 TraceRecorder::adjustCallerTypes(Fragment* f, unsigned* demote_slots, bool& trash)
1706 {
1707 JSTraceMonitor* tm = traceMonitor;
1708 uint8* m = tm->globalTypeMap->data();
1709 uint16* gslots = traceMonitor->globalSlots->data();
1710 unsigned ngslots = traceMonitor->globalSlots->length();
1711 uint8* map = ((TreeInfo*)f->vmprivate)->stackTypeMap.data();
1712 bool ok = true;
1713 trash = false;
1714 FORALL_GLOBAL_SLOTS(cx, ngslots, gslots,
1715 LIns* i = get(vp);
1716 bool isPromote = isPromoteInt(i);
1717 if (isPromote && *m == JSVAL_DOUBLE)
1718 lir->insStorei(get(vp), gp_ins, nativeGlobalOffset(vp));
1719 else if (!isPromote && *m == JSVAL_INT) {
1720 oracle.markGlobalSlotUndemotable(cx->fp->script, nativeGlobalOffset(vp)/sizeof(double));
1721 trash = true;
1722 ok = false;
1723 }
1724 ++m;
1725 );
1726 m = map;
1727 FORALL_SLOTS_IN_PENDING_FRAMES(cx, 0,
1728 LIns* i = get(vp);
1729 bool isPromote = isPromoteInt(i);
1730 if (isPromote && *m == JSVAL_DOUBLE) {
1731 lir->insStorei(get(vp), lirbuf->sp,
1732 -treeInfo->nativeStackBase + nativeStackOffset(vp));
1733 /* Aggressively undo speculation so the inner tree will compile if this fails. */
1734 ADD_UNDEMOTE_SLOT(demote_slots, unsigned(m - map));
1735 } else if (!isPromote && *m == JSVAL_INT) {
1736 debug_only_v(printf("adjusting will fail, %s%d, slot %d\n", vpname, vpnum, m - map);)
1737 ok = false;
1738 ADD_UNDEMOTE_SLOT(demote_slots, unsigned(m - map));
1739 } else if (JSVAL_IS_INT(*vp) && *m == JSVAL_DOUBLE) {
1740 /* Aggressively undo speculation so the inner tree will compile if this fails. */
1741 ADD_UNDEMOTE_SLOT(demote_slots, unsigned(m - map));
1742 }
1743 ++m;
1744 );
1745 /* If this isn't okay, tell the oracle. */
1746 if (!ok) {
1747 for (unsigned i = 1; i <= NUM_UNDEMOTE_SLOTS(demote_slots); i++)
1748 oracle.markStackSlotUndemotable(cx->fp->script, cx->fp->regs->pc, demote_slots[i]);
1749 }
1750 JS_ASSERT(f == f->root);
1751 return ok;
1752 }
1753
1754 uint8
1755 TraceRecorder::determineSlotType(jsval* vp) const
1756 {
1757 uint8 m;
1758 LIns* i = get(vp);
1759 m = isNumber(*vp)
1760 ? (isPromoteInt(i) ? JSVAL_INT : JSVAL_DOUBLE)
1761 : JSVAL_TAG(*vp);
1762 JS_ASSERT((m != JSVAL_INT) || isInt32(*vp));
1763 return m;
1764 }
1765
1766 #define IMACRO_PC_ADJ_BITS 8
1767 #define SCRIPT_PC_ADJ_BITS (32 - IMACRO_PC_ADJ_BITS)
1768
1769 // The stored imacro_pc_adj byte offset is biased by 1.
1770 #define IMACRO_PC_ADJ_LIMIT (JS_BIT(IMACRO_PC_ADJ_BITS) - 1)
1771 #define SCRIPT_PC_ADJ_LIMIT JS_BIT(SCRIPT_PC_ADJ_BITS)
1772
1773 #define IMACRO_PC_ADJ(ip) ((uintptr_t)(ip) >> SCRIPT_PC_ADJ_BITS)
1774 #define SCRIPT_PC_ADJ(ip) ((ip) & JS_BITMASK(SCRIPT_PC_ADJ_BITS))
1775
1776 #define FI_SCRIPT_PC(fi,fp) ((fp)->script->code + SCRIPT_PC_ADJ((fi).ip_adj))
1777
1778 #define FI_IMACRO_PC(fi,fp) (IMACRO_PC_ADJ((fi).ip_adj) \
1779 ? imacro_code[*FI_SCRIPT_PC(fi, fp)] + \
1780 IMACRO_PC_ADJ((fi).ip_adj) \
1781 : NULL)
1782
1783 #define IMACRO_PC_OK(fp,pc) JS_ASSERT(uintN((pc)-imacro_code[*(fp)->imacpc]) \
1784 < JS_BIT(IMACRO_PC_ADJ_BITS))
1785 #define ENCODE_IP_ADJ(fp,pc) ((fp)->imacpc \
1786 ? (IMACRO_PC_OK(fp, pc), \
1787 (((pc) - imacro_code[*(fp)->imacpc]) \
1788 << SCRIPT_PC_ADJ_BITS) + \
1789 (fp)->imacpc - (fp)->script->code) \
1790 : (pc) - (fp)->script->code)
1791
1792 #define DECODE_IP_ADJ(ip,fp) (IMACRO_PC_ADJ(ip) \
1793 ? (fp)->imacpc = (fp)->script->code + \
1794 SCRIPT_PC_ADJ(ip), \
1795 (fp)->regs->pc = imacro_code[*(fp)->imacpc] + \
1796 IMACRO_PC_ADJ(ip) \
1797 : (fp)->regs->pc = (fp)->script->code + (ip))
1798
1799 static jsbytecode* imacro_code[JSOP_LIMIT];
1800
1801 LIns*
1802 TraceRecorder::snapshot(ExitType exitType)
1803 {
1804 JSStackFrame* fp = cx->fp;
1805 JSFrameRegs* regs = fp->regs;
1806 jsbytecode* pc = regs->pc;
1807 if (exitType == BRANCH_EXIT && js_IsLoopExit(pc, (jsbytecode*)fragment->root->ip))
1808 exitType = LOOP_EXIT;
1809
1810 /* Check for a return-value opcode that needs to restart at the next instruction. */
1811 const JSCodeSpec& cs = js_CodeSpec[*pc];
1812
1813 /* WARNING: don't return before restoring the original pc if (resumeAfter). */
1814 bool resumeAfter = (pendingTraceableNative &&
1815 JSTN_ERRTYPE(pendingTraceableNative) == FAIL_JSVAL);
1816 if (resumeAfter) {
1817 JS_ASSERT(*pc == JSOP_CALL || *pc == JSOP_APPLY || *pc == JSOP_NEXTITER);
1818 pc += cs.length;
1819 regs->pc = pc;
1820 MUST_FLOW_THROUGH("restore_pc");
1821 }
1822
1823 /* Generate the entry map for the (possibly advanced) pc and stash it in the trace. */
1824 unsigned stackSlots = js_NativeStackSlots(cx, callDepth);
1825
1826 /* It's sufficient to track the native stack use here since all stores above the
1827 stack watermark defined by guards are killed. */
1828 trackNativeStackUse(stackSlots + 1);
1829
1830 /* Capture the type map into a temporary location. */
1831 unsigned ngslots = traceMonitor->globalSlots->length();
1832 unsigned typemap_size = (stackSlots + ngslots) * sizeof(uint8);
1833 uint8* typemap = (uint8*)alloca(typemap_size);
1834 uint8* m = typemap;
1835
1836 /* Determine the type of a store by looking at the current type of the actual value the
1837 interpreter is using. For numbers we have to check what kind of store we used last
1838 (integer or double) to figure out what the side exit show reflect in its typemap. */
1839 FORALL_SLOTS(cx, ngslots, traceMonitor->globalSlots->data(), callDepth,
1840 *m++ = determineSlotType(vp);
1841 );
1842 JS_ASSERT(unsigned(m - typemap) == ngslots + stackSlots);
1843
1844 /* If we are capturing the stack state on a specific instruction, the value on or near
1845 the top of the stack is a boxed value. Either pc[-cs.length] is JSOP_NEXTITER and we
1846 want one below top of stack, or else it's JSOP_CALL and we want top of stack. */
1847 if (resumeAfter) {
1848 m[(pc[-cs.length] == JSOP_NEXTITER) ? -2 : -1] = JSVAL_BOXED;
1849
1850 /* Now restore the the original pc (after which early returns are ok). */
1851 MUST_FLOW_LABEL(restore_pc);
1852 regs->pc = pc - cs.length;
1853 } else {
1854 /* If we take a snapshot on a goto, advance to the target address. This avoids inner
1855 trees returning on a break goto, which the outer recorder then would confuse with
1856 a break in the outer tree. */
1857 if (*pc == JSOP_GOTO)
1858 pc += GET_JUMP_OFFSET(pc);
1859 else if (*pc == JSOP_GOTOX)
1860 pc += GET_JUMPX_OFFSET(pc);
1861 }
1862 intptr_t ip_adj = ENCODE_IP_ADJ(fp, pc);
1863
1864 /* Check if we already have a matching side exit. If so use that side exit structure,
1865 otherwise we have to create our own. */
1866 VMSideExit** exits = treeInfo->sideExits.data();
1867 unsigned nexits = treeInfo->sideExits.length();
1868 if (exitType == LOOP_EXIT) {
1869 for (unsigned n = 0; n < nexits; ++n) {
1870 VMSideExit* e = exits[n];
1871 if (e->ip_adj == ip_adj &&
1872 !memcmp(getTypeMap(exits[n]), typemap, typemap_size)) {
1873 LIns* data = lir_buf_writer->skip(sizeof(GuardRecord));
1874 GuardRecord* rec = (GuardRecord*)data->payload();
1875 /* setup guard record structure with shared side exit */
1876 memset(rec, 0, sizeof(GuardRecord));
1877 VMSideExit* exit = exits[n];
1878 rec->exit = exit;
1879 exit->addGuard(rec);
1880 AUDIT(mergedLoopExits);
1881 return data;
1882 }
1883 }
1884 }
1885
1886 /* We couldn't find a matching side exit, so create our own side exit structure. */
1887 LIns* data = lir_buf_writer->skip(sizeof(GuardRecord) +
1888 sizeof(VMSideExit) +
1889 (stackSlots + ngslots) * sizeof(uint8));
1890 GuardRecord* rec = (GuardRecord*)data->payload();
1891 VMSideExit* exit = (VMSideExit*)(rec + 1);
1892 /* setup guard record structure */
1893 memset(rec, 0, sizeof(GuardRecord));
1894 rec->exit = exit;
1895 /* setup side exit structure */
1896 memset(exit, 0, sizeof(VMSideExit));
1897 exit->from = fragment;
1898 exit->calldepth = callDepth;
1899 exit->numGlobalSlots = ngslots;
1900 exit->numStackSlots = stackSlots;
1901 exit->numStackSlotsBelowCurrentFrame = cx->fp->callee
1902 ? nativeStackOffset(&cx->fp->argv[-2])/sizeof(double)
1903 : 0;
1904 exit->exitType = exitType;
1905 exit->addGuard(rec);
1906 exit->ip_adj = ip_adj;
1907 exit->sp_adj = (stackSlots * sizeof(double)) - treeInfo->nativeStackBase;
1908 exit->rp_adj = exit->calldepth * sizeof(FrameInfo);
1909 memcpy(getTypeMap(exit), typemap, typemap_size);
1910
1911 /* BIG FAT WARNING: If compilation fails, we currently don't reset the lirbuf so its safe
1912 to keep references to the side exits here. If we ever start rewinding those lirbufs,
1913 we have to make sure we purge the side exits that then no longer will be in valid
1914 memory. */
1915 if (exitType == LOOP_EXIT)
1916 treeInfo->sideExits.add(exit);
1917 return data;
1918 }
1919
1920 /* Emit a guard for condition (cond), expecting to evaluate to boolean result (expected)
1921 and using the supplied side exit if the conditon doesn't hold. */
1922 LIns*
1923 TraceRecorder::guard(bool expected, LIns* cond, LIns* exit)
1924 {
1925 return lir->insGuard(expected ? LIR_xf : LIR_xt, cond, exit);
1926 }
1927
1928 /* Emit a guard for condition (cond), expecting to evaluate to boolean result (expected)
1929 and generate a side exit with type exitType to jump to if the condition does not hold. */
1930 LIns*
1931 TraceRecorder::guard(bool expected, LIns* cond, ExitType exitType)
1932 {
1933 return guard(expected, cond, snapshot(exitType));
1934 }
1935
1936 /* Try to match the type of a slot to type t. checkType is used to verify that the type of
1937 * values flowing into the loop edge is compatible with the type we expect in the loop header.
1938 *
1939 * @param v Value.
1940 * @param t Typemap entry for value.
1941 * @param stage_val Outparam for set() address.
1942 * @param stage_ins Outparam for set() instruction.
1943 * @param stage_count Outparam for set() buffer count.
1944 * @return True if types are compatible, false otherwise.
1945 */
1946 bool
1947 TraceRecorder::checkType(jsval& v, uint8 t, jsval*& stage_val, LIns*& stage_ins,
1948 unsigned& stage_count)
1949 {
1950 if (t == JSVAL_INT) { /* initially all whole numbers cause the slot to be demoted */
1951 debug_only_v(printf("checkType(tag=1, t=%d, isnum=%d, i2f=%d) stage_count=%d\n",
1952 t,
1953 isNumber(v),
1954 isPromoteInt(get(&v)),
1955 stage_count);)
1956 if (!isNumber(v))
1957 return false; /* not a number? type mismatch */
1958 LIns* i = get(&v);
1959 /* This is always a type mismatch, we can't close a double to an int. */
1960 if (!isPromoteInt(i))
1961 return false;
1962 /* Looks good, slot is an int32, the last instruction should be promotable. */
1963 JS_ASSERT(isInt32(v) && isPromoteInt(i));
1964 /* Overwrite the value in this slot with the argument promoted back to an integer. */
1965 stage_val = &v;
1966 stage_ins = f2i(i);
1967 stage_count++;
1968 return true;
1969 }
1970 if (t == JSVAL_DOUBLE) {
1971 debug_only_v(printf("checkType(tag=2, t=%d, isnum=%d, promote=%d) stage_count=%d\n",
1972 t,
1973 isNumber(v),
1974 isPromoteInt(get(&v)),
1975 stage_count);)
1976 if (!isNumber(v))
1977 return false; /* not a number? type mismatch */
1978 LIns* i = get(&v);
1979 /* We sink i2f conversions into the side exit, but at the loop edge we have to make
1980 sure we promote back to double if at loop entry we want a double. */
1981 if (isPromoteInt(i)) {
1982 stage_val = &v;
1983 stage_ins = lir->ins1(LIR_i2f, i);
1984 stage_count++;
1985 }
1986 return true;
1987 }
1988 /* for non-number types we expect a precise match of the type */
1989 #ifdef DEBUG
1990 if (JSVAL_TAG(v) != t) {
1991 debug_only_v(printf("Type mismatch: val %c, map %c ", typeChar[JSVAL_TAG(v)],
1992 typeChar[t]);)
1993 }
1994 #endif
1995 debug_only_v(printf("checkType(tag=%d, t=%d) stage_count=%d\n",
1996 (int) JSVAL_TAG(v), t, stage_count);)
1997 return JSVAL_TAG(v) == t;
1998 }
1999
2000 /**
2001 * Make sure that the current values in the given stack frame and all stack frames
2002 * up and including entryFrame are type-compatible with the entry map.
2003 *
2004 * @param root_peer First fragment in peer list.
2005 * @param stable_peer Outparam for first type stable peer.
2006 * @param trash Whether to trash the tree (demotion).
2007 * @param demotes Array to store demotable stack slots.
2008 * @return True if type stable, false otherwise.
2009 */
2010 bool
2011 TraceRecorder::deduceTypeStability(Fragment* root_peer, Fragment** stable_peer, unsigned* demotes)
2012 {
2013 uint8* m;
2014 uint8* typemap;
2015 unsigned ngslots = traceMonitor->globalSlots->length();
2016 uint16* gslots = traceMonitor->globalSlots->data();
2017 JS_ASSERT(traceMonitor->globalTypeMap->length() == ngslots);
2018
2019 if (stable_peer)
2020 *stable_peer = NULL;
2021
2022 CLEAR_UNDEMOTE_SLOTLIST(demotes);
2023
2024 /*
2025 * Rather than calculate all of this stuff twice, it gets cached locally. The "stage" buffers
2026 * are for calls to set() that will change the exit types.
2027 */
2028 bool success;
2029 bool unstable_from_undemotes;
2030 unsigned stage_count;
2031 jsval** stage_vals = (jsval**)alloca(sizeof(jsval*) * (ngslots + treeInfo->stackTypeMap.length()));
2032 LIns** stage_ins = (LIns**)alloca(sizeof(LIns*) * (ngslots + treeInfo->stackTypeMap.length()));
2033
2034 /* First run through and see if we can close ourselves - best case! */
2035 stage_count = 0;
2036 success = false;
2037 unstable_from_undemotes = false;
2038
2039 debug_only_v(printf("Checking type stability against self=%p\n", fragment);)
2040
2041 m = typemap = traceMonitor->globalTypeMap->data();
2042 FORALL_GLOBAL_SLOTS(cx, ngslots, gslots,
2043 debug_only_v(printf("%s%d ", vpname, vpnum);)
2044 if (!checkType(*vp, *m, stage_vals[stage_count], stage_ins[stage_count], stage_count)) {
2045 /* If the failure was an int->double, tell the oracle. */
2046 if (*m == JSVAL_INT && isNumber(*vp) && !isPromoteInt(get(vp)))
2047 oracle.markGlobalSlotUndemotable(cx->fp->script, gslots[n]);
2048 trashTree = true;
2049 goto checktype_fail_1;
2050 }
2051 ++m;
2052 );
2053 m = typemap = treeInfo->stackTypeMap.data();
2054 FORALL_SLOTS_IN_PENDING_FRAMES(cx, 0,
2055 debug_only_v(printf("%s%d ", vpname, vpnum);)
2056 if (!checkType(*vp, *m, stage_vals[stage_count], stage_ins[stage_count], stage_count)) {
2057 if (*m == JSVAL_INT && isNumber(*vp) && !isPromoteInt(get(vp)))
2058 ADD_UNDEMOTE_SLOT(demotes, unsigned(m - typemap));
2059 else
2060 goto checktype_fail_1;
2061 }
2062 ++m;
2063 );
2064
2065 /*
2066 * If there's an exit that's unstable because of undemotable slots, we want to search for
2067 * peers just in case we can make a connection.
2068 */
2069 if (NUM_UNDEMOTE_SLOTS(demotes))
2070 unstable_from_undemotes = true;
2071 else
2072 success = true;
2073
2074 checktype_fail_1:
2075 /* If we got a success and we don't need to recompile, we should just close here. */
2076 if (success) {
2077 for (unsigned i = 0; i < stage_count; i++)
2078 set(stage_vals[i], stage_ins[i]);
2079 return true;
2080 /* If we need to trash, don't bother checking peers. */
2081 } else if (trashTree) {
2082 return false;
2083 } else {
2084 CLEAR_UNDEMOTE_SLOTLIST(demotes);
2085 }
2086
2087 /* At this point the tree is about to be incomplete, so let's see if we can connect to any
2088 * peer fragment that is type stable.
2089 */
2090 Fragment* f;
2091 TreeInfo* ti;
2092 for (f = root_peer; f != NULL; f = f->peer) {
2093 debug_only_v(printf("Checking type stability against peer=%p (code=%p)\n", f, f->code());)
2094 if (!f->code())
2095 continue;
2096 ti = (TreeInfo*)f->vmprivate;
2097 /* Don't allow varying stack depths */
2098 if (ti->stackTypeMap.length() != treeInfo->stackTypeMap.length())
2099 continue;
2100 stage_count = 0;
2101 success = false;
2102 m = ti->stackTypeMap.data();
2103
2104 FORALL_SLOTS_IN_PENDING_FRAMES(cx, 0,
2105 if (!checkType(*vp, *m, stage_vals[stage_count], stage_ins[stage_count], stage_count))
2106 goto checktype_fail_2;
2107 ++m;
2108 );
2109
2110 success = true;
2111
2112 checktype_fail_2:
2113 if (success) {
2114 /*
2115 * There was a successful match. We don't care about restoring the saved staging, but
2116 * we do need to clear the original undemote list.
2117 */
2118 for (unsigned i = 0; i < stage_count; i++)
2119 set(stage_vals[i], stage_ins[i]);
2120 if (stable_peer)
2121 *stable_peer = f;
2122 return false;
2123 }
2124 }
2125
2126 JS_ASSERT(NUM_UNDEMOTE_SLOTS(demotes) == 0);
2127
2128 /*
2129 * If this is a loop trace and it would be stable with demotions, build an undemote list
2130 * and return true. Our caller should sniff this and trash the tree, recording a new one
2131 * that will assumedly stabilize.
2132 */
2133 if (unstable_from_undemotes && fragment->kind == LoopTrace) {
2134 typemap = m = treeInfo->stackTypeMap.data();
2135 FORALL_SLOTS_IN_PENDING_FRAMES(cx, 0,
2136 if (*m == JSVAL_INT) {
2137 JS_ASSERT(isNumber(*vp));
2138 if (!isPromoteInt(get(vp)))
2139 ADD_UNDEMOTE_SLOT(demotes, unsigned(m - typemap));
2140 } else if (*m == JSVAL_DOUBLE) {
2141 JS_ASSERT(isNumber(*vp));
2142 ADD_UNDEMOTE_SLOT(demotes, unsigned(m - typemap));
2143 } else {
2144 JS_ASSERT(*m == JSVAL_TAG(*vp));
2145 }
2146 m++;
2147 );
2148 return true;
2149 }
2150
2151 return false;
2152 }
2153
2154 /* Check whether the current pc location is the loop header of the loop this recorder records. */
2155 bool
2156 TraceRecorder::isLoopHeader(JSContext* cx) const
2157 {
2158 return cx->fp->regs->pc == fragment->root->ip;
2159 }
2160
2161 /* Compile the current fragment. */
2162 void
2163 TraceRecorder::compile(Fragmento* fragmento)
2164 {
2165 if (treeInfo->maxNativeStackSlots >= MAX_NATIVE_STACK_SLOTS) {
2166 debug_only_v(printf("Trace rejected: excessive stack use.\n"));
2167 js_BlacklistPC(fragmento, fragment);
2168 return;
2169 }
2170 ++treeInfo->branchCount;
2171 if (lirbuf->outOmem()) {
2172 fragmento->assm()->setError(nanojit::OutOMem);
2173 return;
2174 }
2175 ::compile(fragmento->assm(), fragment);
2176 if (anchor)
2177 fragmento->assm()->patch(anchor);
2178 if (fragmento->assm()->error() != nanojit::None)
2179 return;
2180 JS_ASSERT(fragment->code());
2181 JS_ASSERT(!fragment->vmprivate);
2182 if (fragment == fragment->root)
2183 fragment->vmprivate = treeInfo;
2184 /* :TODO: windows support */
2185 #if defined DEBUG && !defined WIN32
2186 const char* filename = cx->fp->script->filename;
2187 char* label = (char*)malloc((filename ? strlen(filename) : 7) + 16);
2188 sprintf(label, "%s:%u", filename ? filename : "<stdin>",
2189 js_FramePCToLineNumber(cx, cx->fp));
2190 fragmento->labels->add(fragment, sizeof(Fragment), 0, label);
2191 free(label);
2192 #endif
2193 AUDIT(traceCompleted);
2194 }
2195
2196 static bool
2197 js_JoinPeersIfCompatible(Fragmento* frago, Fragment* stableFrag, TreeInfo* stableTree,
2198 VMSideExit* exit)
2199 {
2200 JS_ASSERT(exit->numStackSlots == stableTree->stackTypeMap.length());
2201 /* Must have a matching type unstable exit. */
2202 if (memcmp(getTypeMap(exit) + exit->numGlobalSlots,
2203 stableTree->stackTypeMap.data(),
2204 stableTree->stackTypeMap.length()) != 0) {
2205 return false;
2206 }
2207
2208 exit->target = stableFrag;
2209 frago->assm()->patch(exit);
2210
2211 stableTree->dependentTrees.addUnique(exit->from->root);
2212
2213 return true;
2214 }
2215
2216 /* Complete and compile a trace and link it to the existing tree if appropriate. */
2217 bool
2218 TraceRecorder::closeLoop(Fragmento* fragmento, bool& demote, unsigned *demotes)
2219 {
2220 bool stable;
2221 LIns* exitIns;
2222 Fragment* peer;
2223 VMSideExit* exit;
2224 Fragment* peer_root;
2225
2226 demote = false;
2227
2228 exitIns = snapshot(UNSTABLE_LOOP_EXIT);
2229 exit = (VMSideExit*)((GuardRecord*)exitIns->payload())->exit;
2230
2231 if (callDepth != 0) {
2232 debug_only_v(printf("Stack depth mismatch, possible recursion\n");)
2233 js_BlacklistPC(fragmento, fragment);
2234 trashTree = true;
2235 return false;
2236 }
2237
2238 JS_ASSERT(exit->numStackSlots == treeInfo->stackTypeMap.length());
2239
2240 peer_root = fragmento->getLoop(fragment->root->ip);
2241 JS_ASSERT(peer_root != NULL);
2242 stable = deduceTypeStability(peer_root, &peer, demotes);
2243
2244 #if DEBUG
2245 if (!stable || NUM_UNDEMOTE_SLOTS(demotes))
2246 AUDIT(unstableLoopVariable);
2247 #endif
2248
2249 if (trashTree) {
2250 debug_only_v(printf("Trashing tree from type instability.\n");)
2251 return false;
2252 }
2253
2254 if (stable && NUM_UNDEMOTE_SLOTS(demotes)) {
2255 JS_ASSERT(fragment->kind == LoopTrace);
2256 demote = true;
2257 return false;
2258 }
2259
2260 if (!stable) {
2261 fragment->lastIns = lir->insGuard(LIR_x, lir->insImm(1), exitIns);
2262
2263 /*
2264 * If we didn't find a type stable peer, we compile the loop anyway and
2265 * hope it becomes stable later.
2266 */
2267 if (!peer) {
2268 /*
2269 * If such a fragment does not exist, let's compile the loop ahead
2270 * of time anyway. Later, if the loop becomes type stable, we will
2271 * connect these two fragments together.
2272 */
2273 debug_only_v(printf("Trace has unstable loop variable with no stable peer, "
2274 "compiling anyway.\n");)
2275 UnstableExit* uexit = new UnstableExit;
2276 uexit->fragment = fragment;
2277 uexit->exit = exit;
2278 uexit->next = treeInfo->unstableExits;
2279 treeInfo->unstableExits = uexit;
2280
2281 /*
2282 * If we walked out of a loop, this exit is wrong. We need to back
2283 * up to the if operation.
2284 */
2285 if (walkedOutOfLoop())
2286 exit->ip_adj = terminate_ip_adj;
2287
2288 /* If we were trying to stabilize a promotable tree, trash it. */
2289 if (promotedPeer)
2290 js_TrashTree(cx, promotedPeer);
2291 } else {
2292 JS_ASSERT(peer->code());
2293 exit->target = peer;
2294 debug_only_v(printf("Joining type-unstable trace to target fragment %p.\n", peer);)
2295 stable = true;
2296 ((TreeInfo*)peer->vmprivate)->dependentTrees.addUnique(fragment->root);
2297 }
2298
2299 compile(fragmento);
2300 } else {
2301 exit->target = fragment->root;
2302 #if defined(JS_HAS_OPERATION_COUNT) && !JS_HAS_OPERATION_COUNT
2303 exit->exitType = TIMEOUT_EXIT;
2304 guard(false,
2305 lir->ins_eq0(lir->insLoadi(cx_ins,
2306 offsetof(JSContext, operationCount))),
2307 exitIns);
2308 #endif
2309 fragment->lastIns = lir->insGuard(LIR_loop, lir->insImm(1), exitIns);
2310 compile(fragmento);
2311 }
2312
2313 if (fragmento->assm()->error() != nanojit::None)
2314 return false;
2315
2316 joinEdgesToEntry(fragmento, peer_root);
2317
2318 debug_only_v(printf("recording completed at %s:%u@%u via closeLoop\n",
2319 cx->fp->script->filename,
2320 js_FramePCToLineNumber(cx, cx->fp),
2321 FramePCOffset(cx->fp));)
2322 return true;
2323 }
2324
2325 void
2326 TraceRecorder::joinEdgesToEntry(Fragmento* fragmento, Fragment* peer_root)
2327 {
2328 if (fragment->kind == LoopTrace) {
2329 TreeInfo* ti;
2330 Fragment* peer;
2331 uint8* t1, *t2;
2332 UnstableExit* uexit, **unext;
2333
2334 unsigned* demotes = (unsigned*)alloca(treeInfo->stackTypeMap.length() * sizeof(unsigned));
2335 for (peer = peer_root; peer != NULL; peer = peer->peer) {
2336 if (!peer->code())
2337 continue;
2338 ti = (TreeInfo*)peer->vmprivate;
2339 uexit = ti->unstableExits;
2340 unext = &ti->unstableExits;
2341 while (uexit != NULL) {
2342 bool remove = js_JoinPeersIfCompatible(fragmento, fragment, treeInfo, uexit->exit);
2343 JS_ASSERT(!remove || fragment != peer);
2344 debug_only_v(if (remove) {
2345 printf("Joining type-stable trace to target exit %p->%p.\n",
2346 uexit->fragment, uexit->exit); });
2347 if (!remove) {
2348 /* See if this exit contains mismatch demotions, which imply trashing a tree.
2349 This is actually faster than trashing the original tree as soon as the
2350 instability is detected, since we could have compiled a fairly stable
2351 tree that ran faster with integers. */
2352 unsigned count = 0;
2353 t1 = treeInfo->stackTypeMap.data();
2354 t2 = getTypeMap(uexit->exit) + uexit->exit->numGlobalSlots;
2355 for (unsigned i = 0; i < uexit->exit->numStackSlots; i++) {
2356 if (t2[i] == JSVAL_INT && t1[i] == JSVAL_DOUBLE) {
2357 demotes[count++] = i;
2358 } else if (t2[i] != t1[i]) {
2359 count = 0;
2360 break;
2361 }
2362 }
2363 if (count) {
2364 for (unsigned i = 0; i < count; i++)
2365 oracle.markStackSlotUndemotable(cx->fp->script,
2366 cx->fp->regs->pc, demotes[i]);
2367 js_TrashTree(cx, uexit->fragment->root);
2368 break;
2369 }
2370 }
2371 if (remove) {
2372 *unext = uexit->next;
2373 delete uexit;
2374 uexit = *unext;
2375 } else {
2376 unext = &uexit->next;
2377 uexit = uexit->next;
2378 }
2379 }
2380 }
2381 }
2382
2383 debug_only_v(js_DumpPeerStability(fragmento, peer_root->ip);)
2384 }
2385
2386 /* Emit an always-exit guard and compile the tree (used for break statements. */
2387 void
2388 TraceRecorder::endLoop(Fragmento* fragmento)
2389 {
2390 LIns* exitIns = snapshot(LOOP_EXIT);
2391
2392 if (callDepth != 0) {
2393 debug_only_v(printf("Stack depth mismatch, possible recursion\n");)
2394 js_BlacklistPC(fragmento, fragment);
2395 trashTree = true;
2396 return;
2397 }
2398
2399 fragment->lastIns = lir->insGuard(LIR_x, lir->insImm(1), exitIns);
2400 compile(fragmento);
2401
2402 if (fragmento->assm()->error() != nanojit::None)
2403 return;
2404
2405 joinEdgesToEntry(fragmento, fragmento->getLoop(fragment->root->ip));
2406
2407 debug_only_v(printf("recording completed at %s:%u@%u via endLoop\n",
2408 cx->fp->script->filename,
2409 js_FramePCToLineNumber(cx, cx->fp),
2410 FramePCOffset(cx->fp));)
2411 }
2412
2413 /* Emit code to adjust the stack to match the inner tree's stack expectations. */
2414 void
2415 TraceRecorder::prepareTreeCall(Fragment* inner)
2416 {
2417 TreeInfo* ti = (TreeInfo*)inner->vmprivate;
2418 inner_sp_ins = lirbuf->sp;
2419 /* The inner tree expects to be called from the current frame. If the outer tree (this
2420 trace) is currently inside a function inlining code (calldepth > 0), we have to advance
2421 the native stack pointer such that we match what the inner trace expects to see. We
2422 move it back when we come out of the inner tree call. */
2423 if (callDepth > 0) {
2424 /* Calculate the amount we have to lift the native stack pointer by to compensate for
2425 any outer frames that the inner tree doesn't expect but the outer tree has. */
2426 ptrdiff_t sp_adj = nativeStackOffset(&cx->fp->argv[-2]);
2427 /* Calculate the amount we have to lift the call stack by */
2428 ptrdiff_t rp_adj = callDepth * sizeof(FrameInfo);
2429 /* Guard that we have enough stack space for the tree we are trying to call on top
2430 of the new value for sp. */
2431 debug_only_v(printf("sp_adj=%d outer=%d inner=%d\n",
2432 sp_adj, treeInfo->nativeStackBase, ti->nativeStackBase));
2433 LIns* sp_top = lir->ins2i(LIR_piadd, lirbuf->sp,
2434 - treeInfo->nativeStackBase /* rebase sp to beginning of outer tree's stack */
2435 + sp_adj /* adjust for stack in outer frame inner tree can't see */
2436 + ti->maxNativeStackSlots * sizeof(double)); /* plus the inner tree's stack */
2437 guard(true, lir->ins2(LIR_lt, sp_top, eos_ins), OOM_EXIT);
2438 /* Guard that we have enough call stack space. */
2439 LIns* rp_top = lir->ins2i(LIR_piadd, lirbuf->rp, rp_adj +
2440 ti->maxCallDepth * sizeof(FrameInfo));
2441 guard(true, lir->ins2(LIR_lt, rp_top, eor_ins), OOM_EXIT);
2442 /* We have enough space, so adjust sp and rp to their new level. */
2443 lir->insStorei(inner_sp_ins = lir->ins2i(LIR_piadd, lirbuf->sp,
2444 - treeInfo->nativeStackBase /* rebase sp to beginning of outer tree's stack */
2445 + sp_adj /* adjust for stack in outer frame inner tree can't see */
2446 + ti->nativeStackBase), /* plus the inner tree's stack base */
2447 lirbuf->state, offsetof(InterpState, sp));
2448 lir->insStorei(lir->ins2i(LIR_piadd, lirbuf->rp, rp_adj),
2449 lirbuf->state, offsetof(InterpState, rp));
2450 }
2451 }
2452
2453 /* Record a call to an inner tree. */
2454 void
2455 TraceRecorder::emitTreeCall(Fragment* inner, VMSideExit* exit)
2456 {
2457 TreeInfo* ti = (TreeInfo*)inner->vmprivate;
2458 /* Invoke the inner tree. */
2459 LIns* args[] = { INS_CONSTPTR(inner), lirbuf->state }; /* reverse order */
2460 LIns* ret = lir->insCall(&js_CallTree_ci, args);
2461 /* Read back all registers, in case the called tree changed any of them. */
2462 import(ti, inner_sp_ins, exit->numGlobalSlots, exit->calldepth,
2463 getTypeMap(exit), getTypeMap(exit) + exit->numGlobalSlots);
2464 /* Restore sp and rp to their original values (we still have them in a register). */
2465 if (callDepth > 0) {
2466 lir->insStorei(lirbuf->sp, lirbuf->state, offsetof(InterpState, sp));
2467 lir->insStorei(lirbuf->rp, lirbuf->state, offsetof(InterpState, rp));
2468 }
2469 /* Guard that we come out of the inner tree along the same side exit we came out when
2470 we called the inner tree at recording time. */
2471 guard(true, lir->ins2(LIR_eq, ret, INS_CONSTPTR(exit)), NESTED_EXIT);
2472 /* Register us as a dependent tree of the inner tree. */
2473 ((TreeInfo*)inner->vmprivate)->dependentTrees.addUnique(fragment->root);
2474 }
2475
2476 /* Add a if/if-else control-flow merge point to the list of known merge points. */
2477 void
2478 TraceRecorder::trackCfgMerges(jsbytecode* pc)
2479 {
2480 /* If we hit the beginning of an if/if-else, then keep track of the merge point after it. */
2481 JS_ASSERT((*pc == JSOP_IFEQ) || (*pc == JSOP_IFEQX));
2482 jssrcnote* sn = js_GetSrcNote(cx->fp->script, pc);
2483 if (sn != NULL) {
2484 if (SN_TYPE(sn) == SRC_IF) {
2485 cfgMerges.add((*pc == JSOP_IFEQ)
2486 ? pc + GET_JUMP_OFFSET(pc)
2487 : pc + GET_JUMPX_OFFSET(pc));
2488 } else if (SN_TYPE(sn) == SRC_IF_ELSE)
2489 cfgMerges.add(pc + js_GetSrcNoteOffset(sn, 0));
2490 }
2491 }
2492
2493 /* Invert the direction of the guard if this is a loop edge that is not
2494 taken (thin loop). */
2495 void
2496 TraceRecorder::flipIf(jsbytecode* pc, bool& cond)
2497 {
2498 if (js_IsLoopEdge(pc, (jsbytecode*)fragment->root->ip)) {
2499 switch (*pc) {
2500 case JSOP_IFEQ:
2501 case JSOP_IFEQX:
2502 if (!cond)
2503 return;
2504 break;
2505 case JSOP_IFNE:
2506 case JSOP_IFNEX:
2507 if (cond)
2508 return;
2509 break;
2510 default:
2511 JS_NOT_REACHED("flipIf");
2512 }
2513 /* We are about to walk out of the loop, so terminate it with
2514 an inverse loop condition. */
2515 debug_only_v(printf("Walking out of the loop, terminating it anyway.\n");)
2516 cond = !cond;
2517 terminate = true;
2518 /* If when we get to closeLoop the tree is decided to be type unstable, we need to
2519 reverse this logic because the loop won't be closed after all. Store the real
2520 value of the IP the interpreter expects, so we can use it in our final LIR_x.
2521 */
2522 if (*pc == JSOP_IFEQX || *pc == JSOP_IFNEX)
2523 pc += GET_JUMPX_OFFSET(pc);
2524 else
2525 pc += GET_JUMP_OFFSET(pc);
2526 terminate_ip_adj = ENCODE_IP_ADJ(cx->fp, pc);
2527 }
2528 }
2529
2530 /* Emit code for a fused IFEQ/IFNE. */
2531 void
2532 TraceRecorder::fuseIf(jsbytecode* pc, bool cond, LIns* x)
2533 {
2534 if (x->isconst()) // no need to guard if condition is constant
2535 return;
2536 if (*pc == JSOP_IFEQ) {
2537 flipIf(pc, cond);
2538 guard(cond, x, BRANCH_EXIT);
2539 trackCfgMerges(pc);
2540 } else if (*pc == JSOP_IFNE) {
2541 flipIf(pc, cond);
2542 guard(cond, x, BRANCH_EXIT);
2543 }
2544 }
2545
2546 bool
2547 TraceRecorder::hasMethod(JSObject* obj, jsid id)
2548 {
2549 if (!obj)
2550 return false;
2551
2552 JSObject* pobj;
2553 JSProperty* prop;
2554 int protoIndex = OBJ_LOOKUP_PROPERTY(cx, obj, id, &pobj, &prop);
2555 if (protoIndex < 0 || !prop)
2556 return false;
2557
2558 bool found = false;
2559 if (OBJ_IS_NATIVE(pobj)) {
2560 JSScope* scope = OBJ_SCOPE(pobj);
2561 JSScopeProperty* sprop = (JSScopeProperty*) prop;
2562
2563 if (SPROP_HAS_STUB_GETTER(sprop) &&
2564 SPROP_HAS_VALID_SLOT(sprop, scope)) {
2565 jsval v = LOCKED_OBJ_GET_SLOT(pobj, sprop->slot);
2566 if (VALUE_IS_FUNCTION(cx, v)) {
2567 found = true;
2568 if (!SCOPE_IS_BRANDED(scope)) {
2569 SCOPE_MAKE_UNIQUE_SHAPE(cx, scope);
2570 SCOPE_SET_BRANDED(scope);
2571 }
2572 }
2573 }
2574 }
2575
2576 OBJ_DROP_PROPERTY(cx, pobj, prop);
2577 return found;
2578 }
2579
2580 bool
2581 TraceRecorder::hasToStringMethod(JSObject* obj)
2582 {
2583 JS_ASSERT(cx->fp->regs->sp + 1 <= cx->fp->slots + cx->fp->script->nslots);
2584
2585 return hasMethod(obj, ATOM_TO_JSID(cx->runtime->atomState.toStringAtom));
2586 }
2587
2588 bool
2589 TraceRecorder::hasValueOfMethod(JSObject* obj)
2590 {
2591 JS_ASSERT(cx->fp->regs->sp + 2 <= cx->fp->slots + cx->fp->script->nslots);
2592
2593 return hasMethod(obj, ATOM_TO_JSID(cx->runtime->atomState.valueOfAtom));
2594 }
2595
2596 bool
2597 TraceRecorder::hasIteratorMethod(JSObject* obj)
2598 {
2599 JS_ASSERT(cx->fp->regs->sp + 2 <= cx->fp->slots + cx->fp->script->nslots);
2600
2601 return hasMethod(obj, ATOM_TO_JSID(cx->runtime->atomState.iteratorAtom));
2602 }
2603
2604 int
2605 nanojit::StackFilter::getTop(LInsp guard)
2606 {
2607 VMSideExit* e = (VMSideExit*)guard->record()->exit;
2608 if (sp == lirbuf->sp)
2609 return e->sp_adj;
2610 JS_ASSERT(sp == lirbuf->rp);
2611 return e->rp_adj;
2612 }
2613
2614 #if defined NJ_VERBOSE
2615 void
2616 nanojit::LirNameMap::formatGuard(LIns *i, char *out)
2617 {
2618 VMSideExit *x;
2619
2620 x = (VMSideExit *)i->record()->exit;
2621 sprintf(out,
2622 "%s: %s %s -> %lu:%lu sp%+ld rp%+ld",
2623 formatRef(i),
2624 lirNames[i->opcode()],
2625 i->oprnd1()->isCond() ? formatRef(i->oprnd1()) : "",
2626 IMACRO_PC_ADJ(x->ip_adj),
2627 SCRIPT_PC_ADJ(x->ip_adj),
2628 (long int)x->sp_adj,
2629 (long int)x->rp_adj);
2630 }
2631 #endif
2632
2633 void
2634 nanojit::Fragment::onDestroy()
2635 {
2636 if (root == this) {
2637 delete mergeCounts;
2638 delete lirbuf;
2639 }
2640 delete (TreeInfo *)vmprivate;
2641 }
2642
2643 void
2644 js_DeleteRecorder(JSContext* cx)
2645 {
2646 JSTraceMonitor* tm = &JS_TRACE_MONITOR(cx);
2647
2648 /* Aborting and completing a trace end up here. */
2649 JS_ASSERT(tm->onTrace);
2650 tm->onTrace = false;
2651
2652 delete tm->recorder;
2653 tm->recorder = NULL;
2654 }
2655
2656 /**
2657 * Checks whether the shape of the global object has changed.
2658 */
2659 static inline bool
2660 js_CheckGlobalObjectShape(JSContext* cx, JSTraceMonitor* tm, JSObject* globalObj)
2661 {
2662 /* Check the global shape. */
2663 if (OBJ_SHAPE(globalObj) != tm->globalShape) {
2664 AUDIT(globalShapeMismatchAtEntry);
2665 debug_only_v(printf("Global shape mismatch (%u vs. %u), flushing cache.\n",
2666 OBJ_SHAPE(globalObj), tm->globalShape);)
2667 return false;
2668 }
2669 return true;
2670 }
2671
2672 static bool
2673 js_StartRecorder(JSContext* cx, VMSideExit* anchor, Fragment* f, TreeInfo* ti,
2674 unsigned ngslots, uint8* globalTypeMap, uint8* stackTypeMap,
2675 VMSideExit* expectedInnerExit, Fragment* outer)
2676 {
2677 JSTraceMonitor* tm = &JS_TRACE_MONITOR(cx);
2678
2679 /*
2680 * Emulate on-trace semantics and avoid rooting headaches while recording,
2681 * by suppressing last-ditch GC attempts while recording a trace. This does
2682 * means that trace recording must not nest or the following assertion will
2683 * botch.
2684 */
2685 JS_ASSERT(!tm->onTrace);
2686 tm->onTrace = true;
2687
2688 /* start recording if no exception during construction */
2689 tm->recorder = new (&gc) TraceRecorder(cx, anchor, f, ti,
2690 ngslots, globalTypeMap, stackTypeMap,
2691 expectedInnerExit, outer);
2692 if (cx->throwing) {
2693 js_AbortRecording(cx, "setting up recorder failed");
2694 return false;
2695 }
2696 /* clear any leftover error state */
2697 tm->fragmento->assm()->setError(None);
2698 return true;
2699 }
2700
2701 static void
2702 js_TrashTree(JSContext* cx, Fragment* f)
2703 {
2704 JS_ASSERT((!f->code()) == (!f->vmprivate));
2705 JS_ASSERT(f == f->root);
2706 if (!f->code())
2707 return;
2708 AUDIT(treesTrashed);
2709 debug_only_v(printf("Trashing tree info.\n");)
2710 Fragmento* fragmento = JS_TRACE_MONITOR(cx).fragmento;
2711 TreeInfo* ti = (TreeInfo*)f->vmprivate;
2712 f->vmprivate = NULL;
2713 f->releaseCode(fragmento);
2714 Fragment** data = ti->dependentTrees.data();
2715 unsigned length = ti->dependentTrees.length();
2716 for (unsigned n = 0; n < length; ++n)
2717 js_TrashTree(cx, data[n]);
2718 delete ti;
2719 JS_ASSERT(!f->code() && !f->vmprivate);
2720 }
2721
2722 static int
2723 js_SynthesizeFrame(JSContext* cx, const FrameInfo& fi)
2724 {
2725 JS_ASSERT(HAS_FUNCTION_CLASS(fi.callee));
2726
2727 JSFunction* fun = GET_FUNCTION_PRIVATE(cx, fi.callee);
2728 JS_ASSERT(FUN_INTERPRETED(fun));
2729
2730 /* Assert that we have a correct sp distance from cx->fp->slots in fi. */
2731 JS_ASSERT_IF(!FI_IMACRO_PC(fi, cx->fp),
2732 js_ReconstructStackDepth(cx, cx->fp->script, FI_SCRIPT_PC(fi, cx->fp))
2733 == uintN(fi.s.spdist - cx->fp->script->nfixed));
2734
2735 uintN nframeslots = JS_HOWMANY(sizeof(JSInlineFrame), sizeof(jsval));
2736 JSScript* script = fun->u.i.script;
2737 size_t nbytes = (nframeslots + script->nslots) * sizeof(jsval);
2738
2739 /* Code duplicated from inline_call: case in js_Interpret (FIXME). */
2740 JSArena* a = cx->stackPool.current;
2741 void* newmark = (void*) a->avail;
2742 uintN argc = fi.s.argc & 0x7fff;
2743 jsval* vp = cx->fp->slots + fi.s.spdist - (2 + argc);
2744 uintN missing = 0;
2745 jsval* newsp;
2746
2747 if (fun->nargs > argc) {
2748 const JSFrameRegs& regs = *cx->fp->regs;
2749
2750 newsp = vp + 2 + fun->nargs;
2751 JS_ASSERT(newsp > regs.sp);
2752 if ((jsuword) newsp <= a->limit) {
2753 if ((jsuword) newsp > a->avail)
2754 a->avail = (jsuword) newsp;
2755 jsval* argsp = newsp;
2756 do {
2757 *--argsp = JSVAL_VOID;
2758 } while (argsp != regs.sp);
2759 missing = 0;
2760 } else {
2761 missing = fun->nargs - argc;
2762 nbytes += (2 + fun->nargs) * sizeof(jsval);
2763 }
2764 }
2765
2766 /* Allocate the inline frame with its vars and operands. */
2767 if (a->avail + nbytes <= a->limit) {
2768 newsp = (jsval *) a->avail;
2769 a->avail += nbytes;
2770 JS_ASSERT(missing == 0);
2771 } else {
2772 JS_ARENA_ALLOCATE_CAST(newsp, jsval *, &cx->stackPool, nbytes);
2773 if (!newsp) {
2774 js_ReportOutOfScriptQuota(cx);
2775 return 0;
2776 }
2777
2778 /*
2779 * Move args if the missing ones overflow arena a, then push
2780 * undefined for the missing args.
2781 */
2782 if (missing) {
2783 memcpy(newsp, vp, (2 + argc) * sizeof(jsval));
2784 vp = newsp;
2785 newsp = vp + 2 + argc;
2786 do {
2787 *newsp++ = JSVAL_VOID;
2788 } while (--missing != 0);
2789 }
2790 }
2791
2792 /* Claim space for the stack frame and initialize it. */
2793 JSInlineFrame* newifp = (JSInlineFrame *) newsp;
2794 newsp += nframeslots;
2795
2796 newifp->frame.callobj = NULL;
2797 newifp->frame.argsobj = NULL;
2798 newifp->frame.varobj = NULL;
2799 newifp->frame.script = script;
2800 newifp->frame.callee = fi.callee;
2801 newifp->frame.fun = fun;
2802
2803 bool constructing = fi.s.argc & 0x8000;
2804 newifp->frame.argc = argc;
2805
2806 jsbytecode* imacro_pc = FI_IMACRO_PC(fi, cx->fp);
2807 jsbytecode* script_pc = FI_SCRIPT_PC(fi, cx->fp);
2808 newifp->callerRegs.pc = imacro_pc ? imacro_pc : script_pc;
2809 newifp->callerRegs.sp = cx->fp->slots + fi.s.spdist;
2810 cx->fp->imacpc = imacro_pc ? script_pc : NULL;
2811
2812 newifp->frame.argv = newifp->callerRegs.sp - argc;
2813 JS_ASSERT(newifp->frame.argv);
2814 #ifdef DEBUG
2815 // Initialize argv[-1] to a known-bogus value so we'll catch it if
2816 // someone forgets to initialize it later.
2817 newifp->frame.argv[-1] = JSVAL_HOLE;
2818 #endif
2819 JS_ASSERT(newifp->frame.argv >= StackBase(cx->fp) + 2);
2820
2821 newifp->frame.rval = JSVAL_VOID;
2822 newifp->frame.down = cx->fp;
2823 newifp->frame.annotation = NULL;
2824 newifp->frame.scopeChain = OBJ_GET_PARENT(cx, fi.callee);
2825 newifp->frame.sharpDepth = 0;
2826 newifp->frame.sharpArray = NULL;
2827 newifp->frame.flags = constructing ? JSFRAME_CONSTRUCTING : 0;
2828 newifp->frame.dormantNext = NULL;
2829 newifp->frame.xmlNamespace = NULL;
2830 newifp->frame.blockChain = NULL;
2831 newifp->mark = newmark;
2832 newifp->frame.thisp = NULL; // will be set by js_ExecuteTree -> FlushNativeStackFrame
2833
2834 newifp->frame.regs = cx->fp->regs;
2835 newifp->frame.regs->pc = script->code;
2836 newifp->frame.regs->sp = newsp + script->nfixed;
2837 newifp->frame.imacpc = NULL;
2838 newifp->frame.slots = newsp;
2839 if (script->staticDepth < JS_DISPLAY_SIZE) {
2840 JSStackFrame **disp = &cx->display[script->staticDepth];
2841 newifp->frame.displaySave = *disp;
2842 *disp = &newifp->frame;
2843 }
2844 #ifdef DEBUG
2845 newifp->frame.pcDisabledSave = 0;
2846 #endif
2847
2848 /*
2849 * Note that cx->fp->script is still the caller's script; set the callee
2850 * inline frame's idea of caller version from its version.
2851 */
2852 newifp->callerVersion = (JSVersion) cx->fp->script->version;
2853
2854 cx->fp->regs = &newifp->callerRegs;
2855 cx->fp = &newifp->frame;
2856
2857 if (fun->flags & JSFUN_HEAVYWEIGHT) {
2858 /*
2859 * Set hookData to null because the failure case for js_GetCallObject
2860 * involves it calling the debugger hook.
2861 */
2862 newifp->hookData = NULL;
2863 if (!js_GetCallObject(cx, &newifp->frame, newifp->frame.scopeChain))
2864 return -1;
2865 }
2866
2867 /*
2868 * If there's a call hook, invoke it to compute the hookData used by
2869 * debuggers that cooperate with the interpreter.
2870 */
2871 JSInterpreterHook hook = cx->debugHooks->callHook;
2872 if (hook) {
2873 newifp->hookData = hook(cx, &newifp->frame, JS_TRUE, 0,
2874 cx->debugHooks->callHookData);
2875 } else {
2876 newifp->hookData = NULL;
2877 }
2878
2879 // FIXME? we must count stack slots from caller's operand stack up to (but not including)
2880 // callee's, including missing arguments. Could we shift everything down to the caller's
2881 // fp->slots (where vars start) and avoid some of the complexity?
2882 return (fi.s.spdist - cx->fp->down->script->nfixed) +
2883 ((fun->nargs > cx->fp->argc) ? fun->nargs - cx->fp->argc : 0) +
2884 script->nfixed;
2885 }
2886
2887 bool
2888 js_RecordTree(JSContext* cx, JSTraceMonitor* tm, Fragment* f, Fragment* outer, unsigned* demotes)
2889 {
2890 JS_ASSERT(cx->fp->regs->pc == f->ip && f->root == f);
2891
2892 /* Avoid recording loops in overlarge scripts. */
2893 if (cx->fp->script->length >= SCRIPT_PC_ADJ_LIMIT) {
2894 js_AbortRecording(cx, "script too large");
2895 return false;
2896 }
2897
2898 /* Make sure the global type map didn't change on us. */
2899 JSObject* globalObj = JS_GetGlobalForObject(cx, cx->fp->scopeChain);
2900 if (!js_CheckGlobalObjectShape(cx, tm, globalObj)) {
2901 js_FlushJITCache(cx);
2902 return false;
2903 }
2904 TypeMap current;
2905 current.captureGlobalTypes(cx, *tm->globalSlots);
2906 if (!current.matches(*tm->globalTypeMap)) {
2907 js_FlushJITCache(cx);
2908 debug_only_v(printf("Global type map mismatch in RecordTree, flushing cache.\n");)
2909 return false;
2910 }
2911
2912 AUDIT(recorderStarted);
2913
2914 /* Try to find an unused peer fragment, or allocate a new one. */
2915 while (f->code() && f->peer)
2916 f = f->peer;
2917 if (f->code())
2918 f = JS_TRACE_MONITOR(cx).fragmento->getAnchor(f->root->ip);
2919
2920 f->recordAttempts++;
2921 f->root = f;
2922 /* allocate space to store the LIR for this tree */
2923 if (!f->lirbuf) {
2924 f->lirbuf = new (&gc) LirBuffer(tm->fragmento, NULL);
2925 #ifdef DEBUG
2926 f->lirbuf->names = new (&gc) LirNameMap(&gc, NULL, tm->fragmento->labels);
2927 #endif
2928 }
2929
2930 if (f->lirbuf->outOmem()) {
2931 js_FlushJITCache(cx);
2932 debug_only_v(printf("Out of memory recording new tree, flushing cache.\n");)
2933 return false;
2934 }
2935
2936 JS_ASSERT(!f->code() && !f->vmprivate);
2937
2938 /* setup the VM-private treeInfo structure for this fragment */
2939 TreeInfo* ti = new (&gc) TreeInfo(f);
2940
2941 /* capture the coerced type of each active slot in the stack type map */
2942 ti->stackTypeMap.captureStackTypes(cx, 0/*callDepth*/);
2943
2944 if (demotes) {
2945 /* If we get a list of demotions, an outer tree is telling us our types are not callable. */
2946 uint8* typeMap = ti->stackTypeMap.data();
2947 for (unsigned i = 1; i <= NUM_UNDEMOTE_SLOTS(demotes); i++) {
2948 JS_ASSERT(demotes[i] < ti->stackTypeMap.length());
2949 if (typeMap[demotes[i]] == JSVAL_INT)
2950 typeMap[demotes[i]] = JSVAL_DOUBLE;
2951 }
2952 }
2953
2954 /* Check for duplicate entry type maps. This is always wrong and hints at trace explosion
2955 since we are trying to stabilize something without properly connecting peer edges. */
2956 #ifdef DEBUG
2957 TreeInfo* ti_other;
2958 for (Fragment* peer = tm->fragmento->getLoop(f->root->ip); peer != NULL; peer = peer->peer) {
2959 if (!peer->code() || peer == f)
2960 continue;
2961 ti_other = (TreeInfo*)peer->vmprivate;
2962 JS_ASSERT(ti_other);
2963 JS_ASSERT(!ti->stackTypeMap.matches(ti_other->stackTypeMap));
2964 }
2965 #endif
2966
2967 /* determine the native frame layout at the entry point */
2968 unsigned entryNativeStackSlots = ti->stackTypeMap.length();
2969 JS_ASSERT(entryNativeStackSlots == js_NativeStackSlots(cx, 0/*callDepth*/));
2970 ti->nativeStackBase = (entryNativeStackSlots -
2971 (cx->fp->regs->sp - StackBase(cx->fp))) * sizeof(double);
2972 ti->maxNativeStackSlots = entryNativeStackSlots;
2973 ti->maxCallDepth = 0;
2974 ti->script = cx->fp->script;
2975
2976 /* recording primary trace */
2977 if (!js_StartRecorder(cx, NULL, f, ti,
2978 tm->globalSlots->length(), tm->globalTypeMap->data(),
2979 ti->stackTypeMap.data(), NULL, outer)) {
2980 return false;
2981 }
2982
2983 return true;
2984 }
2985
2986 static bool
2987 js_AttemptToStabilizeTree(JSContext* cx, VMSideExit* exit, Fragment* outer)
2988 {
2989 JSTraceMonitor* tm = &JS_TRACE_MONITOR(cx);
2990 Fragment* from = exit->from->root;
2991 unsigned* demotes;
2992
2993 JS_ASSERT(exit->from->root->code());
2994
2995 demotes = ALLOCA_UNDEMOTE_SLOTLIST(exit->numStackSlots);
2996 CLEAR_UNDEMOTE_SLOTLIST(demotes);
2997
2998 uint8* t2 = getTypeMap(exit) + exit->numGlobalSlots;
2999 for (unsigned i = 0; i < exit->numStackSlots; i++) {
3000 if (t2[i] == JSVAL_DOUBLE)
3001 ADD_UNDEMOTE_SLOT(demotes, i);
3002 }
3003
3004 if (!NUM_UNDEMOTE_SLOTS(demotes))
3005 demotes = NULL;
3006
3007 if (!js_RecordTree(cx, tm, from->first, outer, demotes))
3008 return false;
3009
3010 tm->recorder->setPromotedPeer(demotes ? from : NULL);
3011
3012 return true;
3013 }
3014
3015 static bool
3016 js_AttemptToExtendTree(JSContext* cx, VMSideExit* anchor, VMSideExit* exitedFrom, Fragment* outer)
3017 {
3018 Fragment* f = anchor->from->root;
3019 JS_ASSERT(f->vmprivate);
3020 TreeInfo* ti = (TreeInfo*)f->vmprivate;
3021
3022 /* Don't grow trees above a certain size to avoid code explosion due to tail duplication. */
3023 if (ti->branchCount >= MAX_BRANCHES)
3024 return false;
3025
3026 Fragment* c;
3027 if (!(c = anchor->target)) {
3028 c = JS_TRACE_MONITOR(cx).fragmento->createBranch(anchor, cx->fp->regs->pc);
3029 c->spawnedFrom = anchor;
3030 c->parent = f;
3031 anchor->target = c;
3032 c->root = f;
3033 }
3034
3035 debug_only_v(printf("trying to attach another branch to the tree (hits = %d)\n", c->hits());)
3036
3037 if (++c->hits() >= HOTEXIT) {
3038 /* start tracing secondary trace from this point */
3039 c->lirbuf = f->lirbuf;
3040 unsigned ngslots;
3041 uint8* globalTypeMap;
3042 uint8* stackTypeMap;
3043 TypeMap fullMap;
3044 if (exitedFrom == NULL) {
3045 /* If we are coming straight from a simple side exit, just use that exit's type map
3046 as starting point. */
3047 ngslots = anchor->numGlobalSlots;
3048 globalTypeMap = getTypeMap(anchor);
3049 stackTypeMap = globalTypeMap + ngslots;
3050 } else {
3051 /* If we side-exited on a loop exit and continue on a nesting guard, the nesting
3052 guard (anchor) has the type information for everything below the current scope,
3053 and the actual guard we exited from has the types for everything in the current
3054 scope (and whatever it inlined). We have to merge those maps here. */
3055 VMSideExit* e1 = anchor;
3056 VMSideExit* e2 = exitedFrom;
3057 fullMap.add(getTypeMap(e1) + e1->numGlobalSlots, e1->numStackSlotsBelowCurrentFrame);
3058 fullMap.add(getTypeMap(e2) + e2->numGlobalSlots, e2->numStackSlots);
3059 ngslots = e2->numGlobalSlots;
3060 globalTypeMap = getTypeMap(e2);
3061 stackTypeMap = fullMap.data();
3062 }
3063 return js_StartRecorder(cx, anchor, c, (TreeInfo*)f->vmprivate,
3064 ngslots, globalTypeMap, stackTypeMap, exitedFrom, outer);
3065 }
3066 return false;
3067 }
3068
3069 static VMSideExit*
3070 js_ExecuteTree(JSContext* cx, Fragment* f, uintN& inlineCallCount,
3071 VMSideExit** innermostNestedGuardp);
3072
3073 static Fragment*
3074 js_FindVMCompatiblePeer(JSContext* cx, Fragment* f);
3075
3076 static bool
3077 js_CloseLoop(JSContext* cx)
3078 {
3079 JSTraceMonitor* tm = &JS_TRACE_MONITOR(cx);
3080 Fragmento* fragmento = tm->fragmento;
3081 TraceRecorder* r = tm->recorder;
3082 JS_ASSERT(fragmento && r);