/[jscoverage]/trunk/js/jstracer.cpp

Diff of /trunk/js/jstracer.cpp

Parent Directory | Revision Log | View Patch Patch

-revision 459 by siliconforks,
Tue Dec  9 03:37:47 2008 UTC
+revision 460 by siliconforks,
Sat Sep 26 23:15:22 2009 UTC
 Line 1
- /* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 4 -*-
+ /* -*- Mode: C++; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*-
   * vim: set ts=4 sw=4 et tw=99:
   *
   * ***** BEGIN LICENSE BLOCK *****
 Line 50
  #ifdef SOLARIS
  #include <alloca.h>
  #endif
+ #include <limits.h>
  #include "nanojit/nanojit.h"
- #include "jsarray.h"            // higher-level library and API headers
+ #include "jsapi.h"              // higher-level library and API headers
+ #include "jsarray.h"
  #include "jsbool.h"
  #include "jscntxt.h"
  #include "jsdbgapi.h"
-Line 68
+Line 70
  #include "jsdate.h"
  #include "jsstaticcheck.h"
  #include "jstracer.h"
+ #include "jsxml.h"
  #include "jsautooplen.h"        // generated headers last
+ #include "imacros.c.out"
- /* Never use JSVAL_IS_BOOLEAN because it restricts the value (true, false) and
+ #if JS_HAS_XML_SUPPORT
-    the type. What you want to use is JSVAL_TAG(x) == JSVAL_BOOLEAN and then
+ #define ABORT_IF_XML(v)                                                       \
+     JS_BEGIN_MACRO                                                            \
+     if (!JSVAL_IS_PRIMITIVE(v) && OBJECT_IS_XML(BOGUS_CX, JSVAL_TO_OBJECT(v)))\
+         ABORT_TRACE("xml detected");                                          \
+     JS_END_MACRO
+ #else
+ #define ABORT_IF_XML(cx, v) ((void) 0)
+ #endif
+ /* Never use JSVAL_IS_BOOLEAN because it restricts the value (true, false) and
+    the type. What you want to use is JSVAL_TAG(x) == JSVAL_BOOLEAN and then
     handle the undefined case properly (bug 457363). */
  #undef JSVAL_IS_BOOLEAN
  #define JSVAL_IS_BOOLEAN(x) JS_STATIC_ASSERT(0)
  /* Use a fake tag to represent boxed values, borrowing from the integer tag
     range since we only use JSVAL_INT to indicate integers. */
  #define JSVAL_BOXED 3
+ /* Another fake jsval tag, used to distinguish null from object values. */
+ #define JSVAL_TNULL 5
+ /* A last fake jsval tag distinguishing functions from non-function objects. */
+ #define JSVAL_TFUN 7
  /* Map to translate a type tag into a printable representation. */
- static const char typeChar[] = "OIDVS?B?";
+ static const char typeChar[] = "OIDXSNBF";
+ static const char tagChar[]  = "OIDISIBI";
+ /* Blacklist parameters. */
  /* Number of iterations of a loop where we start tracing.  That is, we don't
     start tracing until the beginning of the HOTLOOP-th iteration. */
  #define HOTLOOP 2
+ /* Attempt recording this many times before blacklisting permanently. */
+ #define BL_ATTEMPTS 2
+ /* Skip this many future hits before allowing recording again after blacklisting. */
+ #define BL_BACKOFF 32
  /* Number of times we wait to exit on a side exit before we try to extend the tree. */
  #define HOTEXIT 1
- /* Max call depths for inlining. */
+ /* Number of times we try to extend the tree along a side exit. */
- #define MAX_CALLDEPTH 10
+ #define MAXEXIT 3
- /* Max number of type mismatchs before we trash the tree. */
+ /* Maximum number of peer trees allowed. */
- #define MAX_MISMATCH 20
+ #define MAXPEERS 9
- /* Max blacklist level of inner tree immediate recompiling  */
+ /* Max call depths for inlining. */
- #define MAX_INNER_RECORD_BLACKLIST  -16
+ #define MAX_CALLDEPTH 10
  /* Max native stack size. */
  #define MAX_NATIVE_STACK_SLOTS 1024
-Line 106
+Line 135
  /* Max call stack size. */
  #define MAX_CALL_STACK_ENTRIES 64
+ /* Max global object size. */
+ #define MAX_GLOBAL_SLOTS 4096
+ /* Max memory needed to rebuild the interpreter stack when falling off trace. */
+ #define MAX_INTERP_STACK_BYTES                                                \
+     (MAX_NATIVE_STACK_SLOTS * sizeof(jsval) +                                 \
+      MAX_CALL_STACK_ENTRIES * sizeof(JSInlineFrame) +                         \
+      sizeof(JSInlineFrame)) /* possibly slow native frame at top of stack */
  /* Max number of branches per tree. */
- #define MAX_BRANCHES 16
+ #define MAX_BRANCHES 32
- /* Macros for demote slot lists */
+ #define CHECK_STATUS(expr)                                                    \
- #define ALLOCA_UNDEMOTE_SLOTLIST(num)     (unsigned*)alloca(((num) + 1) * sizeof(unsigned))
+     JS_BEGIN_MACRO                                                            \
- #define ADD_UNDEMOTE_SLOT(list, slot)     list[++list[0]] = slot
+         JSRecordingStatus _status = (expr);                                   \
- #define NUM_UNDEMOTE_SLOTS(list)          list[0]
+         if (_status != JSRS_CONTINUE)                                        \
- #define CLEAR_UNDEMOTE_SLOTLIST(list)     list[0] = 0
+           return _status;                                                     \
+     JS_END_MACRO
  #ifdef JS_JIT_SPEW
- #define ABORT_TRACE(msg)   do { debug_only_v(fprintf(stdout, "abort: %d: %s\n", __LINE__, msg);)  return false; } while (0)
+ #define debug_only_a(x) if (js_verboseAbort || js_verboseDebug ) { x; }
+ #define ABORT_TRACE_RV(msg, value)                                    \
+     JS_BEGIN_MACRO                                                            \
+         debug_only_a(fprintf(stdout, "abort: %d: %s\n", __LINE__, (msg));)    \
+         return (value);                                                       \
+     JS_END_MACRO
  #else
- #define ABORT_TRACE(msg)   return false
+ #define debug_only_a(x)
+ #define ABORT_TRACE_RV(msg, value)   return (value)
  #endif
+ #define ABORT_TRACE(msg)         ABORT_TRACE_RV(msg, JSRS_STOP)
+ #define ABORT_TRACE_ERROR(msg)   ABORT_TRACE_RV(msg, JSRS_ERROR)
  #ifdef JS_JIT_SPEW
  struct __jitstats {
  #define JITSTAT(x) uint64 x;
-Line 201
+Line 249
  #define AUDIT(x) ((void)0)
  #endif /* JS_JIT_SPEW */
- #define INS_CONST(c)    addName(lir->insImm(c), #c)
+ #define INS_CONST(c)        addName(lir->insImm(c), #c)
- #define INS_CONSTPTR(p) addName(lir->insImmPtr((void*) (p)), #p)
+ #define INS_CONSTPTR(p)     addName(lir->insImmPtr(p), #p)
+ #define INS_CONSTFUNPTR(p)  addName(lir->insImmPtr(JS_FUNC_TO_DATA_PTR(void*, p)), #p)
+ #define INS_CONSTWORD(v)    addName(lir->insImmPtr((void *) v), #v)
  using namespace avmplus;
  using namespace nanojit;
-Line 213
+Line 263
  #ifdef JS_JIT_SPEW
  void
- js_DumpPeerStability(Fragmento* frago, const void* ip);
+ js_DumpPeerStability(JSTraceMonitor* tm, const void* ip, JSObject* globalObj, uint32 globalShape, uint32 argc);
  #endif
  /* We really need a better way to configure the JIT. Shaver, where is my fancy JIT object? */
- static bool nesting_enabled = true;
+ static bool did_we_check_processor_features = false;
- #if defined(NANOJIT_IA32)
- static bool did_we_check_sse2 = false;
- #endif
  #ifdef JS_JIT_SPEW
- static bool verbose_debug = getenv("TRACEMONKEY") && strstr(getenv("TRACEMONKEY"), "verbose");
+ bool js_verboseDebug = getenv("TRACEMONKEY") && strstr(getenv("TRACEMONKEY"), "verbose");
- #define debug_only_v(x) if (verbose_debug) { x; }
+ bool js_verboseStats = js_verboseDebug ||
- #else
+     (getenv("TRACEMONKEY") && strstr(getenv("TRACEMONKEY"), "stats"));
- #define debug_only_v(x)
+ bool js_verboseAbort = getenv("TRACEMONKEY") && strstr(getenv("TRACEMONKEY"), "abort");
  #endif
  /* The entire VM shares one oracle. Collisions and concurrent updates are tolerated and worst
     case cause performance regressions. */
  static Oracle oracle;
- /* Blacklists the root peer fragment at a fragment's PC.  This is so blacklisting stays at the
-    top of the peer list and not scattered around. */
- void
- js_BlacklistPC(Fragmento* frago, Fragment* frag);
  Tracker::Tracker()
  {
      pagelist = 0;
-Line 296
+Line 338
  }
  #if defined NANOJIT_64BIT
- #define PAGEMASK        0x7ff
+ #define PAGEMASK 0x7ff
  #else
- #define PAGEMASK        0xfff
+ #define PAGEMASK 0xfff
  #endif
  LIns*
-Line 319
+Line 361
      p->map[(jsuword(v) & PAGEMASK) >> 2] = i;
  }
+ static inline jsuint argSlots(JSStackFrame* fp)
+ {
+     return JS_MAX(fp->argc, fp->fun->nargs);
+ }
  static inline bool isNumber(jsval v)
  {
      return JSVAL_IS_INT(v) || JSVAL_IS_DOUBLE(v);
-Line 341
+Line 388
      return JSDOUBLE_IS_INT(d, i);
  }
+ static inline jsint asInt32(jsval v)
+ {
+     JS_ASSERT(isNumber(v));
+     if (JSVAL_IS_INT(v))
+         return JSVAL_TO_INT(v);
+ #ifdef DEBUG
+     jsint i;
+     JS_ASSERT(JSDOUBLE_IS_INT(*JSVAL_TO_DOUBLE(v), i));
+ #endif
+     return jsint(*JSVAL_TO_DOUBLE(v));
+ }
  /* Return JSVAL_DOUBLE for all numbers (int and double) and the tag otherwise. */
  static inline uint8 getPromotedType(jsval v)
  {
-     return JSVAL_IS_INT(v) ? JSVAL_DOUBLE : uint8(JSVAL_TAG(v));
+     if (JSVAL_IS_INT(v))
+         return JSVAL_DOUBLE;
+     if (JSVAL_IS_OBJECT(v)) {
+         if (JSVAL_IS_NULL(v))
+             return JSVAL_TNULL;
+         if (HAS_FUNCTION_CLASS(JSVAL_TO_OBJECT(v)))
+             return JSVAL_TFUN;
+         return JSVAL_OBJECT;
+     }
+     return uint8(JSVAL_TAG(v));
  }
  /* Return JSVAL_INT for all whole numbers that fit into signed 32-bit and the tag otherwise. */
  static inline uint8 getCoercedType(jsval v)
  {
-     return isInt32(v) ? JSVAL_INT : (uint8) JSVAL_TAG(v);
+     if (isInt32(v))
+         return JSVAL_INT;
+     if (JSVAL_IS_OBJECT(v)) {
+         if (JSVAL_IS_NULL(v))
+             return JSVAL_TNULL;
+         if (HAS_FUNCTION_CLASS(JSVAL_TO_OBJECT(v)))
+             return JSVAL_TFUN;
+         return JSVAL_OBJECT;
+     }
+     return uint8(JSVAL_TAG(v));
+ }
+ /*
+  * Constant seed and accumulate step borrowed from the DJB hash.
+  */
+ #define ORACLE_MASK (ORACLE_SIZE - 1)
+ #define FRAGMENT_TABLE_MASK (FRAGMENT_TABLE_SIZE - 1)
+ #define HASH_SEED 5381
+ static inline void
+ hash_accum(uintptr_t& h, uintptr_t i, uintptr_t mask)
+ {
+     h = ((h << 5) + h + (mask & i)) & mask;
+ }
+ JS_REQUIRES_STACK static inline int
+ stackSlotHash(JSContext* cx, unsigned slot)
+ {
+     uintptr_t h = HASH_SEED;
+     hash_accum(h, uintptr_t(cx->fp->script), ORACLE_MASK);
+     hash_accum(h, uintptr_t(cx->fp->regs->pc), ORACLE_MASK);
+     hash_accum(h, uintptr_t(slot), ORACLE_MASK);
+     return int(h);
+ }
+ JS_REQUIRES_STACK static inline int
+ globalSlotHash(JSContext* cx, unsigned slot)
+ {
+     uintptr_t h = HASH_SEED;
+     JSStackFrame* fp = cx->fp;
+     while (fp->down)
+         fp = fp->down;
+     hash_accum(h, uintptr_t(fp->script), ORACLE_MASK);
+     hash_accum(h, uintptr_t(OBJ_SHAPE(JS_GetGlobalForObject(cx, fp->scopeChain))),
+                ORACLE_MASK);
+     hash_accum(h, uintptr_t(slot), ORACLE_MASK);
+     return int(h);
+ }
+ Oracle::Oracle()
+ {
+     /* Grow the oracle bitsets to their (fixed) size here, once. */
+     _stackDontDemote.set(&gc, ORACLE_SIZE-1);
+     _globalDontDemote.set(&gc, ORACLE_SIZE-1);
+     clear();
  }
  /* Tell the oracle that a certain global variable should not be demoted. */
- void
+ JS_REQUIRES_STACK void
- Oracle::markGlobalSlotUndemotable(JSScript* script, unsigned slot)
+ Oracle::markGlobalSlotUndemotable(JSContext* cx, unsigned slot)
  {
-     _dontDemote.set(&gc, (slot % ORACLE_SIZE));
+     _globalDontDemote.set(&gc, globalSlotHash(cx, slot));
  }
  /* Consult with the oracle whether we shouldn't demote a certain global variable. */
- bool
+ JS_REQUIRES_STACK bool
- Oracle::isGlobalSlotUndemotable(JSScript* script, unsigned slot) const
+ Oracle::isGlobalSlotUndemotable(JSContext* cx, unsigned slot) const
  {
-     return _dontDemote.get(slot % ORACLE_SIZE);
+     return _globalDontDemote.get(globalSlotHash(cx, slot));
  }
  /* Tell the oracle that a certain slot at a certain bytecode location should not be demoted. */
- void
+ JS_REQUIRES_STACK void
- Oracle::markStackSlotUndemotable(JSScript* script, jsbytecode* ip, unsigned slot)
+ Oracle::markStackSlotUndemotable(JSContext* cx, unsigned slot)
  {
-     uint32 hash = uint32(intptr_t(ip)) + (slot << 5);
+     _stackDontDemote.set(&gc, stackSlotHash(cx, slot));
-     hash %= ORACLE_SIZE;
-     _dontDemote.set(&gc, hash);
  }
  /* Consult with the oracle whether we shouldn't demote a certain slot. */
- bool
+ JS_REQUIRES_STACK bool
- Oracle::isStackSlotUndemotable(JSScript* script, jsbytecode* ip, unsigned slot) const
+ Oracle::isStackSlotUndemotable(JSContext* cx, unsigned slot) const
  {
-     uint32 hash = uint32(intptr_t(ip)) + (slot << 5);
+     return _stackDontDemote.get(stackSlotHash(cx, slot));
-     hash %= ORACLE_SIZE;
-     return _dontDemote.get(hash);
  }
- /* Clear the oracle. */
  void
- Oracle::clear()
+ Oracle::clearDemotability()
  {
-     _dontDemote.reset();
+     _stackDontDemote.reset();
+     _globalDontDemote.reset();
+ }
+ struct PCHashEntry : public JSDHashEntryStub {
+     size_t          count;
+ };
+ #define PC_HASH_COUNT 1024
+ static void
+ js_Blacklist(jsbytecode* pc)
+ {
+     JS_ASSERT(*pc == JSOP_LOOP || *pc == JSOP_NOP);
+     *pc = JSOP_NOP;
+ }
+ static void
+ js_Backoff(JSContext *cx, jsbytecode* pc, Fragment* tree=NULL)
+ {
+     JSDHashTable *table = &JS_TRACE_MONITOR(cx).recordAttempts;
+     if (table->ops) {
+         PCHashEntry *entry = (PCHashEntry *)
+             JS_DHashTableOperate(table, pc, JS_DHASH_ADD);
+         if (entry) {
+             if (!entry->key) {
+                 entry->key = pc;
+                 JS_ASSERT(entry->count == 0);
+             }
+             JS_ASSERT(JS_DHASH_ENTRY_IS_LIVE(&(entry->hdr)));
+             if (entry->count++ > (BL_ATTEMPTS * MAXPEERS)) {
+                 entry->count = 0;
+                 js_Blacklist(pc);
+                 return;
+             }
+         }
+     }
+     if (tree) {
+         tree->hits() -= BL_BACKOFF;
+         /*
+          * In case there is no entry or no table (due to OOM) or some
+          * serious imbalance in the recording-attempt distribution on a
+          * multitree, give each tree another chance to blacklist here as
+          * well.
+          */
+         if (++tree->recordAttempts > BL_ATTEMPTS)
+             js_Blacklist(pc);
+     }
+ }
+ static void
+ js_resetRecordingAttempts(JSContext *cx, jsbytecode* pc)
+ {
+     JSDHashTable *table = &JS_TRACE_MONITOR(cx).recordAttempts;
+     if (table->ops) {
+         PCHashEntry *entry = (PCHashEntry *)
+             JS_DHashTableOperate(table, pc, JS_DHASH_LOOKUP);
+         if (JS_DHASH_ENTRY_IS_FREE(&(entry->hdr)))
+             return;
+         JS_ASSERT(JS_DHASH_ENTRY_IS_LIVE(&(entry->hdr)));
+         entry->count = 0;
+     }
+ }
+ static inline size_t
+ fragmentHash(const void *ip, JSObject* globalObj, uint32 globalShape, uint32 argc)
+ {
+     uintptr_t h = HASH_SEED;
+     hash_accum(h, uintptr_t(ip), FRAGMENT_TABLE_MASK);
+     hash_accum(h, uintptr_t(globalObj), FRAGMENT_TABLE_MASK);
+     hash_accum(h, uintptr_t(globalShape), FRAGMENT_TABLE_MASK);
+     hash_accum(h, uintptr_t(argc), FRAGMENT_TABLE_MASK);
+     return size_t(h);
+ }
+ /*
+  * argc is cx->fp->argc at the trace loop header, i.e., the number of arguments
+  * pushed for the innermost JS frame. This is required as part of the fragment
+  * key because the fragment will write those arguments back to the interpreter
+  * stack when it exits, using its typemap, which implicitly incorporates a given
+  * value of argc. Without this feature, a fragment could be called as an inner
+  * tree with two different values of argc, and entry type checking or exit
+  * frame synthesis could crash.
+  */
+ struct VMFragment : public Fragment
+ {
+     VMFragment(const void* _ip, JSObject* _globalObj, uint32 _globalShape, uint32 _argc) :
+         Fragment(_ip),
+         next(NULL),
+         globalObj(_globalObj),
+         globalShape(_globalShape),
+         argc(_argc)
+     {}
+     VMFragment* next;
+     JSObject* globalObj;
+     uint32 globalShape;
+     uint32 argc;
+ };
+ static VMFragment*
+ getVMFragment(JSTraceMonitor* tm, const void *ip, JSObject* globalObj, uint32 globalShape,
+               uint32 argc)
+ {
+     size_t h = fragmentHash(ip, globalObj, globalShape, argc);
+     VMFragment* vf = tm->vmfragments[h];
+     while (vf &&
+            ! (vf->globalObj == globalObj &&
+               vf->globalShape == globalShape &&
+               vf->ip == ip &&
+               vf->argc == argc)) {
+         vf = vf->next;
+     }
+     return vf;
+ }
+ static VMFragment*
+ getLoop(JSTraceMonitor* tm, const void *ip, JSObject* globalObj, uint32 globalShape,
+         uint32 argc)
+ {
+     return getVMFragment(tm, ip, globalObj, globalShape, argc);
+ }
+ static Fragment*
+ getAnchor(JSTraceMonitor* tm, const void *ip, JSObject* globalObj, uint32 globalShape,
+           uint32 argc)
+ {
+     VMFragment *f = new (&gc) VMFragment(ip, globalObj, globalShape, argc);
+     JS_ASSERT(f);
+     Fragment *p = getVMFragment(tm, ip, globalObj, globalShape, argc);
+     if (p) {
+         f->first = p;
+         /* append at the end of the peer list */
+         Fragment* next;
+         while ((next = p->peer) != NULL)
+             p = next;
+         p->peer = f;
+     } else {
+         /* this is the first fragment */
+         f->first = f;
+         size_t h = fragmentHash(ip, globalObj, globalShape, argc);
+         f->next = tm->vmfragments[h];
+         tm->vmfragments[h] = f;
+     }
+     f->anchor = f;
+     f->root = f;
+     f->kind = LoopTrace;
+     return f;
+ }
+ #ifdef DEBUG
+ static void
+ ensureTreeIsUnique(JSTraceMonitor* tm, VMFragment* f, TreeInfo* ti)
+ {
+     JS_ASSERT(f->root == f);
+     /*
+      * Check for duplicate entry type maps.  This is always wrong and hints at
+      * trace explosion since we are trying to stabilize something without
+      * properly connecting peer edges.
+      */
+     TreeInfo* ti_other;
+     for (Fragment* peer = getLoop(tm, f->ip, f->globalObj, f->globalShape, f->argc);
+          peer != NULL;
+          peer = peer->peer) {
+         if (!peer->code() || peer == f)
+             continue;
+         ti_other = (TreeInfo*)peer->vmprivate;
+         JS_ASSERT(ti_other);
+         JS_ASSERT(!ti->typeMap.matches(ti_other->typeMap));
+     }
+ }
+ #endif
+ static void
+ js_AttemptCompilation(JSContext *cx, JSTraceMonitor* tm, JSObject* globalObj, jsbytecode* pc,
+                       uint32 argc)
+ {
+     /*
+      * If we already permanently blacklisted the location, undo that.
+      */
+     JS_ASSERT(*(jsbytecode*)pc == JSOP_NOP || *(jsbytecode*)pc == JSOP_LOOP);
+     *(jsbytecode*)pc = JSOP_LOOP;
+     js_resetRecordingAttempts(cx, pc);
+     /*
+      * Breath new live into all peer fragments at the designated loop header.
+      */
+     Fragment* f = (VMFragment*)getLoop(tm, pc, globalObj, OBJ_SHAPE(globalObj),
+                                        argc);
+     if (!f) {
+         /*
+          * If the global object's shape changed, we can't easily find the
+          * corresponding loop header via a hash table lookup. In this
+          * we simply bail here and hope that the fragment has another
+          * outstanding compilation attempt. This case is extremely rare.
+          */
+         return;
+     }
+     JS_ASSERT(f->root == f);
+     f = f->first;
+     while (f) {
+         JS_ASSERT(f->root == f);
+         --f->recordAttempts;
+         f->hits() = HOTLOOP;
+         f = f->peer;
+     }
  }
- #if defined(NJ_SOFTFLOAT)
  JS_DEFINE_CALLINFO_1(static, DOUBLE,    i2f, INT32,                 1, 1)
  JS_DEFINE_CALLINFO_1(static, DOUBLE,    u2f, UINT32,                1, 1)
- #endif
  static bool isi2f(LInsp i)
  {
      if (i->isop(LIR_i2f))
          return true;
- #if defined(NJ_SOFTFLOAT)
+     if (nanojit::AvmCore::config.soft_float &&
-     if (i->isop(LIR_qjoin) &&
+         i->isop(LIR_qjoin) &&
          i->oprnd1()->isop(LIR_call) &&
          i->oprnd2()->isop(LIR_callh))
      {
          if (i->oprnd1()->callInfo() == &i2f_ci)
              return true;
      }
- #endif
      return false;
  }
-Line 420
+Line 749
      if (i->isop(LIR_u2f))
          return true;
- #if defined(NJ_SOFTFLOAT)
+     if (nanojit::AvmCore::config.soft_float &&
-     if (i->isop(LIR_qjoin) &&
+         i->isop(LIR_qjoin) &&
          i->oprnd1()->isop(LIR_call) &&
          i->oprnd2()->isop(LIR_callh))
      {
          if (i->oprnd1()->callInfo() == &u2f_ci)
              return true;
      }
- #endif
      return false;
  }
  static LInsp iu2fArg(LInsp i)
  {
- #if defined(NJ_SOFTFLOAT)
+     if (nanojit::AvmCore::config.soft_float &&
-     if (i->isop(LIR_qjoin))
+         i->isop(LIR_qjoin))
+     {
          return i->oprnd1()->arg(0);
- #endif
+     }
      return i->oprnd1();
  }
-Line 460
+Line 789
  static bool isPromoteInt(LIns* i)
  {
-     jsdouble d;
+     if (isi2f(i) || i->isconst())
-     return isi2f(i) || i->isconst() ||
+         return true;
-         (i->isconstq() && (d = i->constvalf()) == jsdouble(jsint(d)) && !JSDOUBLE_IS_NEGZERO(d));
+     if (!i->isconstq())
+         return false;
+     jsdouble d = i->constvalf();
+     return d == jsdouble(jsint(d)) && !JSDOUBLE_IS_NEGZERO(d);
  }
  static bool isPromoteUint(LIns* i)
  {
-     jsdouble d;
+     if (isu2f(i) || i->isconst())
-     return isu2f(i) || i->isconst() ||
+         return true;
-         (i->isconstq() && (d = i->constvalf()) == (jsdouble)(jsuint)d && !JSDOUBLE_IS_NEGZERO(d));
+     if (!i->isconstq())
+         return false;
+     jsdouble d = i->constvalf();
+     return d == jsdouble(jsuint(d)) && !JSDOUBLE_IS_NEGZERO(d);
  }
  static bool isPromote(LIns* i)
-Line 491
+Line 826
              ((c->constval() > 0)));
  }
- #if defined(NJ_SOFTFLOAT)
+ /* soft float support */
- /* soft float */
  JS_DEFINE_CALLINFO_1(static, DOUBLE,    fneg, DOUBLE,               1, 1)
  JS_DEFINE_CALLINFO_2(static, INT32,     fcmpeq, DOUBLE, DOUBLE,     1, 1)
-Line 646
+Line 980
      }
  };
- #endif // NJ_SOFTFLOAT
  class FuncFilter: public LirWriter
  {
  public:
-Line 728
+Line 1060
      LInsp insCall(const CallInfo *ci, LInsp args[])
      {
-         LInsp s0 = args[0];
          if (ci == &js_DoubleToUint32_ci) {
+             LInsp s0 = args[0];
              if (s0->isconstq())
                  return out->insImm(js_DoubleToECMAUint32(s0->constvalf()));
              if (isi2f(s0) || isu2f(s0))
                  return iu2fArg(s0);
          } else if (ci == &js_DoubleToInt32_ci) {
+             LInsp s0 = args[0];
              if (s0->isconstq())
                  return out->insImm(js_DoubleToECMAInt32(s0->constvalf()));
              if (s0->isop(LIR_fadd) || s0->isop(LIR_fsub)) {
-Line 748
+Line 1081
              if (isi2f(s0) || isu2f(s0))
                  return iu2fArg(s0);
              // XXX ARM -- check for qjoin(call(UnboxDouble),call(UnboxDouble))
-             if (s0->isCall() && s0->callInfo() == &js_UnboxDouble_ci) {
+             if (s0->isCall()) {
-                 LIns* args2[] = { callArgN(s0, 0) };
+                 const CallInfo* ci2 = s0->callInfo();
-                 return out->insCall(&js_UnboxInt32_ci, args2);
+                 if (ci2 == &js_UnboxDouble_ci) {
-             }
+                     LIns* args2[] = { callArgN(s0, 0) };
-             if (s0->isCall() && s0->callInfo() == &js_StringToNumber_ci) {
+                     return out->insCall(&js_UnboxInt32_ci, args2);
-                 // callArgN's ordering is that as seen by the builtin, not as stored in args here.
+                 } else if (ci2 == &js_StringToNumber_ci) {
-                 // True story!
+                     // callArgN's ordering is that as seen by the builtin, not as stored in
-                 LIns* args2[] = { callArgN(s0, 1), callArgN(s0, 0) };
+                     // args here. True story!
-                 return out->insCall(&js_StringToInt32_ci, args2);
+                     LIns* args2[] = { callArgN(s0, 1), callArgN(s0, 0) };
+                     return out->insCall(&js_StringToInt32_ci, args2);
+                 } else if (ci2 == &js_String_p_charCodeAt0_ci) {
+                     // Use a fast path builtin for a charCodeAt that converts to an int right away.
+                     LIns* args2[] = { callArgN(s0, 0) };
+                     return out->insCall(&js_String_p_charCodeAt0_int_ci, args2);
+                 } else if (ci2 == &js_String_p_charCodeAt_ci) {
+                     LIns* idx = callArgN(s0, 1);
+                     // If the index is not already an integer, force it to be an integer.
+                     idx = isPromote(idx)
+                         ? demote(out, idx)
+                         : out->insCall(&js_DoubleToInt32_ci, &idx);
+                     LIns* args2[] = { idx, callArgN(s0, 0) };
+                     return out->insCall(&js_String_p_charCodeAt_int_ci, args2);
+                 }
              }
          } else if (ci == &js_BoxDouble_ci) {
+             LInsp s0 = args[0];
              JS_ASSERT(s0->isQuad());
-             if (s0->isop(LIR_i2f)) {
+             if (isi2f(s0)) {
-                 LIns* args2[] = { s0->oprnd1(), args[1] };
+                 LIns* args2[] = { iu2fArg(s0), args[1] };
                  return out->insCall(&js_BoxInt32_ci, args2);
              }
              if (s0->isCall() && s0->callInfo() == &js_UnboxDouble_ci)
-Line 772
+Line 1120
  };
  /* In debug mode vpname contains a textual description of the type of the
-    slot during the forall iteration over al slots. */
+    slot during the forall iteration over all slots. If JS_JIT_SPEW is not
+    defined, vpnum is set to a very large integer to catch invalid uses of
+    it. Non-debug code should never use vpnum. */
  #ifdef JS_JIT_SPEW
  #define DEF_VPNAME          const char* vpname; unsigned vpnum
  #define SET_VPNAME(name)    do { vpname = name; vpnum = 0; } while(0)
-Line 780
+Line 1130
  #else
  #define DEF_VPNAME          do {} while (0)
  #define vpname ""
- #define vpnum 0
+ #define vpnum 0x40000000
  #define SET_VPNAME(name)    ((void)0)
  #define INC_VPNUM()         ((void)0)
  #endif
-Line 815
+Line 1165
                  vp = &fp->argv[-1];                                           \
                  { code; }                                                     \
                  SET_VPNAME("argv");                                           \
-                 vp = &fp->argv[0]; vpstop = &fp->argv[fp->fun->nargs];        \
+                 vp = &fp->argv[0]; vpstop = &fp->argv[argSlots(fp)];          \
                  while (vp < vpstop) { code; ++vp; INC_VPNUM(); }              \
              }                                                                 \
              SET_VPNAME("vars");                                               \
-Line 863
+Line 1213
  #define FORALL_SLOTS(cx, ngslots, gslots, callDepth, code)                    \
      JS_BEGIN_MACRO                                                            \
-         FORALL_GLOBAL_SLOTS(cx, ngslots, gslots, code);                       \
          FORALL_SLOTS_IN_PENDING_FRAMES(cx, callDepth, code);                  \
+         FORALL_GLOBAL_SLOTS(cx, ngslots, gslots, code);                       \
      JS_END_MACRO
  /* Calculate the total number of native frame slots we need from this frame
     all the way back to the entry frame, including the current stack usage. */
- unsigned
+ JS_REQUIRES_STACK unsigned
  js_NativeStackSlots(JSContext *cx, unsigned callDepth)
  {
      JSStackFrame* fp = cx->fp;
-Line 884
+Line 1234
              slots += fp->script->nfixed;
          if (callDepth-- == 0) {
              if (fp->callee)
-                 slots += 2/*callee,this*/ + fp->fun->nargs;
+                 slots += 2/*callee,this*/ + argSlots(fp);
  #if defined _DEBUG
              unsigned int m = 0;
              FORALL_SLOTS_IN_PENDING_FRAMES(cx, origCallDepth, m++);
-Line 901
+Line 1251
      JS_NOT_REACHED("js_NativeStackSlots");
  }
- /* Capture the type map for the selected slots of the global object. */
+ /*
- void
+  * Capture the type map for the selected slots of the global object and currently pending
- TypeMap::captureGlobalTypes(JSContext* cx, SlotList& slots)
+  * stack frames.
+  */
+ JS_REQUIRES_STACK void
+ TypeMap::captureTypes(JSContext* cx, SlotList& slots, unsigned callDepth)
  {
      unsigned ngslots = slots.length();
      uint16* gslots = slots.data();
-     setLength(ngslots);
+     setLength(js_NativeStackSlots(cx, callDepth) + ngslots);
      uint8* map = data();
      uint8* m = map;
+     FORALL_SLOTS_IN_PENDING_FRAMES(cx, callDepth,
+         uint8 type = getCoercedType(*vp);
+         if ((type == JSVAL_INT) && oracle.isStackSlotUndemotable(cx, unsigned(m - map)))
+             type = JSVAL_DOUBLE;
+         JS_ASSERT(type != JSVAL_BOXED);
+         debug_only_v(printf("capture stack type %s%d: %d=%c\n", vpname, vpnum, type, typeChar[type]);)
+         JS_ASSERT(uintptr_t(m - map) < length());
+         *m++ = type;
+     );
      FORALL_GLOBAL_SLOTS(cx, ngslots, gslots,
          uint8 type = getCoercedType(*vp);
-         if ((type == JSVAL_INT) && oracle.isGlobalSlotUndemotable(cx->fp->script, gslots[n]))
+         if ((type == JSVAL_INT) && oracle.isGlobalSlotUndemotable(cx, gslots[n]))
              type = JSVAL_DOUBLE;
          JS_ASSERT(type != JSVAL_BOXED);
+         debug_only_v(printf("capture global type %s%d: %d=%c\n", vpname, vpnum, type, typeChar[type]);)
+         JS_ASSERT(uintptr_t(m - map) < length());
          *m++ = type;
      );
+     JS_ASSERT(uintptr_t(m - map) == length());
  }
- /* Capture the type map for the currently pending stack frames. */
+ JS_REQUIRES_STACK void
- void
+ TypeMap::captureMissingGlobalTypes(JSContext* cx, SlotList& slots, unsigned stackSlots)
- TypeMap::captureStackTypes(JSContext* cx, unsigned callDepth)
  {
-     setLength(js_NativeStackSlots(cx, callDepth));
+     unsigned oldSlots = length() - stackSlots;
-     uint8* map = data();
+     int diff = slots.length() - oldSlots;
+     JS_ASSERT(diff >= 0);
+     unsigned ngslots = slots.length();
+     uint16* gslots = slots.data();
+     setLength(length() + diff);
+     uint8* map = data() + stackSlots;
      uint8* m = map;
-     FORALL_SLOTS_IN_PENDING_FRAMES(cx, callDepth,
+     FORALL_GLOBAL_SLOTS(cx, ngslots, gslots,
-         uint8 type = getCoercedType(*vp);
+         if (n >= oldSlots) {
-         if ((type == JSVAL_INT) &&
+             uint8 type = getCoercedType(*vp);
-             oracle.isStackSlotUndemotable(cx->fp->script, cx->fp->regs->pc, unsigned(m - map))) {
+             if ((type == JSVAL_INT) && oracle.isGlobalSlotUndemotable(cx, gslots[n]))
-             type = JSVAL_DOUBLE;
+                 type = JSVAL_DOUBLE;
+             JS_ASSERT(type != JSVAL_BOXED);
+             debug_only_v(printf("capture global type %s%d: %d=%c\n", vpname, vpnum, type, typeChar[type]);)
+             *m = type;
+             JS_ASSERT((m > map + oldSlots) || (*m == type));
          }
-         debug_only_v(printf("capture %s%d: %d\n", vpname, vpnum, type);)
+         m++;
-         *m++ = type;
      );
  }
-Line 959
+Line 1331
      *plength = clength;
  }
+ /* Specializes a tree to any missing globals, including any dependent trees. */
+ static JS_REQUIRES_STACK void
+ specializeTreesToMissingGlobals(JSContext* cx, TreeInfo* root)
+ {
+     TreeInfo* ti = root;
+     ti->typeMap.captureMissingGlobalTypes(cx, *ti->globalSlots, ti->nStackTypes);
+     JS_ASSERT(ti->globalSlots->length() == ti->typeMap.length() - ti->nStackTypes);
+     for (unsigned i = 0; i < root->dependentTrees.length(); i++) {
+         ti = (TreeInfo*)root->dependentTrees.data()[i]->vmprivate;
+         /* ti can be NULL if we hit the recording tree in emitTreeCall; this is harmless. */
+         if (ti && ti->nGlobalTypes() < ti->globalSlots->length())
+             specializeTreesToMissingGlobals(cx, ti);
+     }
+     for (unsigned i = 0; i < root->linkedTrees.length(); i++) {
+         ti = (TreeInfo*)root->linkedTrees.data()[i]->vmprivate;
+         if (ti && ti->nGlobalTypes() < ti->globalSlots->length())
+             specializeTreesToMissingGlobals(cx, ti);
+     }
+ }
  static void
  js_TrashTree(JSContext* cx, Fragment* f);
+ JS_REQUIRES_STACK
  TraceRecorder::TraceRecorder(JSContext* cx, VMSideExit* _anchor, Fragment* _fragment,
-         TreeInfo* ti, unsigned ngslots, uint8* globalTypeMap, uint8* stackTypeMap,
+         TreeInfo* ti, unsigned stackSlots, unsigned ngslots, uint8* typeMap,
-         VMSideExit* innermostNestedGuard, Fragment* outerToBlacklist)
+         VMSideExit* innermostNestedGuard, jsbytecode* outer, uint32 outerArgc)
  {
-     JS_ASSERT(!_fragment->vmprivate && ti);
+     JS_ASSERT(!_fragment->vmprivate && ti && cx->fp->regs->pc == (jsbytecode*)_fragment->ip);
+     /* Reset the fragment state we care about in case we got a recycled fragment. */
+     _fragment->lastIns = NULL;
      this->cx = cx;
      this->traceMonitor = &JS_TRACE_MONITOR(cx);
      this->globalObj = JS_GetGlobalForObject(cx, cx->fp->scopeChain);
+     this->lexicalBlock = cx->fp->blockChain;
      this->anchor = _anchor;
      this->fragment = _fragment;
      this->lirbuf = _fragment->lirbuf;
      this->treeInfo = ti;
      this->callDepth = _anchor ? _anchor->calldepth : 0;
-     this->atoms = cx->fp->script->atomMap.vector;
+     this->atoms = FrameAtomBase(cx, cx->fp);
      this->deepAborted = false;
-     this->applyingArguments = false;
+     this->trashSelf = false;
-     this->trashTree = false;
-     this->whichTreeToTrash = _fragment->root;
      this->global_dslots = this->globalObj->dslots;
-     this->terminate = false;
+     this->loop = true; /* default assumption is we are compiling a loop */
-     this->outerToBlacklist = outerToBlacklist;
      this->wasRootFragment = _fragment == _fragment->root;
+     this->outer = outer;
+     this->outerArgc = outerArgc;
+     this->pendingTraceableNative = NULL;
+     this->newobj_ins = NULL;
+     this->generatedTraceableNative = new JSTraceableNative();
+     JS_ASSERT(generatedTraceableNative);
      debug_only_v(printf("recording starting from %s:%u@%u\n",
-                         cx->fp->script->filename,
+                         ti->treeFileName, ti->treeLineNumber, ti->treePCOffset);)
-                         js_FramePCToLineNumber(cx, cx->fp),
+     debug_only_v(printf("globalObj=%p, shape=%d\n", (void*)this->globalObj, OBJ_SHAPE(this->globalObj));)
-                         FramePCOffset(cx->fp));)
-     debug_only_v(printf("globalObj=%p, shape=%d\n", this->globalObj, OBJ_SHAPE(this->globalObj));)
      lir = lir_buf_writer = new (&gc) LirBufWriter(lirbuf);
- #ifdef DEBUG
+     debug_only_v(lir = verbose_filter = new (&gc) VerboseWriter(&gc, lir, lirbuf->names);)
-     if (verbose_debug)
+     if (nanojit::AvmCore::config.soft_float)
-         lir = verbose_filter = new (&gc) VerboseWriter(&gc, lir, lirbuf->names);
+         lir = float_filter = new (&gc) SoftFloatFilter(lir);
- #endif
+     else
- #ifdef NJ_SOFTFLOAT
+         float_filter = 0;
-     lir = float_filter = new (&gc) SoftFloatFilter(lir);
- #endif
      lir = cse_filter = new (&gc) CseFilter(lir, &gc);
      lir = expr_filter = new (&gc) ExprFilter(lir);
      lir = func_filter = new (&gc) FuncFilter(lir);
      lir->ins0(LIR_start);
      if (!nanojit::AvmCore::config.tree_opt || fragment->root == fragment)
          lirbuf->state = addName(lir->insParam(0, 0), "state");
      lirbuf->sp = addName(lir->insLoad(LIR_ldp, lirbuf->state, (int)offsetof(InterpState, sp)), "sp");
      lirbuf->rp = addName(lir->insLoad(LIR_ldp, lirbuf->state, offsetof(InterpState, rp)), "rp");
      cx_ins = addName(lir->insLoad(LIR_ldp, lirbuf->state, offsetof(InterpState, cx)), "cx");
-     gp_ins = addName(lir->insLoad(LIR_ldp, lirbuf->state, offsetof(InterpState, gp)), "gp");
      eos_ins = addName(lir->insLoad(LIR_ldp, lirbuf->state, offsetof(InterpState, eos)), "eos");
      eor_ins = addName(lir->insLoad(LIR_ldp, lirbuf->state, offsetof(InterpState, eor)), "eor");
+     /* If we came from exit, we might not have enough global types. */
+     if (ti->globalSlots->length() > ti->nGlobalTypes())
+         specializeTreesToMissingGlobals(cx, ti);
      /* read into registers all values on the stack and all globals we know so far */
-     import(treeInfo, lirbuf->sp, ngslots, callDepth, globalTypeMap, stackTypeMap);
+     import(treeInfo, lirbuf->sp, stackSlots, ngslots, callDepth, typeMap);
+     if (fragment == fragment->root) {
+         /*
+          * We poll the operation callback request flag. It is updated asynchronously whenever
+          * the callback is to be invoked.
+          */
+         LIns* x = lir->insLoadi(cx_ins, offsetof(JSContext, operationCallbackFlag));
+         guard(true, lir->ins_eq0(x), snapshot(TIMEOUT_EXIT));
+     }
      /* If we are attached to a tree call guard, make sure the guard the inner tree exited from
         is what we expect it to be. */
      if (_anchor && _anchor->exitType == NESTED_EXIT) {
          LIns* nested_ins = addName(lir->insLoad(LIR_ldp, lirbuf->state,
                                                  offsetof(InterpState, lastTreeExitGuard)),
                                                  "lastTreeExitGuard");
          guard(true, lir->ins2(LIR_eq, nested_ins, INS_CONSTPTR(innermostNestedGuard)), NESTED_EXIT);
      }
-Line 1031
+Line 1441
  TreeInfo::~TreeInfo()
  {
      UnstableExit* temp;
      while (unstableExits) {
          temp = unstableExits->next;
          delete unstableExits;
-Line 1056
+Line 1466
              JS_ASSERT(!fragment->root->vmprivate);
              delete treeInfo;
          }
-         if (trashTree)
-             js_TrashTree(cx, whichTreeToTrash);
+         if (trashSelf)
+             js_TrashTree(cx, fragment->root);
+         for (unsigned int i = 0; i < whichTreesToTrash.length(); i++)
+             js_TrashTree(cx, whichTreesToTrash.get(i));
      } else if (wasRootFragment) {
          delete treeInfo;
      }
-Line 1067
+Line 1481
      delete cse_filter;
      delete expr_filter;
      delete func_filter;
- #ifdef NJ_SOFTFLOAT
      delete float_filter;
- #endif
      delete lir_buf_writer;
+     delete generatedTraceableNative;
  }
  void TraceRecorder::removeFragmentoReferences()
-Line 1078
+Line 1491
      fragment = NULL;
  }
+ void TraceRecorder::deepAbort()
+ {
+     debug_only_v(printf("deep abort");)
+     deepAborted = true;
+ }
  /* Add debug information to a LIR instruction as we emit it. */
  inline LIns*
  TraceRecorder::addName(LIns* ins, const char* name)
  {
- #ifdef DEBUG
+ #ifdef JS_JIT_SPEW
-     lirbuf->names->addName(ins, name);
+     if (js_verboseDebug)
+         lirbuf->names->addName(ins, name);
  #endif
      return ins;
  }
-Line 1101
+Line 1521
  {
      JS_ASSERT(isGlobal(p));
      if (size_t(p - globalObj->fslots) < JS_INITIAL_NSLOTS)
-         return size_t(p - globalObj->fslots) * sizeof(double);
+         return sizeof(InterpState) + size_t(p - globalObj->fslots) * sizeof(double);
-     return ((p - globalObj->dslots) + JS_INITIAL_NSLOTS) * sizeof(double);
+     return sizeof(InterpState) + ((p - globalObj->dslots) + JS_INITIAL_NSLOTS) * sizeof(double);
  }
  /* Determine whether a value is a global stack slot */
-Line 1114
+Line 1534
  }
  /* Determine the offset in the native stack for a jsval we track */
- ptrdiff_t
+ JS_REQUIRES_STACK ptrdiff_t
  TraceRecorder::nativeStackOffset(jsval* p) const
  {
  #ifdef DEBUG
-Line 1152
+Line 1572
          fp = *fsp;
          if (fp->callee) {
              if (fsp == fstack) {
-                 if (size_t(p - &fp->argv[-2]) < size_t(2/*callee,this*/ + fp->fun->nargs))
+                 if (size_t(p - &fp->argv[-2]) < size_t(2/*callee,this*/ + argSlots(fp)))
                      RETURN(offset + size_t(p - &fp->argv[-2]) * sizeof(double));
-                 offset += (2/*callee,this*/ + fp->fun->nargs) * sizeof(double);
+                 offset += (2/*callee,this*/ + argSlots(fp)) * sizeof(double);
              }
              if (size_t(p - &fp->slots[0]) < fp->script->nfixed)
                  RETURN(offset + size_t(p - &fp->slots[0]) * sizeof(double));
-Line 1194
+Line 1614
          treeInfo->maxNativeStackSlots = slots;
  }
  /* Unbox a jsval into a slot. Slots are wide enough to hold double values directly (instead of
     storing a pointer to them). We now assert instead of type checking, the caller must ensure the
     types are compatible. */
  static void
  ValueToNative(JSContext* cx, jsval v, uint8 type, double* slot)
  {
      unsigned tag = JSVAL_TAG(v);
      switch (type) {
+       case JSVAL_OBJECT:
+         JS_ASSERT(tag == JSVAL_OBJECT);
+         JS_ASSERT(!JSVAL_IS_NULL(v) && !HAS_FUNCTION_CLASS(JSVAL_TO_OBJECT(v)));
+         *(JSObject**)slot = JSVAL_TO_OBJECT(v);
+         debug_only_v(printf("object<%p:%s> ", (void*)JSVAL_TO_OBJECT(v),
+                             JSVAL_IS_NULL(v)
+                             ? "null"
+                             : STOBJ_GET_CLASS(JSVAL_TO_OBJECT(v))->name);)
+         return;
        case JSVAL_INT:
          jsint i;
          if (JSVAL_IS_INT(v))
-Line 1222
+Line 1651
          *(jsdouble*)slot = d;
          debug_only_v(printf("double<%g> ", d);)
          return;
-       case JSVAL_BOOLEAN:
+       case JSVAL_BOXED:
-         JS_ASSERT(tag == JSVAL_BOOLEAN);
+         JS_NOT_REACHED("found boxed type in an entry type map");
-         *(JSBool*)slot = JSVAL_TO_BOOLEAN(v);
-         debug_only_v(printf("boolean<%d> ", *(JSBool*)slot);)
          return;
        case JSVAL_STRING:
          JS_ASSERT(tag == JSVAL_STRING);
          *(JSString**)slot = JSVAL_TO_STRING(v);
-         debug_only_v(printf("string<%p> ", *(JSString**)slot);)
+         debug_only_v(printf("string<%p> ", (void*)(*(JSString**)slot));)
          return;
-       default:
+       case JSVAL_TNULL:
-         /* Note: we should never see JSVAL_BOXED in an entry type map. */
-         JS_ASSERT(type == JSVAL_OBJECT);
          JS_ASSERT(tag == JSVAL_OBJECT);
-         *(JSObject**)slot = JSVAL_TO_OBJECT(v);
+         *(JSObject**)slot = NULL;
-         debug_only_v(printf("object<%p:%s> ", JSVAL_TO_OBJECT(v),
+         debug_only_v(printf("null ");)
-                             JSVAL_IS_NULL(v)
+         return;
-                             ? "null"
+       case JSVAL_BOOLEAN:
-                             : STOBJ_GET_CLASS(JSVAL_TO_OBJECT(v))->name);)
+         /* Watch out for pseudo-booleans. */
+         JS_ASSERT(tag == JSVAL_BOOLEAN);
+         *(JSBool*)slot = JSVAL_TO_PSEUDO_BOOLEAN(v);
+         debug_only_v(printf("boolean<%d> ", *(JSBool*)slot);)
+         return;
+       case JSVAL_TFUN: {
+         JS_ASSERT(tag == JSVAL_OBJECT);
+         JSObject* obj = JSVAL_TO_OBJECT(v);
+         *(JSObject**)slot = obj;
+ #ifdef DEBUG
+         JSFunction* fun = GET_FUNCTION_PRIVATE(cx, obj);
+         debug_only_v(printf("function<%p:%s> ", (void*) obj,
+                             fun->atom
+                             ? JS_GetStringBytes(ATOM_TO_STRING(fun->atom))
+                             : "unnamed");)
+ #endif
          return;
+       }
      }
+     JS_NOT_REACHED("unexpected type");
  }
- /* We maintain an emergency recovery pool of doubles so we can recover safely if a trace runs
+ /* We maintain an emergency pool of doubles so we can recover safely if a trace runs
     out of memory (doubles or objects). */
  static jsval
- AllocateDoubleFromRecoveryPool(JSContext* cx)
+ AllocateDoubleFromReservedPool(JSContext* cx)
  {
      JSTraceMonitor* tm = &JS_TRACE_MONITOR(cx);
-     JS_ASSERT(tm->recoveryDoublePoolPtr > tm->recoveryDoublePool);
+     JS_ASSERT(tm->reservedDoublePoolPtr > tm->reservedDoublePool);
-     return *--tm->recoveryDoublePoolPtr;
+     return *--tm->reservedDoublePoolPtr;
  }
  static bool
- js_ReplenishRecoveryPool(JSContext* cx, JSTraceMonitor* tm)
+ js_ReplenishReservedPool(JSContext* cx, JSTraceMonitor* tm)
  {
      /* We should not be called with a full pool. */
-     JS_ASSERT((size_t) (tm->recoveryDoublePoolPtr - tm->recoveryDoublePool) <
+     JS_ASSERT((size_t) (tm->reservedDoublePoolPtr - tm->reservedDoublePool) <
                MAX_NATIVE_STACK_SLOTS);
      /*
       * When the GC runs in js_NewDoubleInRootedValue, it resets
-      * tm->recoveryDoublePoolPtr back to tm->recoveryDoublePool.
+      * tm->reservedDoublePoolPtr back to tm->reservedDoublePool.
       */
      JSRuntime* rt = cx->runtime;
      uintN gcNumber = rt->gcNumber;
-     jsval* ptr = tm->recoveryDoublePoolPtr;
+     uintN lastgcNumber = gcNumber;
-     while (ptr < tm->recoveryDoublePool + MAX_NATIVE_STACK_SLOTS) {
+     jsval* ptr = tm->reservedDoublePoolPtr;
-         if (!js_NewDoubleInRootedValue(cx, 0.0, ptr))
+     while (ptr < tm->reservedDoublePool + MAX_NATIVE_STACK_SLOTS) {
+         if (!js_NewDoubleInRootedValue(cx, 0.0, ptr))
              goto oom;
-         if (rt->gcNumber != gcNumber) {
-             JS_ASSERT(tm->recoveryDoublePoolPtr == tm->recoveryDoublePool);
+         /* Check if the last call to js_NewDoubleInRootedValue GC'd. */
-             ptr = tm->recoveryDoublePool;
+         if (rt->gcNumber != lastgcNumber) {
+             lastgcNumber = rt->gcNumber;
+             JS_ASSERT(tm->reservedDoublePoolPtr == tm->reservedDoublePool);
+             ptr = tm->reservedDoublePool;
+             /*
+              * Have we GC'd more than once? We're probably running really
+              * low on memory, bail now.
+              */
              if (uintN(rt->gcNumber - gcNumber) > uintN(1))
                  goto oom;
              continue;
          }
          ++ptr;
      }
-     tm->recoveryDoublePoolPtr = ptr;
+     tm->reservedDoublePoolPtr = ptr;
      return true;
  oom:
-Line 1289
+Line 1741
       * Already massive GC pressure, no need to hold doubles back.
       * We won't run any native code anyway.
       */
-     tm->recoveryDoublePoolPtr = tm->recoveryDoublePool;
+     tm->reservedDoublePoolPtr = tm->reservedDoublePool;
      return false;
  }
  /* Box a value from the native stack back into the jsval format. Integers
     that are too large to fit into a jsval are automatically boxed into
     heap-allocated doubles. */
- static bool
+ static void
  NativeToValue(JSContext* cx, jsval& v, uint8 type, double* slot)
  {
      jsint i;
      jsdouble d;
      switch (type) {
-       case JSVAL_BOOLEAN:
+       case JSVAL_OBJECT:
-         v = BOOLEAN_TO_JSVAL(*(JSBool*)slot);
+         v = OBJECT_TO_JSVAL(*(JSObject**)slot);
-         debug_only_v(printf("boolean<%d> ", *(JSBool*)slot);)
+         JS_ASSERT(JSVAL_TAG(v) == JSVAL_OBJECT); /* if this fails the pointer was not aligned */
+         JS_ASSERT(v != JSVAL_ERROR_COOKIE); /* don't leak JSVAL_ERROR_COOKIE */
+         debug_only_v(printf("object<%p:%s> ", (void*)JSVAL_TO_OBJECT(v),
+                             JSVAL_IS_NULL(v)
+                             ? "null"
+                             : STOBJ_GET_CLASS(JSVAL_TO_OBJECT(v))->name);)
          break;
        case JSVAL_INT:
          i = *(jsint*)slot;
-Line 1326
+Line 1783
             double boxes. */
          if (cx->doubleFreeList) {
  #ifdef DEBUG
-             bool ok =
+             JSBool ok =
  #endif
                  js_NewDoubleInRootedValue(cx, d, &v);
              JS_ASSERT(ok);
-             return true;
+             return;
          }
-         v = AllocateDoubleFromRecoveryPool(cx);
+         v = AllocateDoubleFromReservedPool(cx);
          JS_ASSERT(JSVAL_IS_DOUBLE(v) && *JSVAL_TO_DOUBLE(v) == 0.0);
          *JSVAL_TO_DOUBLE(v) = d;
-         return true;
+         return;
        }
+       case JSVAL_BOXED:
+         v = *(jsval*)slot;
+         JS_ASSERT(v != JSVAL_ERROR_COOKIE); /* don't leak JSVAL_ERROR_COOKIE */
+         debug_only_v(printf("box<%lx> ", v));
+         break;
        case JSVAL_STRING:
          v = STRING_TO_JSVAL(*(JSString**)slot);
          JS_ASSERT(JSVAL_TAG(v) == JSVAL_STRING); /* if this fails the pointer was not aligned */
          debug_only_v(printf("string<%p> ", *(JSString**)slot);)
          break;
-       case JSVAL_BOXED:
+       case JSVAL_TNULL:
-         v = *(jsval*)slot;
+         JS_ASSERT(*(JSObject**)slot == NULL);
-         debug_only_v(printf("box<%lx> ", v));
+         v = JSVAL_NULL;
+         debug_only_v(printf("null<%p> ", *(JSObject**)slot));
          break;
-       default:
+       case JSVAL_BOOLEAN:
-         JS_ASSERT(type == JSVAL_OBJECT);
+         /* Watch out for pseudo-booleans. */
+         v = PSEUDO_BOOLEAN_TO_JSVAL(*(JSBool*)slot);
+         debug_only_v(printf("boolean<%d> ", *(JSBool*)slot);)
+         break;
+       case JSVAL_TFUN: {
+         JS_ASSERT(HAS_FUNCTION_CLASS(*(JSObject**)slot));
          v = OBJECT_TO_JSVAL(*(JSObject**)slot);
-         JS_ASSERT(JSVAL_TAG(v) == JSVAL_OBJECT); /* if this fails the pointer was not aligned */
+ #ifdef DEBUG
-         debug_only_v(printf("object<%p:%s> ", JSVAL_TO_OBJECT(v),
+         JSFunction* fun = GET_FUNCTION_PRIVATE(cx, JSVAL_TO_OBJECT(v));
-                             JSVAL_IS_NULL(v)
+         debug_only_v(printf("function<%p:%s> ", (void*)JSVAL_TO_OBJECT(v),
-                             ? "null"
+                             fun->atom
-                             : STOBJ_GET_CLASS(JSVAL_TO_OBJECT(v))->name);)
+                             ? JS_GetStringBytes(ATOM_TO_STRING(fun->atom))
+                             : "unnamed");)
+ #endif
          break;
+       }
      }
-     return true;
  }
  /* Attempt to unbox the given list of interned globals onto the native global frame. */
- static void
+ static JS_REQUIRES_STACK void
  BuildNativeGlobalFrame(JSContext* cx, unsigned ngslots, uint16* gslots, uint8* mp, double* np)
  {
      debug_only_v(printf("global: ");)
-Line 1372
+Line 1842
  }
  /* Attempt to unbox the given JS frame onto a native frame. */
- static void
+ static JS_REQUIRES_STACK void
  BuildNativeStackFrame(JSContext* cx, unsigned callDepth, uint8* mp, double* np)
  {
      debug_only_v(printf("stack: ");)
-Line 1384
+Line 1854
      debug_only_v(printf("\n");)
  }
- /* Box the given native frame into a JS frame. This only fails due to a hard error
+ /* Box the given native frame into a JS frame. This is infallible. */
-    (out of memory for example). */
+ static JS_REQUIRES_STACK int
- static int
  FlushNativeGlobalFrame(JSContext* cx, unsigned ngslots, uint16* gslots, uint8* mp, double* np)
  {
      uint8* mp_base = mp;
      FORALL_GLOBAL_SLOTS(cx, ngslots, gslots,
-         if (!NativeToValue(cx, *vp, *mp, np + gslots[n]))
+         debug_only_v(printf("%s%u=", vpname, vpnum);)
-             return -1;
+         NativeToValue(cx, *vp, *mp, np + gslots[n]);
          ++mp;
      );
      debug_only_v(printf("\n");)
      return mp - mp_base;
  }
+ /*
+  * Generic function to read upvars on trace.
+  *     T   Traits type parameter. Must provide static functions:
+  *             interp_get(fp, slot)     Read the value out of an interpreter frame.
+  *             native_slot(argc, slot)  Return the position of the desired value in the on-trace
+  *                                      stack frame (with position 0 being callee).
+  *
+  *     level       Static level of the function containing the upvar definition.
+  *     slot        Identifies the value to get. The meaning is defined by the traits type.
+  *     callDepth   Call depth of current point relative to trace entry
+  */
+ template<typename T>
+ uint32 JS_INLINE
+ js_GetUpvarOnTrace(JSContext* cx, uint32 level, int32 slot, uint32 callDepth, double* result)
+ {
+     InterpState* state = cx->interpState;
+     FrameInfo** fip = state->rp + callDepth;
+     /*
+      * First search the FrameInfo call stack for an entry containing
+      * our upvar, namely one with level == upvarLevel.
+      */
+     while (--fip >= state->callstackBase) {
+         FrameInfo* fi = *fip;
+         JSFunction* fun = GET_FUNCTION_PRIVATE(cx, fi->callee);
+         uintN calleeLevel = fun->u.i.script->staticLevel;
+         if (calleeLevel == level) {
+             /*
+              * Now find the upvar's value in the native stack.
+              * nativeStackFramePos is the offset of the start of the
+              * activation record corresponding to *fip in the native
+              * stack.
+              */
+             int32 nativeStackFramePos = state->callstackBase[0]->spoffset;
+             for (FrameInfo** fip2 = state->callstackBase; fip2 <= fip; fip2++)
+                 nativeStackFramePos += (*fip2)->spdist;
+             nativeStackFramePos -= (2 + (*fip)->get_argc());
+             uint32 native_slot = T::native_slot((*fip)->get_argc(), slot);
+             *result = state->stackBase[nativeStackFramePos + native_slot];
+             return fi->get_typemap()[native_slot];
+         }
+     }
+     // Next search the trace entry frame, which is not in the FrameInfo stack.
+     if (state->outermostTree->script->staticLevel == level) {
+         uint32 argc = ((VMFragment*) state->outermostTree->fragment)->argc;
+         uint32 native_slot = T::native_slot(argc, slot);
+         *result = state->stackBase[native_slot];
+         return state->callstackBase[0]->get_typemap()[native_slot];
+     }
+     /*
+      * If we did not find the upvar in the frames for the active traces,
+      * then we simply get the value from the interpreter state.
+      */
+     JS_ASSERT(level < JS_DISPLAY_SIZE);
+     JSStackFrame* fp = cx->display[level];
+     jsval v = T::interp_get(fp, slot);
+     uint8 type = getCoercedType(v);
+     ValueToNative(cx, v, type, result);
+     return type;
+ }
+ // For this traits type, 'slot' is the argument index, which may be -2 for callee.
+ struct UpvarArgTraits {
+     static jsval interp_get(JSStackFrame* fp, int32 slot) {
+         return fp->argv[slot];
+     }
+     static uint32 native_slot(uint32 argc, int32 slot) {
+         return 2 /*callee,this*/ + slot;
+     }
+ };
+ uint32 JS_FASTCALL
+ js_GetUpvarArgOnTrace(JSContext* cx, uint32 staticLevel, int32 slot, uint32 callDepth, double* result)
+ {
+     return js_GetUpvarOnTrace<UpvarArgTraits>(cx, staticLevel, slot, callDepth, result);
+ }
+ // For this traits type, 'slot' is an index into the local slots array.
+ struct UpvarVarTraits {
+     static jsval interp_get(JSStackFrame* fp, int32 slot) {
+         return fp->slots[slot];
+     }
+     static uint32 native_slot(uint32 argc, int32 slot) {
+         return 2 /*callee,this*/ + argc + slot;
+     }
+ };
+ uint32 JS_FASTCALL
+ js_GetUpvarVarOnTrace(JSContext* cx, uint32 staticLevel, int32 slot, uint32 callDepth, double* result)
+ {
+     return js_GetUpvarOnTrace<UpvarVarTraits>(cx, staticLevel, slot, callDepth, result);
+ }
+ /*
+  * For this traits type, 'slot' is an index into the stack area (within slots, after nfixed)
+  * of a frame with no function. (On trace, the top-level frame is the only one that can have
+  * no function.)
+  */
+ struct UpvarStackTraits {
+     static jsval interp_get(JSStackFrame* fp, int32 slot) {
+         return fp->slots[slot + fp->script->nfixed];
+     }
+     static uint32 native_slot(uint32 argc, int32 slot) {
+         /*
+          * Locals are not imported by the tracer when the frame has no function, so
+          * we do not add fp->script->nfixed.
+          */
+         JS_ASSERT(argc == 0);
+         return slot;
+     }
+ };
+ uint32 JS_FASTCALL
+ js_GetUpvarStackOnTrace(JSContext* cx, uint32 upvarLevel, int32 slot, uint32 callDepth, double* result)
+ {
+     return js_GetUpvarOnTrace<UpvarStackTraits>(cx, upvarLevel, slot, callDepth, result);
+ }
  /**
-  * Box the given native stack frame into the virtual machine stack. This fails
+  * Box the given native stack frame into the virtual machine stack. This
-  * only due to a hard error (out of memory for example).
+  * is infallible.
   *
   * @param callDepth the distance between the entry frame into our trace and
   *                  cx->fp when we make this call.  If this is not called as a
-Line 1414
+Line 2006
   *                  be restored.
   * @return the number of things we popped off of np.
   */
- static int
+ static JS_REQUIRES_STACK int
  FlushNativeStackFrame(JSContext* cx, unsigned callDepth, uint8* mp, double* np,
                        JSStackFrame* stopFrame)
  {
-Line 1424
+Line 2016
      FORALL_SLOTS_IN_PENDING_FRAMES(cx, callDepth,
          if (vp == stopAt) goto skip;
          debug_only_v(printf("%s%u=", vpname, vpnum);)
-         if (!NativeToValue(cx, *vp, *mp, np))
+         NativeToValue(cx, *vp, *mp, np);
-             return -1;
          ++mp; ++np
      );
  skip:
-Line 1447
+Line 2038
          }
          for (; n != 0; fp = fp->down) {
              --n;
-             if (fp->callee) { // might not have it if the entry frame is global
+             if (fp->callee) {
+                 /*
+                  * We might return from trace with a different callee object, but it still
+                  * has to be the same JSFunction (FIXME: bug 471425, eliminate fp->callee).
+                  */
                  JS_ASSERT(JSVAL_IS_OBJECT(fp->argv[-1]));
+                 JS_ASSERT(HAS_FUNCTION_CLASS(JSVAL_TO_OBJECT(fp->argv[-2])));
+                 JS_ASSERT(GET_FUNCTION_PRIVATE(cx, JSVAL_TO_OBJECT(fp->argv[-2])) ==
+                           GET_FUNCTION_PRIVATE(cx, fp->callee));
+                 JS_ASSERT(GET_FUNCTION_PRIVATE(cx, fp->callee) == fp->fun);
+                 fp->callee = JSVAL_TO_OBJECT(fp->argv[-2]);
+                 /*
+                  * SynthesizeFrame sets scopeChain to NULL, because we can't calculate the
+                  * correct scope chain until we have the final callee. Calculate the real
+                  * scope object here.
+                  */
+                 if (!fp->scopeChain) {
+                     fp->scopeChain = OBJ_GET_PARENT(cx, fp->callee);
+                     if (fp->fun->flags & JSFUN_HEAVYWEIGHT) {
+                         /*
+                          * Set hookData to null because the failure case for js_GetCallObject
+                          * involves it calling the debugger hook.
+                          *
+                          * Allocating the Call object must not fail, so use an object
+                          * previously reserved by js_ExecuteTree if needed.
+                          */
+                         void* hookData = ((JSInlineFrame*)fp)->hookData;
+                         ((JSInlineFrame*)fp)->hookData = NULL;
+                         JS_ASSERT(!JS_TRACE_MONITOR(cx).useReservedObjects);
+                         JS_TRACE_MONITOR(cx).useReservedObjects = JS_TRUE;
+ #ifdef DEBUG
+                         JSObject *obj =
+ #endif
+                             js_GetCallObject(cx, fp);
+                         JS_ASSERT(obj);
+                         JS_TRACE_MONITOR(cx).useReservedObjects = JS_FALSE;
+                         ((JSInlineFrame*)fp)->hookData = hookData;
+                     }
+                 }
                  fp->thisp = JSVAL_TO_OBJECT(fp->argv[-1]);
+                 if (fp->flags & JSFRAME_CONSTRUCTING) // constructors always compute 'this'
+                     fp->flags |= JSFRAME_COMPUTED_THIS;
              }
          }
      }
-Line 1458
+Line 2089
  }
  /* Emit load instructions onto the trace that read the initial stack state. */
- void
+ JS_REQUIRES_STACK void
- TraceRecorder::import(LIns* base, ptrdiff_t offset, jsval* p, uint8& t,
+ TraceRecorder::import(LIns* base, ptrdiff_t offset, jsval* p, uint8 t,
                        const char *prefix, uintN index, JSStackFrame *fp)
  {
      LIns* ins;
-Line 1472
+Line 2103
          ins = lir->insLoadi(base, offset);
          ins = lir->ins1(LIR_i2f, ins);
      } else {
-         JS_ASSERT(t == JSVAL_BOXED || isNumber(*p) == (t == JSVAL_DOUBLE));
+         JS_ASSERT_IF(t != JSVAL_BOXED, isNumber(*p) == (t == JSVAL_DOUBLE));
          if (t == JSVAL_DOUBLE) {
              ins = lir->insLoad(LIR_ldq, base, offset);
          } else if (t == JSVAL_BOOLEAN) {
-Line 1481
+Line 2112
              ins = lir->insLoad(LIR_ldp, base, offset);
          }
      }
+     checkForGlobalObjectReallocation();
      tracker.set(p, ins);
  #ifdef DEBUG
      char name[64];
      JS_ASSERT(strlen(prefix) < 10);
-Line 1490
+Line 2123
      const char* funName = NULL;
      if (*prefix == 'a' || *prefix == 'v') {
          mark = JS_ARENA_MARK(&cx->tempPool);
-         if (JS_GET_LOCAL_NAME_COUNT(fp->fun) != 0)
+         if (fp->fun->hasLocalNames())
              localNames = js_GetLocalNameArray(cx, fp->fun, &cx->tempPool);
          funName = fp->fun->atom ? js_AtomToPrintableString(cx, fp->fun->atom) : "<anonymous>";
      }
-Line 1513
+Line 2146
      addName(ins, name);
      static const char* typestr[] = {
-         "object", "int", "double", "3", "string", "5", "boolean", "any"
+         "object", "int", "double", "boxed", "string", "null", "boolean", "function"
      };
      debug_only_v(printf("import vp=%p name=%s type=%s flags=%d\n",
-                         p, name, typestr[t & 7], t >> 3);)
+                         (void*)p, name, typestr[t & 7], t >> 3);)
  #endif
  }
- void
+ JS_REQUIRES_STACK void
- TraceRecorder::import(TreeInfo* treeInfo, LIns* sp, unsigned ngslots, unsigned callDepth,
+ TraceRecorder::import(TreeInfo* treeInfo, LIns* sp, unsigned stackSlots, unsigned ngslots,
-                       uint8* globalTypeMap, uint8* stackTypeMap)
+                       unsigned callDepth, uint8* typeMap)
  {
      /* If we get a partial list that doesn't have all the types (i.e. recording from a side
         exit that was recorded but we added more global slots later), merge the missing types
-Line 1534
+Line 2167
         is if that other trace had at its end a compatible type distribution with the entry
         map. Since thats exactly what we used to fill in the types our current side exit
         didn't provide, this is always safe to do. */
-     unsigned length;
-     if (ngslots < (length = traceMonitor->globalTypeMap->length()))
+     uint8* globalTypeMap = typeMap + stackSlots;
-         mergeTypeMaps(&globalTypeMap, &ngslots,
+     unsigned length = treeInfo->nGlobalTypes();
-                       traceMonitor->globalTypeMap->data(), length,
+     /*
+      * This is potentially the typemap of the side exit and thus shorter than the tree's
+      * global type map.
+      */
+     if (ngslots < length) {
+         mergeTypeMaps(&globalTypeMap/*out param*/, &ngslots/*out param*/,
+                       treeInfo->globalTypeMap(), length,
                        (uint8*)alloca(sizeof(uint8) * length));
-     JS_ASSERT(ngslots == traceMonitor->globalTypeMap->length());
+     }
+     JS_ASSERT(ngslots == treeInfo->nGlobalTypes());
-     /* the first time we compile a tree this will be empty as we add entries lazily */
+     /*
-     uint16* gslots = traceMonitor->globalSlots->data();
+      * Check whether there are any values on the stack we have to unbox and do that first
-     uint8* m = globalTypeMap;
+      * before we waste any time fetching the state from the stack.
+      */
+     ptrdiff_t offset = -treeInfo->nativeStackBase;
+     uint8* m = typeMap;
+     FORALL_SLOTS_IN_PENDING_FRAMES(cx, callDepth,
+         if (*m == JSVAL_BOXED) {
+             import(sp, offset, vp, JSVAL_BOXED, "boxed", vpnum, cx->fp);
+             LIns* vp_ins = get(vp);
+             unbox_jsval(*vp, vp_ins, copy(anchor));
+             set(vp, vp_ins);
+         }
+         m++; offset += sizeof(double);
+     );
+     /*
+      * The first time we compile a tree this will be empty as we add entries lazily.
+      */
+     uint16* gslots = treeInfo->globalSlots->data();
+     m = globalTypeMap;
      FORALL_GLOBAL_SLOTS(cx, ngslots, gslots,
-         import(gp_ins, nativeGlobalOffset(vp), vp, *m, vpname, vpnum, NULL);
+         JS_ASSERT(*m != JSVAL_BOXED);
+         import(lirbuf->state, nativeGlobalOffset(vp), vp, *m, vpname, vpnum, NULL);
          m++;
      );
-     ptrdiff_t offset = -treeInfo->nativeStackBase;
+     offset = -treeInfo->nativeStackBase;
-     m = stackTypeMap;
+     m = typeMap;
      FORALL_SLOTS_IN_PENDING_FRAMES(cx, callDepth,
-         import(sp, offset, vp, *m, vpname, vpnum, fp);
+         if (*m != JSVAL_BOXED)
+             import(sp, offset, vp, *m, vpname, vpnum, fp);
          m++; offset += sizeof(double);
      );
  }
+ JS_REQUIRES_STACK bool
+ TraceRecorder::isValidSlot(JSScope* scope, JSScopeProperty* sprop)
+ {
+     uint32 setflags = (js_CodeSpec[*cx->fp->regs->pc].format & (JOF_SET | JOF_INCDEC | JOF_FOR));
+     if (setflags) {
+         if (!SPROP_HAS_STUB_SETTER(sprop))
+             ABORT_TRACE_RV("non-stub setter", false);
+         if (sprop->attrs & JSPROP_READONLY)
+             ABORT_TRACE_RV("writing to a read-only property", false);
+     }
+     /* This check applies even when setflags == 0. */
+     if (setflags != JOF_SET && !SPROP_HAS_STUB_GETTER(sprop))
+         ABORT_TRACE_RV("non-stub getter", false);
+     if (!SPROP_HAS_VALID_SLOT(sprop, scope))
+         ABORT_TRACE_RV("slotless obj property", false);
+     return true;
+ }
  /* Lazily import a global slot if we don't already have it in the tracker. */
- bool
+ JS_REQUIRES_STACK bool
  TraceRecorder::lazilyImportGlobalSlot(unsigned slot)
  {
      if (slot != uint16(slot)) /* we use a table of 16-bit ints, bail out if that's not enough */
          return false;
+     /*
+      * If the global object grows too large, alloca in js_ExecuteTree might fail, so
+      * abort tracing on global objects with unreasonably many slots.
+      */
+     if (STOBJ_NSLOTS(globalObj) > MAX_GLOBAL_SLOTS)
+         return false;
      jsval* vp = &STOBJ_GET_SLOT(globalObj, slot);
-     if (tracker.has(vp))
+     if (known(vp))
          return true; /* we already have it */
-     unsigned index = traceMonitor->globalSlots->length();
+     unsigned index = treeInfo->globalSlots->length();
-     /* If this the first global we are adding, remember the shape of the global object. */
-     if (index == 0)
-         traceMonitor->globalShape = OBJ_SHAPE(JS_GetGlobalForObject(cx, cx->fp->scopeChain));
      /* Add the slot to the list of interned global slots. */
-     traceMonitor->globalSlots->add(slot);
+     JS_ASSERT(treeInfo->nGlobalTypes() == treeInfo->globalSlots->length());
+     treeInfo->globalSlots->add(slot);
      uint8 type = getCoercedType(*vp);
-     if ((type == JSVAL_INT) && oracle.isGlobalSlotUndemotable(cx->fp->script, slot))
+     if ((type == JSVAL_INT) && oracle.isGlobalSlotUndemotable(cx, slot))
          type = JSVAL_DOUBLE;
-     traceMonitor->globalTypeMap->add(type);
+     treeInfo->typeMap.add(type);
-     import(gp_ins, slot*sizeof(double), vp, type, "global", index, NULL);
+     import(lirbuf->state, sizeof(struct InterpState) + slot*sizeof(double),
+            vp, type, "global", index, NULL);
+     specializeTreesToMissingGlobals(cx, treeInfo);
      return true;
  }
-Line 1592
+Line 2280
  }
  /* Update the tracker, then issue a write back store. */
- void
+ JS_REQUIRES_STACK void
  TraceRecorder::set(jsval* p, LIns* i, bool initializing)
  {
-     JS_ASSERT(initializing || tracker.has(p));
+     JS_ASSERT(i != NULL);
+     JS_ASSERT(initializing || known(p));
+     checkForGlobalObjectReallocation();
      tracker.set(p, i);
      /* If we are writing to this location for the first time, calculate the offset into the
         native frame manually, otherwise just look up the last load or store associated with
-Line 1603
+Line 2293
      LIns* x = nativeFrameTracker.get(p);
      if (!x) {
          if (isGlobal(p))
-             x = writeBack(i, gp_ins, nativeGlobalOffset(p));
+             x = writeBack(i, lirbuf->state, nativeGlobalOffset(p));
          else
              x = writeBack(i, lirbuf->sp, -treeInfo->nativeStackBase + nativeStackOffset(p));
          nativeFrameTracker.set(p, x);
      } else {
  #define ASSERT_VALID_CACHE_HIT(base, offset)                                  \
-     JS_ASSERT(base == lirbuf->sp || base == gp_ins);                          \
+     JS_ASSERT(base == lirbuf->sp || base == lirbuf->state);                   \
      JS_ASSERT(offset == ((base == lirbuf->sp)                                 \
          ? -treeInfo->nativeStackBase + nativeStackOffset(p)                   \
          : nativeGlobalOffset(p)));                                            \
-Line 1626
+Line 2316
  #undef ASSERT_VALID_CACHE_HIT
  }
- LIns*
+ JS_REQUIRES_STACK LIns*
- TraceRecorder::get(jsval* p) const
+ TraceRecorder::get(jsval* p)
  {
+     checkForGlobalObjectReallocation();
      return tracker.get(p);
  }
- /* Determine whether the current branch instruction terminates the loop. */
+ JS_REQUIRES_STACK bool
- static bool
+ TraceRecorder::known(jsval* p)
- js_IsLoopExit(jsbytecode* pc, jsbytecode* header)
  {
-     switch (*pc) {
+     checkForGlobalObjectReallocation();
-       case JSOP_LT:
+     return tracker.has(p);
-       case JSOP_GT:
+ }
-       case JSOP_LE:
-       case JSOP_GE:
-       case JSOP_NE:
-       case JSOP_EQ:
-         /* These ops try to dispatch a JSOP_IFEQ or JSOP_IFNE that follows. */
-         JS_ASSERT(js_CodeSpec[*pc].length == 1);
-         pc++;
-         break;
-       default:
-         for (;;) {
-             if (*pc == JSOP_AND || *pc == JSOP_OR)
-                 pc += GET_JUMP_OFFSET(pc);
-             else if (*pc == JSOP_ANDX || *pc == JSOP_ORX)
-                 pc += GET_JUMPX_OFFSET(pc);
-             else
-                 break;
-         }
-     }
-     switch (*pc) {
-       case JSOP_IFEQ:
-       case JSOP_IFNE:
-         /*
-          * Forward jumps are usually intra-branch, but for-in loops jump to the
-          * trailing enditer to clean up, so check for that case here.
-          */
-         if (pc[GET_JUMP_OFFSET(pc)] == JSOP_ENDITER)
-             return true;
-         return pc + GET_JUMP_OFFSET(pc) == header;
-       case JSOP_IFEQX:
-       case JSOP_IFNEX:
-         if (pc[GET_JUMPX_OFFSET(pc)] == JSOP_ENDITER)
-             return true;
-         return pc + GET_JUMPX_OFFSET(pc) == header;
-       default:;
+ /*
+  * The dslots of the global object are sometimes reallocated by the interpreter.
+  * This function check for that condition and re-maps the entries of the tracker
+  * accordingly.
+  */
+ JS_REQUIRES_STACK void
+ TraceRecorder::checkForGlobalObjectReallocation()
+ {
+     if (global_dslots != globalObj->dslots) {
+         debug_only_v(printf("globalObj->dslots relocated, updating tracker\n");)
+         jsval* src = global_dslots;
+         jsval* dst = globalObj->dslots;
+         jsuint length = globalObj->dslots[-1] - JS_INITIAL_NSLOTS;
+         LIns** map = (LIns**)alloca(sizeof(LIns*) * length);
+         for (jsuint n = 0; n < length; ++n) {
+             map[n] = tracker.get(src);
+             tracker.set(src++, NULL);
+         }
+         for (jsuint n = 0; n < length; ++n)
+             tracker.set(dst++, map[n]);
+         global_dslots = globalObj->dslots;
      }
-     return false;
  }
  /* Determine whether the current branch is a loop edge (taken or not taken). */
- static bool
+ static JS_REQUIRES_STACK bool
  js_IsLoopEdge(jsbytecode* pc, jsbytecode* header)
  {
      switch (*pc) {
-Line 1693
+Line 2366
        case JSOP_IFNEX:
          return ((pc + GET_JUMPX_OFFSET(pc)) == header);
        default:
          JS_ASSERT((*pc == JSOP_AND) || (*pc == JSOP_ANDX) ||
                    (*pc == JSOP_OR) || (*pc == JSOP_ORX));
      }
      return false;
  }
- /* Promote slots if necessary to match the called tree' type map and report error if thats
+ /*
-    impossible. */
+  * Promote slots if necessary to match the called tree's type map. This function is
- bool
+  * infallible and must only be called if we are certain that it is possible to
- TraceRecorder::adjustCallerTypes(Fragment* f, unsigned* demote_slots, bool& trash)
+  * reconcile the types for each slot in the inner and outer trees.
+  */
+ JS_REQUIRES_STACK void
+ TraceRecorder::adjustCallerTypes(Fragment* f)
  {
-     JSTraceMonitor* tm = traceMonitor;
+     uint16* gslots = treeInfo->globalSlots->data();
-     uint8* m = tm->globalTypeMap->data();
+     unsigned ngslots = treeInfo->globalSlots->length();
-     uint16* gslots = traceMonitor->globalSlots->data();
+     JS_ASSERT(ngslots == treeInfo->nGlobalTypes());
-     unsigned ngslots = traceMonitor->globalSlots->length();
+     TreeInfo* ti = (TreeInfo*)f->vmprivate;
-     uint8* map = ((TreeInfo*)f->vmprivate)->stackTypeMap.data();
+     uint8* map = ti->globalTypeMap();
-     bool ok = true;
+     uint8* m = map;
-     trash = false;
+     FORALL_GLOBAL_SLOTS(cx, ngslots, gslots,
-     FORALL_GLOBAL_SLOTS(cx, ngslots, gslots,
          LIns* i = get(vp);
          bool isPromote = isPromoteInt(i);
          if (isPromote && *m == JSVAL_DOUBLE)
-             lir->insStorei(get(vp), gp_ins, nativeGlobalOffset(vp));
+             lir->insStorei(get(vp), lirbuf->state, nativeGlobalOffset(vp));
-         else if (!isPromote && *m == JSVAL_INT) {
+         JS_ASSERT(!(!isPromote && *m == JSVAL_INT));
-             oracle.markGlobalSlotUndemotable(cx->fp->script, nativeGlobalOffset(vp)/sizeof(double));
-             trash = true;
-             ok = false;
-         }
          ++m;
      );
+     JS_ASSERT(unsigned(m - map) == ti->nGlobalTypes());
+     map = ti->stackTypeMap();
      m = map;
      FORALL_SLOTS_IN_PENDING_FRAMES(cx, 0,
          LIns* i = get(vp);
          bool isPromote = isPromoteInt(i);
          if (isPromote && *m == JSVAL_DOUBLE) {
              lir->insStorei(get(vp), lirbuf->sp,
                             -treeInfo->nativeStackBase + nativeStackOffset(vp));
              /* Aggressively undo speculation so the inner tree will compile if this fails. */
-             ADD_UNDEMOTE_SLOT(demote_slots, unsigned(m - map));
+             oracle.markStackSlotUndemotable(cx, unsigned(m - map));
-         } else if (!isPromote && *m == JSVAL_INT) {
-             debug_only_v(printf("adjusting will fail, %s%d, slot %d\n", vpname, vpnum, m - map);)
-             ok = false;
-             ADD_UNDEMOTE_SLOT(demote_slots, unsigned(m - map));
-         } else if (JSVAL_IS_INT(*vp) && *m == JSVAL_DOUBLE) {
-             /* Aggressively undo speculation so the inner tree will compile if this fails. */
-             ADD_UNDEMOTE_SLOT(demote_slots, unsigned(m - map));
          }
+         JS_ASSERT(!(!isPromote && *m == JSVAL_INT));
          ++m;
      );
-     /* If this isn't okay, tell the oracle. */
+     JS_ASSERT(unsigned(m - map) == ti->nStackTypes);
-     if (!ok) {
-         for (unsigned i = 1; i <= NUM_UNDEMOTE_SLOTS(demote_slots); i++)
-             oracle.markStackSlotUndemotable(cx->fp->script, cx->fp->regs->pc, demote_slots[i]);
-     }
      JS_ASSERT(f == f->root);
-     return ok;
  }
- uint8
+ JS_REQUIRES_STACK uint8
- TraceRecorder::determineSlotType(jsval* vp) const
+ TraceRecorder::determineSlotType(jsval* vp)
  {
      uint8 m;
      LIns* i = get(vp);
-     m = isNumber(*vp)
+     if (isNumber(*vp)) {
-         ? (isPromoteInt(i) ? JSVAL_INT : JSVAL_DOUBLE)
+         m = isPromoteInt(i) ? JSVAL_INT : JSVAL_DOUBLE;
-         : JSVAL_TAG(*vp);
+     } else if (JSVAL_IS_OBJECT(*vp)) {
+         if (JSVAL_IS_NULL(*vp))
+             m = JSVAL_TNULL;
+         else if (HAS_FUNCTION_CLASS(JSVAL_TO_OBJECT(*vp)))
+             m = JSVAL_TFUN;
+         else
+             m = JSVAL_OBJECT;
+     } else {
+         m = JSVAL_TAG(*vp);
+     }
      JS_ASSERT((m != JSVAL_INT) || isInt32(*vp));
      return m;
  }
- #define IMACRO_PC_ADJ_BITS   8
+ JS_REQUIRES_STACK VMSideExit*
- #define SCRIPT_PC_ADJ_BITS   (32 - IMACRO_PC_ADJ_BITS)
- // The stored imacro_pc_adj byte offset is biased by 1.
- #define IMACRO_PC_ADJ_LIMIT  (JS_BIT(IMACRO_PC_ADJ_BITS) - 1)
- #define SCRIPT_PC_ADJ_LIMIT  JS_BIT(SCRIPT_PC_ADJ_BITS)
- #define IMACRO_PC_ADJ(ip)    ((uintptr_t)(ip) >> SCRIPT_PC_ADJ_BITS)
- #define SCRIPT_PC_ADJ(ip)    ((ip) & JS_BITMASK(SCRIPT_PC_ADJ_BITS))
- #define FI_SCRIPT_PC(fi,fp)  ((fp)->script->code + SCRIPT_PC_ADJ((fi).ip_adj))
- #define FI_IMACRO_PC(fi,fp)  (IMACRO_PC_ADJ((fi).ip_adj)                      \
-                               ? imacro_code[*FI_SCRIPT_PC(fi, fp)] +          \
-                                 IMACRO_PC_ADJ((fi).ip_adj)                    \
-                               : NULL)
- #define IMACRO_PC_OK(fp,pc)  JS_ASSERT(uintN((pc)-imacro_code[*(fp)->imacpc]) \
-                                        < JS_BIT(IMACRO_PC_ADJ_BITS))
- #define ENCODE_IP_ADJ(fp,pc) ((fp)->imacpc                                    \
-                               ? (IMACRO_PC_OK(fp, pc),                        \
-                                  (((pc) - imacro_code[*(fp)->imacpc])         \
-                                   << SCRIPT_PC_ADJ_BITS) +                    \
-                                  (fp)->imacpc - (fp)->script->code)           \
-                               : (pc) - (fp)->script->code)
- #define DECODE_IP_ADJ(ip,fp) (IMACRO_PC_ADJ(ip)                               \
-                               ? (fp)->imacpc = (fp)->script->code +           \
-                                                SCRIPT_PC_ADJ(ip),             \
-                                 (fp)->regs->pc = imacro_code[*(fp)->imacpc] + \
-                                                  IMACRO_PC_ADJ(ip)            \
-                               : (fp)->regs->pc = (fp)->script->code + (ip))
- static jsbytecode* imacro_code[JSOP_LIMIT];
- LIns*
  TraceRecorder::snapshot(ExitType exitType)
  {
      JSStackFrame* fp = cx->fp;
      JSFrameRegs* regs = fp->regs;
      jsbytecode* pc = regs->pc;
-     if (exitType == BRANCH_EXIT && js_IsLoopExit(pc, (jsbytecode*)fragment->root->ip))
-         exitType = LOOP_EXIT;
      /* Check for a return-value opcode that needs to restart at the next instruction. */
      const JSCodeSpec& cs = js_CodeSpec[*pc];
-     /* WARNING: don't return before restoring the original pc if (resumeAfter). */
+     /*
+      * When calling a _FAIL native, make the snapshot's pc point to the next
+      * instruction after the CALL or APPLY. Even on failure, a _FAIL native must not
+      * be called again from the interpreter.
+      */
      bool resumeAfter = (pendingTraceableNative &&
-                         JSTN_ERRTYPE(pendingTraceableNative) == FAIL_JSVAL);
+                         JSTN_ERRTYPE(pendingTraceableNative) == FAIL_STATUS);
      if (resumeAfter) {
-         JS_ASSERT(*pc == JSOP_CALL || *pc == JSOP_APPLY || *pc == JSOP_NEXTITER);
+         JS_ASSERT(*pc == JSOP_CALL || *pc == JSOP_APPLY || *pc == JSOP_NEW);
          pc += cs.length;
          regs->pc = pc;
          MUST_FLOW_THROUGH("restore_pc");
-Line 1828
+Line 2466
      trackNativeStackUse(stackSlots + 1);
      /* Capture the type map into a temporary location. */
-     unsigned ngslots = traceMonitor->globalSlots->length();
+     unsigned ngslots = treeInfo->globalSlots->length();
      unsigned typemap_size = (stackSlots + ngslots) * sizeof(uint8);
-     uint8* typemap = (uint8*)alloca(typemap_size);
+     void *mark = JS_ARENA_MARK(&cx->tempPool);
+     uint8* typemap;
+     JS_ARENA_ALLOCATE_CAST(typemap, uint8*, &cx->tempPool, typemap_size);
      uint8* m = typemap;
      /* Determine the type of a store by looking at the current type of the actual value the
         interpreter is using. For numbers we have to check what kind of store we used last
         (integer or double) to figure out what the side exit show reflect in its typemap. */
-     FORALL_SLOTS(cx, ngslots, traceMonitor->globalSlots->data(), callDepth,
+     FORALL_SLOTS(cx, ngslots, treeInfo->globalSlots->data(), callDepth,
          *m++ = determineSlotType(vp);
      );
      JS_ASSERT(unsigned(m - typemap) == ngslots + stackSlots);
-     /* If we are capturing the stack state on a specific instruction, the value on or near
+     /*
-        the top of the stack is a boxed value. Either pc[-cs.length] is JSOP_NEXTITER and we
+      * If we are currently executing a traceable native or we are attaching a second trace
-        want one below top of stack, or else it's JSOP_CALL and we want top of stack. */
+      * to it, the value on top of the stack is boxed. Make a note of this in the typemap.
-     if (resumeAfter) {
+      */
-         m[(pc[-cs.length] == JSOP_NEXTITER) ? -2 : -1] = JSVAL_BOXED;
+     if (pendingTraceableNative && (pendingTraceableNative->flags & JSTN_UNBOX_AFTER))
+         typemap[stackSlots - 1] = JSVAL_BOXED;
-         /* Now restore the the original pc (after which early returns are ok). */
+     /* Now restore the the original pc (after which early returns are ok). */
+     if (resumeAfter) {
          MUST_FLOW_LABEL(restore_pc);
          regs->pc = pc - cs.length;
      } else {
          /* If we take a snapshot on a goto, advance to the target address. This avoids inner
             trees returning on a break goto, which the outer recorder then would confuse with
             a break in the outer tree. */
          if (*pc == JSOP_GOTO)
              pc += GET_JUMP_OFFSET(pc);
          else if (*pc == JSOP_GOTOX)
              pc += GET_JUMPX_OFFSET(pc);
      }
-     intptr_t ip_adj = ENCODE_IP_ADJ(fp, pc);
-     /* Check if we already have a matching side exit. If so use that side exit structure,
+     /*
-        otherwise we have to create our own. */
+      * Check if we already have a matching side exit; if so we can return that
+      * side exit instead of creating a new one.
+      */
      VMSideExit** exits = treeInfo->sideExits.data();
      unsigned nexits = treeInfo->sideExits.length();
      if (exitType == LOOP_EXIT) {
          for (unsigned n = 0; n < nexits; ++n) {
              VMSideExit* e = exits[n];
-             if (e->ip_adj == ip_adj &&
+             if (e->pc == pc && e->imacpc == fp->imacpc &&
-                 !memcmp(getTypeMap(exits[n]), typemap, typemap_size)) {
+                 ngslots == e->numGlobalSlots &&
-                 LIns* data = lir_buf_writer->skip(sizeof(GuardRecord));
+                 !memcmp(getFullTypeMap(exits[n]), typemap, typemap_size)) {
-                 GuardRecord* rec = (GuardRecord*)data->payload();
-                 /* setup guard record structure with shared side exit */
-                 memset(rec, 0, sizeof(GuardRecord));
-                 VMSideExit* exit = exits[n];
-                 rec->exit = exit;
-                 exit->addGuard(rec);
                  AUDIT(mergedLoopExits);
-                 return data;
+                 JS_ARENA_RELEASE(&cx->tempPool, mark);
+                 return e;
              }
          }
      }
-     /* We couldn't find a matching side exit, so create our own side exit structure. */
+     if (sizeof(VMSideExit) + (stackSlots + ngslots) * sizeof(uint8) >= MAX_SKIP_BYTES) {
-     LIns* data = lir_buf_writer->skip(sizeof(GuardRecord) +
+         /*
-                                       sizeof(VMSideExit) +
+          * ::snapshot() is infallible in the sense that callers don't
-                                       (stackSlots + ngslots) * sizeof(uint8));
+          * expect errors; but this is a trace-aborting error condition. So
-     GuardRecord* rec = (GuardRecord*)data->payload();
+          * mangle the request to consume zero slots, and mark the tree as
-     VMSideExit* exit = (VMSideExit*)(rec + 1);
+          * to-be-trashed. This should be safe as the trace will be aborted
-     /* setup guard record structure */
+          * before assembly or execution due to the call to
-     memset(rec, 0, sizeof(GuardRecord));
+          * trackNativeStackUse above.
-     rec->exit = exit;
+          */
-     /* setup side exit structure */
+         stackSlots = 0;
+         ngslots = 0;
+         typemap_size = 0;
+         trashSelf = true;
+     }
+     /* We couldn't find a matching side exit, so create a new one. */
+     LIns* data = lir->skip(sizeof(VMSideExit) + (stackSlots + ngslots) * sizeof(uint8));
+     VMSideExit* exit = (VMSideExit*) data->payload();
+     /* Setup side exit structure. */
      memset(exit, 0, sizeof(VMSideExit));
      exit->from = fragment;
      exit->calldepth = callDepth;
-Line 1902
+Line 2550
          ? nativeStackOffset(&cx->fp->argv[-2])/sizeof(double)
          : 0;
      exit->exitType = exitType;
-     exit->addGuard(rec);
+     exit->block = fp->blockChain;
-     exit->ip_adj = ip_adj;
+     exit->pc = pc;
+     exit->imacpc = fp->imacpc;
      exit->sp_adj = (stackSlots * sizeof(double)) - treeInfo->nativeStackBase;
-     exit->rp_adj = exit->calldepth * sizeof(FrameInfo);
+     exit->rp_adj = exit->calldepth * sizeof(FrameInfo*);
-     memcpy(getTypeMap(exit), typemap, typemap_size);
+     exit->nativeCalleeWord = 0;
+     memcpy(getFullTypeMap(exit), typemap, typemap_size);
-     /* BIG FAT WARNING: If compilation fails, we currently don't reset the lirbuf so its safe
+     JS_ARENA_RELEASE(&cx->tempPool, mark);
-        to keep references to the side exits here. If we ever start rewinding those lirbufs,
+     return exit;
-        we have to make sure we purge the side exits that then no longer will be in valid
+ }
-        memory. */
-     if (exitType == LOOP_EXIT)
+ JS_REQUIRES_STACK LIns*
+ TraceRecorder::createGuardRecord(VMSideExit* exit)
+ {
+     LIns* guardRec = lir->skip(sizeof(GuardRecord));
+     GuardRecord* gr = (GuardRecord*) guardRec->payload();
+     memset(gr, 0, sizeof(GuardRecord));
+     gr->exit = exit;
+     exit->addGuard(gr);
+     return guardRec;
+ }
+ /*
+  * Emit a guard for condition (cond), expecting to evaluate to boolean result
+  * (expected) and using the supplied side exit if the conditon doesn't hold.
+  */
+ JS_REQUIRES_STACK void
+ TraceRecorder::guard(bool expected, LIns* cond, VMSideExit* exit)
+ {
+     LIns* guardRec = createGuardRecord(exit);
+     /*
+      * BIG FAT WARNING: If compilation fails we don't reset the lirbuf, so it's
+      * safe to keep references to the side exits here. If we ever start
+      * rewinding those lirbufs, we have to make sure we purge the side exits
+      * that then no longer will be in valid memory.
+      */
+     if (exit->exitType == LOOP_EXIT)
          treeInfo->sideExits.add(exit);
-     return data;
+     if (!cond->isCond()) {
+         expected = !expected;
+         cond = lir->ins_eq0(cond);
+     }
+     LIns* guardIns =
+         lir->insGuard(expected ? LIR_xf : LIR_xt, cond, guardRec);
+     if (guardIns) {
+         debug_only_v(printf("    SideExit=%p exitType=%d\n", (void*)exit, exit->exitType);)
+     } else {
+         debug_only_v(printf("    redundant guard, eliminated\n");)
+     }
  }
- /* Emit a guard for condition (cond), expecting to evaluate to boolean result (expected)
+ JS_REQUIRES_STACK VMSideExit*
-    and using the supplied side exit if the conditon doesn't hold. */
+ TraceRecorder::copy(VMSideExit* copy)
- LIns*
- TraceRecorder::guard(bool expected, LIns* cond, LIns* exit)
  {
-     return lir->insGuard(expected ? LIR_xf : LIR_xt, cond, exit);
+     size_t typemap_size = copy->numGlobalSlots + copy->numStackSlots;
+     LIns* data = lir->skip(sizeof(VMSideExit) +
+                            typemap_size * sizeof(uint8));
+     VMSideExit* exit = (VMSideExit*) data->payload();
+     /* Copy side exit structure. */
+     memcpy(exit, copy, sizeof(VMSideExit) + typemap_size * sizeof(uint8));
+     exit->guards = NULL;
+     exit->from = fragment;
+     exit->target = NULL;
+     /*
+      * BIG FAT WARNING: If compilation fails we don't reset the lirbuf, so it's
+      * safe to keep references to the side exits here. If we ever start
+      * rewinding those lirbufs, we have to make sure we purge the side exits
+      * that then no longer will be in valid memory.
+      */
+     if (exit->exitType == LOOP_EXIT)
+         treeInfo->sideExits.add(exit);
+     return exit;
  }
  /* Emit a guard for condition (cond), expecting to evaluate to boolean result (expected)
     and generate a side exit with type exitType to jump to if the condition does not hold. */
- LIns*
+ JS_REQUIRES_STACK void
  TraceRecorder::guard(bool expected, LIns* cond, ExitType exitType)
  {
-     return guard(expected, cond, snapshot(exitType));
+     guard(expected, cond, snapshot(exitType));
  }
  /* Try to match the type of a slot to type t. checkType is used to verify that the type of
-Line 1943
+Line 2650
   * @param stage_count   Outparam for set() buffer count.
   * @return              True if types are compatible, false otherwise.
   */
- bool
+ JS_REQUIRES_STACK bool
  TraceRecorder::checkType(jsval& v, uint8 t, jsval*& stage_val, LIns*& stage_ins,
                           unsigned& stage_count)
  {
      if (t == JSVAL_INT) { /* initially all whole numbers cause the slot to be demoted */
          debug_only_v(printf("checkType(tag=1, t=%d, isnum=%d, i2f=%d) stage_count=%d\n",
                              t,
                              isNumber(v),
                              isPromoteInt(get(&v)),
-Line 1985
+Line 2692
          }
          return true;
      }
+     if (t == JSVAL_TNULL)
+         return JSVAL_IS_NULL(v);
+     if (t == JSVAL_TFUN)
+         return !JSVAL_IS_PRIMITIVE(v) && HAS_FUNCTION_CLASS(JSVAL_TO_OBJECT(v));
+     if (t == JSVAL_OBJECT)
+         return !JSVAL_IS_PRIMITIVE(v) && !HAS_FUNCTION_CLASS(JSVAL_TO_OBJECT(v));
      /* for non-number types we expect a precise match of the type */
+     uint8 vt = getCoercedType(v);
  #ifdef DEBUG
-     if (JSVAL_TAG(v) != t) {
+     if (vt != t) {
-         debug_only_v(printf("Type mismatch: val %c, map %c ", typeChar[JSVAL_TAG(v)],
+         debug_only_v(printf("Type mismatch: val %c, map %c ", typeChar[vt],
                              typeChar[t]);)
      }
  #endif
-     debug_only_v(printf("checkType(tag=%d, t=%d) stage_count=%d\n",
+     debug_only_v(printf("checkType(vt=%d, t=%d) stage_count=%d\n",
-                         (int) JSVAL_TAG(v), t, stage_count);)
+                         (int) vt, t, stage_count);)
-     return JSVAL_TAG(v) == t;
+     return vt == t;
  }
  /**
-Line 2003
+Line 2718
   *
   * @param root_peer         First fragment in peer list.
   * @param stable_peer       Outparam for first type stable peer.
-  * @param trash             Whether to trash the tree (demotion).
+  * @param demote            True if stability was achieved through demotion.
-  * @param demotes           Array to store demotable stack slots.
   * @return                  True if type stable, false otherwise.
   */
- bool
+ JS_REQUIRES_STACK bool
- TraceRecorder::deduceTypeStability(Fragment* root_peer, Fragment** stable_peer, unsigned* demotes)
+ TraceRecorder::deduceTypeStability(Fragment* root_peer, Fragment** stable_peer, bool& demote)
  {
      uint8* m;
      uint8* typemap;
-     unsigned ngslots = traceMonitor->globalSlots->length();
+     unsigned ngslots = treeInfo->globalSlots->length();
-     uint16* gslots = traceMonitor->globalSlots->data();
+     uint16* gslots = treeInfo->globalSlots->data();
-     JS_ASSERT(traceMonitor->globalTypeMap->length() == ngslots);
+     JS_ASSERT(ngslots == treeInfo->nGlobalTypes());
      if (stable_peer)
          *stable_peer = NULL;
-     CLEAR_UNDEMOTE_SLOTLIST(demotes);
      /*
       * Rather than calculate all of this stuff twice, it gets cached locally.  The "stage" buffers
       * are for calls to set() that will change the exit types.
       */
      bool success;
-     bool unstable_from_undemotes;
      unsigned stage_count;
-     jsval** stage_vals = (jsval**)alloca(sizeof(jsval*) * (ngslots + treeInfo->stackTypeMap.length()));
+     jsval** stage_vals = (jsval**)alloca(sizeof(jsval*) * (treeInfo->typeMap.length()));
-     LIns** stage_ins = (LIns**)alloca(sizeof(LIns*) * (ngslots + treeInfo->stackTypeMap.length()));
+     LIns** stage_ins = (LIns**)alloca(sizeof(LIns*) * (treeInfo->typeMap.length()));
      /* First run through and see if we can close ourselves - best case! */
      stage_count = 0;
      success = false;
-     unstable_from_undemotes = false;
-     debug_only_v(printf("Checking type stability against self=%p\n", fragment);)
+     debug_only_v(printf("Checking type stability against self=%p\n", (void*)fragment);)
-     m = typemap = traceMonitor->globalTypeMap->data();
+     m = typemap = treeInfo->globalTypeMap();
      FORALL_GLOBAL_SLOTS(cx, ngslots, gslots,
          debug_only_v(printf("%s%d ", vpname, vpnum);)
          if (!checkType(*vp, *m, stage_vals[stage_count], stage_ins[stage_count], stage_count)) {
              /* If the failure was an int->double, tell the oracle. */
-             if (*m == JSVAL_INT && isNumber(*vp) && !isPromoteInt(get(vp)))
+             if (*m == JSVAL_INT && isNumber(*vp) && !isPromoteInt(get(vp))) {
-                 oracle.markGlobalSlotUndemotable(cx->fp->script, gslots[n]);
+                 oracle.markGlobalSlotUndemotable(cx, gslots[n]);
-             trashTree = true;
+                 demote = true;
-             goto checktype_fail_1;
+             } else {
+                 goto checktype_fail_1;
+             }
          }
          ++m;
      );
-     m = typemap = treeInfo->stackTypeMap.data();
+     m = typemap = treeInfo->stackTypeMap();
      FORALL_SLOTS_IN_PENDING_FRAMES(cx, 0,
          debug_only_v(printf("%s%d ", vpname, vpnum);)
          if (!checkType(*vp, *m, stage_vals[stage_count], stage_ins[stage_count], stage_count)) {
-             if (*m == JSVAL_INT && isNumber(*vp) && !isPromoteInt(get(vp)))
+             if (*m == JSVAL_INT && isNumber(*vp) && !isPromoteInt(get(vp))) {
-                 ADD_UNDEMOTE_SLOT(demotes, unsigned(m - typemap));
+                 oracle.markStackSlotUndemotable(cx, unsigned(m - typemap));
-             else
+                 demote = true;
+             } else {
                  goto checktype_fail_1;
+             }
          }
          ++m;
      );
-     /*
+     success = true;
-      * If there's an exit that's unstable because of undemotable slots, we want to search for
-      * peers just in case we can make a connection.
-      */
-     if (NUM_UNDEMOTE_SLOTS(demotes))
-         unstable_from_undemotes = true;
-     else
-         success = true;
  checktype_fail_1:
      /* If we got a success and we don't need to recompile, we should just close here. */
-     if (success) {
+     if (success && !demote) {
          for (unsigned i = 0; i < stage_count; i++)
              set(stage_vals[i], stage_ins[i]);
          return true;
      /* If we need to trash, don't bother checking peers. */
-     } else if (trashTree) {
+     } else if (trashSelf) {
          return false;
-     } else {
-         CLEAR_UNDEMOTE_SLOTLIST(demotes);
      }
-     /* At this point the tree is about to be incomplete, so let's see if we can connect to any
+     demote = false;
+     /* At this point the tree is about to be incomplete, so let's see if we can connect to any
       * peer fragment that is type stable.
       */
      Fragment* f;
      TreeInfo* ti;
      for (f = root_peer; f != NULL; f = f->peer) {
-         debug_only_v(printf("Checking type stability against peer=%p (code=%p)\n", f, f->code());)
+         debug_only_v(printf("Checking type stability against peer=%p (code=%p)\n", (void*)f, f->code());)
          if (!f->code())
              continue;
          ti = (TreeInfo*)f->vmprivate;
          /* Don't allow varying stack depths */
-         if (ti->stackTypeMap.length() != treeInfo->stackTypeMap.length())
+         if ((ti->nStackTypes != treeInfo->nStackTypes) ||
+             (ti->typeMap.length() != treeInfo->typeMap.length()) ||
+             (ti->globalSlots->length() != treeInfo->globalSlots->length()))
              continue;
          stage_count = 0;
          success = false;
-         m = ti->stackTypeMap.data();
+         m = ti->globalTypeMap();
+         FORALL_GLOBAL_SLOTS(cx, treeInfo->globalSlots->length(), treeInfo->globalSlots->data(),
+                 if (!checkType(*vp, *m, stage_vals[stage_count], stage_ins[stage_count], stage_count))
+                     goto checktype_fail_2;
+                 ++m;
+             );
+         m = ti->stackTypeMap();
          FORALL_SLOTS_IN_PENDING_FRAMES(cx, 0,
-             if (!checkType(*vp, *m, stage_vals[stage_count], stage_ins[stage_count], stage_count))
+                 if (!checkType(*vp, *m, stage_vals[stage_count], stage_ins[stage_count], stage_count))
-                 goto checktype_fail_2;
+                     goto checktype_fail_2;
-             ++m;
+                 ++m;
-         );
+             );
          success = true;
  checktype_fail_2:
          if (success) {
              /*
               * There was a successful match.  We don't care about restoring the saved staging, but
               * we do need to clear the original undemote list.
               */
              for (unsigned i = 0; i < stage_count; i++)
                  set(stage_vals[i], stage_ins[i]);
              if (stable_peer)
                  *stable_peer = f;
+             demote = false;
              return false;
          }
      }
-     JS_ASSERT(NUM_UNDEMOTE_SLOTS(demotes) == 0);
      /*
       * If this is a loop trace and it would be stable with demotions, build an undemote list
       * and return true.  Our caller should sniff this and trash the tree, recording a new one
       * that will assumedly stabilize.
       */
-     if (unstable_from_undemotes && fragment->kind == LoopTrace) {
+     if (demote && fragment->kind == LoopTrace) {
-         typemap = m = treeInfo->stackTypeMap.data();
+         typemap = m = treeInfo->globalTypeMap();
-         FORALL_SLOTS_IN_PENDING_FRAMES(cx, 0,
+         FORALL_GLOBAL_SLOTS(cx, treeInfo->globalSlots->length(), treeInfo->globalSlots->data(),
              if (*m == JSVAL_INT) {
                  JS_ASSERT(isNumber(*vp));
                  if (!isPromoteInt(get(vp)))
-                     ADD_UNDEMOTE_SLOT(demotes, unsigned(m - typemap));
+                     oracle.markGlobalSlotUndemotable(cx, gslots[n]);
              } else if (*m == JSVAL_DOUBLE) {
                  JS_ASSERT(isNumber(*vp));
-                 ADD_UNDEMOTE_SLOT(demotes, unsigned(m - typemap));
+                 oracle.markGlobalSlotUndemotable(cx, gslots[n]);
              } else {
                  JS_ASSERT(*m == JSVAL_TAG(*vp));
              }
              m++;
          );
+         typemap = m = treeInfo->stackTypeMap();
+         FORALL_SLOTS_IN_PENDING_FRAMES(cx, 0,
+             if (*m == JSVAL_INT) {
+                 JS_ASSERT(isNumber(*vp));
+                 if (!isPromoteInt(get(vp)))
+                     oracle.markStackSlotUndemotable(cx, unsigned(m - typemap));
+             } else if (*m == JSVAL_DOUBLE) {
+                 JS_ASSERT(isNumber(*vp));
+                 oracle.markStackSlotUndemotable(cx, unsigned(m - typemap));
+             } else {
+                 JS_ASSERT((*m == JSVAL_TNULL)
+                           ? JSVAL_IS_NULL(*vp)
+                           : *m == JSVAL_TFUN
+                           ? !JSVAL_IS_PRIMITIVE(*vp) && HAS_FUNCTION_CLASS(JSVAL_TO_OBJECT(*vp))
+                           : *m == JSVAL_TAG(*vp));
+             }
+             m++;
+         );
          return true;
+     } else {
+         demote = false;
      }
      return false;
  }
- /* Check whether the current pc location is the loop header of the loop this recorder records. */
+ static JS_REQUIRES_STACK void
- bool
+ FlushJITCache(JSContext* cx)
- TraceRecorder::isLoopHeader(JSContext* cx) const
  {
-     return cx->fp->regs->pc == fragment->root->ip;
+     if (!TRACING_ENABLED(cx))
+         return;
+     JSTraceMonitor* tm = &JS_TRACE_MONITOR(cx);
+     debug_only_v(printf("Flushing cache.\n");)
+     if (tm->recorder)
+         js_AbortRecording(cx, "flush cache");
+     TraceRecorder* tr;
+     while ((tr = tm->abortStack) != NULL) {
+         tr->removeFragmentoReferences();
+         tr->deepAbort();
+         tr->popAbortStack();
+     }
+     Fragmento* fragmento = tm->fragmento;
+     if (fragmento) {
+         if (tm->prohibitFlush) {
+             debug_only_v(printf("Deferring fragmento flush due to deep bail.\n");)
+             tm->needFlush = JS_TRUE;
+             return;
+         }
+         fragmento->clearFrags();
+ #ifdef DEBUG
+         JS_ASSERT(fragmento->labels);
+         fragmento->labels->clear();
+ #endif
+         tm->lirbuf->rewind();
+         for (size_t i = 0; i < FRAGMENT_TABLE_SIZE; ++i) {
+             VMFragment* f = tm->vmfragments[i];
+             while (f) {
+                 VMFragment* next = f->next;
+                 fragmento->clearFragment(f);
+                 f = next;
+             }
+             tm->vmfragments[i] = NULL;
+         }
+         for (size_t i = 0; i < MONITOR_N_GLOBAL_STATES; ++i) {
+             tm->globalStates[i].globalShape = -1;
+             tm->globalStates[i].globalSlots->clear();
+         }
+     }
+     tm->needFlush = JS_FALSE;
  }
  /* Compile the current fragment. */
- void
+ JS_REQUIRES_STACK void
- TraceRecorder::compile(Fragmento* fragmento)
+ TraceRecorder::compile(JSTraceMonitor* tm)
  {
+     if (tm->needFlush) {
+         FlushJITCache(cx);
+         return;
+     }
+     Fragmento* fragmento = tm->fragmento;
      if (treeInfo->maxNativeStackSlots >= MAX_NATIVE_STACK_SLOTS) {
-         debug_only_v(printf("Trace rejected: excessive stack use.\n"));
+         debug_only_v(printf("Blacklist: excessive stack use.\n"));
-         js_BlacklistPC(fragmento, fragment);
+         js_Blacklist((jsbytecode*) fragment->root->ip);
          return;
      }
-     ++treeInfo->branchCount;
+     if (anchor && anchor->exitType != CASE_EXIT)
-     if (lirbuf->outOmem()) {
+         ++treeInfo->branchCount;
+     if (lirbuf->outOMem()) {
          fragmento->assm()->setError(nanojit::OutOMem);
          return;
      }
      ::compile(fragmento->assm(), fragment);
-     if (anchor)
+     if (fragmento->assm()->error() == nanojit::OutOMem)
-         fragmento->assm()->patch(anchor);
-     if (fragmento->assm()->error() != nanojit::None)
          return;
+     if (fragmento->assm()->error() != nanojit::None) {
+         debug_only_v(printf("Blacklisted: error during compilation\n");)
+         js_Blacklist((jsbytecode*) fragment->root->ip);
+         return;
+     }
+     js_resetRecordingAttempts(cx, (jsbytecode*) fragment->ip);
+     js_resetRecordingAttempts(cx, (jsbytecode*) fragment->root->ip);
+     if (anchor) {
+ #ifdef NANOJIT_IA32
+         if (anchor->exitType == CASE_EXIT)
+             fragmento->assm()->patch(anchor, anchor->switchInfo);
+         else
+ #endif
+             fragmento->assm()->patch(anchor);
+     }
      JS_ASSERT(fragment->code());
      JS_ASSERT(!fragment->vmprivate);
      if (fragment == fragment->root)
-Line 2194
+Line 2988
  }
  static bool
  js_JoinPeersIfCompatible(Fragmento* frago, Fragment* stableFrag, TreeInfo* stableTree,
                           VMSideExit* exit)
  {
-     JS_ASSERT(exit->numStackSlots == stableTree->stackTypeMap.length());
+     JS_ASSERT(exit->numStackSlots == stableTree->nStackTypes);
      /* Must have a matching type unstable exit. */
-     if (memcmp(getTypeMap(exit) + exit->numGlobalSlots,
+     if ((exit->numGlobalSlots + exit->numStackSlots != stableTree->typeMap.length()) ||
-                stableTree->stackTypeMap.data(),
+         memcmp(getFullTypeMap(exit), stableTree->typeMap.data(), stableTree->typeMap.length())) {
-                stableTree->stackTypeMap.length()) != 0) {
+        return false;
-        return false;
      }
      exit->target = stableFrag;
      frago->assm()->patch(exit);
      stableTree->dependentTrees.addUnique(exit->from->root);
+     ((TreeInfo*)exit->from->root->vmprivate)->linkedTrees.addUnique(stableFrag);
      return true;
  }
  /* Complete and compile a trace and link it to the existing tree if appropriate. */
- bool
+ JS_REQUIRES_STACK void
- TraceRecorder::closeLoop(Fragmento* fragmento, bool& demote, unsigned *demotes)
+ TraceRecorder::closeLoop(JSTraceMonitor* tm, bool& demote)
  {
+     /*
+      * We should have arrived back at the loop header, and hence we don't want to be in an imacro
+      * here and the opcode should be either JSOP_LOOP, or in case this loop was blacklisted in the
+      * meantime JSOP_NOP.
+      */
+     JS_ASSERT((*cx->fp->regs->pc == JSOP_LOOP || *cx->fp->regs->pc == JSOP_NOP) && !cx->fp->imacpc);
      bool stable;
-     LIns* exitIns;
      Fragment* peer;
-     VMSideExit* exit;
+     VMFragment* peer_root;
-     Fragment* peer_root;
+     Fragmento* fragmento = tm->fragmento;
-     demote = false;
-     exitIns = snapshot(UNSTABLE_LOOP_EXIT);
-     exit = (VMSideExit*)((GuardRecord*)exitIns->payload())->exit;
      if (callDepth != 0) {
-         debug_only_v(printf("Stack depth mismatch, possible recursion\n");)
+         debug_only_v(printf("Blacklisted: stack depth mismatch, possible recursion.\n");)
-         js_BlacklistPC(fragmento, fragment);
+         js_Blacklist((jsbytecode*) fragment->root->ip);
-         trashTree = true;
+         trashSelf = true;
-         return false;
+         return;
      }
-     JS_ASSERT(exit->numStackSlots == treeInfo->stackTypeMap.length());
+     VMSideExit* exit = snapshot(UNSTABLE_LOOP_EXIT);
+     JS_ASSERT(exit->numStackSlots == treeInfo->nStackTypes);
-     peer_root = fragmento->getLoop(fragment->root->ip);
+     VMFragment* root = (VMFragment*)fragment->root;
+     peer_root = getLoop(traceMonitor, root->ip, root->globalObj, root->globalShape, root->argc);
      JS_ASSERT(peer_root != NULL);
-     stable = deduceTypeStability(peer_root, &peer, demotes);
-     #if DEBUG
+     stable = deduceTypeStability(peer_root, &peer, demote);
-     if (!stable || NUM_UNDEMOTE_SLOTS(demotes))
+ #if DEBUG
+     if (!stable)
          AUDIT(unstableLoopVariable);
-     #endif
+ #endif
-     if (trashTree) {
+     if (trashSelf) {
          debug_only_v(printf("Trashing tree from type instability.\n");)
-         return false;
+         return;
      }
-     if (stable && NUM_UNDEMOTE_SLOTS(demotes)) {
+     if (stable && demote) {
          JS_ASSERT(fragment->kind == LoopTrace);
-         demote = true;
+         return;
-         return false;
      }
      if (!stable) {
-         fragment->lastIns = lir->insGuard(LIR_x, lir->insImm(1), exitIns);
+         fragment->lastIns = lir->insGuard(LIR_x, lir->insImm(1), createGuardRecord(exit));
          /*
           * If we didn't find a type stable peer, we compile the loop anyway and
-Line 2277
+Line 3075
              uexit->exit = exit;
              uexit->next = treeInfo->unstableExits;
              treeInfo->unstableExits = uexit;
-             /*
-              * If we walked out of a loop, this exit is wrong. We need to back
-              * up to the if operation.
-              */
-             if (walkedOutOfLoop())
-                 exit->ip_adj = terminate_ip_adj;
-             /* If we were trying to stabilize a promotable tree, trash it. */
-             if (promotedPeer)
-                 js_TrashTree(cx, promotedPeer);
          } else {
              JS_ASSERT(peer->code());
              exit->target = peer;
-             debug_only_v(printf("Joining type-unstable trace to target fragment %p.\n", peer);)
+             debug_only_v(printf("Joining type-unstable trace to target fragment %p.\n", (void*)peer);)
              stable = true;
              ((TreeInfo*)peer->vmprivate)->dependentTrees.addUnique(fragment->root);
+             treeInfo->linkedTrees.addUnique(peer);
          }
-         compile(fragmento);
      } else {
          exit->target = fragment->root;
- #if defined(JS_HAS_OPERATION_COUNT) && !JS_HAS_OPERATION_COUNT
+         fragment->lastIns = lir->insGuard(LIR_loop, lir->insImm(1), createGuardRecord(exit));
-         exit->exitType = TIMEOUT_EXIT;
-         guard(false,
-               lir->ins_eq0(lir->insLoadi(cx_ins,
-                                          offsetof(JSContext, operationCount))),
-               exitIns);
- #endif
-         fragment->lastIns = lir->insGuard(LIR_loop, lir->insImm(1), exitIns);
-         compile(fragmento);
      }
+     compile(tm);
      if (fragmento->assm()->error() != nanojit::None)
-         return false;
+         return;
      joinEdgesToEntry(fragmento, peer_root);
+     debug_only_v(printf("updating specializations on dependent and linked trees\n"))
+     if (fragment->root->vmprivate)
+         specializeTreesToMissingGlobals(cx, (TreeInfo*)fragment->root->vmprivate);
+     /*
+      * If this is a newly formed tree, and the outer tree has not been compiled yet, we
+      * should try to compile the outer tree again.
+      */
+     if (outer)
+         js_AttemptCompilation(cx, tm, globalObj, outer, outerArgc);
      debug_only_v(printf("recording completed at %s:%u@%u via closeLoop\n",
                          cx->fp->script->filename,
                          js_FramePCToLineNumber(cx, cx->fp),
                          FramePCOffset(cx->fp));)
-     return true;
  }
- void
+ JS_REQUIRES_STACK void
- TraceRecorder::joinEdgesToEntry(Fragmento* fragmento, Fragment* peer_root)
+ TraceRecorder::joinEdgesToEntry(Fragmento* fragmento, VMFragment* peer_root)
  {
      if (fragment->kind == LoopTrace) {
          TreeInfo* ti;
          Fragment* peer;
          uint8* t1, *t2;
          UnstableExit* uexit, **unext;
+         uint32* stackDemotes = (uint32*)alloca(sizeof(uint32) * treeInfo->nStackTypes);
+         uint32* globalDemotes = (uint32*)alloca(sizeof(uint32) * treeInfo->nGlobalTypes());
-         unsigned* demotes = (unsigned*)alloca(treeInfo->stackTypeMap.length() * sizeof(unsigned));
          for (peer = peer_root; peer != NULL; peer = peer->peer) {
              if (!peer->code())
                  continue;
-Line 2341
+Line 3131
              while (uexit != NULL) {
                  bool remove = js_JoinPeersIfCompatible(fragmento, fragment, treeInfo, uexit->exit);
                  JS_ASSERT(!remove || fragment != peer);
                  debug_only_v(if (remove) {
                               printf("Joining type-stable trace to target exit %p->%p.\n",
-                                     uexit->fragment, uexit->exit); });
+                                     (void*)uexit->fragment, (void*)uexit->exit); });
                  if (!remove) {
                      /* See if this exit contains mismatch demotions, which imply trashing a tree.
                         This is actually faster than trashing the original tree as soon as the
                         instability is detected, since we could have compiled a fairly stable
                         tree that ran faster with integers. */
-                     unsigned count = 0;
+                     unsigned stackCount = 0;
-                     t1 = treeInfo->stackTypeMap.data();
+                     unsigned globalCount = 0;
-                     t2 = getTypeMap(uexit->exit) + uexit->exit->numGlobalSlots;
+                     t1 = treeInfo->stackTypeMap();
+                     t2 = getStackTypeMap(uexit->exit);
                      for (unsigned i = 0; i < uexit->exit->numStackSlots; i++) {
                          if (t2[i] == JSVAL_INT && t1[i] == JSVAL_DOUBLE) {
-                             demotes[count++] = i;
+                             stackDemotes[stackCount++] = i;
                          } else if (t2[i] != t1[i]) {
-                             count = 0;
+                             stackCount = 0;
                              break;
                          }
                      }
-                     if (count) {
+                     t1 = treeInfo->globalTypeMap();
-                         for (unsigned i = 0; i < count; i++)
+                     t2 = getGlobalTypeMap(uexit->exit);
-                             oracle.markStackSlotUndemotable(cx->fp->script,
+                     for (unsigned i = 0; i < uexit->exit->numGlobalSlots; i++) {
-                                                             cx->fp->regs->pc, demotes[i]);
+                         if (t2[i] == JSVAL_INT && t1[i] == JSVAL_DOUBLE) {
-                         js_TrashTree(cx, uexit->fragment->root);
+                             globalDemotes[globalCount++] = i;
+                         } else if (t2[i] != t1[i]) {
+                             globalCount = 0;
+                             stackCount = 0;
+                             break;
+                         }
+                     }
+                     if (stackCount || globalCount) {
+                         for (unsigned i = 0; i < stackCount; i++)
+                             oracle.markStackSlotUndemotable(cx, stackDemotes[i]);
+                         for (unsigned i = 0; i < globalCount; i++)
+                             oracle.markGlobalSlotUndemotable(cx, ti->globalSlots->data()[globalDemotes[i]]);
+                         JS_ASSERT(peer == uexit->fragment->root);
+                         if (fragment == peer)
+                             trashSelf = true;
+                         else
+                             whichTreesToTrash.addUnique(uexit->fragment->root);
                          break;
                      }
                  }
-Line 2376
+Line 3183
                      unext = &uexit->next;
                      uexit = uexit->next;
                  }
              }
          }
      }
-     debug_only_v(js_DumpPeerStability(fragmento, peer_root->ip);)
+     debug_only_v(js_DumpPeerStability(traceMonitor, peer_root->ip, peer_root->globalObj,
+                                       peer_root->globalShape, peer_root->argc);)
  }
  /* Emit an always-exit guard and compile the tree (used for break statements. */
- void
+ JS_REQUIRES_STACK void
- TraceRecorder::endLoop(Fragmento* fragmento)
+ TraceRecorder::endLoop(JSTraceMonitor* tm)
  {
-     LIns* exitIns = snapshot(LOOP_EXIT);
      if (callDepth != 0) {
-         debug_only_v(printf("Stack depth mismatch, possible recursion\n");)
+         debug_only_v(printf("Blacklisted: stack depth mismatch, possible recursion.\n");)
-         js_BlacklistPC(fragmento, fragment);
+         js_Blacklist((jsbytecode*) fragment->root->ip);
-         trashTree = true;
+         trashSelf = true;
          return;
      }
-     fragment->lastIns = lir->insGuard(LIR_x, lir->insImm(1), exitIns);
+     fragment->lastIns =
-     compile(fragmento);
+         lir->insGuard(LIR_x, lir->insImm(1), createGuardRecord(snapshot(LOOP_EXIT)));
+     compile(tm);
-     if (fragmento->assm()->error() != nanojit::None)
+     if (tm->fragmento->assm()->error() != nanojit::None)
          return;
-     joinEdgesToEntry(fragmento, fragmento->getLoop(fragment->root->ip));
+     VMFragment* root = (VMFragment*)fragment->root;
+     joinEdgesToEntry(tm->fragmento, getLoop(tm, root->ip, root->globalObj, root->globalShape, root->argc));
+     /* Note: this must always be done, in case we added new globals on trace and haven't yet
+        propagated those to linked and dependent trees. */
+     debug_only_v(printf("updating specializations on dependent and linked trees\n"))
+     if (fragment->root->vmprivate)
+         specializeTreesToMissingGlobals(cx, (TreeInfo*)fragment->root->vmprivate);
+     /*
+      * If this is a newly formed tree, and the outer tree has not been compiled yet, we
+      * should try to compile the outer tree again.
+      */
+     if (outer)
+         js_AttemptCompilation(cx, tm, globalObj, outer, outerArgc);
      debug_only_v(printf("recording completed at %s:%u@%u via endLoop\n",
                          cx->fp->script->filename,
                          js_FramePCToLineNumber(cx, cx->fp),
-Line 2411
+Line 3232
  }
  /* Emit code to adjust the stack to match the inner tree's stack expectations. */
- void
+ JS_REQUIRES_STACK void
  TraceRecorder::prepareTreeCall(Fragment* inner)
  {
      TreeInfo* ti = (TreeInfo*)inner->vmprivate;
-Line 2425
+Line 3246
             any outer frames that the inner tree doesn't expect but the outer tree has. */
          ptrdiff_t sp_adj = nativeStackOffset(&cx->fp->argv[-2]);
          /* Calculate the amount we have to lift the call stack by */
-         ptrdiff_t rp_adj = callDepth * sizeof(FrameInfo);
+         ptrdiff_t rp_adj = callDepth * sizeof(FrameInfo*);
          /* Guard that we have enough stack space for the tree we are trying to call on top
             of the new value for sp. */
          debug_only_v(printf("sp_adj=%d outer=%d inner=%d\n",
-Line 2437
+Line 3258
          guard(true, lir->ins2(LIR_lt, sp_top, eos_ins), OOM_EXIT);
          /* Guard that we have enough call stack space. */
          LIns* rp_top = lir->ins2i(LIR_piadd, lirbuf->rp, rp_adj +
-                 ti->maxCallDepth * sizeof(FrameInfo));
+                 ti->maxCallDepth * sizeof(FrameInfo*));
          guard(true, lir->ins2(LIR_lt, rp_top, eor_ins), OOM_EXIT);
          /* We have enough space, so adjust sp and rp to their new level. */
          lir->insStorei(inner_sp_ins = lir->ins2i(LIR_piadd, lirbuf->sp,
-Line 2451
+Line 3272
  }
  /* Record a call to an inner tree. */
- void
+ JS_REQUIRES_STACK void
  TraceRecorder::emitTreeCall(Fragment* inner, VMSideExit* exit)
  {
      TreeInfo* ti = (TreeInfo*)inner->vmprivate;
      /* Invoke the inner tree. */
      LIns* args[] = { INS_CONSTPTR(inner), lirbuf->state }; /* reverse order */
      LIns* ret = lir->insCall(&js_CallTree_ci, args);
      /* Read back all registers, in case the called tree changed any of them. */
-     import(ti, inner_sp_ins, exit->numGlobalSlots, exit->calldepth,
+     JS_ASSERT(!memchr(getGlobalTypeMap(exit), JSVAL_BOXED, exit->numGlobalSlots) &&
-            getTypeMap(exit), getTypeMap(exit) + exit->numGlobalSlots);
+               !memchr(getStackTypeMap(exit), JSVAL_BOXED, exit->numStackSlots));
+     /* bug 502604 - It is illegal to extend from the outer typemap without first extending from the
+      * inner. Make a new typemap here.
+      */
+     TypeMap fullMap;
+     fullMap.add(getStackTypeMap(exit), exit->numStackSlots);
+     fullMap.add(getGlobalTypeMap(exit), exit->numGlobalSlots);
+     TreeInfo* innerTree = (TreeInfo*)exit->from->root->vmprivate;
+     if (exit->numGlobalSlots < innerTree->nGlobalTypes()) {
+         fullMap.add(innerTree->globalTypeMap() + exit->numGlobalSlots,
+                     innerTree->nGlobalTypes() - exit->numGlobalSlots);
+     }
+     import(ti, inner_sp_ins, exit->numStackSlots, fullMap.length() - exit->numStackSlots,
+            exit->calldepth, fullMap.data());
      /* Restore sp and rp to their original values (we still have them in a register). */
      if (callDepth > 0) {
          lir->insStorei(lirbuf->sp, lirbuf->state, offsetof(InterpState, sp));
          lir->insStorei(lirbuf->rp, lirbuf->state, offsetof(InterpState, rp));
      }
-     /* Guard that we come out of the inner tree along the same side exit we came out when
-        we called the inner tree at recording time. */
+     /*
+      * Guard that we come out of the inner tree along the same side exit we came out when
+      * we called the inner tree at recording time.
+      */
      guard(true, lir->ins2(LIR_eq, ret, INS_CONSTPTR(exit)), NESTED_EXIT);
      /* Register us as a dependent tree of the inner tree. */
      ((TreeInfo*)inner->vmprivate)->dependentTrees.addUnique(fragment->root);
+     treeInfo->linkedTrees.addUnique(inner);
  }
  /* Add a if/if-else control-flow merge point to the list of known merge points. */
- void
+ JS_REQUIRES_STACK void
  TraceRecorder::trackCfgMerges(jsbytecode* pc)
  {
      /* If we hit the beginning of an if/if-else, then keep track of the merge point after it. */
-Line 2482
+Line 3323
      jssrcnote* sn = js_GetSrcNote(cx->fp->script, pc);
      if (sn != NULL) {
          if (SN_TYPE(sn) == SRC_IF) {
              cfgMerges.add((*pc == JSOP_IFEQ)
                            ? pc + GET_JUMP_OFFSET(pc)
                            : pc + GET_JUMPX_OFFSET(pc));
          } else if (SN_TYPE(sn) == SRC_IF_ELSE)
              cfgMerges.add(pc + js_GetSrcNoteOffset(sn, 0));
      }
  }
  /* Invert the direction of the guard if this is a loop edge that is not
     taken (thin loop). */
- void
+ JS_REQUIRES_STACK void
- TraceRecorder::flipIf(jsbytecode* pc, bool& cond)
+ TraceRecorder::emitIf(jsbytecode* pc, bool cond, LIns* x)
  {
+     ExitType exitType;
      if (js_IsLoopEdge(pc, (jsbytecode*)fragment->root->ip)) {
-         switch (*pc) {
+         exitType = LOOP_EXIT;
-           case JSOP_IFEQ:
-           case JSOP_IFEQX:
+         /*
-             if (!cond)
+          * If we are about to walk out of the loop, generate code for the inverse loop
-                 return;
+          * condition, pretending we recorded the case that stays on trace.
-             break;
+          */
-           case JSOP_IFNE:
+         if ((*pc == JSOP_IFEQ || *pc == JSOP_IFEQX) == cond) {
-           case JSOP_IFNEX:
+             JS_ASSERT(*pc == JSOP_IFNE || *pc == JSOP_IFNEX || *pc == JSOP_IFEQ || *pc == JSOP_IFEQX);
-             if (cond)
+             debug_only_v(printf("Walking out of the loop, terminating it anyway.\n");)
-                 return;
+             cond = !cond;
-             break;
-           default:
-             JS_NOT_REACHED("flipIf");
          }
-         /* We are about to walk out of the loop, so terminate it with
-            an inverse loop condition. */
+         /*
-         debug_only_v(printf("Walking out of the loop, terminating it anyway.\n");)
+          * Conditional guards do not have to be emitted if the condition is constant. We
-         cond = !cond;
+          * make a note whether the loop condition is true or false here, so we later know
-         terminate = true;
+          * whether to emit a loop edge or a loop end.
-         /* If when we get to closeLoop the tree is decided to be type unstable, we need to
-            reverse this logic because the loop won't be closed after all.  Store the real
-            value of the IP the interpreter expects, so we can use it in our final LIR_x.
           */
-         if (*pc == JSOP_IFEQX || *pc == JSOP_IFNEX)
+         if (x->isconst()) {
-             pc += GET_JUMPX_OFFSET(pc);
+             loop = (x->constval() == cond);
-         else
+             return;
-             pc += GET_JUMP_OFFSET(pc);
+         }
-         terminate_ip_adj = ENCODE_IP_ADJ(cx->fp, pc);
+     } else {
+         exitType = BRANCH_EXIT;
      }
+     if (!x->isconst())
+         guard(cond, x, exitType);
  }
  /* Emit code for a fused IFEQ/IFNE. */
- void
+ JS_REQUIRES_STACK void
  TraceRecorder::fuseIf(jsbytecode* pc, bool cond, LIns* x)
  {
-     if (x->isconst()) // no need to guard if condition is constant
+     if (*pc == JSOP_IFEQ || *pc == JSOP_IFNE) {
-         return;
+         emitIf(pc, cond, x);
-     if (*pc == JSOP_IFEQ) {
+         if (*pc == JSOP_IFEQ)
-         flipIf(pc, cond);
+             trackCfgMerges(pc);
-         guard(cond, x, BRANCH_EXIT);
-         trackCfgMerges(pc);
-     } else if (*pc == JSOP_IFNE) {
-         flipIf(pc, cond);
-         guard(cond, x, BRANCH_EXIT);
      }
  }
+ /* Check whether we have reached the end of the trace. */
+ JS_REQUIRES_STACK JSRecordingStatus
+ TraceRecorder::checkTraceEnd(jsbytecode *pc)
+ {
+     if (js_IsLoopEdge(pc, (jsbytecode*)fragment->root->ip)) {
+         /*
+          * If we compile a loop, the trace should have a zero stack balance at the loop
+          * edge. Currently we are parked on a comparison op or IFNE/IFEQ, so advance
+          * pc to the loop header and adjust the stack pointer and pretend we have
+          * reached the loop header.
+          */
+         if (loop) {
+             JS_ASSERT(!cx->fp->imacpc && (pc == cx->fp->regs->pc || pc == cx->fp->regs->pc + 1));
+             bool fused = pc != cx->fp->regs->pc;
+             JSFrameRegs orig = *cx->fp->regs;
+             cx->fp->regs->pc = (jsbytecode*)fragment->root->ip;
+             cx->fp->regs->sp -= fused ? 2 : 1;
+             bool demote = false;
+             closeLoop(traceMonitor, demote);
+             *cx->fp->regs = orig;
+             /*
+              * If compiling this loop generated new oracle information which will likely
+              * lead to a different compilation result, immediately trigger another
+              * compiler run. This is guaranteed to converge since the oracle only
+              * accumulates adverse information but never drops it (except when we
+              * flush it during garbage collection.)
+              */
+             if (demote)
+                 js_AttemptCompilation(cx, traceMonitor, globalObj, outer, outerArgc);
+         } else {
+             endLoop(traceMonitor);
+         }
+         return JSRS_STOP;
+     }
+     return JSRS_CONTINUE;
+ }
  bool
  TraceRecorder::hasMethod(JSObject* obj, jsid id)
  {
-Line 2566
+Line 3441
              if (VALUE_IS_FUNCTION(cx, v)) {
                  found = true;
                  if (!SCOPE_IS_BRANDED(scope)) {
-                     SCOPE_MAKE_UNIQUE_SHAPE(cx, scope);
+                     js_MakeScopeShapeUnique(cx, scope);
                      SCOPE_SET_BRANDED(scope);
                  }
              }
-Line 2577
+Line 3452
      return found;
  }
- bool
+ JS_REQUIRES_STACK bool
- TraceRecorder::hasToStringMethod(JSObject* obj)
- {
-     JS_ASSERT(cx->fp->regs->sp + 1 <= cx->fp->slots + cx->fp->script->nslots);
-     return hasMethod(obj, ATOM_TO_JSID(cx->runtime->atomState.toStringAtom));
- }
- bool
- TraceRecorder::hasValueOfMethod(JSObject* obj)
- {
-     JS_ASSERT(cx->fp->regs->sp + 2 <= cx->fp->slots + cx->fp->script->nslots);
-     return hasMethod(obj, ATOM_TO_JSID(cx->runtime->atomState.valueOfAtom));
- }
- bool
  TraceRecorder::hasIteratorMethod(JSObject* obj)
  {
      JS_ASSERT(cx->fp->regs->sp + 2 <= cx->fp->slots + cx->fp->script->nslots);
-Line 2619
+Line 3478
      x = (VMSideExit *)i->record()->exit;
      sprintf(out,
-             "%s: %s %s -> %lu:%lu sp%+ld rp%+ld",
+             "%s: %s %s -> pc=%p imacpc=%p sp%+ld rp%+ld",
              formatRef(i),
              lirNames[i->opcode()],
              i->oprnd1()->isCond() ? formatRef(i->oprnd1()) : "",
-             IMACRO_PC_ADJ(x->ip_adj),
+             (void *)x->pc,
-             SCRIPT_PC_ADJ(x->ip_adj),
+             (void *)x->imacpc,
              (long int)x->sp_adj,
              (long int)x->rp_adj);
  }
-Line 2633
+Line 3492
  void
  nanojit::Fragment::onDestroy()
  {
-     if (root == this) {
-         delete mergeCounts;
-         delete lirbuf;
-     }
      delete (TreeInfo *)vmprivate;
  }
- void
+ static JS_REQUIRES_STACK bool
  js_DeleteRecorder(JSContext* cx)
  {
      JSTraceMonitor* tm = &JS_TRACE_MONITOR(cx);
      /* Aborting and completing a trace end up here. */
-     JS_ASSERT(tm->onTrace);
-     tm->onTrace = false;
      delete tm->recorder;
      tm->recorder = NULL;
+     /*
+      * If we ran out of memory, flush the code cache.
+      */
+     if (JS_TRACE_MONITOR(cx).fragmento->assm()->error() == OutOMem ||
+         js_OverfullFragmento(tm, tm->fragmento)) {
+         FlushJITCache(cx);
+         return false;
+     }
+     return true;
  }
  /**
   * Checks whether the shape of the global object has changed.
   */
- static inline bool
+ static JS_REQUIRES_STACK bool
- js_CheckGlobalObjectShape(JSContext* cx, JSTraceMonitor* tm, JSObject* globalObj)
+ CheckGlobalObjectShape(JSContext* cx, JSTraceMonitor* tm, JSObject* globalObj,
+                        uint32 *shape=NULL, SlotList** slots=NULL)
  {
-     /* Check the global shape. */
+     if (tm->needFlush) {
-     if (OBJ_SHAPE(globalObj) != tm->globalShape) {
+         FlushJITCache(cx);
-         AUDIT(globalShapeMismatchAtEntry);
-         debug_only_v(printf("Global shape mismatch (%u vs. %u), flushing cache.\n",
-                             OBJ_SHAPE(globalObj), tm->globalShape);)
          return false;
      }
-     return true;
+     if (STOBJ_NSLOTS(globalObj) > MAX_GLOBAL_SLOTS)
+         return false;
+     uint32 globalShape = OBJ_SHAPE(globalObj);
+     if (tm->recorder) {
+         VMFragment* root = (VMFragment*)tm->recorder->getFragment()->root;
+         TreeInfo* ti = tm->recorder->getTreeInfo();
+         /* Check the global shape matches the recorder's treeinfo's shape. */
+         if (globalObj != root->globalObj || globalShape != root->globalShape) {
+             AUDIT(globalShapeMismatchAtEntry);
+             debug_only_v(printf("Global object/shape mismatch (%p/%u vs. %p/%u), flushing cache.\n",
+                                 (void*)globalObj, globalShape, (void*)root->globalObj,
+                                 root->globalShape);)
+             js_Backoff(cx, (jsbytecode*) root->ip);
+             FlushJITCache(cx);
+             return false;
+         }
+         if (shape)
+             *shape = globalShape;
+         if (slots)
+             *slots = ti->globalSlots;
+         return true;
+     }
+     /* No recorder, search for a tracked global-state (or allocate one). */
+     for (size_t i = 0; i < MONITOR_N_GLOBAL_STATES; ++i) {
+         GlobalState &state = tm->globalStates[i];
+         if (state.globalShape == uint32(-1)) {
+             state.globalObj = globalObj;
+             state.globalShape = globalShape;
+             JS_ASSERT(state.globalSlots);
+             JS_ASSERT(state.globalSlots->length() == 0);
+         }
+         if (state.globalObj == globalObj && state.globalShape == globalShape) {
+             if (shape)
+                 *shape = globalShape;
+             if (slots)
+                 *slots = state.globalSlots;
+             return true;
+         }
+     }
+     /* No currently-tracked-global found and no room to allocate, abort. */
+     AUDIT(globalShapeMismatchAtEntry);
+     debug_only_v(printf("No global slotlist for global shape %u, flushing cache.\n",
+                         globalShape));
+     FlushJITCache(cx);
+     return false;
  }
- static bool
+ static JS_REQUIRES_STACK bool
  js_StartRecorder(JSContext* cx, VMSideExit* anchor, Fragment* f, TreeInfo* ti,
-                  unsigned ngslots, uint8* globalTypeMap, uint8* stackTypeMap,
+                  unsigned stackSlots, unsigned ngslots, uint8* typeMap,
-                  VMSideExit* expectedInnerExit, Fragment* outer)
+                  VMSideExit* expectedInnerExit, jsbytecode* outer, uint32 outerArgc)
  {
      JSTraceMonitor* tm = &JS_TRACE_MONITOR(cx);
+     if (JS_TRACE_MONITOR(cx).needFlush) {
+         FlushJITCache(cx);
+         return false;
+     }
-     /*
+     JS_ASSERT(f->root != f || !cx->fp->imacpc);
-      * Emulate on-trace semantics and avoid rooting headaches while recording,
-      * by suppressing last-ditch GC attempts while recording a trace. This does
-      * means that trace recording must not nest or the following assertion will
-      * botch.
-      */
-     JS_ASSERT(!tm->onTrace);
-     tm->onTrace = true;
      /* start recording if no exception during construction */
      tm->recorder = new (&gc) TraceRecorder(cx, anchor, f, ti,
-                                            ngslots, globalTypeMap, stackTypeMap,
+                                            stackSlots, ngslots, typeMap,
-                                            expectedInnerExit, outer);
+                                            expectedInnerExit, outer, outerArgc);
      if (cx->throwing) {
          js_AbortRecording(cx, "setting up recorder failed");
          return false;
-Line 2715
+Line 3625
      unsigned length = ti->dependentTrees.length();
      for (unsigned n = 0; n < length; ++n)
          js_TrashTree(cx, data[n]);
+     data = ti->linkedTrees.data();
+     length = ti->linkedTrees.length();
+     for (unsigned n = 0; n < length; ++n)
+         js_TrashTree(cx, data[n]);
      delete ti;
      JS_ASSERT(!f->code() && !f->vmprivate);
  }
-Line 2722
+Line 3636
  static int
  js_SynthesizeFrame(JSContext* cx, const FrameInfo& fi)
  {
+     VOUCH_DOES_NOT_REQUIRE_STACK();
      JS_ASSERT(HAS_FUNCTION_CLASS(fi.callee));
      JSFunction* fun = GET_FUNCTION_PRIVATE(cx, fi.callee);
      JS_ASSERT(FUN_INTERPRETED(fun));
      /* Assert that we have a correct sp distance from cx->fp->slots in fi. */
-     JS_ASSERT_IF(!FI_IMACRO_PC(fi, cx->fp),
+     JSStackFrame* fp = cx->fp;
-                  js_ReconstructStackDepth(cx, cx->fp->script, FI_SCRIPT_PC(fi, cx->fp))
+     JS_ASSERT_IF(!fi.imacpc,
-                  == uintN(fi.s.spdist - cx->fp->script->nfixed));
+                  js_ReconstructStackDepth(cx, fp->script, fi.pc)
+                  == uintN(fi.spdist - fp->script->nfixed));
      uintN nframeslots = JS_HOWMANY(sizeof(JSInlineFrame), sizeof(jsval));
      JSScript* script = fun->u.i.script;
-Line 2739
+Line 3656
      /* Code duplicated from inline_call: case in js_Interpret (FIXME). */
      JSArena* a = cx->stackPool.current;
      void* newmark = (void*) a->avail;
-     uintN argc = fi.s.argc & 0x7fff;
+     uintN argc = fi.get_argc();
-     jsval* vp = cx->fp->slots + fi.s.spdist - (2 + argc);
+     jsval* vp = fp->slots + fi.spdist - (2 + argc);
      uintN missing = 0;
      jsval* newsp;
      if (fun->nargs > argc) {
-         const JSFrameRegs& regs = *cx->fp->regs;
+         const JSFrameRegs& regs = *fp->regs;
          newsp = vp + 2 + fun->nargs;
          JS_ASSERT(newsp > regs.sp);
-Line 2769
+Line 3686
          a->avail += nbytes;
          JS_ASSERT(missing == 0);
      } else {
+         /* This allocation is infallible: js_ExecuteTree reserved enough stack. */
          JS_ARENA_ALLOCATE_CAST(newsp, jsval *, &cx->stackPool, nbytes);
-         if (!newsp) {
+         JS_ASSERT(newsp);
-             js_ReportOutOfScriptQuota(cx);
-             return 0;
-         }
          /*
           * Move args if the missing ones overflow arena a, then push
-Line 2797
+Line 3712
      newifp->frame.argsobj = NULL;
      newifp->frame.varobj = NULL;
      newifp->frame.script = script;
-     newifp->frame.callee = fi.callee;
+     newifp->frame.callee = fi.callee; // Roll with a potentially stale callee for now.
      newifp->frame.fun = fun;
-     bool constructing = fi.s.argc & 0x8000;
+     bool constructing = fi.is_constructing();
      newifp->frame.argc = argc;
+     newifp->callerRegs.pc = fi.pc;
+     newifp->callerRegs.sp = fp->slots + fi.spdist;
+     fp->imacpc = fi.imacpc;
-     jsbytecode* imacro_pc = FI_IMACRO_PC(fi, cx->fp);
+ #ifdef DEBUG
-     jsbytecode* script_pc = FI_SCRIPT_PC(fi, cx->fp);
+     if (fi.block != fp->blockChain) {
-     newifp->callerRegs.pc = imacro_pc ? imacro_pc : script_pc;
+         for (JSObject* obj = fi.block; obj != fp->blockChain; obj = STOBJ_GET_PARENT(obj))
-     newifp->callerRegs.sp = cx->fp->slots + fi.s.spdist;
+             JS_ASSERT(obj);
-     cx->fp->imacpc = imacro_pc ? script_pc : NULL;
+     }
+ #endif
+     fp->blockChain = fi.block;
      newifp->frame.argv = newifp->callerRegs.sp - argc;
      JS_ASSERT(newifp->frame.argv);
-Line 2816
+Line 3736
      // someone forgets to initialize it later.
      newifp->frame.argv[-1] = JSVAL_HOLE;
  #endif
-     JS_ASSERT(newifp->frame.argv >= StackBase(cx->fp) + 2);
+     JS_ASSERT(newifp->frame.argv >= StackBase(fp) + 2);
      newifp->frame.rval = JSVAL_VOID;
-     newifp->frame.down = cx->fp;
+     newifp->frame.down = fp;
      newifp->frame.annotation = NULL;
-     newifp->frame.scopeChain = OBJ_GET_PARENT(cx, fi.callee);
+     newifp->frame.scopeChain = NULL; // will be updated in FlushNativeStackFrame
      newifp->frame.sharpDepth = 0;
      newifp->frame.sharpArray = NULL;
      newifp->frame.flags = constructing ? JSFRAME_CONSTRUCTING : 0;
-Line 2829
+Line 3749
      newifp->frame.xmlNamespace = NULL;
      newifp->frame.blockChain = NULL;
      newifp->mark = newmark;
-     newifp->frame.thisp = NULL; // will be set by js_ExecuteTree -> FlushNativeStackFrame
+     newifp->frame.thisp = NULL; // will be updated in FlushNativeStackFrame
-     newifp->frame.regs = cx->fp->regs;
+     newifp->frame.regs = fp->regs;
      newifp->frame.regs->pc = script->code;
      newifp->frame.regs->sp = newsp + script->nfixed;
      newifp->frame.imacpc = NULL;
      newifp->frame.slots = newsp;
-     if (script->staticDepth < JS_DISPLAY_SIZE) {
+     if (script->staticLevel < JS_DISPLAY_SIZE) {
-         JSStackFrame **disp = &cx->display[script->staticDepth];
+         JSStackFrame **disp = &cx->display[script->staticLevel];
          newifp->frame.displaySave = *disp;
          *disp = &newifp->frame;
      }
- #ifdef DEBUG
-     newifp->frame.pcDisabledSave = 0;
- #endif
      /*
-      * Note that cx->fp->script is still the caller's script; set the callee
+      * Note that fp->script is still the caller's script; set the callee
       * inline frame's idea of caller version from its version.
       */
-     newifp->callerVersion = (JSVersion) cx->fp->script->version;
+     newifp->callerVersion = (JSVersion) fp->script->version;
-     cx->fp->regs = &newifp->callerRegs;
-     cx->fp = &newifp->frame;
-     if (fun->flags & JSFUN_HEAVYWEIGHT) {
+     // After this paragraph, fp and cx->fp point to the newly synthesized frame.
-         /*
+     fp->regs = &newifp->callerRegs;
-          * Set hookData to null because the failure case for js_GetCallObject
+     fp = cx->fp = &newifp->frame;
-          * involves it calling the debugger hook.
-          */
-         newifp->hookData = NULL;
-         if (!js_GetCallObject(cx, &newifp->frame, newifp->frame.scopeChain))
-             return -1;
-     }
      /*
       * If there's a call hook, invoke it to compute the hookData used by
-Line 2870
+Line 3778
       */
      JSInterpreterHook hook = cx->debugHooks->callHook;
      if (hook) {
-         newifp->hookData = hook(cx, &newifp->frame, JS_TRUE, 0,
+         newifp->hookData = hook(cx, fp, JS_TRUE, 0, cx->debugHooks->callHookData);
-                                 cx->debugHooks->callHookData);
      } else {
          newifp->hookData = NULL;
      }
-Line 2879
+Line 3786
      // FIXME? we must count stack slots from caller's operand stack up to (but not including)
      // callee's, including missing arguments. Could we shift everything down to the caller's
      // fp->slots (where vars start) and avoid some of the complexity?
-     return (fi.s.spdist - cx->fp->down->script->nfixed) +
+     return (fi.spdist - fp->down->script->nfixed) +
-            ((fun->nargs > cx->fp->argc) ? fun->nargs - cx->fp->argc : 0) +
+            ((fun->nargs > fp->argc) ? fun->nargs - fp->argc : 0) +
             script->nfixed;
  }
- bool
+ static void
- js_RecordTree(JSContext* cx, JSTraceMonitor* tm, Fragment* f, Fragment* outer, unsigned* demotes)
+ SynthesizeSlowNativeFrame(JSContext *cx, VMSideExit *exit)
  {
-     JS_ASSERT(cx->fp->regs->pc == f->ip && f->root == f);
+     void *mark;
+     JSInlineFrame *ifp;
-     /* Avoid recording loops in overlarge scripts. */
-     if (cx->fp->script->length >= SCRIPT_PC_ADJ_LIMIT) {
+     /* This allocation is infallible: js_ExecuteTree reserved enough stack. */
-         js_AbortRecording(cx, "script too large");
+     mark = JS_ARENA_MARK(&cx->stackPool);
-         return false;
+     JS_ARENA_ALLOCATE_CAST(ifp, JSInlineFrame *, &cx->stackPool, sizeof(JSInlineFrame));
-     }
+     JS_ASSERT(ifp);
+     JSStackFrame *fp = &ifp->frame;
+     fp->regs = NULL;
+     fp->imacpc = NULL;
+     fp->slots = NULL;
+     fp->callobj = NULL;
+     fp->argsobj = NULL;
+     fp->varobj = cx->fp->varobj;
+     fp->callee = exit->nativeCallee();
+     fp->script = NULL;
+     fp->fun = GET_FUNCTION_PRIVATE(cx, fp->callee);
+     // fp->thisp is really a jsval, so reinterpret_cast here, not JSVAL_TO_OBJECT.
+     fp->thisp = (JSObject *) cx->nativeVp[1];
+     fp->argc = cx->nativeVpLen - 2;
+     fp->argv = cx->