/*
 * Copyright 2004-2019 Cray Inc.
 * Other additional copyright holders may be indicated within.
 *
 * The entirety of this work is licensed under the Apache License,
 * Version 2.0 (the "License"); you may not use this file except
 * in compliance with the License.
 *
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

#include "astutil.h"
#include "AstVisitorTraverse.h"
#include "CForLoop.h"
#include "driver.h"
#include "errorHandling.h"
#include "expr.h"
#include "ForallStmt.h"
#include "ForLoop.h"
#include "iterator.h"
#include "oldCollectors.h"
#include "optimizations.h"
#include "passes.h"
#include "resolution.h"
#include "resolveIntents.h"
#include "stlUtil.h"
#include "stmt.h"
#include "stringutil.h"
#include "symbol.h"
#include "view.h"
#include "wellknown.h"

bool iteratorsLowered = false;

//
// getTheIteratorFn(): get the original (user-written) iterator function
// that corresponds to an _iteratorClass type or symbol
// or a CallExpr therewith.
//

// 'ic' must be an instance of _iteratorClass
FnSymbol* getTheIteratorFn(Symbol* ic) {
  return getTheIteratorFn(ic->type);
}

// icType is either an _iteratorClass type or a tuple thereof
// When icType is a tuple, this function returns
//  the getIterator function for the first tuple element.
// When icType is an _iteratorClass, this function returns
//  the original iterator function.
FnSymbol* getTheIteratorFn(Type* icType)
{
  // the asserts document the current state
  AggregateType* icTypeAgg = toAggregateType(icType);
  INT_ASSERT(icTypeAgg);

  if (icType->symbol->hasFlag(FLAG_TUPLE)) {
    // A tuple of iterator classes -> get the first of them.
    Type* firstIcType = icTypeAgg->getField(1)->type;
    icTypeAgg = toAggregateType(firstIcType);
  }

  INT_ASSERT(icTypeAgg->symbol->hasFlag(FLAG_ITERATOR_CLASS) ||
             icTypeAgg->symbol->hasFlag(FLAG_ITERATOR_RECORD) );

  return icTypeAgg->iteratorInfo->iterator;
}

// When debugging, avoid assertion failures of the production version.
FnSymbol* debugGetTheIteratorFn(Type* type) {
  FnSymbol* result = NULL;
  if (AggregateType* agg = toAggregateType(type)) {
    if (agg->symbol->hasFlag(FLAG_TUPLE)) {
      // Through the end of resolution, the first field is the tuple size.
      if (!strcmp(agg->getField(1)->name, "size")) {
        if (!strcmp(agg->getField(2)->name, "x1"))
          result = getTheIteratorFn(agg->getField(2)->type);
      } else {
        result = getTheIteratorFn(agg);
      }
    } else if (agg->symbol->hasFlag(FLAG_ITERATOR_CLASS))
      result = getTheIteratorFn(agg);
    else if (agg->symbol->hasFlag(FLAG_ITERATOR_RECORD))
      if (IteratorInfo* ii = agg->iteratorInfo)
        result = ii->iterator;
  }
  return result;
}

FnSymbol* debugGetTheIteratorFn(ForLoop* forLoop) {
  if (forLoop)
    if (SymExpr* iteratorSE = forLoop->iteratorGet())
      return debugGetTheIteratorFn(iteratorSE->symbol()->type);
  // otherwise
  return NULL;
}


// This consistency check should probably be moved earlier in the compilation.
// It needs to be after resolution because it sets FLAG_INLINE_ITERATOR.
// Does it need to be recursive? (Currently, it is not.)
static void nonLeaderParCheckInt(FnSymbol* origfn, FnSymbol* fn, bool allowYields);

static void nonLeaderParCheck()
{
  //
  // check for parallel constructs in non-leader iterators
  //
  forv_Vec(FnSymbol, fn, gFnSymbols) {
    if (fn->isIterator() && !fn->hasFlag(FLAG_INLINE_ITERATOR)) {
      nonLeaderParCheckInt(fn, fn, false);
    }
  }
  USR_STOP();
}

bool isVirtualIterator(FnSymbol* iterFn) {
  bool retval = false;
  Type* IRtype = NULL;

  if (IteratorInfo* info = iterFn->iteratorInfo) {
    // A proper iterator.
    // Using info->iclass instead of ->irecord gives identical result.
    IRtype = info->irecord;

  } else {
    // An iterator forwarder. Converted to return-by-ref.
    // Use the former return type, now the type of the ret-arg formal.
    INT_ASSERT(iterFn->hasFlag(FLAG_FN_RETURNS_ITERATOR));
    INT_ASSERT(iterFn->hasFlag(FLAG_FN_RETARG));
    INT_ASSERT(iterFn->retType == dtVoid);
    for_formals(formal, iterFn)
      if (formal->hasFlag(FLAG_RETARG)) {
        INT_ASSERT(formal->isRef());
        IRtype = formal->getValType();
      }
  }
  
  if (AggregateType* at = toAggregateType(IRtype)) {
    Vec<AggregateType*>* children = &(at->dispatchChildren);

    if (children->n == 0) {
      retval = false;

    } else if (children->n == 1 && children->v[0] == dtObject) {
      retval = false;

    } else {
      retval = true;
    }
  }

  return retval;
}

static void nonLeaderParCheckInt(FnSymbol* origfn, FnSymbol* fn, bool markYields)
{
  std::vector<CallExpr*> calls;

  collectCallExprs(fn, calls);

  for_vector(CallExpr, call, calls) {
    if ((call->isPrimitive(PRIM_BLOCK_BEGIN)) ||
        (call->isPrimitive(PRIM_BLOCK_COBEGIN)) ||
        (call->isPrimitive(PRIM_BLOCK_COFORALL))) {
      // begin/cobegin/coforall *blocks* are eliminated earlier.
      // If they are not, need issue the USR_FATAL_CONT like below.
      INT_ASSERT(false);
    }

    FnSymbol* taskFn = resolvedToTaskFun(call);

    bool isParallelConstruct = taskFn &&
                               (taskFn->hasFlag(FLAG_BEGIN) ||
                                taskFn->hasFlag(FLAG_COBEGIN_OR_COFORALL));

    if (isParallelConstruct ||
        call->isNamed("_toLeader") ||
        call->isNamed("_toStandalone")) {
      USR_FATAL_CONT(call,
                     "invalid use of parallel construct in serial iterator");
    }

    if ((call->isPrimitive(PRIM_BLOCK_ON)) ||
        (call->isPrimitive(PRIM_BLOCK_BEGIN_ON)) ||
        (call->isPrimitive(PRIM_BLOCK_COBEGIN_ON)) ||
        (call->isPrimitive(PRIM_BLOCK_COFORALL_ON))) {
      // begin/cobegin/coforall *blocks* are eliminated earlier.
      // If they are not, check for PRIM_YIELD like below.
      INT_ASSERT(false);
    }
    if (markYields) {
      if (call->isPrimitive(PRIM_YIELD)) {
        origfn->addFlag(FLAG_YIELD_WITHIN_ON);
      }
    }
    if (taskFn) {
      // This used to be the body of the parallel or 'on' construct
      // so need to descend into it.
      nonLeaderParCheckInt(origfn, taskFn, taskFn->hasFlag(FLAG_ON));
    }
  }
}

static bool isCallVectorHazard(CallExpr* call,
                               std::map<FnSymbol*, bool> &fnHasVectorHazard);

namespace {
  class VectorHazardVisitor : public AstVisitorTraverse {
    public:
      VectorHazardVisitor (std::map<FnSymbol*, bool> &fnHasVectorHazard);
      virtual bool enterCallExpr (CallExpr*  node);

      bool hazard;
      CallExpr* reason;
      std::map<FnSymbol*, bool> &fnHasVectorHazard;
  };

  VectorHazardVisitor::VectorHazardVisitor(std::map<FnSymbol*, bool>
      &fnHasVectorHazard) : hazard(false), reason(NULL), fnHasVectorHazard(fnHasVectorHazard)
  {
  }

  bool VectorHazardVisitor::enterCallExpr(CallExpr* node) {
    bool foundHazard = isCallVectorHazard(node, fnHasVectorHazard);
    if (foundHazard && !hazard)
      reason = node;
    hazard |= foundHazard;
    return true;
  }
};

static bool doesFnHaveVectorHazard(FnSymbol* fn,
                                   std::map<FnSymbol*, bool> &fnHasVectorHazard)
{
  if (fnHasVectorHazard.count(fn) != 0)
    return fnHasVectorHazard[fn];

  // Mark it as a hazard if there is recursion
  fnHasVectorHazard[fn] = true;

  TypeSymbol* thisTypeSymbol = NULL;
  if (fn->_this != NULL)
    thisTypeSymbol = fn->_this->getValType()->symbol;

  CallExpr* reason = NULL;
  bool hazard = false;
  if (fn->hasFlag(FLAG_LLVM_READNONE) ||
      fn->hasFlag(FLAG_FN_SYNCHRONIZATION_FREE) ||
      fn->hasFlag(FLAG_ALLOCATOR) ||
      fn->hasFlag(FLAG_LOCALE_MODEL_ALLOC) ||
      fn->hasFlag(FLAG_LOCALE_MODEL_FREE) ||
      fn->hasFlag(FLAG_FUNCTION_TERMINATES_PROGRAM)) {
    // these methods shouldn't inhibit vectorization
    // (but of course not all vectorizers will be able to handle them)
  } else if (fn->hasFlag(FLAG_EXTERN)) {
    // Who knows what extern functions do!
    // To allow an extern function, mark it with FLAG_FN_SYNCHRONIZATION_FREE.
    hazard = true;
  } else if (thisTypeSymbol != NULL &&
             (thisTypeSymbol->hasFlag(FLAG_ATOMIC_TYPE) ||
              thisTypeSymbol->hasFlag(FLAG_SYNC) ||
              thisTypeSymbol->hasFlag(FLAG_SINGLE))) {
    // methods on synchronization constructs do synchronization!
    hazard = true;
  } else {
    // Check what the function body includes.
    VectorHazardVisitor v(fnHasVectorHazard);
    fn->body->accept(&v);

    hazard = v.hazard;
    reason = v.reason;
  }

  bool report = false;
  if (fReportVectorizedLoops) {
    ModuleSymbol *mod = toModuleSymbol(fn->getModule());
    INT_ASSERT(mod);

    report = (developer || mod->modTag == MOD_USER);
  }

  if (report && hazard) {
    FnSymbol *calledFn = NULL;
    if (reason)
      calledFn = reason->resolvedFunction();
    if (developer && calledFn)
      USR_PRINT(fn, "fn %s [%i] hazard -- calls synchronizing function %s [%i]", fn->name, fn->id, calledFn->name, calledFn->id);
    else if (calledFn)
      USR_PRINT(fn, "fn %s hazard -- calls synchronizing function %s", fn->name, calledFn->name);
    else if (reason && reason->isPrimitive(PRIM_VIRTUAL_METHOD_CALL))
      USR_PRINT(fn, "fn %s hazard -- calls virtual function", fn->name);
    else
      USR_PRINT(fn, "fn %s hazard -- other", fn->name);
  }

  fnHasVectorHazard[fn] = hazard;

  if (hazard == false)
    fn->addFlag(FLAG_FN_SYNCHRONIZATION_FREE);

  return hazard;
}

static bool isCallVectorHazard(CallExpr* call,
                               std::map<FnSymbol*, bool> &fnHasVectorHazard)
{
  bool hazard = false;
  if (call->isPrimitive(PRIM_VIRTUAL_METHOD_CALL)) {
    // Could be supported in the future by investigating all
    // overrides.
    hazard = true;
  } if (FnSymbol* calledFn = call->resolvedFunction()) {
    hazard |= doesFnHaveVectorHazard(calledFn, fnHasVectorHazard);
  }
  return hazard;
}

static void markVectorizableForallLoops()
{
  std::map<FnSymbol*, bool> fnHasVectorHazard;

  // The --force-vectorize flag exists mainly for testing and
  // disables this logic. Instead of disabling vectorization for
  // loops with vectorization hazards, all hazards will be ignored.
  if (fForceVectorize)
    return;

  // Check for loops over vectorize-only iterators
  forv_Vec(BlockStmt, block, gBlockStmts) {
    if (block->isLoopStmt()) {
      LoopStmt* loop = toLoopStmt(block);

      // Don't try to vectorize coforalls.
      if (loop->isCoforallLoop()) {
        loop->setHasVectorizationHazard(true);
      } else {

        bool hazard = false;

        // Does the loop body have any calls that are vectorization hazards?
        VectorHazardVisitor v(fnHasVectorHazard);
        loop->accept(&v);
        hazard = v.hazard;
        loop->setHasVectorizationHazard(hazard);

        bool report = false;
        if (fReportVectorizedLoops) {
          ModuleSymbol *mod = toModuleSymbol(loop->getModule());
          INT_ASSERT(mod);

          report = (developer || mod->modTag == MOD_USER);
        }

        if (report && hazard) {
          FnSymbol *fn = NULL;
          if (v.reason)
            fn = v.reason->resolvedFunction();
          if (developer && fn && v.hazard)
            USR_PRINT(loop, "Vectorization hazard -- calls synchronizing function %s [%i]", fn->name, fn->id);
          else if (fn && v.hazard)
            USR_PRINT(loop, "Vectorization hazard -- calls synchronizing function %s", fn->name);
          else if (v.hazard && v.reason && v.reason->isPrimitive(PRIM_VIRTUAL_METHOD_CALL))
            USR_PRINT(loop, "Vectorization hazard -- calls virtual function");
          else
            USR_PRINT(loop, "Vectorization hazard -- other");
        }
      }
    }
  }

  forv_Vec(ForallStmt, forall, gForallStmts) {
    // Consider functions called in the loop body.
    // Do they present any of the following hazards?
    //   * synchronization (use of sync variables or atomics)
    //   * reductions that aren't known to work with the vectorizer

    bool hazard = false;

    // Does the loop body have any calls that are vectorization hazards?
    VectorHazardVisitor v(fnHasVectorHazard);
    forall->loopBody()->accept(&v);
    hazard = v.hazard;

    // Another potential issue is defs inside the loop.
    // These are later, since many of these might be folded
    // away and/or inlined.

    // Does the loop use a reduction type that is not vectorizable yet?
    for_shadow_vars (shadow, temp, forall) {
      if (shadow->isReduce()) {
        if (ShadowVarSymbol* op = shadow->ReduceOpForAccumState()) {
          Type* accumType = shadow->type;
          Type* opType = op->type;
          bool ok = false;
          // Only vectorize + reductions on numbers
          if (is_int_type(accumType) ||
              is_uint_type(accumType) ||
              is_imag_type(accumType) ||
              is_real_type(accumType)
              // TODO: is_complex_type
             ) {
            if (startsWith(opType->symbol->name, "SumReduceScanOp"))
              ok = true;
          }
          if (ok == false)
            hazard = true;
        }
      }
    }
    forall->setHasVectorizationHazard(hazard);

    bool report = false;
    if (fReportVectorizedLoops) {
      ModuleSymbol *mod = toModuleSymbol(forall->getModule());
      INT_ASSERT(mod);

      report = (developer || mod->modTag == MOD_USER);
    }

    if (report && hazard) {
      FnSymbol *fn = NULL;
      if (v.reason)
        fn = v.reason->resolvedFunction();
      if (developer && fn && v.hazard)
        USR_PRINT(forall, "Vectorization hazard -- calls synchronizing function %s [%i]", fn->name, fn->id);
      else if (fn && v.hazard)
        USR_PRINT(forall, "Vectorization hazard -- calls synchronizing function %s", fn->name);
       else if (v.hazard && v.reason && v.reason->isPrimitive(PRIM_VIRTUAL_METHOD_CALL))
        USR_PRINT(forall, "Vectorization hazard -- calls virtual function");
      else
        USR_PRINT(forall, "Vectorization hazard -- other");
    }
  }
}


// Traverse all callers of this function, to see if it calls the base iterator recursively.
// Return true if so; false otherwise.
// This version uses a depth-first search.
static bool find_recursive_caller(FnSymbol* fn, FnSymbol* iter, Vec<FnSymbol*>& fnSet)
{
  // We've already seen this fn, and if we're still searching it means
  // we didn't return true last time, so return false this time too.
  if (fnSet.set_in(fn))
    return false;

  // OK, first time.  Mark this function as visited.
  fnSet.set_add(fn);

  // Traverse all its call sites.
  forv_Vec(CallExpr, call, *fn->calledBy)
  {
    // Extract the symbol representing the calling function.
    FnSymbol* caller = call->getFunction();

    // If the caller is the same as the iterator we started with,
    // it calls itself recursively.  Further searching is unnecessary.
    if (caller == iter)
      return true;

    // Ignore recursion through non-iterators.
    if (!(caller->isIterator() ||
          caller->retType->symbol->hasFlag(FLAG_ITERATOR_RECORD) ||
          isTaskFun(caller)))
      continue;

    // Otherwise, search recursively up the call chain.
    // If recursion was detected, no further searching is required.
    if (find_recursive_caller(caller, iter, fnSet))
      return true;

    // Otherwise, try the next call site.
  }

  // OK, no recursive callers at all.
  return false;
}


//
// Task functions in this set are like their enclosing recursive iterators -
// they are to be ignored when inlining.
// This property of course does not transfer upon function->copy().
//
static Vec<Symbol*> taskFunInRecursiveIteratorSet;


//
// Mark as recursive those iterators which call themselves (directly or indirectly).
// When dealing with recursive iterators, we only inline the body of the loop,
// not the iterator as a whole.
// Also mark the task functions in those iterators.
//
static void computeRecursiveIteratorSet() {
  compute_call_sites();

  // Walk all functions
  forv_Vec(FnSymbol, iter, gFnSymbols)
  {
    // And select just the iterators.
    if (!iter->isIterator())
      continue;

    // Determine if the iterator calls itself, either directly or indirectly.
    Vec<FnSymbol*> fnSet;   // Used to avoid recursion
    if (find_recursive_caller(iter, iter, fnSet)) {
      // If so, add the recursive iterator flag.
      iter->addFlag(FLAG_RECURSIVE_ITERATOR);
      if (iter->hasFlag(FLAG_YIELD_WITHIN_ON)) {
        USR_FATAL_CONT(iter, "'yield' within 'on' not currently supported within recursive serial iterators");
      }
    }
  }

  // Mark task functions, too, by adding to taskFunInRecursiveIteratorSet
  forv_Vec(FnSymbol, taskFn, gFnSymbols) {
    if (!isTaskFun(taskFn))
      continue;

    // Ascend the callers until we find a non-task function.
    FnSymbol* currFn = taskFn;
    while (true) {
      if (currFn->calledBy->n == 0) {
        // currFn has no callers, so by inference neither does taskFn.
        // That being the case, we can bail -- no harm done.
        break;
      }

      // If there are multiple callers, they should be clones of
      // each other, so any one will do(??).  We grab the first one.
      currFn = toFnSymbol(currFn->calledBy->v[0]->parentSymbol);
      INT_ASSERT(currFn);

      if (isTaskFun(currFn)) {
        // Task functions are not recursive by construction.
        // Otherwise this is an infinite loop.
        INT_ASSERT(currFn != taskFn);
      } else {
        // Found it!
        if (currFn && currFn->hasFlag(FLAG_RECURSIVE_ITERATOR))
          taskFunInRecursiveIteratorSet.set_add(taskFn);
        break;
      }
    }
  }
}


static bool containsYield(Expr* arg) {
  std::vector<CallExpr*> calls;
  collectCallExprs(arg, calls);
  for_vector(CallExpr, call, calls) {
    if (call->isPrimitive()) {
      if (call->isPrimitive(PRIM_YIELD))
        return true;
    } else if (FnSymbol* tfn = resolvedToTaskFun(call)) { // vass need this?
      if (containsYield(tfn->body))
        return true;
    }
  }
  return false;
}


//
// If a local block has no yields, returns, gotos or labels
// then it can safely be left unfragmented.
// A block whose blockInfo is PRIM_BLOCK_UNLOCAL must also be fragmented.
//
static bool leaveLocalBlockUnfragmented(BlockStmt* block) {
  std::vector<BaseAST*> asts;
  collect_asts(block, asts);

  for_vector(BaseAST, ast, asts) {

    // Check if the AST element is a GOTO.
    if (isGotoStmt(ast))
      return false;     // Yes, FAIL.

    // See if it is a yield or return statement.
    if (CallExpr* call = toCallExpr(ast)) {
      if (call->isPrimitive(PRIM_YIELD) || call->isPrimitive(PRIM_RETURN))
        return false;     // Yes, FAIL.

      // Check coforall et al. recursively.
      if (FnSymbol* taskFn = resolvedToTaskFun(call))
        if (!leaveLocalBlockUnfragmented(taskFn->body))
          return false;
    }

    // See if it is an unlocal block.
    if (BlockStmt* block = toBlockStmt(ast)) {
      // NOAKES 2014/11/25. Transitional.  Avoid calling blockInfoGet
      if (block->isLoopStmt() == true) {

      } else if (block->blockInfoGet() &&
                 block->blockInfoGet()->isPrimitive(PRIM_BLOCK_UNLOCAL)) {
        return false;     // Yes, FAIL.
      }
    }

    // See if it is a label.
    if (DefExpr* def = toDefExpr(ast)) {
      if (def && isLabelSymbol(def->sym))
        return false;     // Yes, FAIL.
    }
  }

  // OK.  No undesirable statements found.
  return true;
}


//
// replace a local block by smaller local blocks that do not contain
// goto statements, yields, returns labels or unlocal blocks.
//
static void
fragmentLocalBlocks() {
  Vec<Expr*> preVec; // stmts to be inserted before new local blocks
  Vec<Expr*> inVec;  // stmts to be inserted in new local blocks

  //
  // collect all local blocks which need to be fragmented
  //
  Vec<BlockStmt*> localBlocks; // old local blocks

  forv_Vec(BlockStmt, block, gBlockStmts) {
    // NOAKES 2014/11/25 Transitional.  Avoid calling blockInfoGet() on loops
    if (block->isLoopStmt() == true) {

    } else if (block->inTree()       &&
               block->blockInfoGet() &&
               block->blockInfoGet()->isPrimitive(PRIM_BLOCK_LOCAL) &&
               !leaveLocalBlockUnfragmented(block)) {
      localBlocks.add(block);
    }
  }

  //
  // collect first statements of local blocks into queue vector
  //
  Vec<Expr*> queue;

  forv_Vec(BlockStmt, block, localBlocks) {
    if (block->body.head)
      queue.add(block->body.head);
  }

  forv_Vec(Expr, expr, queue) {
    SET_LINENO(expr);

    for (Expr* current = expr; current; current = current->next) {
      bool      insertNewLocal = false;
      CallExpr* call           = toCallExpr(current);
      DefExpr*  def            = toDefExpr(current);

      //
      // If this statement is a yield, a return, a label definition, a
      // goto, a conditional, or a block, insert a new local block
      // that contains all the statements seen to this point (by
      // setting insertNewLocal to true) and add the first statements
      // of blocks and conditional blocks to the queue; otherwise, if
      // this statement is a definition, add it to preVec; otherwise,
      // add this statement to inVec.
      // Blocks and conditionals are visited in turn (breadth-first).
      //
      if ((call && (call->isPrimitive(PRIM_YIELD) ||
                    call->isPrimitive(PRIM_RETURN))) ||
          (def && isLabelSymbol(def->sym)) ||
          isGotoStmt(current) ||
          isCondStmt(current) ||
          isBlockStmt(current)) {
        insertNewLocal = true;

        if (BlockStmt* block = toBlockStmt(current)) {
          // NOAKES 2014/11/25 Transitional.
          // Avoid calling blockInfoGet() on loops
          if (block->isLoopStmt()   == false &&
              block->blockInfoGet() != NULL  &&
              block->blockInfoGet()->isPrimitive(PRIM_BLOCK_UNLOCAL)) {
            // UNLOCAL applies to a single LOCAL
            block->blockInfoGet()->remove();

          } else if (block->body.head) {
            queue.add(block->body.head);
          }
        } else if (CondStmt* cond = toCondStmt(current)) {
          if (cond->thenStmt && cond->thenStmt->body.head)
            queue.add(cond->thenStmt->body.head);
          if (cond->elseStmt && cond->elseStmt->body.head)
            queue.add(cond->elseStmt->body.head);
        }
      } else if (call && resolvedToTaskFun(call)) {
        // Do what the above would have done to a coforall/etc. block.
        insertNewLocal = true;

        Expr* taskfnBodyHead = call->resolvedFunction()->body->body.head;

        if (taskfnBodyHead) {
          queue.add(taskfnBodyHead);
        }

      } else if (isDefExpr(current)) {
        preVec.add(current);

      } else {
        inVec.add(current);
      }

      //
      // If ready to insert a new local block or we are on the last
      // statement in a block, insert a new local block containing all
      // the statements in inVec; move all the statements in preVec to
      // a point just before this new local block.
      //
      if (insertNewLocal || !current->next) {
        if (preVec.n || inVec.n) {
          BlockStmt* newLocalBlock = new BlockStmt();
          newLocalBlock->blockInfoSet(new CallExpr(PRIM_BLOCK_LOCAL));
          current->insertBefore(newLocalBlock);
          forv_Vec(Expr, expr, preVec) {
            newLocalBlock->insertBefore(expr->remove());
          }
          preVec.clear();
          forv_Vec(Expr, expr, inVec) {
            newLocalBlock->insertAtTail(expr->remove());
          }
          inVec.clear();
        }
      }
    }
  }

  //
  // remove old local blocks
  //
  forv_Vec(BlockStmt, block, localBlocks) {
    block->blockInfoGet()->remove();
  }
}


// In the body of the iterator, replace references to the iterator formals
// with references to fields in the iterator class.
//
// Multiple temps may be created for each formal.
static void
replaceIteratorFormalsWithIteratorFields(FnSymbol* iterator, Symbol* ic,
                                         SymExpr* se) {
  int count = 1;
  for_formals(formal, iterator) {
    if (formal->hasFlag(FLAG_RETARG) == false &&
        se->symbol() == formal) {
      // count is used to get the nth field out of the iterator class;
      // it is replaced by the field once the iterator class is created
      Expr* stmt = se->getStmtExpr();

      // Error variable arguments should have already been handled.
      INT_ASSERT(! (formal->defPoint->parentSymbol != se->parentSymbol &&
                     formal->hasFlag(FLAG_ERROR_VARIABLE)));

      QualifiedType qt = formal->qualType();
      // Workaround: use a ref type here
      // In the future, the Qualifier should be sufficient
      qt = qt.refToRefType();

      VarSymbol* tmp = newTemp(formal->name, qt);

      stmt->insertBefore(new DefExpr(tmp));

      // fixNumericalGetMemberPrims changes some of these
      // to PRIM_GET_MEMBER_VALUE
      stmt->insertBefore(new CallExpr(PRIM_MOVE, tmp, new CallExpr(PRIM_GET_MEMBER, ic, new_IntSymbol(count))));
      se->setSymbol(tmp);
    }
    count++;
  }
}

// If 'se' is an arg to PRIM_ITERATOR_RECORD_FIELD_VALUE_BY_FORMAL,
// replaceIteratorFormalsWithIteratorFields() would prevent this PRIM
// from being lowered in addLocalsToClassAndRecord().
static bool isPrimIRFieldByFormalArg(SymExpr* se) {
  if (CallExpr* parent = toCallExpr(se->parentExpr))
    if (parent->isPrimitive(PRIM_ITERATOR_RECORD_FIELD_VALUE_BY_FORMAL))
      if (se == parent->get(2))
        return true;
  return false;
}

static void replaceErrorFormalWithEnclosingError(SymExpr* se);

static void
replaceIteratorFormals(FnSymbol* iterator, Symbol* ic,
                       std::vector<SymExpr*> & symExprs) {
  bool throws = iterator->throwsError();

  for_vector(SymExpr, se, symExprs) {
    // For an inlined iterator, we don't want to track the error
    // argument through the iterator class. Instead of setting
    // a formal error argument, set an enclosing error variable.
    if (throws)
      replaceErrorFormalWithEnclosingError(se);
    // if se was not replaced by the above call...
    if (se->inTree() && ! isPrimIRFieldByFormalArg(se))
      replaceIteratorFormalsWithIteratorFields(iterator, ic, se);
  }
}

static Map<FnSymbol*,FnSymbol*> iteratorFnMap;
static FnSymbol* argBundleCopyFn = NULL;
static FnSymbol* argBundleFreeFn = NULL;
static AggregateType*  argBundleType = NULL;


//
// Build the definition for the abstract recursive iterator argument bundle
// class, if one does not exist yet.
//
static void
ensureArgBundleType(AggregateType* ct) {
  if (argBundleType == NULL) {
    argBundleType = new AggregateType(AGGREGATE_CLASS);
    TypeSymbol* aabSym = new TypeSymbol("chpl_argBundle", argBundleType);
    aabSym->addFlag(FLAG_NO_OBJECT);
    ct->symbol->defPoint->insertBefore(new DefExpr(aabSym));
  }
}


// Creates a function to free the arg bundle associated with a recursive iterator.
//
// This is currently done using just one free function for all arg bundles.
// There is (effectively) a switch statement inside the function that selects
// the appropriate bundle based on a function ID inserted into the last field
//  and frees its memory.
//
// The function looks approximately like:
// void chpl__freeRecursiveIteratorArgumentBundle(int loopBodyFnID, chpl_argBundle loopBodyFnArgs)
// {
//   chpl_free(loopBodyFnArgs);
//   // potentially more
//   return void;
// }
//
static void
createArgBundleFreeFn(AggregateType* ct, FnSymbol* loopBodyFnWrapper) {
  if (argBundleFreeFn == NULL) {
    // Create the shared function that frees recursive argument bundles.
    argBundleFreeFn = new FnSymbol("chpl__freeRecursiveIteratorArgumentBundle");
    argBundleFreeFn->insertFormalAtTail(new ArgSymbol(INTENT_CONST_IN, "loopBodyFnID", dtInt[INT_SIZE_DEFAULT]));
    argBundleFreeFn->insertFormalAtTail(new ArgSymbol(INTENT_CONST_IN, "loopBodyFnArgs", argBundleType));
    argBundleFreeFn->insertAtTail(callChplHereFree(argBundleFreeFn->getFormal(2)));
    argBundleFreeFn->insertAtTail(new CallExpr(PRIM_RETURN, gVoid));
    argBundleFreeFn->retType = dtVoid;
    ct->symbol->defPoint->insertBefore(new DefExpr(argBundleFreeFn));
  }

  bool addedRecursiveCall = false;
  ArgSymbol* loopBodyFnIDArg = argBundleFreeFn->getFormal(1);
  ArgSymbol* loopBodyFnArgsArg = argBundleFreeFn->getFormal(2);

  // Add a block for each concrete arg bundle type.
  BlockStmt* block = new BlockStmt();

  // Get the bundle arg and cast it to the concrete type.
  Symbol* loopBodyFnArgsTmp = newTemp("loopBodyFnArgsTmp", ct);
  block->insertAtTail(new DefExpr(loopBodyFnArgsTmp));
  block->insertAtTail(new CallExpr(PRIM_MOVE, loopBodyFnArgsTmp, new CallExpr(PRIM_CAST, ct->symbol, loopBodyFnArgsArg)));

  // The last field contains the index used to select the recursive iterator argument bundle.
  Symbol* lastField = NULL; // assume function pointer for function argument bundle
  for_fields(field, ct) {
    // If the bundle contains a pointer to another bundle,
    // its clause must call this free function recursively.
    if (field->type->symbol->hasFlag(FLAG_REF) &&
        field->getValType()->symbol->hasFlag(FLAG_LOOP_BODY_ARGUMENT_CLASS)) {
      addedRecursiveCall = true;

      // Load the function ID ref into a temp.
      Symbol* recursiveFnIDRef = newTemp("recursiveFnIDRef", lastField->type);
      block->insertAtTail(new DefExpr(recursiveFnIDRef));
      block->insertAtTail(new CallExpr(PRIM_MOVE, recursiveFnIDRef, new CallExpr(PRIM_GET_MEMBER_VALUE, loopBodyFnArgsTmp, lastField)));

      // Load the function ID for the recursive call.
      VarSymbol* recursiveFnID = newTemp("recursiveFnID", dtInt[INT_SIZE_DEFAULT]);
      block->insertAtTail(new DefExpr(recursiveFnID));
      block->insertAtTail(new CallExpr(PRIM_MOVE, recursiveFnID, new CallExpr(PRIM_DEREF, recursiveFnIDRef)));

      // Load the recursive bundle pointer into a temp.
      Symbol* recursiveArgsRef = newTemp("recursiveArgsRef", field->type);
      block->insertAtTail(new DefExpr(recursiveArgsRef));
      block->insertAtTail(new CallExpr(PRIM_MOVE, recursiveArgsRef, new CallExpr(PRIM_GET_MEMBER_VALUE, loopBodyFnArgsTmp, field)));

      // Load the recursive bundle.
      VarSymbol* recursiveArgsTmp = newTemp("recursiveArgs", argBundleType);
      block->insertAtTail(new DefExpr(recursiveArgsTmp));
      block->insertAtTail(new CallExpr(PRIM_MOVE, recursiveArgsTmp,
                                       new CallExpr(PRIM_CAST, argBundleType->symbol,
                                                    new CallExpr(PRIM_DEREF, recursiveArgsRef))));

      // Generate the recursive call.
      block->insertAtTail(new CallExpr(argBundleFreeFn, recursiveFnID, recursiveArgsTmp));
    }
    lastField = field;
  }

  // If we added a recursive call, we have to add a new clause to the free function:
  //   if (loopBodyFnIDArg == ftable_index(loopBodyFnWrapper))
  //   {
  //     ...
  //     chpl__freeRecursiveIteratorArgumentBundle(recursive_ID, recursive_bundle);
  //   }
  if (addedRecursiveCall) {
    Symbol* cond = newTemp("cond", dtBool);
    argBundleFreeFn->insertAtHead(new CondStmt(new SymExpr(cond), block));
    argBundleFreeFn->insertAtHead(new CallExpr(PRIM_MOVE, cond, new CallExpr(PRIM_EQUAL, loopBodyFnIDArg, new_IntSymbol(ftableMap[loopBodyFnWrapper]))));
    argBundleFreeFn->insertAtHead(new DefExpr(cond));
  }
}


// Creates a function to copy the arg bundle associated with a recursive iterator.
//
// This is currently done using just one copy function for all arg bundles.
// There is (effectively) a switch statement inside the function that selects
// the appropriate bundle based on loopBodyFnID and copies that part.
//
// The created function has the signature:
//  chpl_argBundle chpl__copyRecursiveIteratorArgumentBundle(int loopBodyFnID, chpl_argBundle loopBodyFnArgs)
//
static void
createArgBundleCopyFn(AggregateType* ct, FnSymbol* loopBodyFnWrapper) {
  if (argBundleCopyFn == NULL) {
    // Create the shared function that copies recursive argument bundles.
    // The initial implementation is just a shell.  After creation, it looks like:
    //  chpl_argBundle chpl__copyRecursiveIteratorArgumentBundle(int loopBodyFnID, chpl_argBundle loopBodyFnArgs)
    //  {
    //    chpl_argBundle tmp = nil;
    //    return tmp;
    //  }
    argBundleCopyFn = new FnSymbol("chpl__copyRecursiveIteratorArgumentBundle");
    argBundleCopyFn->insertFormalAtTail(new ArgSymbol(INTENT_CONST_IN, "loopBodyFnID", dtInt[INT_SIZE_DEFAULT]));
    argBundleCopyFn->insertFormalAtTail(new ArgSymbol(INTENT_CONST_IN, "loopBodyFnArgs", argBundleType));
    Symbol* tmp = newTemp("dummyBundle", argBundleType);
    argBundleCopyFn->insertAtTail(new DefExpr(tmp));
    argBundleCopyFn->insertAtTail(new CallExpr(PRIM_MOVE, tmp, gNil));
    argBundleCopyFn->insertAtTail(new CallExpr(PRIM_RETURN, tmp));
    argBundleCopyFn->retType = argBundleType;
    ct->symbol->defPoint->insertBefore(new DefExpr(argBundleCopyFn));
  }

  // Get the arguments to the (shared) copy function.
  ArgSymbol* loopBodyFnIDArg = argBundleCopyFn->getFormal(1);
  ArgSymbol* loopBodyFnArgsArg = argBundleCopyFn->getFormal(2);

  // Create a new body for this call to the copy function.
  //{
  //  args = (ct*)malloc(sizeof(ct));
  //  fnArgsTmp = (ct*)_loopBodyFnArgs;
  //  <field_type> tmp = fnArgsTmp->field;  // for each field
  //  int fnTmp = *lastTmp;                 // for each loop body arg field only
  //  <field_val_tmp> argsTmp = *tmp;       // for each loop body arg field only
  //  tmp = (<field_val_type>)
  //    chpl__copyRecursiveIteratorArgumentBundle(fnTmp, argsTmp);  // ditto
  //  args->field = tmp;                    // for each field
  //  ret = args;
  //}
  BlockStmt* block = new BlockStmt();
  Symbol* argBundle = newTemp("argBundle", ct);
  block->insertAtTail(new DefExpr(argBundle));
  insertChplHereAlloc(block->body.tail, true /*insertAfter*/,
                      argBundle, ct, newMemDesc("bundled args"));
  Symbol* loopBodyFnArgsTmp = newTemp("loopBodyFnArgsTmp", ct);
  block->insertAtTail(new DefExpr(loopBodyFnArgsTmp));
  block->insertAtTail(new CallExpr(PRIM_MOVE, loopBodyFnArgsTmp, new CallExpr(PRIM_CAST, ct->symbol, loopBodyFnArgsArg)));
  Symbol* lastTmp = NULL; // assume function pointer for function argument bundle
  for_fields(field, ct) {
    Symbol* tmp = newTemp(field->name, field->type);
    block->insertAtTail(new DefExpr(tmp));
    block->insertAtTail(new CallExpr(PRIM_MOVE, tmp, new CallExpr(PRIM_GET_MEMBER_VALUE, loopBodyFnArgsTmp, field)));

    if (field->type->symbol->hasFlag(FLAG_REF) &&
        field->getValType()->symbol->hasFlag(FLAG_LOOP_BODY_ARGUMENT_CLASS)) {
      VarSymbol* copy = newTemp(field->name, field->type);
      VarSymbol* recursiveFnID = newTemp("recursiveFnID", dtInt[INT_SIZE_DEFAULT]);
      VarSymbol* recursiveArgs = newTemp("recursiveArgs", argBundleType);
      block->insertAtTail(new DefExpr(copy));
      block->insertAtTail(new DefExpr(recursiveFnID));
      block->insertAtTail(new DefExpr(recursiveArgs));
      block->insertAtTail(new CallExpr(PRIM_MOVE, recursiveFnID, new CallExpr(PRIM_DEREF, lastTmp)));
      block->insertAtTail(new CallExpr(PRIM_MOVE, recursiveArgs,
                                       new CallExpr(PRIM_CAST, argBundleType->symbol,
                                                    new CallExpr(PRIM_DEREF, tmp))));
      VarSymbol* retTmp = newTemp("retTmp", argBundleCopyFn->retType);
      block->insertAtTail(new DefExpr(retTmp));
      VarSymbol* castTmp = newTemp("castTmp", field->getValType());
      block->insertAtTail(new DefExpr(castTmp));
      block->insertAtTail(new CallExpr(PRIM_MOVE, retTmp, new CallExpr(argBundleCopyFn, recursiveFnID, recursiveArgs)));
      block->insertAtTail(new CallExpr(PRIM_MOVE, castTmp, new CallExpr(PRIM_CAST, castTmp->type->symbol, retTmp)));
      block->insertAtTail(new CallExpr(PRIM_MOVE, copy, new CallExpr(PRIM_ADDR_OF, castTmp)));
      tmp = copy;
    }

    block->insertAtTail(new CallExpr(PRIM_SET_MEMBER, argBundle, field, tmp));
    lastTmp = tmp;
  }
  Symbol* ret = argBundleCopyFn->getReturnSymbol();
  block->insertAtTail(new CallExpr(PRIM_MOVE, ret, new CallExpr(PRIM_CAST, ret->type->symbol, argBundle)));

  // The body is placed in a conditional, so only the body corresponding to the current
  // fnArg is actually executed when the copy function is called:
  //  if (fnArg == ftable[loopBodyFnWrapper])
  //  { <body_as_defined_above> }
  Symbol* cond = newTemp(dtBool);
  argBundleCopyFn->insertBeforeEpilogue(new DefExpr(cond));
  argBundleCopyFn->insertBeforeEpilogue(new CallExpr(PRIM_MOVE, cond, new CallExpr(PRIM_EQUAL, loopBodyFnIDArg, new_IntSymbol(ftableMap[loopBodyFnWrapper]))));
  argBundleCopyFn->insertBeforeEpilogue(new CondStmt(new SymExpr(cond), block));
}


// This creates the class type carrying the arguments to a (possibly recursive)
// iterator function call and fills a temp with the passed-in values.
//
static AggregateType*
bundleLoopBodyFnArgsForIteratorFnCall(CallExpr* iteratorFnCall,
                                      CallExpr* loopBodyFnCall,
                                      FnSymbol* loopBodyFnWrapper) {
  FnSymbol* iteratorFn = iteratorFnCall->resolvedFunction();
  FnSymbol* loopBodyFn = loopBodyFnCall->resolvedFunction();

  // Create the argument bundle type
  AggregateType* ct = new AggregateType(AGGREGATE_CLASS);
  TypeSymbol* ts = new TypeSymbol("_fn_arg_bundle", ct);
  ts->addFlag(FLAG_NO_OBJECT);
  ts->addFlag(FLAG_LOOP_BODY_ARGUMENT_CLASS);
  iteratorFnCall->parentSymbol->defPoint->insertBefore(new DefExpr(ts));

  // And its corresponding reference type.
  AggregateType* rct = new AggregateType(AGGREGATE_CLASS);
  TypeSymbol* rts = new TypeSymbol("_ref_fn_arg_bundle", rct);
  rts->addFlag(FLAG_NO_OBJECT);
  rts->addFlag(FLAG_REF);
  iteratorFnCall->parentSymbol->defPoint->insertBefore(new DefExpr(rts));
  rct->fields.insertAtTail(new DefExpr(new VarSymbol("_val", ct)));
  ct->refType = rct;

  // Create the argument bundle.
  // args = (ct*)malloc(sizeof(ct));
  VarSymbol* argBundle = newTemp("argBundle", ct);
  iteratorFnCall->insertBefore(new DefExpr(argBundle));
  insertChplHereAlloc(iteratorFnCall, false /*insertAfter*/, argBundle,
                      ct, newMemDesc("bundled args"));
  iteratorFnCall->insertAtTail(argBundle);
  iteratorFnCall->insertAfter(callChplHereFree(argBundle));

  // loopBodyWrapper(int index, ct* fn_args) {
  //   loopBodyFn(index);
  // }
  ArgSymbol* wrapperIndexArg = loopBodyFn->getFormal(1)->copy();
  loopBodyFnWrapper->insertFormalAtTail(wrapperIndexArg);
  ArgSymbol* wrapperArgsArg = new ArgSymbol(INTENT_CONST_IN, "fn_args", ct);
  loopBodyFnWrapper->insertFormalAtTail(wrapperArgsArg);
  CallExpr* loopBodyFnWrapperCall = new CallExpr(loopBodyFn, wrapperIndexArg);

  int i = 1;
  Expr* lastActual = NULL;
  while (loopBodyFnCall->numActuals() >= 2) {
    // Skip the index arg.
    Expr* actual = loopBodyFnCall->get(2)->remove();
    ArgSymbol* formal = loopBodyFn->getFormal(i+1);
    // The formal's ref-ness must transfer to 'field' below.
    bool fieldIsRef = formal->isRef();
    // Todo: replace ref type with QUAL_REF.
    Type* fieldType = fieldIsRef ? formal->type->getRefType() : formal->type;

    // Create a field for this arg.
    VarSymbol* field = new VarSymbol(astr("_arg", istr(i++)), fieldType);
    ct->fields.insertAtTail(new DefExpr(field));
    // Make sure the older code is aligned with newer representation.
    INT_ASSERT(fieldIsRef == fieldType->isRef());
    INT_ASSERT(fieldIsRef == field->type->symbol->hasFlag(FLAG_REF));

    if (fieldIsRef) {
      // Does anything need to be done here if the iterator invokes
      // task function(s)?
      if (iteratorFn->hasFlag(FLAG_ITERATOR_WITH_ON) &&
          field->getValType()->symbol->hasFlag(FLAG_LOOP_BODY_ARGUMENT_CLASS)) {

        // For recursive args in forked bodies,
        // build up a local copy of the argument bundle (recursive copy).

        // This code assumes that the depth arg for the recursive loopBodyFnCall
        // precedes the recursive arg bundle in the list of actual arguments.
        //
        // int index = *lastActual;
        // ct* rec_args = *actual;
        // ct* rec_args_copy = (ct*)chpl__copyRecursiveIteratorArgumentBundle(index, rec_args);
        //
        VarSymbol* tmp = newTemp(field->name, field->type);
        VarSymbol* recursiveFnID = newTemp("recursiveFnID", dtInt[INT_SIZE_DEFAULT]);
        VarSymbol* loopBodyFnArgsTmp = newTemp("loopBodyFnArgsTmp", argBundleType);
        iteratorFnCall->insertBefore(new DefExpr(tmp));
        iteratorFnCall->insertBefore(new DefExpr(recursiveFnID));
        iteratorFnCall->insertBefore(new DefExpr(loopBodyFnArgsTmp));
        iteratorFnCall->insertBefore(new CallExpr(PRIM_MOVE, recursiveFnID, new CallExpr(PRIM_DEREF, lastActual->copy())));
        iteratorFnCall->insertBefore(new CallExpr(PRIM_MOVE, loopBodyFnArgsTmp,
                                                  new CallExpr(PRIM_CAST, argBundleType->symbol,
                                                               new CallExpr(PRIM_DEREF, actual))));
        VarSymbol* retTmp = newTemp("retTmp", argBundleCopyFn->retType);
        iteratorFnCall->insertBefore(new DefExpr(retTmp));
        VarSymbol* castTmp = newTemp("castTmp", field->getValType());
        iteratorFnCall->insertBefore(new DefExpr(castTmp));
        iteratorFnCall->insertBefore(new CallExpr(PRIM_MOVE, retTmp, new CallExpr(argBundleCopyFn, recursiveFnID, loopBodyFnArgsTmp)));
        iteratorFnCall->insertBefore(new CallExpr(PRIM_MOVE, castTmp, new CallExpr(PRIM_CAST, castTmp->type->symbol, retTmp)));
        iteratorFnCall->insertBefore(new CallExpr(PRIM_MOVE, tmp, new CallExpr(PRIM_ADDR_OF, castTmp)));
        iteratorFnCall->insertAfter(new CallExpr(argBundleFreeFn, recursiveFnID, retTmp));
        actual = new SymExpr(tmp);
      }
      else if (SymExpr* actualSE = toSymExpr(actual)) {
        Symbol* actualSym = actualSE->symbol();
        if (actualSym->isConstValWillNotChange())
          field->addFlag(FLAG_REF_TO_IMMUTABLE);
        if (!actualSym->isRef()) {
          // We are passing the actual by reference. Add a ref temp.
          VarSymbol* refTmp = newTemp("refTmp", fieldType);
          iteratorFnCall->insertBefore(new DefExpr(refTmp));
          iteratorFnCall->insertBefore(new CallExpr(PRIM_MOVE, refTmp, new CallExpr(PRIM_ADDR_OF, actual)));
          actual = new SymExpr(refTmp);
        }
      }
    }

    iteratorFnCall->insertBefore(new CallExpr(PRIM_SET_MEMBER, argBundle, field, actual));

    VarSymbol* tmp = newTemp(field->name, field->type);

    // In the wrapper function, moves the current arg bundle field into a temp
    // (unbundles it) and adds it to the args used  to call the loop body function.
    loopBodyFnWrapper->insertAtTail(new DefExpr(tmp));
    loopBodyFnWrapper->insertAtTail(new CallExpr(PRIM_MOVE, tmp, new CallExpr(PRIM_GET_MEMBER_VALUE, wrapperArgsArg, field)));
    loopBodyFnWrapperCall->insertAtTail(tmp);
    lastActual = actual; // assume function pointer for function argument bundle
  }
  loopBodyFnCall->remove();
  loopBodyFnWrapper->insertAtTail(loopBodyFnWrapperCall);
  loopBodyFnWrapper->retType = dtVoid;
  loopBodyFnWrapper->insertAtTail(new CallExpr(PRIM_RETURN, gVoid));
  ensureArgBundleType(ct);
  createArgBundleCopyFn(ct, loopBodyFnWrapper);
  createArgBundleFreeFn(ct, loopBodyFnWrapper);
  return ct;
}


//
// return the number of local blocks that lexically enclose 'expr',
// stopping the search outward when encountering the outer block
// todo: should this count carry across calls to task functions?
//
static int
countEnclosingLocalBlocks(Expr* expr, BlockStmt* outer = NULL) {
  int count = 0;

  for (Expr* tmp = expr; tmp && tmp != outer; tmp = tmp->parentExpr) {
    if (BlockStmt* blk = toBlockStmt(tmp)) {
      // NOAKES 2014/11/25 Transitional.  Avoid calling blockInfoGet() on loops
      if (blk->isLoopStmt() == true) {

      } else if (blk->blockInfoGet() == NULL) {

      } else if (blk->blockInfoGet()->isPrimitive(PRIM_BLOCK_LOCAL)) {
        count++;

      } else if (blk->blockInfoGet()->isPrimitive(PRIM_BLOCK_UNLOCAL)) {
        count--;
      }
    }
  }
  return count;
}


static Map<FnSymbol*,FnSymbol*> loopBodyFnArgsSuppliedMap;

//
// Thread loopBodyFnIDArg, loopBodyFnArgArgs through calls to task functions.
// We assume that threadLoopBodyFnArgs() sees each call at most once.
// The call needs to be augmented to pass these two things as actuals.
//
// The callee taskFn needs to be augmented with two new formals.
// Note that the original taskFn needs to be left around unmodified.
// That's because it is likely invoked from the original recursive iterator
// before (in the AST nesting sense, that is) the point where we
// implement recursion with (loopBodyFnIDArg, loopBodyFnArgArgs).
//
// The call needs to be redirected to this augmented taskFn copy.
// 'taskFnP' is updated to point to this copy as well.
//
static void threadLoopBodyFnArgs(CallExpr* call,
                                 ArgSymbol* loopBodyFnIDArg,
                                 ArgSymbol* loopBodyFnArgArgs,
                                 FnSymbol**  taskFnP,
                                 ArgSymbol** newIdArgP,
                                 ArgSymbol** newArgArgsP)
{
  FnSymbol* origTFn = *taskFnP; // same as call->isResolved()
  FnSymbol* copyTFn = loopBodyFnArgsSuppliedMap.get(origTFn);

  if (copyTFn) {
    // This might happen. It should work by removing the assert,
    // but has not been tested.
    INT_ASSERT(false);

    int nf = copyTFn->numFormals();
    *newIdArgP = copyTFn->getFormal(nf-1);
    *newArgArgsP = copyTFn->getFormal(nf);

  } else {
    // Created an augmented copy.
    copyTFn = origTFn->copy();
    copyTFn->name = astr("rec_iter_task_fn_", copyTFn->name);
    origTFn->defPoint->insertBefore(new DefExpr(copyTFn));
    loopBodyFnArgsSuppliedMap.put(origTFn, copyTFn);

    // Add the formals, created just like in createIteratorFn().
    ArgSymbol* newIdArg = new ArgSymbol(INTENT_CONST_IN, loopBodyFnIDArg->name, loopBodyFnIDArg->type);
    copyTFn->insertFormalAtTail(newIdArg);
    ArgSymbol* newArgArgs = new ArgSymbol(INTENT_CONST_IN, loopBodyFnArgArgs->name, loopBodyFnArgArgs->type);
    copyTFn->insertFormalAtTail(newArgArgs);

    *newIdArgP = newIdArg;
    *newArgArgsP = newArgArgs;
  }

  // This 'call' has not been processed yet. If it has been,
  // it should not be modified again.
  INT_ASSERT(call->numActuals() + 2 == copyTFn->numFormals());

  // Augment the call as well.
  call->insertAtTail(loopBodyFnIDArg);
  call->insertAtTail(loopBodyFnArgArgs);

  // Redirect call to the taskFn copy.
  call->baseExpr->replace(new SymExpr(copyTFn));
  *taskFnP = copyTFn;
}


// Replace each yield with a call to the loop body function,
// and remove return statements.
// If 'asts' are the body of a task function, the return statement
// must stay because it is a required part of a function.
static void convertYieldsAndReturns(std::vector<CallExpr*>& calls, Symbol* index,
                                    ArgSymbol* loopBodyFnIDArg, ArgSymbol* loopBodyFnArgArgs)
{
  for_vector(CallExpr, call, calls) {
    if (call->isPrimitive(PRIM_YIELD)) {
      SET_LINENO(call);
      Symbol* yieldedIndex = newTemp("_yieldedIndex", index->type);
      call->insertBefore(new DefExpr(yieldedIndex));
      call->insertBefore(new CallExpr(PRIM_MOVE, yieldedIndex, call->get(1)->remove()));
      Expr* loopBodyFnCall = new CallExpr(PRIM_FTABLE_CALL, loopBodyFnIDArg, yieldedIndex, loopBodyFnArgArgs);

      // The inserted loop body function call is surrounded by a number of
      // unlocal blocks equal to the number of local blocks surrounding the original call.
      int count = countEnclosingLocalBlocks(call);
      Expr* callOrBlk = loopBodyFnCall;
      while (count-- > 0)
      {
        BlockStmt* blk = new BlockStmt(callOrBlk);
        blk->blockInfoSet(new CallExpr(PRIM_BLOCK_UNLOCAL));
        callOrBlk = blk;
      }
      call->replace(callOrBlk);
    } else if (call->isPrimitive(PRIM_RETURN) && !isTaskFun(toFnSymbol(call->parentSymbol))) {
      // Just remove a return.
      call->remove();
    } else if (FnSymbol* taskFn = resolvedToTaskFun(call)) {
      // Process yields within (a clone of) taskFn.
      // Todo: do this only if taskFn has yields (directly or not).
      ArgSymbol *tIDArg, *tArgArgs;
      threadLoopBodyFnArgs(call, loopBodyFnIDArg, loopBodyFnArgArgs,
                           &taskFn, &tIDArg, &tArgArgs);
      // both 'taskFn' and 'call' are updated by threadLoopBodyFnArgs()
      INT_ASSERT(call->resolvedFunction() == taskFn);
      std::vector<CallExpr*> taskCallExprs;
      collectCallExprs(taskFn, taskCallExprs);
      convertYieldsAndReturns(taskCallExprs, index, tIDArg, tArgArgs);
    }
  }
}


// Returns true if the given function contains an on statement; false otherwise.
// "Contains" includes "had a coforall/etc. block, now replaced with a call".
static bool fnContainsOn(FnSymbol* fn)
{
  std::vector<CallExpr*> calls;

  collectCallExprs(fn, calls);

  for_vector(CallExpr, call, calls) {
    if (call->isPrimitive(PRIM_BLOCK_ON) ||
        call->isPrimitive(PRIM_BLOCK_BEGIN_ON) ||
        call->isPrimitive(PRIM_BLOCK_COBEGIN_ON) ||
        call->isPrimitive(PRIM_BLOCK_COFORALL_ON))
      return true;

    if (FnSymbol* taskFn = resolvedToTaskFun(call))
      if (fnContainsOn(taskFn))
        return true;
  }

  return false;
}


static FnSymbol*
createIteratorFn(FnSymbol* iterator, CallExpr* iteratorFnCall, Symbol* index,
                 CallExpr* loopBodyFnCall, FnSymbol* loopBodyFnWrapper,
                 Symbol* ic)
{
  SET_LINENO(iterator);
  FnSymbol* iteratorFn = new FnSymbol(astr("_rec_iter_fn_", iterator->name));

  // identify and mark iterators with on-statements
  if (fnContainsOn(iterator))
    iteratorFn->addFlag(FLAG_ITERATOR_WITH_ON);

  // Now calls the newly-created iterator function.
  iteratorFnCall->baseExpr->replace(new SymExpr(iteratorFn));
  AggregateType* argsBundleType =
    bundleLoopBodyFnArgsForIteratorFnCall(iteratorFnCall, loopBodyFnCall, loopBodyFnWrapper);

  iteratorFn->body = iterator->body->copy();
  iterator->defPoint->insertBefore(new DefExpr(iteratorFn));
  std::vector<SymExpr*> symExprs;
  std::vector<CallExpr*> calls;
  collectSymExprs(iteratorFn, symExprs);
  collectCallExprs(iteratorFn, calls);
  ArgSymbol* icArg = new ArgSymbol(blankIntentForType(ic->type), "_ic", ic->type);
  iteratorFn->insertFormalAtTail(icArg);

  replaceIteratorFormals(iterator, icArg, symExprs);

  ArgSymbol* loopBodyFnIDArg = new ArgSymbol(INTENT_CONST_IN, "_loopBodyFnID", dtInt[INT_SIZE_DEFAULT]);
  iteratorFn->insertFormalAtTail(loopBodyFnIDArg);
  ArgSymbol* loopBodyFnArgArgs = new ArgSymbol(INTENT_CONST_IN, "_loopBodyFnArgs", argsBundleType);
  iteratorFn->insertFormalAtTail(loopBodyFnArgArgs);

  convertYieldsAndReturns(calls, index, loopBodyFnIDArg, loopBodyFnArgArgs);

  iteratorFn->retType = dtVoid;
  iteratorFn->insertAtTail(new CallExpr(PRIM_RETURN, gVoid));
  iteratorFn->removeFlag(FLAG_INLINE_ITERATOR);
  iteratorFn->removeFlag(FLAG_ITERATOR_FN);

  return iteratorFn;
}

/// \param call A for loop block primitive.
static void
expandRecursiveIteratorInline(ForLoop* forLoop)
{
  SET_LINENO(forLoop);

  FnSymbol*  parent            = toFnSymbol(forLoop->parentSymbol);

  // create a nested function for the loop body (call->parentExpr), and then transform
  // the iterator into a function that takes this nested function as an argument
  FnSymbol*  loopBodyFn        = new FnSymbol(astr("_rec_iter_loop_", parent->name));

  // The index is passed to the loop body function as its first argument.
  Symbol*    index             = forLoop->indexGet()->symbol();
  ArgSymbol* indexArg          = new ArgSymbol(blankIntentForType(index->type), "_index", index->type);

  // The recursive iterator loop wrapper is ... .
  FnSymbol*  loopBodyFnWrapper = new FnSymbol(astr("_rec_iter_loop_wrapper_", parent->name));

  loopBodyFn->insertFormalAtTail(indexArg);

  forLoop->insertBefore(new DefExpr(loopBodyFn));

  parent->defPoint->insertBefore(new DefExpr(loopBodyFnWrapper));

  ftableVec.push_back(loopBodyFnWrapper);
  ftableMap[loopBodyFnWrapper] = ftableVec.size()-1;

  //
  // insert a call to the iterator function (using iterator as a
  // placeholder); build a call to loopBodyFn which will be
  // removed later and its arguments passed to the iteratorFn (we
  // build this to capture the actual arguments that should be
  // passed to it when this function is flattened)
  //
  Symbol*    ic             = forLoop->iteratorGet()->symbol();
  FnSymbol*  iterator       = getTheIteratorFn(ic);
  CallExpr*  iteratorFnCall = new CallExpr(iterator, ic, new_IntSymbol(ftableMap[loopBodyFnWrapper]));

  // replace function in iteratorFnCall with iterator function once that is created
  CallExpr*  loopBodyFnCall = new CallExpr(loopBodyFn, gNone);

  // use and remove loopBodyFnCall later
  // We expect this call to cause the loop body function to be converted like a
  // "normal" iterator function body.
  forLoop->insertBefore(loopBodyFnCall);
  forLoop->insertBefore(iteratorFnCall);

  // Copy the body of forLoop into the (new) loop body function
  // and remove forLoop.
  loopBodyFn->insertAtTail(forLoop->copyBody());
  forLoop->remove();

  // Now populate the loop body function.
  // Load the index arg.
  loopBodyFn->insertAtHead(new CallExpr(PRIM_MOVE, index, indexArg));
  loopBodyFn->insertAtHead(index->defPoint->remove());

  // Return nothing.
  loopBodyFn->insertAtTail(new CallExpr(PRIM_RETURN, gVoid));
  loopBodyFn->retType = dtVoid;

  // Move the loop body function out to the module level.
  flattenNestedFunction(loopBodyFn);

  FnSymbol* iteratorFn = iteratorFnMap.get(iterator);

  if (!iteratorFn) {
    iteratorFn = createIteratorFn(iterator,
                                  iteratorFnCall,
                                  index,
                                  loopBodyFnCall,
                                  loopBodyFnWrapper,
                                  ic);

    iteratorFnMap.put(iterator, iteratorFn);
  } else {
    iteratorFnCall->baseExpr->replace(new SymExpr(iteratorFn));
    bundleLoopBodyFnArgsForIteratorFnCall(iteratorFnCall, loopBodyFnCall, loopBodyFnWrapper);

    Symbol* argBundleTmp = newTemp("argBundleTmp", iteratorFn->getFormal(3)->type);

    iteratorFnCall->insertBefore(new DefExpr(argBundleTmp));
    iteratorFnCall->insertBefore(new CallExpr(PRIM_MOVE,
                                              argBundleTmp,
                                              new CallExpr(PRIM_CAST,
                                                           argBundleTmp->type->symbol,
                                                           iteratorFnCall->get(3)->remove())));

    iteratorFnCall->insertAtTail(argBundleTmp);
  }
}


typedef Map<FnSymbol*,FnSymbol*> TaskFnCopyMap;

static void
expandBodyForIteratorInline(ForLoop*       forLoop,
                            BlockStmt*     ibody,
                            Symbol*        index);


static void
expandBodyForIteratorInline(ForLoop*       forLoop,
                            BlockStmt*     ibody,
                            Symbol*        index,
                            bool           inTaskFn,
                            TaskFnCopyMap& taskFnCopies,
                            bool&          addErrorArgToCall);

static void markLoopProperties(ForLoop* forLoop, BlockStmt* ibody) {
  bool isOrderIndependent = forLoop->isOrderIndependent();
  bool hasVectorHazard = forLoop->hasVectorizationHazard();

  // if the loop being expanded was order independent, all of the yielding
  // loops in the body are also order independent. Note that this must occur
  // after the ibody replaces the forLoop since findEnclosingLoop() requires
  // that its argument be in the AST. It must occur before yields are
  // replaced in the functions below though.
  if (isOrderIndependent || hasVectorHazard) {
    std::vector<CallExpr*> callExprs;

    collectCallExprs(ibody, callExprs);

    for_vector(CallExpr, call, callExprs) {
      if (call->isPrimitive(PRIM_YIELD)) {
        if (LoopStmt* loop = LoopStmt::findEnclosingLoop(call)) {
          if (loop->isCoforallLoop() == false) {
            loop->orderIndependentSet(isOrderIndependent);
            loop->setHasVectorizationHazard(hasVectorHazard);
          }
        }
      }
    }
  }
}

/// \param call A for loop block primitive.
static bool
// Returns true if the given ForLoop was handled (converted and removed from
// the tree); false otherwise.
expandIteratorInline(ForLoop* forLoop) {
  Symbol*   ic       = forLoop->iteratorGet()->symbol();
  FnSymbol* iterator = getTheIteratorFn(ic);

  if (fReportInlinedIterators) {
    ModuleSymbol *mod = iterator->getModule();

    if (developer || mod->modTag == MOD_USER) {
      printf("Inlined iterator (%s) in module %s (%s:%d)\n", iterator->cname,
              mod->name, iterator->fname(), iterator->linenum());
    }
  }


  if (iterator->hasFlag(FLAG_RECURSIVE_ITERATOR)) {
    // NOAKES 2014/11/30  Only 6 tests, some with minor variations, use this path

    // The iterand of the forLoop is marked as a recursive iterator.  This is
    // the case we are looking for.  But first, we have to weed out a few
    // troublesome cases.

    // sjd:
    // loops over recursive iterators in recursive iterators only need
    // to be handled in the recursive iterator function
    //
    if (forLoop->parentSymbol->hasFlag(FLAG_RECURSIVE_ITERATOR)) {
      return false;
    // vass: ditto for task functions called from recursive iterators
    } else if (taskFunInRecursiveIteratorSet.set_in(forLoop->parentSymbol)) {
      return false;
    } else if (containsYield(forLoop)) {
      // Inlining a recursive iterator into a loop with a 'yield' pushes
      // that 'yield' into one of _rec_ functions, where it dangles. Ex.:
      // test/library/standard/FileSystem/filerator/bradc/findfiles-par.chpl
      return false;
    } else {
      expandRecursiveIteratorInline(forLoop);
      INT_ASSERT(!forLoop->inTree());
      return true;
    }

  } else {
    SET_LINENO(forLoop);

    Symbol*       index = forLoop->indexGet()->symbol();
    BlockStmt*    ibody = iterator->body->copy();

    if (! preserveInlinedLineNumbers)
      reset_ast_loc(ibody, forLoop);

    // and the entire for loop block is replaced by the iterator body.
    forLoop->replace(ibody);

    markLoopProperties(forLoop, ibody);

    // Replace yield statements in the inlined iterator body with copies
    // of the body of the For Loop that invoked the iterator, substituting
    // the yielded index for the iterator formal.
    expandBodyForIteratorInline(forLoop, ibody, index);

    std::vector<SymExpr*> symExprs;
    collectSymExprs(ibody, symExprs);
    replaceIteratorFormals(iterator, ic, symExprs);

    // We can return true if forLoop has been removed from the tree.
    INT_ASSERT(!forLoop->inTree());
    return true;
  }
}

static void
expandBodyForIteratorInline(ForLoop*       forLoop,
                            BlockStmt*     ibody,
                            Symbol*        index) {
  TaskFnCopyMap taskFnCopies;
  bool addErrorArgToCall = false;
  expandBodyForIteratorInline(forLoop, ibody, index, false,
                              taskFnCopies, addErrorArgToCall);
  INT_ASSERT(addErrorArgToCall == false); // case not handled
}

/*
 A Goto ErrorHandling label in the body of a loop will not
 be properly handled when the iterator is inlined into a task
 function because the label will be outside of the task function.

 These functions help to compensate by discovering any GotoStmt
 that needs to be adjusted.
 */

// For task functions, we need an error argument for
// parallel to process them correctly, but the error
// is actually passed through the end-count error list.
void
addDummyErrorArgumentToCall(CallExpr* call)
{
  VarSymbol* errorTmp = newTemp("dummy_error", dtError);
  errorTmp->addFlag(FLAG_ERROR_VARIABLE);
  call->insertBefore(new DefExpr(errorTmp));
  call->insertAtTail(errorTmp);
}

static ArgSymbol*
findErrorFormalForFn(FnSymbol* fn)
{
  ArgSymbol* errorArg = NULL;
  if (fn->throwsError()) {
    for_formals(formal, fn) {
      // TODO: should this check FLAG_ERROR_VARIABLE?
      if (formal->getValType() == dtError)
        errorArg = formal;
    }
  }
  return errorArg;
}

static ArgSymbol*
addDummyErrorFormalToFn(FnSymbol* fn)
{
  ArgSymbol* errorArg = new ArgSymbol(INTENT_REF, "error_out", dtError);
  errorArg->addFlag(FLAG_ERROR_VARIABLE);
  fn->insertFormalAtTail(errorArg);
  fn->throwsErrorInit();
  return errorArg;
}

// Adjust an error handling exit in two cases:
// 1. the GotoStmt points outside of this function
// 2. the error variable updated is outside of this function.
// In both cases, we need to change it to update this function's
// error_out argument and exit this function.
void
fixupErrorHandlingExits(BlockStmt* body, bool& adjustCaller) {
  FnSymbol* fn = toFnSymbol(body->parentSymbol);
  INT_ASSERT(fn);

  std::vector<GotoStmt*> gotos;
  collectGotoStmts(body, gotos);

  for_vector(GotoStmt, g, gotos) {

    if (g->gotoTag == GOTO_ERROR_HANDLING ||
        g->gotoTag == GOTO_BREAK_ERROR_HANDLING ||
        g->gotoTag == GOTO_RETURN) {
      // Does the target of this Goto exist within the same function?
      LabelSymbol* target = g->gotoTarget();
      INT_ASSERT(target->defPoint->parentSymbol);

      // Do we need to fix the Goto because it points outside the function?
      bool fixTarget = (target->defPoint->parentSymbol != body->parentSymbol);
      // Do we need to fix the error handling because it uses an Error
      // variable outside of this function?
      bool fixError = false;

      Expr* oldErrorSrc = NULL;
      Expr* oldErrorDst = NULL;

      // Find the PRIM_MOVE/PRIM_ASSIGN setting the error variable
      CallExpr* prevMoveErr = NULL;
      Expr* cur = g->prev;
      while (cur != NULL) {
        if (CallExpr* call = toCallExpr(cur)) {
          if ((call->isPrimitive(PRIM_MOVE) ||
               call->isPrimitive(PRIM_ASSIGN)) &&
              call->get(1)->typeInfo() == dtError) {
            prevMoveErr = call;
            break;
          }
        }
        cur = cur->prev;
      }

      // TODO -- only consider errors with FLAG_ERROR_VAR?
      // Is it possible for user code to look like this?

      if (prevMoveErr != NULL) {
        oldErrorDst = prevMoveErr->get(1);
        oldErrorSrc = prevMoveErr->get(2);
        INT_ASSERT(oldErrorDst);
        INT_ASSERT(oldErrorDst->typeInfo() == dtError);
        SymExpr* dstSe = toSymExpr(oldErrorDst);
        INT_ASSERT(dstSe);
        Symbol* dstSym = dstSe->symbol();
        INT_ASSERT(dstSym->defPoint->parentSymbol);
        if (dstSym->defPoint->parentSymbol != fn)
          fixError = true;
      }

      if (fixTarget || fixError) {
        // Find the function epilogue
        LabelSymbol* epilogue = fn->getOrCreateEpilogueLabel();

        // Find the formal to store the error
        // ... find the (last) error argument
        ArgSymbol* errorArg = findErrorFormalForFn(fn);
        if (errorArg == NULL) {
          errorArg = addDummyErrorFormalToFn(fn);
          adjustCaller = true;
        }
        INT_ASSERT(errorArg != NULL);

        // Replace it with assign / goto return
        if (oldErrorDst && oldErrorSrc) {
          oldErrorDst->remove();
          oldErrorSrc->remove();
          prevMoveErr->replace(new CallExpr(PRIM_ASSIGN, errorArg, oldErrorSrc));
        }
        GotoStmt* newGoto = new GotoStmt(GOTO_RETURN, epilogue);
        g->replace(newGoto);
      }
    }
  }
}

static bool
findFollowingCheckErrorBlock(SymExpr* se, LabelSymbol*& outHandlerLabel,
    Symbol*& outErrorSymbol) {
  Expr* stmt = se->getStmtExpr(); // aka last scope
  Expr* scope = stmt->parentExpr;

  while (scope) {
    if (isBlockStmt(scope)) {
      // Consider statements that appear before stmt
      // We are looking for a DefExpr of an error label
      for(Expr* cur = stmt->next; cur != NULL; cur = cur->next) {
        if (DefExpr* def = toDefExpr(cur))
          if (LabelSymbol* label = toLabelSymbol(def->sym))
            if (label->hasFlag(FLAG_ERROR_LABEL)) {
              outHandlerLabel = label;
              // find the error that this block is working with
              for(Expr* e = def->next; e != NULL; e = e->next) {
                std::vector<CallExpr*> calls;
                collectCallExprs(e, calls);
                for_vector(CallExpr, call, calls) {
                  if (call->isPrimitive(PRIM_CHECK_ERROR)) {
                    SymExpr* se = toSymExpr(call->get(1));
                    INT_ASSERT(se->symbol()->hasFlag(FLAG_ERROR_VARIABLE));
                    outErrorSymbol = se->symbol();
                    return true;
                  }
                }
              }
              INT_FATAL("Could not find error variable for handler");
            }
      }
    }
    stmt = scope;
    scope = scope->parentExpr;
  }

  return false;
}

void handleChplPropagateErrorCall(CallExpr* call) {
  SymExpr* errSe = toSymExpr(call->get(1));
  INT_ASSERT(errSe && errSe->typeInfo() == dtError);
  LabelSymbol* label = NULL;
  Symbol* error = NULL;
  if (findFollowingCheckErrorBlock(errSe, label, error)) {
    errSe->remove();
    call->insertBefore(new CallExpr(PRIM_MOVE, error, errSe));
    call->insertBefore(new GotoStmt(GOTO_ERROR_HANDLING, label));
    call->remove();
  }
}

/* When inlining an iterator, the iterator might throw
   an error. If it does, instead of updating the iterator's
   out error argument, the error should propagate to
   a following error-handling block if one exists.
 */
static void
replaceErrorFormalWithEnclosingError(SymExpr* se) {

  Symbol* oldSymbol = se->symbol();
  if (oldSymbol->defPoint->parentSymbol != se->parentSymbol &&
      oldSymbol->hasFlag(FLAG_ERROR_VARIABLE)) {

    LabelSymbol* newLabel = NULL;
    Symbol* newError = NULL;

    // This code is only written to handle a PRIM_MOVE/ASSIGN
    // that is setting the out error argument of the iterator.
    // This out error argument is no longer available since the
    // iterator is being inlined.
    CallExpr* parentCall = toCallExpr(se->parentExpr);
    INT_ASSERT(parentCall->isPrimitive(PRIM_MOVE) ||
               parentCall->isPrimitive(PRIM_ASSIGN));
    INT_ASSERT(se == parentCall->get(1));

    FnSymbol* inFn = toFnSymbol(se->parentSymbol);

    // find the Goto we need to replace
    GotoStmt* fixGoto = NULL;
    for(Expr* cur = se->getStmtExpr(); cur != NULL; cur = cur->next) {
      fixGoto = toGotoStmt(cur);
      if (fixGoto != NULL)
        break;
    }
    INT_ASSERT(fixGoto);

    if (findFollowingCheckErrorBlock(se, newLabel, newError)) {
      // Adjust the current error handling block.
      // 1. Change the use of the error argument to use the
      // identified error variable.
      // 2. Change the Goto to point to the identified error label.

      se->setSymbol(newError);
      fixGoto->gotoTag = GOTO_ERROR_HANDLING;
      fixGoto->label->replace(new SymExpr(newLabel));
    } else if(canFunctionImplicitlyThrow(inFn)) {
      // 1. Make sure that the task function has an error argument.
      // 2. Set that out argument instead of the invalid one.
      ArgSymbol* errorArg = findErrorFormalForFn(inFn);
      if (errorArg == NULL) {
        errorArg = addDummyErrorFormalToFn(inFn);
        // And adjust callers of this function
        for_SymbolSymExprs(se, inFn) {
          CallExpr* call = toCallExpr(se->parentExpr);
          addDummyErrorArgumentToCall(call);
        }
      }
      se->setSymbol(errorArg);
      INT_ASSERT(fixGoto->gotoTag == GOTO_RETURN);
    } else {
      // Just call gChplUncaughtError
      VarSymbol* tmp = newTemp("error", dtError);
      parentCall->insertBefore(new DefExpr(tmp));
      se->setSymbol(tmp);
      fixGoto->replace(new CallExpr(gChplUncaughtError, tmp));
    }
  }
}

//
// Handle IBB - Iterator Break Block.
//
// An IBB contains the defer actions to be taken when breaking
// out of the enclosing loop. See also the PR message for #12963.
//

// Return an appropriate IBB insertion point for an outbound goto 'gt'.
// 'loopRef' is the forLoop or its copy for lowering, whichever is inTree().
// 'IC' is the forLoop's _iteratorClass, or NULL if lowering a ForallStmt.
//
static Expr* ibbInsertPoint(Expr* loopRef, Symbol* IC, GotoStmt* gt) {
  DefExpr* gtTarget = toSymExpr(gt->label)->symbol()->defPoint;
  // Sanity: the goto's target is in the forLoop's function.
  INT_ASSERT(gtTarget->parentSymbol == loopRef->parentSymbol);

  // When lowering a ForallStmt, there is no IC.
  // Insert the IBB right before the goto.
  if (!IC) {
    return gt;
  }
  
  // If we are breaking out from this loop, the IC is freed
  // at the break target. Insert the IBB right before the goto.
  // Cf. if gt is a GOTO_RETURN, the IC is freed at the goto.
  if (gt->gotoTag == GOTO_BREAK                   &&
      gtTarget->parentExpr == loopRef->parentExpr ) {
    return gt;
  }

  // The insertion point is the _freeIterator of IC.
  // We want to give the iterator an opportunity to clean up
  // **before** freeing its iterator class.
  for (Expr* stmt = gt->prev; stmt; stmt = stmt->prev)
    if (CallExpr* call = toCallExpr(stmt))
      if (call->isNamed("_freeIterator"))
        if (toSymExpr(call->get(1))->symbol() == IC)
          // Insert the IBB right before freeing the IC.
          return stmt;

  INT_FATAL(gt, "did not find the insertion point");
  return NULL;
}

// Insert the IBB 'breakBlock' right before each goto that exits 'loopBody'.
// 'loopRef' and 'IC' are passed through to ibbInsertPoint().
static void addIteratorBreakBlocks(Expr* loopRef, Symbol* IC,
                                   BlockStmt* loopBody, BlockStmt* breakBlock)
{
  INT_ASSERT(!loopBody->inTree()); // caller responsibility
  std::vector<GotoStmt*> exits;
  std::vector<CondStmt*> IBBs;

  // We judge outbound-ness by checking whether the goto's target is inTree().
  // Given that 'loopBody' is not inTree(), a target that IS inTree()
  // is surely outside of 'loopBody'.
  collectTreeBoundGotosAndIteratorBreakBlocks(loopBody, exits, IBBs);

  for_vector(GotoStmt, gt, exits) {
    BlockStmt* bbcopy = breakBlock->copy();
    ibbInsertPoint(loopRef, IC, gt)->insertBefore(bbcopy);
    bbcopy->flattenAndRemove(); // otherwise later ibbInsertPoint may fail
  }

  for_vector(CondStmt, cs, IBBs) {
    // Run IBB for the invoked iterator before IBB for the invoking iterator.
    BlockStmt* bbcopy = breakBlock->copy();
    cs->thenStmt->insertAtHead(bbcopy);
    bbcopy->flattenAndRemove(); // might not be necessary
  }
}

//
// Call addIteratorBreakBlocks(). The breakBlock is the block
// that createIteratorBreakBlocks() inserted for this yield
// and callDestructors filled with the appropriate defer actions.
//
void addIteratorBreakBlocksInline(Expr* loopRef, Symbol* IC,
                                  BlockStmt* loopBody, CallExpr* yield,
                                  std::vector<Expr*>* delayedRemoval)
{
  BlockStmt* breakBlock = getAndRemoveIteratorBreakBlockForYield(delayedRemoval,
                                                                 yield);
  // Remove the last goto in the breakBlock. The corresponding goto
  // in 'loopBody' will branch to the exit instead.
  toGotoStmt(breakBlock->body.tail)->remove();

  addIteratorBreakBlocks(loopRef, IC, loopBody, breakBlock);
}

//
// Call addIteratorBreakBlocks(). The break block does this:
//   ic.more = -ic.more; ic.advance();
//
// In the zippered case, do the above for each zippered 'ic'.
//
// The _jump_break_N label inserted by buildJumpTables() into ic.advance()
// is dispatched to when ic.more contains the negative value -N.
//
static void addIteratorBreakBlocksJumptable(Expr* loopRef, Symbol* IC,
                                            BlockStmt* loopBody,
                                            Vec<Symbol*> iterators) {
  // foreach ic in iterators:
  //   def moreRef : ref(int)
  //   move moreRef, .(ic, more)
  //   def moreVal : int
  //   assign moreVal, moreRef
  //   def moreValM : int
  //   move moreValM, - moreVal
  //   assign moreRef, moreValM
  //   call advance(ic)

  BlockStmt* breakBlk  = new BlockStmt();
  int idx = 0;

  forv_Vec(Symbol, ic, iterators) {
    idx++;
    const char* idxs    = istr(idx);
    Symbol* moreField   = ic->type->getField("more");
    Type*   moreType    = moreField->type;
    Type*   moreTypeRef = moreType->getRefType();
    VarSymbol* moreRef  = newTemp(astr("moreRef", idxs), moreTypeRef);
    VarSymbol* moreVal  = newTempConst(astr("moreVal", idxs), moreType);
    VarSymbol* moreValM = newTempConst(astr("moreValNeg", idxs), moreType);
    FnSymbol*  advanceF = toAggregateType(ic->type)->iteratorInfo->advance;

    breakBlk->insertAtTail(new DefExpr(moreRef));
    breakBlk->insertAtTail("'move'(%S,'.'(%S,%S))", moreRef, ic, moreField);
    breakBlk->insertAtTail(new DefExpr(moreVal));
    breakBlk->insertAtTail("'='(%S,%S)", moreVal, moreRef);
    breakBlk->insertAtTail(new DefExpr(moreValM));
    breakBlk->insertAtTail("'move'(%S,'u-'(%S))", moreValM, moreVal);
    breakBlk->insertAtTail("'='(%S,%S)", moreRef, moreValM);
    breakBlk->insertAtTail(new CallExpr(advanceF, ic));
  }

  addIteratorBreakBlocks(loopRef, IC, loopBody, breakBlk);

  // Future work: implement the bulk of the breakBlk contents to a function
  // in ChapelIteratorSupport.chpl. Bonus: in that function, also do
  // "if boundsChecking then assert ic.more != 0".
  // Things to work out:
  // * Store that function in another field of iteratorInfo so we can get
  //   to it here.
  // * Prevent that function from being pruned at resolution.
  // * How to access ic.advance? ic.more?
  // * How to handle zippering? or handle just one ic at a time?
}


static void
expandBodyForIteratorInline(ForLoop*       forLoop,
                            BlockStmt*     ibody,
                            Symbol*        index,
                            bool           inTaskFn,
                            TaskFnCopyMap& taskFnCopies,
                            bool&          addErrorArgToCall) {
  bool removeReturn = !inTaskFn;
  std::vector<CallExpr*> bodyCalls;
  collectCallExprs(ibody, bodyCalls);

  for_vector(CallExpr, call, bodyCalls)
  {
      if (call->isPrimitive(PRIM_YIELD)) {
        Symbol*    yieldedIndex  = newTemp("_yieldedIndex", index->type);
        Symbol*    yieldedSymbol = toSymExpr(call->get(1))->symbol();
        bool       inserted      = false;

        if (forLoop->isCoforallLoop()) {
          // parallel.cpp wants to know about these when considering whether
          // or not to insert autoCopies
          yieldedIndex->addFlag(FLAG_COFORALL_INDEX_VAR);
        }

        SymbolMap  map;
        map.put(index, yieldedIndex);

        BlockStmt* bodyCopy = forLoop->copyBody(&map);
        addIteratorBreakBlocksInline(ibody, forLoop->iteratorGet()->symbol(),
                                     bodyCopy, call, NULL);
        
        if (int count = countEnclosingLocalBlocks(call, ibody)) {
          for (int i = 0; i < count; i++) {
            bodyCopy = new BlockStmt(bodyCopy);

            bodyCopy->blockInfoSet(new CallExpr(PRIM_BLOCK_UNLOCAL));
          }
        }

        // remove move to return-temp that is defined at top of iterator
        if (CallExpr* prev = toCallExpr(call->prev)) {
          if (prev->isPrimitive(PRIM_MOVE)) {
            if (SymExpr* lhs = toSymExpr(prev->get(1))) {
              if (lhs->symbol() == yieldedSymbol) {
                lhs->setSymbol(yieldedIndex);

                prev->insertBefore(new DefExpr(yieldedIndex));

                inserted = true;
              }
            }
          }
        }

        call->replace(bodyCopy);

        if (inserted == false) {
          bodyCopy->insertBefore(new DefExpr(yieldedIndex));
          bodyCopy->insertBefore(new CallExpr(PRIM_MOVE, yieldedIndex, call->get(1)));
        }

        // Fix an error-handling sequence that now refers
        // outside of its function.
        if (inTaskFn)
          fixupErrorHandlingExits(bodyCopy, addErrorArgToCall);

      }


      // Remove returns if the command-line argument indicates to
      if (call->isPrimitive(PRIM_RETURN)) {
        if (removeReturn)
          call->remove();
      }

      // Adjust calls to chpl_propagate_error
      if (FnSymbol* calledFn = call->resolvedFunction()) {
        if (calledFn == gChplPropagateError)
          handleChplPropagateErrorCall(call);
      }

      // Adjust task functions within the iterator
      if (FnSymbol* cfn = resolvedToTaskFun(call)) {
        // Todo: skip this handling of 'cfn' if it does not have yields
        // in itself or any other taskFns it may call.

        // This holds because we flatten everything right away.
        // We need it so that we can place the def of 'fcopy' anywhere
        // while preserving correct scoping of its SymExprs.
        INT_ASSERT(isGlobal(cfn));

        bool addErrorArgToSubCall = false;
        FnSymbol* fcopy = taskFnCopies.get(cfn);

        if (!fcopy) {
          // Clone the function. Just once per 'body' should suffice.
          fcopy = cfn->copy();
          if (!preserveInlinedLineNumbers)
            reset_ast_loc(fcopy, call);

          // Note that 'fcopy' will likely get a copy of 'body',
          // so we need to preserve correct scoping of its SymExprs.
          call->insertBefore(new DefExpr(fcopy));

          // I don't expect invocation of expandBodyForIteratorInline() below
          // to encounter another call to 'cfn'. But even if does, 'fcopy'
          // will be fetched from 'taskFnCopies', avoiding recursion.
          taskFnCopies.put(cfn, fcopy);

          // Repeat, recursively.
          expandBodyForIteratorInline(forLoop, fcopy->body, index, true,
              taskFnCopies, addErrorArgToSubCall);

        } else {
          // Indeed, 'cfn' is encountered only once per 'body',
          // although I cannot explain why this is *always* the case.
          // ('cfn' may be encountered more than once overall.)
          // If it *is* seen more than once, everything should still work,
          // so this assertion should be OK to remove.
          // (If using flattenOneFunction() below, need to get more calls
          // to its actual argument.)
          INT_ASSERT(false);
        }

        // Call 'fcopy' instead
        call->baseExpr->replace(new SymExpr(fcopy));

        if (addErrorArgToSubCall) {
          addDummyErrorArgumentToCall(call);
        }

        // Note: this is an expensive operation due to compute_call_sites().
        // We do it because it may eliminate further cloning of 'fcopy'
        // e.g. when the enclosing fn or block are copied for any reason.
        // Ideally, replace with flattenOneFunction().
        flattenNestedFunction(fcopy);
      }
  }
}


// Returns the number of yields in the given function.
//
// If any call expressions call a task function, this functions is called
// recursively on those task functions.  Task functions are out-lined in
// createTaskFunctions, but contain statements -- including yields -- that
// actually "belong" to the calling function.  Recursive calls of this function
// effectively inline those out-lined functions so we get an accurate count.
static int
countYieldsInFn(FnSymbol* fn)

{
  int count = 0;

  std::vector<CallExpr*> calls;

  collectCallExprs(fn, calls);

  for_vector(CallExpr, call, calls)
  {
    if (call->isPrimitive(PRIM_YIELD))
      count++;

    if (FnSymbol* taskFn = resolvedToTaskFun(call))
      count += countYieldsInFn(taskFn);
  }
  return count;
}

// Returns true if the iterator can be inlined; false otherwise.
//
// Iterators can be inlined if they contains between 1 and
// inline_iter_yield_limit yield statements
static bool
canInlineIterator(FnSymbol* iterator) {
  int count = countYieldsInFn(iterator);
  return count <= inline_iter_yield_limit;
}

static void
setupSimultaneousIterators(Vec<Symbol*>& iterators,
                           Vec<Symbol*>& indices,
                           Symbol*       iterator,
                           Symbol*       index,
                           ForLoop*      loop) {
  if (iterator->type->symbol->hasFlag(FLAG_TUPLE)) {
    AggregateType* iteratorType = toAggregateType(iterator->type);
    AggregateType* indexType    = toAggregateType(index->type);

    for (int i = 1; i <= iteratorType->fields.length; i++) {
      Symbol* tmpIterator = newTemp("_iterator", iteratorType->getField(i)->type);
      Symbol* tmpIndex    = newTemp("_index",    indexType->getField(i)->type);

      loop->insertBefore(new DefExpr(tmpIterator));
      loop->insertBefore(new CallExpr(PRIM_MOVE,
                                      tmpIterator,
                                      new CallExpr(PRIM_GET_MEMBER_VALUE,
                                                   iterator,
                                                   iteratorType->getField(i))));

      loop->insertAtHead(new CallExpr(PRIM_SET_MEMBER,
                                      index,
                                      indexType->getField(i),
                                      tmpIndex));

      loop->insertAtHead(new DefExpr(tmpIndex));

      setupSimultaneousIterators(iterators, indices, tmpIterator, tmpIndex, loop);
    }
  } else {
    iterators.add(iterator);
    indices.add(index);
  }
}


static bool
isBoundedIterator(FnSymbol* fn) {
  if (fn->_this) {
    Type* type = fn->_this->getValType();
    if (type->symbol->hasFlag(FLAG_RANGE)) {
      if (!strcmp(type->substitutions.v[1].value->name, "bounded"))
        return true;
      else
        return false;
    }
  }
  return true;
}


static void getIteratorChildren(Vec<Type*>& children, Type* type) {
  if (AggregateType* at = toAggregateType(type)) {
    forv_Vec(AggregateType, child, at->dispatchChildren) {
      if (child && child != dtObject) {
        children.add_exclusive(child);
        getIteratorChildren(children, child);
      }
    }
  }
}

#define ZIP1     1
#define ZIP2     2
#define ZIP3     3
#define ZIP4     4
#define HASMORE  5
#define GETVALUE 6
#define INIT     7
#define INCR     8

static void buildIteratorCallInner(BlockStmt* block,
                                   Symbol*    ret,
                                   int        fnid,
                                   Symbol*    iterator) {
  IteratorInfo* ii = getTheIteratorFn(iterator)->iteratorInfo;
  FnSymbol*     fn = NULL;

  switch (fnid) {
  case ZIP1:     fn = ii->zip1;     break;
  case ZIP2:     fn = ii->zip2;     break;
  case ZIP3:     fn = ii->zip3;     break;
  case ZIP4:     fn = ii->zip4;     break;
  case HASMORE:  fn = ii->hasMore;  break;
  case GETVALUE: fn = ii->getValue; break;
  case INIT:     fn = ii->init;     break;
  case INCR:     fn = ii->incr;     break;
  }

  CallExpr* call = new CallExpr(fn, iterator);

  if (ret) {
    if (fn->retType->getValType() == ret->type->getValType()) {
      INT_ASSERT(fn->retType == ret->type);
      block->insertAtTail(new CallExpr(PRIM_MOVE, ret, call));
    } else {
      VarSymbol* tmp = newTemp("retTmp", fn->retType);
      block->insertAtTail(new DefExpr(tmp));
      block->insertAtTail(new CallExpr(PRIM_MOVE, tmp, call));
      block->insertAtTail(new CallExpr(PRIM_MOVE, ret, new CallExpr(PRIM_CAST, ret->type->symbol, tmp)));
    }
  } else {
    block->insertAtTail(call);
  }
}


static BlockStmt*
buildIteratorCall(Symbol* ret, int fnid, Symbol* iterator, Vec<Type*>& children) {
  BlockStmt* block = new BlockStmt();
  BlockStmt* outerBlock = block;
  forv_Vec(Type, type, children) {
    VarSymbol* cid = newTemp("cidSelector", dtBool);
    block->insertAtTail(new DefExpr(cid));
    block->insertAtTail(new CallExpr(PRIM_MOVE, cid,
                          new CallExpr(PRIM_TESTCID,
                                       iterator, type->symbol)));
    BlockStmt* thenStmt = new BlockStmt();
    BlockStmt* elseStmt = new BlockStmt();
    VarSymbol* childIterator = newTemp("childIterator", type);
    thenStmt->insertAtTail(new DefExpr(childIterator));
    thenStmt->insertAtTail(new CallExpr(PRIM_MOVE, childIterator, new CallExpr(PRIM_CAST, type->symbol, iterator)));
    buildIteratorCallInner(thenStmt, ret, fnid, childIterator);
    block->insertAtTail(new CondStmt(new SymExpr(cid), thenStmt, elseStmt));
    block = elseStmt;
  }
  buildIteratorCallInner(block, ret, fnid, iterator);
  return outerBlock;
}

// Replace a ForLoop with its inline equivalent, if possible.
// Otherwise, convert it into a C-style for loop.
// The given forLoop is converted unconditionally.
static void
expandForLoop(ForLoop* forLoop) {
  SymExpr*   se2      = forLoop->iteratorGet();
  VarSymbol* iterator = toVarSymbol(se2->symbol());
  bool converted = false;

  if (!fNoInlineIterators)
  {
    FnSymbol* iterFn = getTheIteratorFn(iterator->type);
    if (iterFn->iteratorInfo                          &&
        !iterator->type->symbol->hasFlag(FLAG_TUPLE)  &&
        canInlineIterator(iterFn)                     &&
        ! isVirtualIterator(iterFn)                   ) {
      converted = expandIteratorInline(forLoop);
    }
  }

  if (! converted)
  {
    // This code handles zippered iterators, dynamic iterators, and any other
    // iterator that cannot be inlined.

    // When expanding a single-loop iterator with a single yield, the generated
    // code looks like:
    //   zip1(_iterator1); zip1(_iterator2);
    //   idx2 = getValue(_iterator2); idx1 = getValue(_iterator1);
    //   bool more1 = hasMore(iterator1);
    //   for (init(_iterator1), init(_iterator2); more1; incr(_iterator1), incr(_iterator2)) {
    //     zip2(_iterator);
    //     // Bounds checks inserted here.
    //     zip3(_iterator1); zip3(_iterator2);
    //     more = hasMore(iterator1);
    //   }
    //   zip4(_iterator2); zip4(_iterator1);
    // In zippered iterators, each clause may contain multiple calls to zip1(),
    // getValue(), etc.  These are inserted in the order shown.

    SET_LINENO(forLoop);

    Vec<Symbol*> iterators;
    Vec<Symbol*> indices;

    FnSymbol* iterFn = getTheIteratorFn(iterator->type);
    if (iterFn->throwsError()) {
      // In this event, the error handling pass added a PRIM_CHECK_ERROR
      // after the call to the iterator function.
      // Scroll backwards to find the error handling block.

      // TODO: finish this case, see GitHub #7134
      //       I think we need to use the ForLoop's break label

      USR_FATAL_CONT(forLoop,
        "throwing non-inlined iterators are not yet supported");
      USR_PRINT(iterFn, "the invoked iterator is here");
      USR_STOP();
    }

    SymExpr*     se1       = toSymExpr(forLoop->indexGet());
    VarSymbol*   index     = toVarSymbol(se1->symbol());

    BlockStmt*   initBlock = new BlockStmt();
    BlockStmt*   testBlock = NULL;
    BlockStmt*   incrBlock = new BlockStmt();

    setupSimultaneousIterators(iterators, indices, iterator, index, forLoop);

    bool allOrderIndependent = true;
    // For each iterator we add the zip* functions in the appropriate place and
    // if bounds checking was on, we insert the code for that. Note that this
    // code handles iterators that have regular loops, c for loops, and
    // dynamically dispatched iterators. The ordering is VERY important!
    for (int i = 0; i < iterators.n; i++) {
      Vec<Type*> children;
      VarSymbol* cond         = newTemp("_cond", dtBool);
      bool       isNotDynIter = false;

      getIteratorChildren(children, iterators.v[i]->type);

      // Add zip1 before the loop
      forLoop->insertBefore(buildIteratorCall(NULL,         ZIP1,     iterators.v[i], children));

      // Get the current value of the iterator.
      forLoop->insertAtHead(buildIteratorCall(indices.v[i], GETVALUE, iterators.v[i], children));

      isNotDynIter = (children.n == 0);

      if (isNotDynIter) {
        // add the init, and incr functions to the init, and incr blocks of the
        // c for loop. If the underlying iterator does not have a c for loop,
        // these blocks will be empty
        initBlock->insertAtTail(buildIteratorCall(NULL, INIT, iterators.v[i], children));
        incrBlock->insertAtTail(buildIteratorCall(NULL, INCR, iterators.v[i], children));

      } else {
        // for dynamically dispatched iterators, conditional checks and other
        // code are added in buildIteratorCall. These constructs aren't legal
        // in a c for loop, so instead of creating a well formed c for loop we
        // add the init before the loop, and the incr at the bottom of it.
        forLoop->insertBefore(buildIteratorCall(NULL, INIT, iterators.v[i], children));
        forLoop->insertAtTail(buildIteratorCall(NULL, INCR, iterators.v[i], children));
      }

      // Add zip3 and zip4 at tail and after the loop respectively.
      forLoop->insertAtTail(buildIteratorCall(NULL, ZIP3, iterators.v[i], children));
      forLoop->insertAfter (buildIteratorCall(NULL, ZIP4, iterators.v[i], children));

      FnSymbol* iterFn = getTheIteratorFn(iterators.v[i]);
      if (iterFn->hasFlag(FLAG_YIELD_WITHIN_ON)) {
        USR_FATAL_CONT(forLoop, "'yield' statements within 'on' clauses are not currently supported for iterators that are not inlined (e.g., within zippered loops)");
        break;
      }

      if (isBoundedIterator(iterFn)) {
        if (testBlock == NULL) {
          if (isNotDynIter) {
            // note that we have found the first test
            testBlock = buildIteratorCall(NULL, HASMORE, iterators.v[i], children);

          } else {
            // note that we have found the first test block and add checks for
            // more before and at the end of the loop. As mentioned above,
            // dynamic iterators generate things that can't be in the header of
            // the c for loop, so we generate a simple bool variable to put at
            // the test of the c for loop, and update that condition var before
            // the loop is run, and at the end of each iteration.
            forLoop->insertBefore(new DefExpr(cond));
            forLoop->insertBefore(buildIteratorCall(cond, HASMORE, iterators.v[i], children));
            forLoop->insertAtTail(buildIteratorCall(cond, HASMORE, iterators.v[i], children));

            testBlock = new BlockStmt(new SymExpr(cond));
          }

        } else if (!fNoBoundsChecks) {
          // for all but the first iterator add checks at the beginning of each loop run
          // and a final one after to make sure the other iterators don't finish before
          // the "leader" and they don't have more afterwards.
          VarSymbol* hasMore    = newTemp("hasMore",    dtBool);
          VarSymbol* isFinished = newTemp("isFinished", dtBool);

          forLoop->insertBefore(new DefExpr(isFinished));
          forLoop->insertBefore(new DefExpr(hasMore));

          forLoop->insertAtHead(new CondStmt(new SymExpr(isFinished),
                                             new CallExpr(PRIM_RT_ERROR,
                                                          new_CStringSymbol("zippered iterations have non-equal lengths"))));

          forLoop->insertAtHead(new CallExpr(PRIM_MOVE, isFinished, new CallExpr(PRIM_UNARY_LNOT, hasMore)));

          forLoop->insertAtHead(buildIteratorCall(hasMore, HASMORE, iterators.v[i], children));

          forLoop->insertAfter(new CondStmt(new SymExpr(hasMore),
                                            new CallExpr(PRIM_RT_ERROR,
                                                         new_CStringSymbol("zippered iterations have non-equal lengths"))));

          forLoop->insertAfter(buildIteratorCall(hasMore, HASMORE, iterators.v[i], children));
        }
      }

      forLoop->insertAtHead(buildIteratorCall(NULL, ZIP2, iterators.v[i], children));

      // Need to check if iterator will be inlined, isSingleLoopIterator()
      // doesn't handle arbitrary blockstmts well, so we collapse them first
      iterFn->collapseBlocks();
      Vec<BaseAST*> asts;
      collect_asts_postorder(iterFn, asts);

      // If the iterator cannot be inlined a re-entrant advance function will
      // be built. This function maintains state and must be called in order.
      // If inlined, the iterator's loop will be order independent if it was
      // independent prior to inlining or the forLoop is order independent
      bool curOrderIndependent = false;
      if (CallExpr* singleLoopYield = isSingleLoopIterator(iterFn, asts)) {
        if (LoopStmt* loop = LoopStmt::findEnclosingLoop(singleLoopYield)) {
          curOrderIndependent = loop->isOrderIndependent() || forLoop->isOrderIndependent();
        }
      }
      allOrderIndependent = allOrderIndependent && curOrderIndependent;
    }

    // The loop will be order independent if all the iters it zips are order
    // independent (all iters will be inlined and were all marked order
    // independent or the forLoop was marked independent prior zippering.)
    forLoop->orderIndependentSet(allOrderIndependent);

    // 2015-02-23 hilde:
    // TODO: I think this wants to be insertBefore, and moved before the call
    // to getValue is inserted.  Check the order in the generated code to see
    // if this is the case.  Avoid moving the global void value when it is
    // the loop index.
    if (index != gNone)
      forLoop->insertAtHead(index->defPoint->remove());

    // Ensure that the test clause for completely unbounded loops contains
    // something.
    // testBlock is only non-NULL if isBoundedIterator() evaluates to true for
    // at least one of the iterators being zippered together.
    if (testBlock == NULL) {
      testBlock = new BlockStmt();

      testBlock->insertAtTail(new SymExpr(gTrue));
    }

    // NOAKES 2014/11/19: An error occurs if the replacement is moved to
    // earlier in the pass.  I have yet to identify the issue but suspect
    // that doing the copy too soon causes variables to cross from one
    // scope to another if done in mid-transformation.
    CForLoop* cforLoop = CForLoop::buildWithBodyFrom(forLoop);

    addIteratorBreakBlocksJumptable(forLoop, iterator,
                                    (BlockStmt*)cforLoop, iterators);

    // Even for zippered iterators we only have one conditional test for the
    // loop. This takes that conditional and puts it into the test segment of
    // the c for loop.  Code in the initBlock and incrBlock clauses are each a
    // concatenation of the init() and incr() calls for all iterators being
    // zippered together.  The init() and incr() calls appear in the same order
    // as the iterators appear in the zip(,,) tuple.
    cforLoop->loopHeaderSet(initBlock, testBlock, incrBlock);

    forLoop->replace(cforLoop);
  }
}


// Find all iterator constructs
// Select those whose _getIterator() functions have the FLAG_INLINE_ITERATOR.
// Inline the selected iterators at their call sites.
static void
inlineIterators() {
  forv_Vec(BlockStmt, block, gBlockStmts) {
    // Skip blocks that are no longer in the tree.
    if (!block->inTree())
      continue;

    if (ForLoop* forLoop = toForLoop(block)) {
      Symbol*   iterator = toSymExpr(forLoop->iteratorGet())->symbol();
      FnSymbol* ifn = getTheIteratorFn(iterator);
      if (ifn->hasFlag(FLAG_INLINE_ITERATOR)) {
        // The Boolean return value from expandIteratorInline() is being
        // ignored here, which means that forLoop might not have been replaced.
        // However, all ForLoops that remain in the tree after the call to
        // inlineIterators() are passed through expandForLoop() which *does*
        // replace them.
        expandIteratorInline(forLoop);
      }
    }
  }
}


static void fixNumericalGetMemberPrims()
{
  // fix GET_MEMBER primitives that access fields of an iterator class
  // via a number
  forv_Vec(CallExpr, call, gCallExprs) {
    if (call->inTree() && call->isPrimitive(PRIM_GET_MEMBER)) {
      AggregateType* ct = toAggregateType(call->get(1)->getValType());
      int64_t num;
      if (get_int(call->get(2), &num)) {
        Symbol* field = ct->getField(num+2); // skip fields: super, more
        SET_LINENO(call);
        call->get(2)->replace(new SymExpr(field));
        CallExpr* parent = toCallExpr(call->parentExpr);
        INT_ASSERT(parent->isPrimitive(PRIM_MOVE));
        Symbol* local = toSymExpr(parent->get(1))->symbol();
        if (local->type == field->type)
          call->primitive = primitives[PRIM_GET_MEMBER_VALUE];
        else if (local->type != field->type->refType)
          INT_FATAL(call, "unexpected case");
      }
    }
  }
}


//
// For _toLeader fns, convert out-of-scope references to prim calls.
// cleanupLeaderFollowerIteratorCalls() wouldn't handle such references
// if a _toLeader call is inlined.
//
static void cleanupLeaderIteratorCalls()
{
  forv_Vec(FnSymbol, fn, gFnSymbols) {
    if (fn->inTree() && !strcmp(fn->name, "_toLeader")) {
      ArgSymbol* toLeaderArg = fn->getFormal(1);
      if (! toLeaderArg->type->symbol->hasFlag(FLAG_ITERATOR_RECORD))
        continue;

      std::vector<SymExpr*> symExprs;
      collectSymExprs(fn, symExprs);
      for_vector(SymExpr, se, symExprs) {
        ArgSymbol* iteratorArg = toArgSymbol(se->symbol());
        if (iteratorArg == NULL) continue;
        if (iteratorArg->defPoint->parentSymbol == fn) continue;

        SET_LINENO(se);
        VarSymbol* ftemp = newTemp("ftemp", iteratorArg->qualType());
        PrimitiveTag irf = PRIM_ITERATOR_RECORD_FIELD_VALUE_BY_FORMAL;
        Expr*       stmt = se->getStmtExpr();
        stmt->insertBefore(new DefExpr(ftemp));
        stmt->insertBefore(new CallExpr(PRIM_MOVE, ftemp,
                            new CallExpr(irf, toLeaderArg, iteratorArg)));
        se->replace(new SymExpr(ftemp));
      }
    }
  }
}


static void cleanupLeaderFollowerIteratorCalls()
{
  //
  // cleanup leader and follower iterator calls
  //
  // Fixes uses of formals outside of their function.
  // Such formals were temporarily added (e.g. in preFold for PRIM_TO_FOLLOWER)
  //
  forv_Vec(CallExpr, call, gCallExprs) {
    if (call->inTree()) {
      if (FnSymbol* fn = call->resolvedFunction()) {
        if (fn->retType->symbol->hasFlag(FLAG_ITERATOR_RECORD) ||
            (isDefExpr(fn->formals.tail) &&
             !strcmp(toDefExpr(fn->formals.tail)->sym->name, "_retArg") &&
             toDefExpr(fn->formals.tail)->sym->getValType() &&
             toDefExpr(fn->formals.tail)->sym->getValType()->symbol->hasFlag(FLAG_ITERATOR_RECORD))) {
          if (// "_toLeader" case is handled in cleanupLeaderIteratorCalls()
              !strcmp(call->parentSymbol->name, "_toFollower") ||
              !strcmp(call->parentSymbol->name, "_toFastFollower") ||
              !strcmp(call->parentSymbol->name, "_toStandalone")) {
            ArgSymbol* iterator = toFnSymbol(call->parentSymbol)->getFormal(1);
            Type* iteratorType = iterator->getValType();
            int i = 3; // skip fields: super, more
            for_actuals(actual, call) {
              SymExpr* se = toSymExpr(actual);
              if (isArgSymbol(se->symbol()) &&
                  call->parentSymbol != se->symbol()->defPoint->parentSymbol) {
                Symbol* field = toAggregateType(iteratorType)->getField(i);
                VarSymbol* tmp = NULL;
                SET_LINENO(call);
                if (field->type->refType == se->symbol()->type) {
                  tmp = newTemp(field->name, field->type->refType);
                  call->getStmtExpr()->insertBefore(new DefExpr(tmp));
                  call->getStmtExpr()->insertBefore(new CallExpr(PRIM_MOVE, tmp, new CallExpr(PRIM_GET_MEMBER, iterator, field)));
                } else {
                  tmp = newTemp(field->name, field->qualType());
                  call->getStmtExpr()->insertBefore(new DefExpr(tmp));
                  call->getStmtExpr()->insertBefore(new CallExpr(PRIM_MOVE, tmp, new CallExpr(PRIM_GET_MEMBER_VALUE, iterator, field)));
                }
                actual->replace(new SymExpr(tmp));
                i++;
              }
            }
          }
        }
      }
    }
  }
}


static void handlePolymorphicIterators()
{
  forv_Vec(FnSymbol, fn, gFnSymbols) {
    // Find iterator functions that are not in the AST tree.
    if (fn->inTree()     &&
        fn->iteratorInfo &&
        !fn->hasFlag(FLAG_TASK_FN_FROM_ITERATOR_FN)) {
      // Assert that the getIterator() function *is* in the tree.
      FnSymbol* getIterator = fn->iteratorInfo->getIterator;
      INT_ASSERT(getIterator->defPoint->parentSymbol);

      // See if the iterator record is polymorphic.
      AggregateType* irecord = fn->iteratorInfo->irecord;

      if (irecord->dispatchChildren.n > 0) {
        // If so, then simulate dynamic dispatch by adding one conditional
        // block for each possible subtype.
        SET_LINENO(getIterator);

        LabelSymbol* label = new LabelSymbol("end");

        getIterator->insertBeforeEpilogue(new DefExpr(label));

        Symbol* ret = getIterator->getReturnSymbol();

        forv_Vec(AggregateType, subTypeAgg, irecord->dispatchChildren) {
          VarSymbol* tmp       = newTemp(irecord->getField(1)->type);
          VarSymbol* cid       = newTemp(dtBool);
          BlockStmt* thenStmt  = new BlockStmt();
          VarSymbol* recordTmp = newTemp("recordTmp", subTypeAgg);
          VarSymbol* classTmp  = newTemp("classTmp", subTypeAgg->iteratorInfo->getIterator->retType);

          thenStmt->insertAtTail(new DefExpr(recordTmp));
          thenStmt->insertAtTail(new DefExpr(classTmp));


          AggregateType* ct = subTypeAgg;

          for_fields(field, ct) {
            VarSymbol* ftmp = newTemp("ftmp", getIterator->getFormal(1)->type->getField(field->name)->type);
            thenStmt->insertAtTail(new DefExpr(ftmp));
            thenStmt->insertAtTail(new CallExpr(PRIM_MOVE, ftmp, new CallExpr(PRIM_GET_MEMBER_VALUE, getIterator->getFormal(1), getIterator->getFormal(1)->type->getField(field->name))));
            // Store temp in record field.
            if (ftmp->type == field->type) {
              thenStmt->insertAtTail(new CallExpr(PRIM_SET_MEMBER, recordTmp, field, ftmp));
            } else {
              VarSymbol* ftmp2 = newTemp(field->type);
              thenStmt->insertAtTail(new DefExpr(ftmp2));
              thenStmt->insertAtTail(new CallExpr(PRIM_MOVE, ftmp2, new CallExpr(PRIM_CAST, field->type->symbol, ftmp)));
              thenStmt->insertAtTail(new CallExpr(PRIM_SET_MEMBER, recordTmp, field, ftmp2));
            }
          }
          thenStmt->insertAtTail(new CallExpr(PRIM_MOVE, classTmp, new CallExpr(subTypeAgg->iteratorInfo->getIterator, recordTmp)));
          thenStmt->insertAtTail(new CallExpr(PRIM_MOVE, ret, new CallExpr(PRIM_CAST, ret->type->symbol, classTmp)));
          thenStmt->insertAtTail(new GotoStmt(GOTO_GETITER_END, label));
          ret->defPoint->insertAfter(new CondStmt(new SymExpr(cid), thenStmt));
          ret->defPoint->insertAfter(new CallExpr(PRIM_MOVE, cid, new CallExpr(PRIM_TESTCID, tmp, subTypeAgg->iteratorInfo->irecord->getField(1)->type->symbol)));
          ret->defPoint->insertAfter(new DefExpr(cid));
          ret->defPoint->insertAfter(new CallExpr(PRIM_MOVE, tmp, new CallExpr(PRIM_GET_MEMBER_VALUE, getIterator->getFormal(1), irecord->getField(1))));
          ret->defPoint->insertAfter(new DefExpr(tmp));
        }
      }
    }
  }
}


static void reconstructIRAutoCopy(FnSymbol* fn)
{
  Symbol* arg = fn->getFormal(1);
  Symbol* ret = fn->getReturnSymbol();
  BlockStmt* block = new BlockStmt();
  block->insertAtTail(ret->defPoint->remove());
  AggregateType* irt = toAggregateType(arg->type);
  for_fields(field, irt) {
    SET_LINENO(field);

    Symbol* fieldValue = newTemp(field->name, field->qualType());
    block->insertAtTail(new DefExpr(fieldValue));

    // Read the field
    block->insertAtTail(new CallExpr(PRIM_MOVE, fieldValue, new CallExpr(PRIM_GET_MEMBER_VALUE, arg, field)));

    // Now auto-copy it if appropriate
    Symbol* copyResult = fieldValue;
    if (isUserDefinedRecord(field->type) && !field->isRef() ) {
      FnSymbol* autoCopy = getAutoCopyForType(field->type);
      Symbol* valueToCopy = fieldValue;
      Type* copyArgType = autoCopy->getFormal(1)->type;
      // If the copy function is expecting a reference type that is
      // a reference to the value we have, do a PRIM_ADDR_OF to pass it.
      if (isReferenceType(copyArgType) &&
          copyArgType->getValType() == fieldValue->type) {
        valueToCopy = newTemp(copyArgType);
        block->insertAtTail(new DefExpr(valueToCopy));
        block->insertAtTail(new CallExpr(PRIM_MOVE, valueToCopy, new CallExpr(PRIM_ADDR_OF, fieldValue)));
      }
      copyResult = newTemp(autoCopy->retType);
      block->insertAtTail(new DefExpr(copyResult));
      block->insertAtTail(new CallExpr(PRIM_MOVE, copyResult, new CallExpr(autoCopy, valueToCopy)));
    }

    // Now set the field
    block->insertAtTail(new CallExpr(PRIM_SET_MEMBER, ret, field, copyResult));
  }
  block->insertAtTail(new CallExpr(PRIM_RETURN, ret));
  fn->body->replace(block);
}


static void reconstructIRAutoDestroy(FnSymbol* fn)
{
  Symbol* arg = fn->getFormal(1);
  BlockStmt* block = new BlockStmt();
  AggregateType* irt = toAggregateType(arg->type);
  for_fields(field, irt) {
    SET_LINENO(field);
    if (isUserDefinedRecord(field->type) && !field->isRef() ) {
      if (FnSymbol* autoDestroy = autoDestroyMap.get(field->type)) {
        Symbol* tmp = newTemp(field->name, field->type);
        block->insertAtTail(new DefExpr(tmp));
        block->insertAtTail(new CallExpr(PRIM_MOVE, tmp, new CallExpr(PRIM_GET_MEMBER_VALUE, arg, field)));
        block->insertAtTail(new CallExpr(autoDestroy, tmp));
      }
    }
  }
  block->insertAtTail(new CallExpr(PRIM_RETURN, gVoid));
  fn->body->replace(block);
}


static void reconstructIRautoCopyAutoDestroy()
{
  //
  // reconstruct autoCopy and autoDestroy for iterator records
  //
  forv_Vec(FnSymbol, fn, gFnSymbols) {
    if (!fn->inTree()) continue;

    if (fn->numFormals() == 1 &&
        fn->getFormal(1)->type->symbol->hasFlag(FLAG_ITERATOR_RECORD))
    {
      SET_LINENO(fn);
      if (fn->hasFlag(FLAG_AUTO_COPY_FN))
        reconstructIRAutoCopy(fn);
      if (fn->hasFlag(FLAG_AUTO_DESTROY_FN))
        reconstructIRAutoDestroy(fn);
    }
  }
}


// release some memory
static void cleanupTemporaryVectors() {
  taskFunInRecursiveIteratorSet.clear();
  iteratorFnMap.clear();
  loopBodyFnArgsSuppliedMap.clear();
}

// These are most likely due to iterators getting inlined.
// Todo: instead execute the thenStmt block upon 'break' from
// the corresponding forLoops.
static void cleanupIteratorBreakToken() {
  for_alive_in_Vec(CondStmt, cond, gCondStmts)
    if (SymExpr* se = toSymExpr(cond->condExpr))
      if (se->symbol() == gIteratorBreakToken) {
        FnSymbol* parent = toFnSymbol(cond->parentSymbol);
        // Proper IBB handling in recursive iterators is TODO.
        if (! strncmp(parent->name, "_rec_iter_", 10)        ||
            ! strncmp(parent->name, "rec_iter_task_fn_", 17) )
          cond->remove();
        else
          INT_FATAL(cond, "should not remain in the tree");
      }
}


// 'depth' is a heuristic to avoid the risk of unbounded recursion.
// Ex. what if 'parentSym' is a recursive function?
static bool maybeCalled(Symbol* parentSym, int depth) {
  if (!parentSym || !parentSym->inTree())
    return false;

  FnSymbol* fn = toFnSymbol(parentSym);
  if (!fn)
    return true; // conservative

  if (ftableMap.count(fn))
    return true;

  if (!fn->calledBy || fn->calledBy->n == 0)
    return false;

  if (depth <= 0)
    return true; // conservative

  depth--;

  forv_Vec(CallExpr, call, *fn->calledBy)
    if (maybeCalled(call->parentSymbol, depth))
      return true;

  return false;
}

// WORKAROUND:
// When the body of a for loop is moved into the loop body function, yield
// primitives remain in it (if the for loop itself appears in an iterator
// function.
// When the containing iterator is expanded in a for loop, yields in the loop
// body will be replaced appropriately (through the recursive call to
// expandBodyForIteratorInline() that occurs when a call to a task function is
// encountered).
// Any yields remaining in the tree at this point are apparently dead code.
// Thus the functions containing them can be removed from the tree and calls to
// them stubbed out.  As a assertion that these calls are never reached, they
// are replaced by internal error primitives.
static void removeUncalledIterators()
{
  compute_call_sites();

  forv_Vec(CallExpr, call, gCallExprs)
  {
    // We only care about calls that are still in the tree.
    if (! call->parentSymbol)
      continue;
    // We only care about yields.
    if (! call->isPrimitive(PRIM_YIELD))
      continue;

    // This is a backup check to ensure PRIM_INT_ERROR is never executed.
    if (fVerify && maybeCalled(call->parentSymbol, 4))
      INT_FATAL(call, "unexpected leftover PRIM_YIELD");

    // If this function contains a yield, it was never expanded, so the static
    // analysis used in lowerIterators says it was never invoked through a for
    // loop.
    FnSymbol* fn = toFnSymbol(call->parentSymbol);
    if (!fn->defPoint->parentSymbol)
      // Only bother with functions that are still in the tree.
      continue;

    // Replace invocations of this function with a runtime error.
    if (fn->calledBy)
      forv_Vec(CallExpr, invcn, *fn->calledBy)
      {
        if (!invcn->parentSymbol)
          continue;

        SET_LINENO(invcn);
        invcn->replace(new CallExpr(PRIM_INT_ERROR));
      }

    // and yank the function itself.
    fn->defPoint->remove();
  }
}

void lowerIterators() {
  nonLeaderParCheck();

  markVectorizableForallLoops();

  computeRecursiveIteratorSet();

  cleanupLeaderIteratorCalls();

  lowerForallStmtsInline();

  for_alive_in_Vec(FnSymbol, fn, gFnSymbols) {
    if (fn->isIterator()) {
      fn->collapseBlocks();

      removeUnnecessaryGotos(fn);

#if DEBUG_CP < 2    // That is, disabled if DEBUG_CP >= 2
      // Run localCopyPropagation to remove any fields we can so they will not
      // be added to the iterator class
      if (!fNoCopyPropagation)
        localCopyPropagation(fn);
#endif
    }
  }

  // TODO: The AST is not valid between inlineIterators and
  // fixNumericalGetMemberPrims because of
  // replaceIteratorFormalsWithIteratorFields.
  // We should look at creating the iterator class before here so we don't have
  // this problem.
  inlineIterators();

  for_alive_in_Vec(FnSymbol, fn, gFnSymbols) {
    if (fn->isIterator()) {
      fn->collapseBlocks();
      removeUnnecessaryGotos(fn);
    }
  }

  for_alive_in_Vec(BlockStmt, block, gBlockStmts) {
    if (ForLoop* loop = toForLoop(block))
      expandForLoop(loop);
  }

  if (fVerify) {
    for_alive_in_Vec(BlockStmt, block, gBlockStmts)
      if (block->isForLoop())
        // All forLoops should have been removed from the tree by now.
        INT_FATAL(block, "Unexpected forLoop in tree.");
  }

  fragmentLocalBlocks();

  for_alive_in_Vec(FnSymbol, fn, gFnSymbols) {
    if (fn->isIterator()) {
      // This collapseBlocks call is required for lowerIterator to inline
      // advance() into zip[1-4]
      fn->collapseBlocks();
      lowerIterator(fn);
    }
  }

  removeUncalledIterators();

  fixNumericalGetMemberPrims();

  cleanupLeaderFollowerIteratorCalls();

  handlePolymorphicIterators();

  reconstructIRautoCopyAutoDestroy();

  cleanupTemporaryVectors();
  cleanupIteratorBreakToken();

  iteratorsLowered = true;
}
