/* itgdec-cyk-hook.cpp
 *
 * hook cyk decoder
 *
 * $Id: itgdec-cyk-hook.cpp,v 1.3 2011/10/16 22:36:43 gildea Exp $
 */

#include "itgdec-cyk-hook.h"
#include "itggrmr.h"

hash_map<int, hash_map <LMSTATE, ITGDCell> > * CYKDecoder::matchinghookspan(int s, int t, int thn, DINT v, bool left) {
  hash_map <LMSTATE, ITGDCell> *lcol;
  hash_map <LMSTATE, ITGDCell>::iterator litr;
  DINT ul[2], vl[2];
  ITGDCell *pd;

  //to be extended
  int offset = s * (dchart->T + 1) - s * (s + 1) / 2 + t;

  //iterate over all matching second right hand side nonterminals
  //  rhsr_lhs_inv_erhs_rule_hash::iterator rhsr_itr;
  //  for (rhsr_itr = grammar->rule_table[nl].begin(); rhsr_itr != grammar->rule_table[nl].end(); ++rhsr_itr){
  //  int rhsr = rhsr_itr->first;

  hash_map <int, hash_map <LMSTATE, ITGDCell> > *lspan;
  hash_map <int, hash_map <LMSTATE, ITGDCell> >::iterator lcitr;

  lspan = dchart->span(s, t);
  for (lcitr = lspan->begin(); lcitr != lspan->end(); ++lcitr){
    int rhsr = lcitr->first;

    //    if (left && !grammar->rule_table[thn].count(rhsr))
    //      continue;

    //    if (!left && !grammar->rule_table[rhsr].count(thn))
    //      continue;

    if (dchart->indexedhooks[offset][v].count(rhsr) == 0){ //hooks do not exist, build them
      lcol = dchart->column(rhsr, s, t);

      for (litr = lcol->begin(); litr != lcol->end(); litr++){

	ITGDCell* lcell = &litr->second;

	// the frontiers of the left constituent
	ul[0] = dchart->u0of(litr->first);
	ul[1] = dchart->u1of(litr->first);
	vl[0] = dchart->v0of(litr->first);
	vl[1] = dchart->v1of(litr->first);

	if (ul[0] == 0)
	  continue;

	double trg = LMWEIGHT * ngram->bigram(v, ul[0]);

	pd = dchart->indexedhook(rhsr, s, t, v, 0, vl[1], vl[0]); //used to be vl[0] vl[1], 

	double newsita = lcell->prob + trg;
	double oldsita = pd->prob;

	itemcount++;
	hookitemcount++;

	//maximizing the hook and put onto the agenda
	if (newsita > oldsita){//maximizing
	  pd->N = rhsr;
	  pd->s = s;
	  pd->t = t;
	  pd->prob = newsita;

	  //	  DUEXASSIGN(pd->lword, uchoice);
	  //	  pd->lword[0]=v;
	  //	  pd->lword[1]=0;
	  DUEXASSIGN(pd->lword, ul);
	  DUEXASSIGN(pd->rword, vl);

	  pd->rule = NULL;		// backpointers
	  pd->lchild = lcell; //like a unary rule
	  pd->rchild = lcell;  //special treatment //NULL; 

	}//end of update

	backpointer thebkptr;
	thebkptr.lchild=lcell;
	thebkptr.rchild=NULL;
	thebkptr.rule=NULL;

	thebkptr.tri =  (ngram->N==3);
	pd->bkptrs.push_back(thebkptr);

      }//end for lcol
    }//end building hooks
    
  }//end of matching rhsrs

  return &dchart->indexedhooks[offset][v];
    
}

void CYKDecoder::cyk_hook_bottom_up() {
  
  edgecount = 0;
  if (Globals::Get<int>("verbosity") == 1) {
    cout << "* running CYK bottom up...";
    fflush(stdout);
  }
  string prune_method = Globals::Get<string>("prune_method");

  hash_map <LMSTATE, ITGDCell> *lcol, *rcol;
  hash_map <LMSTATE, ITGDCell>::iterator litr, ritr;

  // stext holds the dp result for the extended span (s, t) - with insertions
  // stext is indexed by stage, nonterm, lm state
  hash_map<int, hash_map <LMSTATE, ITGDCell> > stext[3];

  /* There are three stages: 
   *
   * - stage 1 handles the normal case when two adjacent nonzero
   *   blocks are combined
   * - stage 2 handles the case when the left block is an insertion (0
   *   span in source lang) 
   * - stage 3 handles the case when the right block is an insertion
   *
   * The reason Hao broke this into three stages is because insertions
   * create a new block over the exact same span -- in other words,
   * you update what you're looking at. Thus a separate record has to
   * be kept of it, since you can't alter the data structure you're
   * enumerating.  After all three stages have been gone through, a
   * single pass over everything is made in order to handle the
   * pruning.
   */

  itemcount = 0;
  hookitemcount = 0;

  for (int span = 1; span <= T; span++) {
    
    // cerr << "span: " << span << endl;

    for (int s = 0; s <= T - span; s++) {
      int t = s + span;

     // keep track of max item created over span; allows some pruning at creation
      // time rather than just as an afterstep
      double maxscore = -INFINITY;



//       cout << "\rSPAN(" << s << "," << t << ")";
//       fflush(stdout);

      //dp for (s, t)
      //for binary rules
      for (int stage = 0; stage < 3; stage++) {

        // clear the map for each nonterminal for the current stage
        stext[stage].clear();//prepare for holding new results

        // stage 0: deltas = 1..span-1 (combine nonzero blocks)
        // stage 1: deltas = 0 (insertion on left side)
        // stage 2: deltas = 0 (insertion on right side)
        int from = 0, to = 0;
        switch(stage) {
        case 0: // combine two nonempty blocks
          from = 1;
          to = span;
          break;
        case 1: // insertion on left side
          from = 0;
          to = 1;
          break;
        case 2: // insertion on right side
          from = 0;
          to = 1;
          break;
        }

        for (int deltas = from; deltas < to; deltas++) {
          int S;
          if (stage == 0) {
            S = s + deltas; 
          }
          else if (stage == 1) {
            S = s; 
            // right column same as
            // whole span, so must
            // update parallel data
            // structure instead
          } else { //stage==2
            S = t; 
            // left column same as
            // whole span, so must
            // update parallel data
            // structure instead
          }
          if (Globals::Get<int>("debuglevel") >= 2) {
            fprintf(logfile, "stage = %d, span = %d-%d-%d\n", stage, s, S, t);
          }
	  
          hash_map<int, hash_map <LMSTATE, ITGDCell> > *lspan, *rspan;
          hash_map<int, hash_map <LMSTATE, ITGDCell> >::iterator lnt_itr, rnt_itr;

          lspan = dchart->span(s, S); //non-hook
          rspan = dchart->span(S, t); //non-hook

          // see which nonterm pairs exist in chart
          for (lnt_itr = lspan->begin(); lnt_itr != lspan->end(); lnt_itr++) {
	    int nl = lnt_itr->first;


	    lcol = dchart->column(nl, s, S);
	    
	    for (litr = lcol->begin(); litr != lcol->end(); litr++) {
	      int ul[2], vl[2];


	      ITGDCell* lcell = &litr->second;
	      
	      // the frontiers of the left constituent
	      ul[0] = dchart->u0of(litr->first);
	      ul[1] = dchart->u1of(litr->first);
	      vl[0] = dchart->v0of(litr->first);
	      vl[1] = dchart->v1of(litr->first);
	      
	      double sita3 = lcell->prob;


	      if (vl[0] == 0){ //deletion
		for (rnt_itr = rspan->begin(); rnt_itr != rspan->end(); rnt_itr++) {
	      
		  int nr = rnt_itr->first;

		  // lookahead, avoid enumerating one column while the other is empty
		  if (!dchart->has_column(nl, s, S) || !dchart->has_column(nr, S, t))
		    continue;
	      
		  if (!grammar->rule_table[nl].count(nr))
		    continue;

		  rcol = dchart->column(nr, S, t);

	      
		  // for each binary rule having specified children
		  lhs_inv_erhs_rule_hash *rnt = &grammar->rule_table[nl][nr];
	      
		  lhs_inv_erhs_rule_hash::iterator lhs_itr;
		  inv_erhs_rule_hash::iterator inv_itr;
		  erhs_rule_hash::iterator e_rhs_itr;

		  for (lhs_itr = rnt->begin(); lhs_itr != rnt->end(); lhs_itr++)
		    for (inv_itr = lhs_itr->second.begin(); inv_itr != lhs_itr->second.end(); inv_itr++)
		      for (e_rhs_itr = inv_itr->second.begin(); e_rhs_itr != inv_itr->second.end(); e_rhs_itr++) {
			Rule* rule = e_rhs_itr->second;

			int n = rule->lhs_int;

			if (Globals::Get<int>("debuglevel") >= 3) {
			  fprintf(logfile, " rule %s  ", rule->str().c_str());
			  if (rule->e_rhs)
			    fprint_array(logfile, rule->e_rhs);
			  fprintf(logfile, "\n");
			}

		      
			for (ritr = rcol->begin(); ritr != rcol->end(); ritr++){
			  int ur[2], vr[2], uchoice[2], vchoice[2];

			  ITGDCell* rcell = &ritr->second;

			  // the frontiers of the right constituent
			  ur[0] = dchart->u0of(ritr->first);
			  ur[1] = dchart->u1of(ritr->first);
			  vr[0] = dchart->v0of(ritr->first);
			  vr[1] = dchart->v1of(ritr->first);
			
			  double sita4 = rcell->prob;
			
			  double sita34 = sita3 + sita4; //log +
			
			  double lm_prob = 0.0;
			
			  uchoice[0] = uchoice[1] = 0;
			  vchoice[0] = vchoice[1] = 0;
			
			  if (rule->inverted) {
			    lm_prob = LMWEIGHT * compute_lm(ur, ul, vr, vl, rule, uchoice, vchoice);
			  } else {
			    lm_prob = LMWEIGHT * compute_lm(ul, ur, vl, vr, rule, uchoice, vchoice);
			  }
			  double newsita = rule->prob + sita34;
			  //+ lm_prob;
			
                        // this saves a *ton* of memory because we
                        // don't waste space populating stext with
                        // entries that are going to get pruned
                        // anyway. it can be the difference between
                        // out-of-memory errors and successful runs

			  //this makes a difference to the two-pass decoder
			  //because fewer bad constituents are stored in the coarse-level forest
			  
                        if (newsita >= maxscore - LOGRATIOBEAM) {

			 
                          // update new high score for span
                          maxscore = max(newsita, maxscore);
 

			  ITGDCell* pd = &(stext[stage][n][dchart->u01v01(uchoice[0], uchoice[1], vchoice[1], vchoice[0])]);
			  double oldsita = pd->prob;
			  itemcount++;

//                         pd->backptrs.insert(backpointer(rule, lcell, rcell, newsita));

			  //added by Hao
			  backpointer thebkptr;
			  thebkptr.lchild=lcell;
			  thebkptr.rchild=rcell;
			  thebkptr.rule=rule;

			  thebkptr.tri =  (ngram->N==3);
			  pd->bkptrs.push_back(thebkptr);

			  if (newsita > oldsita){//maximizing
			    
			    pd->N = rule->lhs_int;
			    pd->s = s;
			    pd->t = t;
			    DUEXASSIGN(pd->lword, uchoice);
			    DUEXASSIGN(pd->rword, vchoice);

			    pd->prob = newsita;

			    // best backpointers
			    pd->lchild = lcell;
			    pd->rchild = rcell;
			    pd->rule = rule;

			    // deprecated backpointers
//                           pd->S = S;
//                           DUEXASSIGN(pd->ul, ul);
//                           DUEXASSIGN(pd->vl, vl);
//                           DUEXASSIGN(pd->ur, ur);
//                           DUEXASSIGN(pd->vr, vr);
			  
			    if (Globals::Get<int>("debuglevel") >= 3) {
			      fprintf(logfile, "chart ");
			      fprintf(logfile, " %d %d %s[%s %s ... %s %s]  %g   lm: %g\n",
				      s, t, rule->lhs.data(), 
				      l1vocab[uchoice[0]].data(), l1vocab[uchoice[1]].data(), 
				      l1vocab[vchoice[1]].data(), l1vocab[vchoice[0]].data(),
				      newsita, lm_prob);
			    }
			  }//end of update

			}//end of maxscore pruning

			}//end of rcol
		      }//end for all binary rules 
		} // end for right child nonterminal	     

	      }//end if deletions
	      else{ //non deletions
	      
	      hash_map <int, hash_map <LMSTATE, ITGDCell > > *rhookspan;
	      hash_map <int, hash_map <LMSTATE, ITGDCell > >::iterator  rhookitr;
	      //find matching hooks
	      rhookspan = matchinghookspan(S, t, nl, vl[0], true);  //return (build if do not exist) the column of hooks on the right

	      for (rhookitr = rhookspan->begin(); rhookitr != rhookspan->end(); rhookitr++){
		int nr = rhookitr->first; 

	      
		  
		rcol = dchart->hookcolumn(S, t, nr, vl[0]);

		// for each binary rule having specified children
		lhs_inv_erhs_rule_hash *rnt = &grammar->rule_table[nl][nr];
	      
		lhs_inv_erhs_rule_hash::iterator lhs_itr;
		inv_erhs_rule_hash::iterator inv_itr;
		erhs_rule_hash::iterator e_rhs_itr;
	      
		for (lhs_itr = rnt->begin(); lhs_itr != rnt->end(); lhs_itr++)
		  for (inv_itr = lhs_itr->second.begin(); inv_itr != lhs_itr->second.end(); inv_itr++)
		    for (e_rhs_itr = inv_itr->second.begin(); e_rhs_itr != inv_itr->second.end(); e_rhs_itr++) {
		      Rule* rule = e_rhs_itr->second;

		      if (rule->inverted)
			continue;

		      int n = rule->lhs_int;

		      if (Globals::Get<int>("debuglevel") >= 3) {
			fprintf(logfile, " rule %s  ", rule->str().c_str());
			if (rule->e_rhs)
			  fprint_array(logfile, rule->e_rhs);
			fprintf(logfile, "\n");
		      }

		      
                      for (ritr = rcol->begin(); ritr != rcol->end(); ritr++){
                        int vr[2], uchoice[2], vchoice[2];

                        ITGDCell* rcell = &ritr->second;

                        // the frontiers of the right constituent
			//                        ur[0] = dchart->u0of(ritr->first);
			//                        ur[1] = dchart->u1of(ritr->first);
			//                        vr[0] = dchart->v0of(ritr->first);
			//                        vr[1] = dchart->v1of(ritr->first);

			DUEXASSIGN(vr, rcell->rword);

                        double sita4 = rcell->prob;
			
                        double sita34 = sita3 + sita4; //log +
			
                        double lm_prob = 0.0;
			

			DUEXASSIGN(uchoice, ul);
			DUEXASSIGN(vchoice, vr);
			//                        uchoice[0] = uchoice[1] = 0;
			//                        vchoice[0] = vchoice[1] = 0;

			/*			
			if (rule->inverted) {
			  lm_prob = LMWEIGHT * compute_lm(ur, ul, vr, vl, rule, uchoice, vchoice);
                        } else {
                          lm_prob = LMWEIGHT * compute_lm(ul, ur, vl, vr, rule, uchoice, vchoice);
                        }
			*/

                        double newsita = rule->prob + sita34 ;
			  //+ lm_prob;

			
                        // this saves a *ton* of memory because we
                        // don't waste space populating stext with
                        // entries that are going to get pruned
                        // anyway. it can be the difference between
                        // out-of-memory errors and successful runs

			  //this makes a difference to the two-pass decoder
			  //because fewer bad constituents are stored in the coarse-level forest
			  
                        if (newsita >= maxscore - LOGRATIOBEAM) {
                          // update new high score for span
                          maxscore = max(newsita, maxscore);


			
                        ITGDCell* pd = &(stext[stage][n][dchart->u01v01(uchoice[0], uchoice[1], vchoice[1], vchoice[0])]);
                        double oldsita = pd->prob;
                        itemcount++;

//                         pd->backptrs.insert(backpointer(rule, lcell, rcell, newsita));

			//added by Hao
			backpointer thebkptr;
			thebkptr.lchild=lcell;
			thebkptr.rchild=rcell;
			thebkptr.rule=rule;

			thebkptr.tri =  (ngram->N==3);

			pd->bkptrs.push_back(thebkptr);

			if (newsita > oldsita){//maximizing

                          pd->N = rule->lhs_int;
                          pd->s = s;
                          pd->t = t;
                          DUEXASSIGN(pd->lword, uchoice);
                          DUEXASSIGN(pd->rword, vchoice);

                          pd->prob = newsita;

                          // best backpointers
                          pd->lchild = lcell;
                          pd->rchild = rcell;
                          pd->rule = rule;

                          // deprecated backpointers
//                           pd->S = S;
//                           DUEXASSIGN(pd->ul, ul);
//                           DUEXASSIGN(pd->vl, vl);
//                           DUEXASSIGN(pd->ur, ur);
//                           DUEXASSIGN(pd->vr, vr);
			  
                          if (Globals::Get<int>("debuglevel") >= 3) {
                            fprintf(logfile, "chart ");
                            fprintf(logfile, " %d %d %s[%s %s ... %s %s]  %g   lm: %g\n",
                                    s, t, rule->lhs.data(), 
                                    l1vocab[uchoice[0]].data(), l1vocab[uchoice[1]].data(), 
                                    l1vocab[vchoice[1]].data(), l1vocab[vchoice[0]].data(),
                                    newsita, lm_prob);
                          }
                        }//end of updating

			}//end of maxscore

                      }//end of rcol
		    }//end for all binary rules 
	      } // end for right child nonterminal

	      } //end if non-deletion


          } // end for left child nonerminal
	    
	  }//end of lcol

	  

          // see which nonterm pairs exist in chart
          for (rnt_itr = rspan->begin(); rnt_itr != rspan->end(); rnt_itr++) {
	    int nr = rnt_itr->first;


	    rcol = dchart->column(nr, S, t);
	    
	    for (ritr = rcol->begin(); ritr != rcol->end(); ritr++) {
	      int ur[2], vr[2];


	      ITGDCell* rcell = &ritr->second;
	      
	      // the frontiers of the left constituent
	      ur[0] = dchart->u0of(ritr->first);
	      ur[1] = dchart->u1of(ritr->first);
	      vr[0] = dchart->v0of(ritr->first);
	      vr[1] = dchart->v1of(ritr->first);
	      
	      double sita4 = rcell->prob;

	      if (vr[0] == 0){

		for (lnt_itr = lspan->begin(); lnt_itr != lspan->end(); lnt_itr++) {
	      
		  int nl = lnt_itr->first;

		  // lookahead, avoid enumerating one column while the other is empty
		  if (!dchart->has_column(nl, s, S) || !dchart->has_column(nr, S, t))
		    continue;
	      
		  if (!grammar->rule_table[nl].count(nr))
		    continue;
		  
		  lcol = dchart->column(nl, s, S);

	      
		  // for each binary rule having specified children
		  lhs_inv_erhs_rule_hash *rnt = &grammar->rule_table[nl][nr];
	      
		  lhs_inv_erhs_rule_hash::iterator lhs_itr;
		  inv_erhs_rule_hash::iterator inv_itr;
		  erhs_rule_hash::iterator e_rhs_itr;
		  
		  for (lhs_itr = rnt->begin(); lhs_itr != rnt->end(); lhs_itr++)
		    for (inv_itr = lhs_itr->second.begin(); inv_itr != lhs_itr->second.end(); inv_itr++)
		      for (e_rhs_itr = inv_itr->second.begin(); e_rhs_itr != inv_itr->second.end(); e_rhs_itr++) {
			Rule* rule = e_rhs_itr->second;

			int n = rule->lhs_int;

			if (Globals::Get<int>("debuglevel") >= 3) {
			  fprintf(logfile, " rule %s  ", rule->str().c_str());
			  if (rule->e_rhs)
			    fprint_array(logfile, rule->e_rhs);
			  fprintf(logfile, "\n");
			}

		      
			for (litr = lcol->begin(); litr != lcol->end(); litr++){
			  int ul[2], vl[2], uchoice[2], vchoice[2];

			  ITGDCell* lcell = &litr->second;

			  // the frontiers of the right constituent
			  ul[0] = dchart->u0of(litr->first);
			  ul[1] = dchart->u1of(litr->first);
			  vl[0] = dchart->v0of(litr->first);
			  vl[1] = dchart->v1of(litr->first);
			
			  double sita3 = lcell->prob;
			
			  double sita34 = sita3 + sita4; //log +
			
			  double lm_prob = 0.0;
			
			  uchoice[0] = uchoice[1] = 0;
			  vchoice[0] = vchoice[1] = 0;
			
			  if (rule->inverted) {
			    lm_prob = LMWEIGHT * compute_lm(ur, ul, vr, vl, rule, uchoice, vchoice);
			  } else {
			    lm_prob = LMWEIGHT * compute_lm(ul, ur, vl, vr, rule, uchoice, vchoice);
			  }
			  double newsita = rule->prob + sita34;
			  //+ lm_prob;
			
			
                        // this saves a *ton* of memory because we
                        // don't waste space populating stext with
                        // entries that are going to get pruned
                        // anyway. it can be the difference between
                        // out-of-memory errors and successful runs

			  //this makes a difference to the two-pass decoder
			  //because fewer bad constituents are stored in the coarse-level forest
			  
                        if (newsita >= maxscore - LOGRATIOBEAM) {

                          // update new high score for span
                          maxscore = max(newsita, maxscore);



			  ITGDCell* pd = &(stext[stage][n][dchart->u01v01(uchoice[0], uchoice[1], vchoice[1], vchoice[0])]);
			  double oldsita = pd->prob;
			  itemcount++;

//                         pd->backptrs.insert(backpointer(rule, lcell, rcell, newsita));

			  //added by Hao
			  backpointer thebkptr;
			  thebkptr.lchild=lcell;
			  thebkptr.rchild=rcell;
			  thebkptr.rule=rule;

			  thebkptr.tri =  (ngram->N==3);
			  pd->bkptrs.push_back(thebkptr);

			  if (newsita > oldsita){//maximizing
			    
			    pd->N = rule->lhs_int;
			    pd->s = s;
			    pd->t = t;
			    DUEXASSIGN(pd->lword, uchoice);
			    DUEXASSIGN(pd->rword, vchoice);

			    pd->prob = newsita;

			    // best backpointers
			    pd->lchild = lcell;
			    pd->rchild = rcell;
			    pd->rule = rule;

			    // deprecated backpointers
//                           pd->S = S;
//                           DUEXASSIGN(pd->ul, ul);
//                           DUEXASSIGN(pd->vl, vl);
//                           DUEXASSIGN(pd->ur, ur);
//                           DUEXASSIGN(pd->vr, vr);
			  
			    if (Globals::Get<int>("debuglevel") >= 3) {
			      fprintf(logfile, "chart ");
			      fprintf(logfile, " %d %d %s[%s %s ... %s %s]  %g   lm: %g\n",
				      s, t, rule->lhs.data(), 
				      l1vocab[uchoice[0]].data(), l1vocab[uchoice[1]].data(), 
				      l1vocab[vchoice[1]].data(), l1vocab[vchoice[0]].data(),
				      newsita, lm_prob);
			    }
			  }//end of update

			}//end of maxscore
			}//end of lcol
		      }//end for all binary rules 
		} // end for left child nonterminal	     
	       
	      }//end if deletions
	      else{

	      hash_map <int, hash_map <LMSTATE, ITGDCell > > *lhookspan;
	      hash_map <int, hash_map <LMSTATE, ITGDCell > >::iterator  lhookitr;
	      //find matching hooks
	      lhookspan = matchinghookspan(s, S, nr, vr[0], false);  //return (build if do not exist) the column of hooks on the right

	      for (lhookitr = lhookspan->begin(); lhookitr != lhookspan->end(); lhookitr++){
		int nl = lhookitr->first; 

		lcol = dchart->hookcolumn(s, S, nl, vr[0]);


		// for each binary rule having specified children
		lhs_inv_erhs_rule_hash *rnt = &grammar->rule_table[nl][nr];
	      
		lhs_inv_erhs_rule_hash::iterator lhs_itr;
		inv_erhs_rule_hash::iterator inv_itr;
		erhs_rule_hash::iterator e_rhs_itr;
	      
		for (lhs_itr = rnt->begin(); lhs_itr != rnt->end(); lhs_itr++)
		  for (inv_itr = lhs_itr->second.begin(); inv_itr != lhs_itr->second.end(); inv_itr++)
		    for (e_rhs_itr = inv_itr->second.begin(); e_rhs_itr != inv_itr->second.end(); e_rhs_itr++) {
		      Rule* rule = e_rhs_itr->second;

		      if (!rule->inverted)
			continue;

		      int n = rule->lhs_int;

		      if (Globals::Get<int>("debuglevel") >= 3) {
			fprintf(logfile, " rule %s  ", rule->str().c_str());
			if (rule->e_rhs)
			  fprint_array(logfile, rule->e_rhs);
			fprintf(logfile, "\n");
		      }

		      
                      for (litr = lcol->begin(); litr != lcol->end(); litr++){
                        int vl[2], uchoice[2], vchoice[2];

                        ITGDCell* lcell = &litr->second;

                        // the frontiers of the right constituent
			//                        ur[0] = dchart->u0of(ritr->first);
			//                        ur[1] = dchart->u1of(ritr->first);
			//                        vr[0] = dchart->v0of(ritr->first);
			//                        vr[1] = dchart->v1of(ritr->first);

			DUEXASSIGN(vl, lcell->rword);

                        double sita3 = lcell->prob;
			
                        double sita34 = sita3 + sita4; //log +
			
                        double lm_prob = 0.0;

			DUEXASSIGN(uchoice, ur);
			DUEXASSIGN(vchoice, vl);

			/*
                        uchoice[0] = uchoice[1] = 0;
                        vchoice[0] = vchoice[1] = 0;
			
			if (rule->inverted) {
			  lm_prob = LMWEIGHT * compute_lm(ur, ul, vr, vl, rule, uchoice, vchoice);
                        } else {
                          lm_prob = LMWEIGHT * compute_lm(ul, ur, vl, vr, rule, uchoice, vchoice);
                        }
			*/

                        double newsita = rule->prob + sita34 ;
			  //+ lm_prob;
			
			
                        // this saves a *ton* of memory because we
                        // don't waste space populating stext with
                        // entries that are going to get pruned
                        // anyway. it can be the difference between
                        // out-of-memory errors and successful runs

			  //this makes a difference to the two-pass decoder
			  //because fewer bad constituents are stored in the coarse-level forest
			  
                        if (newsita >= maxscore - LOGRATIOBEAM) {

                          // update new high score for span
                          maxscore = max(newsita, maxscore);


                        ITGDCell* pd = &(stext[stage][n][dchart->u01v01(uchoice[0], uchoice[1], vchoice[1], vchoice[0])]);
                        double oldsita = pd->prob;
                        itemcount++;

//                         pd->backptrs.insert(backpointer(rule, lcell, rcell, newsita));

			//added by Hao
			backpointer thebkptr;
			thebkptr.lchild=lcell;
			thebkptr.rchild=rcell;
			thebkptr.rule=rule;

			thebkptr.tri = (ngram->N==3);
			pd->bkptrs.push_back(thebkptr);

			if (newsita > oldsita){//maximizing

                          pd->N = rule->lhs_int;
                          pd->s = s;
                          pd->t = t;
                          DUEXASSIGN(pd->lword, uchoice);
                          DUEXASSIGN(pd->rword, vchoice);

                          pd->prob = newsita;

                          // best backpointers
                          pd->lchild = lcell;
                          pd->rchild = rcell;
                          pd->rule = rule;

                          // deprecated backpointers
//                           pd->S = S;
//                           DUEXASSIGN(pd->ul, ul);
//                           DUEXASSIGN(pd->vl, vl);
//                           DUEXASSIGN(pd->ur, ur);
//                           DUEXASSIGN(pd->vr, vr);
			  
                          if (Globals::Get<int>("debuglevel") >= 3) {
                            fprintf(logfile, "chart ");
                            fprintf(logfile, " %d %d %s[%s %s ... %s %s]  %g   lm: %g\n",
                                    s, t, rule->lhs.data(), 
                                    l1vocab[uchoice[0]].data(), l1vocab[uchoice[1]].data(), 
                                    l1vocab[vchoice[1]].data(), l1vocab[vchoice[0]].data(),
                                    newsita, lm_prob);
                          }
                        }//end of updating

			}//end of maxscore
                      }//end of lcol
		    }//end for all binary rules 
	      } // end for right child nonterminal

	      }//end if non-deletions

          } // end for left child nonerminal

	  }//end of rcol
	  


	  
        }//end for deltas, splitting S




        /* pruning */

        double maxsita = -1e10;
        // with-ratio pruning
  
        //find the max prob
        double thesita;
        //find the max

        /* find the highest probability entry in the relevant portion (= ??) of the chart */
        hash_map<int, hash_map <LMSTATE, ITGDCell> >::iterator nt_itr;
        for (nt_itr = stext[stage].begin(); nt_itr != stext[stage].end(); nt_itr++){
          lcol = &(nt_itr->second);
          for (litr = lcol->begin(); litr != lcol->end(); litr++){
            thesita = litr->second.prob;
            if (thesita < 0 && thesita > maxsita) {
              maxsita = thesita;
	    }
          }
        }

	// const bool TOPK_PRUNING = true;
	hash_map <LMSTATE, ITGDCell>::iterator bklitr;
	for (nt_itr = stext[stage].begin(); nt_itr != stext[stage].end(); nt_itr++) {
	  int N = nt_itr->first;
	  lcol = &(nt_itr->second);
	  
	  // preparing for topk pruning
	  priority_queue<double> topk_pq_1;
	  priority_queue<double> topk_pq_2;
	  int count1 = 0;
	  int count2 = 0;
	  double mymax = -INFINITY;
	  if (prune_method == "topk") {
	    for (litr = lcol->begin(); litr != lcol->end();litr++){
	      double thesita = litr->second.prob;
	      ++count1;
	      if (thesita > mymax) {
		mymax = thesita;
	      }
	      topk_pq_1.push(-thesita);
	      if (topk_pq_1.size() > Globals::Get<double>("prune_topk_other_nonterm")) {
		topk_pq_1.pop();
	      }
	      topk_pq_2.push(-thesita);
	      if (topk_pq_2.size() > Globals::Get<double>("prune_topk_universal_nonterm")) {
		topk_pq_2.pop();
	      }
	    }
	  }
	  
	  if (prune_method == "topk") {
	    for (litr = lcol->begin(); litr != lcol->end();){
	      double thesita = litr->second.prob;
	      if (N != 3) {
		if (-thesita > topk_pq_1.top()) {
		  bklitr=litr;
		  litr++;
		  lcol->erase(bklitr);
		  ++count2;
		} else {
		  litr++;
		}
	      } else {
		if (-thesita > topk_pq_2.top()) {
		  bklitr=litr;
		  litr++;
		  lcol->erase(bklitr);
		  ++count2;
		} else {
		  litr++;
		}
	      }
	    }
	  } else if (prune_method == "ratio") {
	    // use the highest prob to prune everything that falls outside the ratio beam
	    // logratio beam pruning
	    for (litr = lcol->begin(); litr != lcol->end();){
	      double thesita = litr->second.prob;
	      if (thesita < (maxsita - LOGRATIOBEAM)) { //log +
		bklitr=litr;
		litr++;
		lcol->erase(bklitr);
	      } else {
		litr++;
	      }
	    }
	  } // else
	  /*
	  if (N == 3) {
	    cerr << endl;
	  }
	  if (count2)  {
	    cerr << "**";
	  }
	  cerr << "for [" << N << "] ";
	  cerr << " total: " << count1 
	       << " (" << s << ", " << t << ") "
	       << " pruned: " << count2 
	       << " remain: " << count1 - count2 << endl;
	  if (N != 3) {
	    for(; topk_pq_1.size() ;) {
	      cerr << "\t" << exp(-topk_pq_1.top()) << endl;
	      topk_pq_1.pop();
	    }
	  } else {
	    for(; topk_pq_2.size() ;) {
	      cerr << "\t" << exp(-topk_pq_2.top()) << endl;
	      topk_pq_2.pop();
	    }
	  }
	  cerr << " threshold: " << exp(mythreshold)
	       << " max: " << exp(mymax)
	       << endl;
	  */
	}
	
        if (Globals::Get<int>("debuglevel") >= 2) {
          int entries=0;

          for (nt_itr = stext[stage].begin(); nt_itr != stext[stage].end(); nt_itr++){
            lcol = &(nt_itr->second);
            entries += lcol->size();
          }

          fprintf(logfile,"Entries over (%d,%d) AFTER PRUNING: %d\n", s, t, entries);
        }

        // copy from stext to chart
        for (nt_itr = stext[stage].begin(); nt_itr != stext[stage].end(); nt_itr++){
          int n = nt_itr->first;
          lcol = &(nt_itr->second);

          for (litr = lcol->begin(); litr != lcol->end(); litr++){
            LMSTATE index = litr->first;
            itemcount++;
            if ( ((*dchart->column(n, s, t))[index].prob == 0.0) 
                 || ((*dchart->column(n, s, t))[index].prob < litr->second.prob) ) {
              (*dchart->column(n, s, t))[index] = litr->second;
            }
          }
        }
      }	// end for stage

      hash_map<int, hash_map <LMSTATE, ITGDCell> > *span = dchart->span(s, t);
      hash_map<int, hash_map <LMSTATE, ITGDCell> >::iterator nt_itr;

      if (Globals::Get<int>("debuglevel") >= 2) {
        int entries = 0;
      
        for (nt_itr = span->begin(); nt_itr != span->end(); nt_itr++){
          lcol = &(nt_itr->second);

          //pruning
          entries += lcol->size();
        }

        fprintf(logfile,"Entries over (%d,%d) BEFORE PRUNING: %d\n", s, t, entries);
      }
         
      double maxsita = -1e10;

      // with-ratio pruning
      for (nt_itr = span->begin(); nt_itr != span->end(); nt_itr++) {
        lcol = &(*nt_itr).second;
        for (litr = lcol->begin(); litr != lcol->end(); litr++){
          double thesita = litr->second.prob;
          if (thesita < 0 && thesita > maxsita) {
            maxsita = thesita;
	  }
        }
      }

      // const bool TOPK_PRUNING = true;
      hash_map <LMSTATE, ITGDCell>::iterator bklitr;
      for (nt_itr = span->begin(); nt_itr != span->end(); nt_itr++) {
	int N = nt_itr->first;
        lcol = &(nt_itr->second);

	// preparing for topk pruning
	priority_queue<double> topk_pq_1;
	priority_queue<double> topk_pq_2;
	int count1 = 0;
	int count2 = 0;
	double mymax = -INFINITY;
	if (prune_method == "topk") {
	  for (litr = lcol->begin(); litr != lcol->end();litr++){
	    double thesita = litr->second.prob;
	    ++count1;
	    if (thesita > mymax) {
	      mymax = thesita;
	    }
	    topk_pq_1.push(-thesita);
	    if (topk_pq_1.size() > Globals::Get<double>("prune_topk_other_nonterm")) {
	      topk_pq_1.pop();
	    }
	    topk_pq_2.push(-thesita);
	    if (topk_pq_2.size() > Globals::Get<double>("prune_topk_universal_nonterm")) {
	      topk_pq_2.pop();
	    }
	  }
	}

	if (prune_method == "topk") {
	  for (litr = lcol->begin(); litr != lcol->end();){
	    double thesita = litr->second.prob;
	    if (N != 3) {
	      if (-thesita > topk_pq_1.top()) {
		bklitr=litr;
		litr++;
		lcol->erase(bklitr);
		++count2;
	      } else {
		litr++;
	      }
	    } else {
	      if (-thesita > topk_pq_2.top()) {
		bklitr=litr;
		litr++;
		lcol->erase(bklitr);
		++count2;
	      } else {
		litr++;
	      }
	    }
	  }
	} else if (prune_method == "ratio") {
	  // use the highest prob to prune everything that falls outside the ratio beam
	  // logratio beam pruning
	  hash_map <LMSTATE, ITGDCell>::iterator bklitr;
	  for (litr = lcol->begin(); litr != lcol->end();){
	    double thesita = litr->second.prob;
	    if (thesita < (maxsita - LOGRATIOBEAM)) { //log +
	      bklitr=litr;
	      litr++;
	      lcol->erase(bklitr);
	    } else {
	      litr++;
	    }
	  }
	} // else
	/*
	if (N == 3) {
	  cerr << endl;
	}
	if (count2)  {
	  cerr << "**";
	}
	cerr << "FOR [" << N << "] ";
	cerr << " total: " << count1 
	     << " (" << s << ", " << t << ") "
	     << " pruned: " << count2 
	     << " remain: " << count1 - count2 << endl;
	if (N != 3) {
	  for(; topk_pq_1.size() ;) {
	    cerr << "\t" << exp(-topk_pq_1.top()) << endl;
	    topk_pq_1.pop();
	  }
	} else {
	  for(; topk_pq_2.size() ;) {
	    cerr << "\t" << exp(-topk_pq_2.top()) << endl;
	    topk_pq_2.pop();
	  }
	}
	cerr << " threshold: " << exp(mythreshold)
	     << " max: " << exp(mymax)
	     << endl;
	*/
      }
      
      if (Globals::Get<int>("debuglevel") >=2 ) {
        int entries=0;

        for (nt_itr = span->begin(); nt_itr != span->end(); nt_itr++) {
          lcol = &(*nt_itr).second;
          entries += lcol->size();
        }

        fprintf(logfile,"Entries over (%d,%d) AFTER PRUNING: %d\n", s, t, entries);
      }
     
      // unary rules
      // (no split point)

      // for unary rules, there is a order related issule.
      // We can not control the order because the rules are stored in a hash table, 
      // even worse, it will be altered inside the loop
      // So, we repeately apply the unary rules until no more changes.
      // Practically, we stop after 99 iterations to avoid infinite loop.
      // Another option is to preprocess the grammar to get a topological order.
      // -Zhao

      int LOOP_COUNT = 99;
      bool changed = true;
      for (int loop_count = 0; loop_count <= LOOP_COUNT; ++loop_count) {
	if (loop_count >= LOOP_COUNT) {
	  cerr << "WARNING: Your grammar may have loop for unary rules!" << endl;
	  break;
	}
	if (!changed) {
	  break;
	}
	changed = false;
	// cerr << endl << "loop... " << endl;

	for (nt_itr = span->begin(); nt_itr != span->end(); nt_itr++) {
	  int rhs = nt_itr->first;
	  lhs_inv_erhs_rule_hash *rnt = &grammar->rule_table[rhs][-1];
	  
	  lhs_inv_erhs_rule_hash::iterator lhs_itr;
	  inv_erhs_rule_hash::iterator inv_itr;
	  erhs_rule_hash::iterator e_rhs_itr;
	  
	  for (lhs_itr = rnt->begin(); lhs_itr != rnt->end(); lhs_itr++)
	    for (inv_itr = lhs_itr->second.begin(); inv_itr != lhs_itr->second.end(); inv_itr++)
	      for (e_rhs_itr = inv_itr->second.begin(); e_rhs_itr != inv_itr->second.end(); e_rhs_itr++) {
		Rule* rule = e_rhs_itr->second;
		
		int lhs = rule->lhs_int;
		
		//filtering non-root S-es except at the end
		if ((lhs == grammar->start_symbol) && (s > 0 || t < T))
		  break;
		
		if (!dchart->has_column(rhs, s, t))
		  continue;
		
		lcol = dchart->column(rhs, s, t);
		for (litr = lcol->begin(); litr != lcol->end(); litr++){
		  int u[2], v[2], uchoice[2], vchoice[2];
		  
		  ITGDCell* child = &litr->second;
		  
		  u[0] = dchart->u0of(litr->first);
		  u[1] = dchart->u1of(litr->first);
		  v[0] = dchart->v0of(litr->first);
		  v[1] = dchart->v1of(litr->first);
		  
		  double lm_prob;
		  uchoice[0] = uchoice[1] = 0;
		  vchoice[0] = vchoice[1] = 0;
		  
		  lm_prob = LMWEIGHT * compute_lm(u, NULL, v, NULL, rule, uchoice, vchoice);
		  
		  double prob = rule->prob;
		  double newsita = prob + lm_prob + child->prob; //log +

		  // uchoice and vchoice include any target language terminals
		  // that were added by the unary rule.
		  ITGDCell* pd = dchart->cell(lhs, s, t, uchoice[0], uchoice[1], vchoice[1], vchoice[0]);

		  double oldsita = pd->prob;
		  itemcount++;
		  
// 		  pd->backptrs.insert(backpointer(rule, child, NULL, newsita));
		  
		  // added by Hao
		  backpointer thebkptr;
		  thebkptr.lchild=child;
		  thebkptr.rchild=NULL;
		  thebkptr.rule=rule;

		  thebkptr.tri =  (ngram->N==3);
		  pd->bkptrs.push_back(thebkptr);
		  
		  
		  if (newsita > oldsita){ //maximizing
		    // cerr << lhs << "  ->  " << rhs << "\t";
		    // cerr << "(" << newsita << " > " << oldsita << ")" << endl;
		    changed = true;
		    
		    pd->N = rule->lhs_int;
		    pd->s = s;
		    pd->t = t;
		    DUEXASSIGN(pd->lword, uchoice);
		    DUEXASSIGN(pd->rword, vchoice);
		    pd->prob = newsita;


		    // best backpointers
		    pd->lchild = child;
		    pd->rchild = NULL;
		    pd->rule = rule;

                    // deprecated backpointers
// 		    pd->S = -1;
// 		    DUEXASSIGN(pd->ul, u);
// 		    DUEXASSIGN(pd->vl, v);
// 		    DUEXASSIGN(pd->ur, u);
// 		    DUEXASSIGN(pd->vr, v);
		    
		    if (Globals::Get<int>("debuglevel") >= 3) {
		      fprintf(logfile, "chart ");
		      fprintf(logfile, " %d %d %d[%s %s ... %s %s]  %g   \n", s, t, lhs, 
			      l1vocab[uchoice[0]].data(), l1vocab[uchoice[1]].data(), 
			      l1vocab[vchoice[1]].data(), l1vocab[vchoice[0]].data(), newsita);
		    }
		    
		  }//end of updating viterbi
		}//end for lcol   
	      }//end for unary rules with specified rhs
	}//end of for nt_itr over rhs
      } // loop until no change

      for (nt_itr = span->begin(); nt_itr != span->end(); nt_itr++) {
	int N = nt_itr->first;
	// speed up for generalized ITG grammar
	// do not create empty entries in the hash table, that wastes a huge mount of time!
	if (dchart->has_column(N,s,t)) {
	  edgecount += dchart->column(N,s,t)->size();
	}
      }

    }//end for s, t
  }//end for span

  /* find the best cell */
  /* 1. find the best entry that covers the entire span */
  lcol = dchart->column(grammar->start_symbol, 0, T);
  for (litr = lcol->begin(); litr != lcol->end(); litr++){
    ITGDCell* cell = &litr->second;

    int u[2], v[2];
    u[0] = dchart->u0of(litr->first);
    u[1] = dchart->u1of(litr->first);
    v[0] = dchart->v0of(litr->first);
    v[1] = dchart->v1of(litr->first);

    //  if (n>0)
    //    continue;//only looking at the "S"
    if ( (u[0] > 0 && v[0] > 0) || ((u[0] + v[0]) == 0)) {
      double lm_prob = 0.0;
      if (ngram->N == 2) {
        lm_prob = LMWEIGHT * (ngram->bigram(-1, u[0]) + ngram->bigram(v[0], -2));
      } else { // trigram
        lm_prob = LMWEIGHT * (ngram->trigram(-1, u[0], u[1]) 
                              + ngram->trigram(v[1], v[0], -2));
        lm_prob += LMWEIGHT * (ngram->bigram(-1, u[0])); //adding the leading bigram probability
      }

      double prob = cell->prob + lm_prob;

      ITGDCell* top = dchart->cell(grammar->_nonterminal_cnt, 0, T, 0, 0, 0, 0); 

      top->bkptrs.push_back(backpointer(NULL, cell, NULL, prob));
//       top->backptrs.insert(backpointer(NULL, cell, NULL, prob));

      if (prob > top->prob) {
        top->N = grammar->start_symbol;
        top->s = 0;
        top->t = T;
        top->prob = prob;
        top->lword[0] = -1;
        if (ngram->N == 2) {
          top->lword[1] = 0;
          top->rword[1] = 0;
        } else {
          top->lword[1] = cell->lword[0];
          top->rword[1] = cell->rword[0];
        }
        top->rword[0] = -2;

        // best backpointers
        top->rule = NULL;
        top->lchild = cell;
        top->rchild = NULL;

        // deprecated backpointers
//         top->S = -1;
//         DUEXASSIGN(pd->ul, u);
//         DUEXASSIGN(pd->vl, v);
//         DUEXASSIGN(pd->ur, u);
//         DUEXASSIGN(pd->vr, v);
      }
    }
  }

  //  fprintf(logfile, "arcs popped %d\n", itemcount);

  if (Globals::Get<int>("verbosity") == 1) {
    cout << "produced forest of " << edgecount << " edges\n";
    fflush(stdout);
  }

  /* dump chart */

  if (Globals::Get<int>("debuglevel") >= 3) {
    dchart->dump(_sentno);
  }
}
