/* itgdec-cyk-hook-agenda.cpp
 * use hooks to lower the complexity
 * $Id: itgdec-cyk-hook-agenda.cpp,v 1.3 2011/10/16 22:33:51 gildea Exp $
 */

#include "itgdec-cyk-agenda.h"




/*
 *  Decoding with an agenda, using hooks.
 */
string CYKDecoder::decode_hook_agenda(bool standalone) {

  int ul[2], vl[2], u[2], v[2];
  double sita0, sita3, sita4,  sita30, sita40;
  ITGDCell *pd = NULL; //pointer to a decoding chart cell
  int uchoice[2];
  int vchoice[2];
  int _prune_agenda = Globals::Get<bool>("prune_agenda");


  //best in column for both hooks and non-hooks
  hash_map<int, double> ***bc = new hash_map<int, double> **[2];
  for (int h = 0; h < 2; h++){
    bc[h]=new hash_map<int, double> *[T+1];
    for (int iter = 0; iter <= T; iter++){
      bc[h][iter] = new hash_map<int, double> [T+1];
    }
  }

  agenda_t agenda; //for agenda-based decoding

  // neighbors' index
  // neighbors[left/right][hook/non-hook][boundary_location] [u][nonterm_id]

  hash_map<int, hash_map<int, neighbor_t> > ***neighbors;

  // neighbor index allocation
  neighbors = new hash_map<int, hash_map<int, neighbor_t> > **[2];
  for (int d = 0; d < 2; d++){
    neighbors[d] = new hash_map<int, hash_map<int, neighbor_t> > *[2];
    for (int h = 0; h < 2; h++){
      neighbors[d][h] = new hash_map<int, hash_map<int, neighbor_t> >  [T+1];
    }
  }

  //bottom-up
  //initialize the agenda

  /* add phrase translations to the agenda */

  for (int s = 0; s <= T; s++){
    if (s < T) {
      for (int lspan = 1; lspan <= min(T - s, MAXPHR); lspan++) {
        int t = s + lspan;
        for (int uidx = 0; enclosure[s][lspan][uidx]; uidx++){
          WORDTYPE * ephr = enclosure[s][lspan][uidx];
          int phrlen = phraselen(ephr);

          int u[2], v[2];
          u[0] = *ephr;
          v[0] = last_word(ephr);
          u[1] = v[1] = 0;
          // update the inside words for 3+ grams
	  if (ngram->N > 2 && phrlen > 1) {
            u[1] = ephr[1];
            v[1] = penultimate_word(ephr);
	  }

	  WORDTYPE* cphr_t = splice(cn, s, s + lspan);
	  WORDTYPE* cphr = grammar->phrase_symbol_table_entry(cphr_t);
	  delete[] cphr_t;
	  
	  if (lspan == 1 && !grammar->probs_table.count(cphr))
	    cphr = grammar->unkphr;
	  
	  lhs_to_prob_table yitr = grammar->probs_table[cphr][ephr];
	  lhs_to_prob_table::iterator l_iter;
	  for (l_iter = yitr.begin(); l_iter != yitr.end(); l_iter++) {
	    // l_iter->first is preterminal id
	    // l_iter->second is rule prob
	    int preT = l_iter->first;
	    pd = dchart->cell(preT, s, t, u[0], u[1], v[1], v[0]);
	    pd->N = preT;
	    pd->s = s;
	    pd->t = t;
	    pd->rule = NULL;
	    pd->lchild = pd->rchild = NULL;
	    pd->closed = 0;
	    pd->inserted[0] = 0;
// 	    pd->S = -1;
	    pd->ephr = ephr;
	    double est = dchart->estimate(preT, s, t, u[0], u[1], v[1], v[0]);
	    double insout = pd->prob + est;
	    //  printf("access [%d\t%d]\t[%d\t%d]\n", 0, s, s+1, T-s-1);
	    agenda_node_t a = agenda_node_t(s, t, u[0], u[1], v[0], v[1], preT, 0, insout);
	    agenda.push(a);
	    
	    if (Globals::Get<int>("debuglevel") >= 3) 
	      fprintf(logfile, "Adding %s to the agenda (prob = %.10e, est = %.10e)\n", 
		      a.data().data(), pd->prob, est);
	  }
        }
      }
    } // if (s < T)

    /* add insertions to the agenda */

    //epsilon to word
    if (Globals::Get<bool>("allow_insertions")) {
      for (int uidx = 0; enclosure[s][0][uidx]; uidx++){
        WORDTYPE * ephr = enclosure[s][0][uidx];
        int phrlen = phraselen(ephr);

        int u[2], v[2];
        u[0] = *ephr;
        v[0] = last_word(ephr);
        u[1] = v[1] = 0;

        // update the inside words for 3+ grams
        if (ngram->N > 2 && phrlen > 1) {
          u[1] = ephr[1];
          v[1] = penultimate_word(ephr);
        }

        //s, s, u, u
        pd = dchart->cell(grammar->T, s, s, u[0], u[1], v[1], v[0]);
        pd->N = grammar->T;
        pd->s = s;
        pd->t = s;
        pd->closed = 0;
        pd->inserted[0] = 0;
//         pd->S = -1;
        pd->ephr = ephr;
        DUEXASSIGN(pd->lword,u);
        DUEXASSIGN(pd->rword,v);
        double est = dchart->estimate(grammar->T, s, s, u[0], u[1], v[1], v[0]);
        double insout = pd->prob + est;
        agenda_node_t a = agenda_node_t(s, s, u[0], u[1], v[0], v[1], grammar->T, 0, insout);
        agenda.push(a);

        if (Globals::Get<int>("debuglevel") >= 3) 
          fprintf(logfile, "Adding %s to the agenda (prob = %.10e, est = %.10e)\n", 
                  a.data().data(), pd->prob, est);
      }
    }
  }

  bool include_itgscore = ! Globals::Get<bool>("suppress_itgscore");

  itemcount = 0;
  hookitemcount = 0;

  int agenda_count = 0;
  //  int curins; 
  //searching
  while (!agenda.empty()) {
    agenda_node_t anode = agenda.top();
    int s = anode.s;
    int t = anode.t;
    u[0] = anode.u[0];
    u[1] = anode.u[1];
    v[0] = anode.v[0];
    v[1] = anode.v[1];
    int n = anode.n;
    
    //added field
    int hk = anode.h;

    int hl[2];
    hl[0]=hl[1]=0;

    agenda.pop();

    agenda_count++;

    /* GOAL 
     * The goal is distinguished by setting the nonterminal
     * spanning it to grammar->N.  We can't use S, because when
     * we see the chart entry (S,0,T), we need to add in the LM
     * probabilities for the end and beginning of the sentence.
     * When that is done, the entry is put back onto the agenda
     * with the special nonterminal grammar->N.
     */
    if ( (s == 0) && (t == T ) && (n == grammar->_nonterminal_cnt) ) {
      fprintf(logfile, "Agenda: popped goal %s\n", anode.data().data());
      break;
    }

    //    ITGDCell *popped_pd = dchart->cell(n, s, t, u[0], u[1], v[1], v[0]);
    ITGDCell *popped_pd = dchart->cook(hk, n, s, t, u[0], u[1], v[1], v[0]); //cell or hook

    if (popped_pd->closed)
      continue;

    if (Globals::Get<int>("debuglevel") >= 3) {
      fprintf(logfile, "Agenda: popped  n : %s hk : %d s: %d t: %d, lm: %d %d %d %d (%.5f)\n", 
              grammar->nonterminals[n].data(), hk, s, t, u[0], u[1], v[0], v[1], popped_pd->prob);
    }

    popped_pd->closed = 1;
    sita0 = popped_pd->prob; //focus
    //    if (hk)
    //      DUEXASSIGN(hl, popped_pd->lchild->lword);  //reach the words hidden under the hook

    //what's inside in hook

    if (!bc[hk][s][t].count(n) || sita0 > bc[hk][s][t][n])
      bc[hk][s][t][n] = sita0;

    if (_prune_agenda)
      if (!hk && 
	  (sita0 < (bc[hk][s][t][n] - LOGRATIOBEAM) ) ) { //only for non-hooks?
        if (Globals::Get<int>("debuglevel") >= 3) 
          fprintf(logfile, "sita0: %g, bc %g, logratiobeam %g\n",
                  sita0, bc[hk][s][t][n], LOGRATIOBEAM);
        continue;
      }

    //here, neighbors are not indexed by the boundary words
    //hooks are indexed by u, non-hooks are by v
    //for bigrams only
    neighbors[0][hk][s][hk?u[0]:v[0]][n].push_back(neighbor_node_t(t, u[0], u[1], v[0], v[1]));
    neighbors[1][hk][t][hk?u[0]:v[0]][n].push_back(neighbor_node_t(s, u[0], u[1], v[0], v[1]));


    //for non-deletions
    if (u[0] != 0)
    for (int iter=0; iter<(hk?1:2); iter++){
      //there are two iterations for non-hooks
      //in the second iteration, we combine epsilon non-hooks
      
      if (iter==1){
	DUEXASSIGN(hl, u);
	u[0]=u[1]=0;
	hk=1; //pretend it is a hook, looking for an epsilon non-hook
      }


    // exploring, expanding the open set
    // left neighbors end at position s:
    hash_map<int, neighbor_t>::iterator nl_itr;
    for (nl_itr = neighbors[1][hk?0:1][s][hk?u[0]:v[0]].begin(); nl_itr != neighbors[1][hk?0:1][s][hk?u[0]:v[0]].end();
         nl_itr++) {

      int nl = nl_itr->first;

      neighbor_t * theneighbor = &(neighbors[1][hk?0:1][s][hk?u[0]:v[0]][nl]);
      int nbr_neighbors = theneighbor->size();

      for (int ni = 0; ni < nbr_neighbors; ni++){

        // for each binary rule having specified children
        lhs_inv_erhs_rule_hash *rnt = &grammar->rule_table[nl][n];
        lhs_inv_erhs_rule_hash::iterator lhs_itr;
        inv_erhs_rule_hash::iterator inv_itr;
        erhs_rule_hash::iterator e_rhs_itr;

        for (lhs_itr = rnt->begin(); lhs_itr != rnt->end(); lhs_itr++)
          for (inv_itr = lhs_itr->second.begin(); inv_itr != lhs_itr->second.end(); inv_itr++)
            for (e_rhs_itr = inv_itr->second.begin(); e_rhs_itr != inv_itr->second.end(); 
                 e_rhs_itr++) {
	       
              Rule *rule = e_rhs_itr->second;
              int lhs = rule->lhs_int;


	      //hooks are left-extending
	      if (iter==0) //in iter 1, the pretended hook do not have the matching constraint
		if ( ! ( (rule->inverted && !hk ) || (!rule->inverted && hk) ) )
		  continue; 


              if (Globals::Get<int>("debuglevel") >= 3)
                fprintf(logfile, "looking for left neighbor: 1 s: %d nl: %d\n", s, nl);
	     
              int S = (*theneighbor)[ni].S;

	      ITGDCell *neighbor_pd = dchart->cook(hk?0:1, nl, S, s, (*theneighbor)[ni].u[0], (*theneighbor)[ni].u[1],
						   (*theneighbor)[ni].v[1], (*theneighbor)[ni].v[0]); 

	      if (hk){
		//the left neighbor is not a hook
		ul[0] = (*theneighbor)[ni].u[0];
		ul[1] = (*theneighbor)[ni].u[1];
		vl[0] = (*theneighbor)[ni].v[0];
		vl[1] = (*theneighbor)[ni].v[1];
	     
	      }
	      else{
		//the left neighbor is a hook
		ul[0] = neighbor_pd->lchild->lword[0];
		ul[1] = neighbor_pd->lchild->lword[1]; //needs the words under the hook
		vl[0] = (*theneighbor)[ni].v[0];
		vl[1] = (*theneighbor)[ni].v[1];
	     
		
	      }
	     
              if (Globals::Get<int>("debuglevel") >= 3) 
                fprintf(logfile, "left neighbor: %d lm %d %d %d %d\n", S, ul[0], ul[1], vl[1], vl[0]);
	     

              sita3 = neighbor_pd->prob;
	     
              if (_prune_agenda)
                if (hk &&  
		    ( bc[hk?0:1][S][s].count(nl) && sita3 < (bc[hk?0:1][S][s][nl] - LOGRATIOBEAM)) )
                  continue;
	     
              sita30 = sita3 + sita0;//log+
	      
              uchoice[0] = uchoice[1] = 0;
              vchoice[0] = vchoice[1] = 0;
	     
              pd = dchart->cell(lhs, S, t, uchoice[0], uchoice[1], vchoice[1], vchoice[0]);
	     
              double newsita = //lm_prob + 
		sita30; //lm prob already included in the hook
              if (include_itgscore)
                newsita += rule->prob;
	     
              double oldsita = pd->prob;
	     
              itemcount++;

	      if (newsita > oldsita){//maximizing
                pd->N = lhs;
                pd->s = S;
                pd->t = t;
                pd->prob = newsita;
                DUEXASSIGN(pd->lword, uchoice);
                DUEXASSIGN(pd->rword, vchoice);
	       
                pd->rule = rule;	// backpointers
		pd->lchild = (hk)?neighbor_pd:neighbor_pd->lchild;   //needs to point to the cells
		pd->rchild = popped_pd;

                double insout = pd->prob + dchart->estimate(lhs, S, t, uchoice[0], uchoice[1], vchoice[1], vchoice[0]);
                //put on agenda
                agenda_node_t a(S, t, uchoice[0], uchoice[1], vchoice[0], vchoice[1], lhs, 0, insout); //non-hook
                agenda.push(a);
                if (Globals::Get<int>("debuglevel") >= 3) 
                  fprintf(logfile, "Adding %s to the agenda (prob = %.5f, insout = %.5f)\n", a.data().data(), pd->prob, insout);

              } // if max

	      
	      backpointer thebkptr;
	      thebkptr.lchild=neighbor_pd;         //(hk)?neighbor_pd:neighbor_pd->lchild;
	      thebkptr.rchild=popped_pd;
	      thebkptr.rule=rule;		

	      thebkptr.tri =  (ngram->N==3);
	      pd->bkptrs.push_back(thebkptr);


            } // e_rhs
      } //end of enumerating neighbors
    } // left side nonterms
   
      // right neighbors begin at position t:
    hash_map<int, neighbor_t>::iterator nr_itr;
    for (nr_itr = neighbors[0][hk?0:1][t][hk?u[0]:v[0]].begin(); nr_itr != neighbors[0][hk?0:1][t][hk?u[0]:v[0]].end(); 
         nr_itr++) {

      int nr = nr_itr->first;

      neighbor_t * theneighbor = &(neighbors[0][hk?0:1][t][hk?u[0]:v[0]][nr]);
      int nbr_neighbors = theneighbor->size();

      for (int ni = 0; ni < nbr_neighbors; ni++){

        // for each binary rule having specified children
        lhs_inv_erhs_rule_hash *rnt = &grammar->rule_table[n][nr];
        lhs_inv_erhs_rule_hash::iterator lhs_itr;
        inv_erhs_rule_hash::iterator inv_itr;
        erhs_rule_hash::iterator e_rhs_itr;
	 
        for (lhs_itr = rnt->begin(); lhs_itr != rnt->end(); lhs_itr++)
          for (inv_itr = lhs_itr->second.begin(); inv_itr != lhs_itr->second.end(); inv_itr++)
            for (e_rhs_itr = inv_itr->second.begin(); e_rhs_itr != inv_itr->second.end(); 
                 e_rhs_itr++) {

              Rule *rule = e_rhs_itr->second;
              int lhs = rule->lhs_int;

	      if (iter==0)
		if (! ( (rule->inverted && hk) || (!rule->inverted && !hk) ) )
		  continue;


              int S = (*theneighbor)[ni].S;

	      ITGDCell *neighbor_pd = dchart->cook(hk?0:1, nr, t, S, (*theneighbor)[ni].u[0], (*theneighbor)[ni].u[1], 
						   (*theneighbor)[ni].v[1], (*theneighbor)[ni].v[0]); 
	      //ur[0], ur[1], vr[1], vr[0]);

	      if (hk){
		//the right neighbor is not a hook
		ul[0] = (iter)?popped_pd->lword[0]:popped_pd->lchild->lword[0]; //hl[0];
		ul[1] = (iter)?popped_pd->lword[1]:popped_pd->lchild->lword[1]; //hl[1];
		vl[0] = v[0];
		vl[1] = v[1];
	    	
	      }
	      else{
		
		//the right neighbor is a hook
		ul[0] = u[0];
		ul[1] = u[1];
		vl[0] = v[0];
		vl[1] = v[1];
	      }

              sita4 = neighbor_pd->prob;

              if (_prune_agenda) {
                if (hk && 
		    ( bc[hk?0:1][t][S].count(nr) && (sita4 < (bc[hk?0:1][t][S][nr] - LOGRATIOBEAM) ) ) )  {
                  continue;
		}
              }

              sita40 = sita4 + sita0;//log+

              uchoice[0] = uchoice[1] = 0;
              vchoice[0] = vchoice[1] = 0;
			
              pd = dchart->cell(lhs, s, S, uchoice[0], uchoice[1], vchoice[1], vchoice[0]);
       
              double newsita = //lm_prob + 
		sita40; //lm prob already included
              if (include_itgscore)
                newsita += rule->prob;
        
              double oldsita = pd->prob;

              itemcount++;

	      if (newsita > oldsita){//maximizing
                pd->N = lhs;
                pd->s = s;
                pd->t = S;
                pd->prob = newsita;

                DUEXASSIGN(pd->lword, uchoice);
                DUEXASSIGN(pd->rword, vchoice);

                pd->rule = rule;		// backpointers
		pd->lchild = popped_pd;
		pd->rchild = (hk)?neighbor_pd:neighbor_pd->lchild; //neighbor_pd;

                double insout = pd->prob + dchart->estimate(lhs, s, S, uchoice[0], uchoice[1], vchoice[1], vchoice[0]);
                //put on agenda
                agenda_node_t a(s, S, uchoice[0], uchoice[1], vchoice[0], vchoice[1], lhs, 0, insout); //non-hook
                agenda.push(a);
                if (Globals::Get<int>("debuglevel") >= 3) 
                  fprintf(logfile, "Adding %s to the agenda (prob = %.10e, insout = %.10e)\n", a.data().data(), pd->prob, insout);
              }//end max

	      backpointer thebkptr;
	      thebkptr.lchild=popped_pd;
	      thebkptr.rchild=neighbor_pd; //(hk)?neighbor_pd:neighbor_pd->lchild;
	      thebkptr.rule=rule;		

	      thebkptr.tri =  (ngram->N==3);
	      pd->bkptrs.push_back(thebkptr);


	    } // e_rhs
      } //end of enumerating neighbors
    } //end of right side nonterms

    if (iter==1){
      hk=0;
      DUEXASSIGN(u, hl);
    }
    }//end of two stages



    //epsilon non-hooks

    if (!hk && (u[0] == 0) ){

    // exploring, expanding the open set
    // left neighbors end at position s:
      hash_map<int, hash_map<int, neighbor_t> >::iterator nll_itr;
      hash_map<int, neighbor_t >::iterator nl_itr;


      //nlll, first level iterator
      for (nll_itr = neighbors[1][0][s].begin(); nll_itr != neighbors[1][0][s].end();
	   nll_itr++) {
	for (nl_itr = (*nll_itr).second.begin(); nl_itr != (*nll_itr).second.end(); 
	     nl_itr++){

	  int nl = nl_itr->first;
	  
	  neighbor_t * theneighbor = &(neighbors[1][0][s][nll_itr->first][nl]);

	  int nbr_neighbors = theneighbor->size();

	  for (int ni = 0; ni < nbr_neighbors; ni++){

	    // for each binary rule having specified children
	    lhs_inv_erhs_rule_hash *rnt = &grammar->rule_table[nl][n];
	    lhs_inv_erhs_rule_hash::iterator lhs_itr;
	    inv_erhs_rule_hash::iterator inv_itr;
	    erhs_rule_hash::iterator e_rhs_itr;

	    for (lhs_itr = rnt->begin(); lhs_itr != rnt->end(); lhs_itr++)
	      for (inv_itr = lhs_itr->second.begin(); inv_itr != lhs_itr->second.end(); inv_itr++)
		for (e_rhs_itr = inv_itr->second.begin(); e_rhs_itr != inv_itr->second.end(); 
		     e_rhs_itr++) {
	       
		  Rule *rule = e_rhs_itr->second;
		  int lhs = rule->lhs_int;


		  //		  //hooks are left-extending
		  //		  if ( ! ( (rule->inverted && !hk ) || (!rule->inverted && hk) ) )
		  //		    continue; 


		  if (Globals::Get<int>("debuglevel") >= 3)
		    fprintf(logfile, "looking for left neighbor: 1 s: %d nl: %d\n", s, nl);
	     
		  int S = (*theneighbor)[ni].S;

		  ITGDCell *neighbor_pd = dchart->cook(0, nl, S, s, (*theneighbor)[ni].u[0], (*theneighbor)[ni].u[1],
						       (*theneighbor)[ni].v[1], (*theneighbor)[ni].v[0]); 

		  //the left neighbor is not a hook
		  ul[0] = (*theneighbor)[ni].u[0];
		  ul[1] = (*theneighbor)[ni].u[1];
		  vl[0] = (*theneighbor)[ni].v[0];
		  vl[1] = (*theneighbor)[ni].v[1];
	     
		  if (Globals::Get<int>("debuglevel") >= 3) 
		    fprintf(logfile, "left neighbor: %d lm %d %d %d %d\n", S, ul[0], ul[1], vl[1], vl[0]);
	     

		  sita3 = neighbor_pd->prob;
	     
		  if (_prune_agenda)
		    if (( bc[0][S][s].count(nl) && sita3 < (bc[0][S][s][nl] - LOGRATIOBEAM)) )
		      continue;
	     
		  sita30 = sita3 + sita0;//log+
	      
		  uchoice[0] = uchoice[1] = 0;
		  vchoice[0] = vchoice[1] = 0;
	     
		  pd = dchart->cell(lhs, S, t, uchoice[0], uchoice[1], vchoice[1], vchoice[0]);
	     
		  double newsita = //lm_prob + 
		    sita30; //lm prob already included in the hook
		  if (include_itgscore)
		    newsita += rule->prob;
	     
		  double oldsita = pd->prob;
	     
		  itemcount++;

		  if (newsita > oldsita){//maximizing
		    pd->N = lhs;
		    pd->s = S;
		    pd->t = t;
		    pd->prob = newsita;
		    DUEXASSIGN(pd->lword, uchoice);
		    DUEXASSIGN(pd->rword, vchoice);
	       
		    pd->rule = rule;	// backpointers
		    pd->lchild = neighbor_pd;   //needs to point to the cells
		    pd->rchild = popped_pd;

		    double insout = pd->prob + dchart->estimate(lhs, S, t, uchoice[0], uchoice[1], vchoice[1], vchoice[0]);
		    //put on agenda
		    agenda_node_t a(S, t, uchoice[0], uchoice[1], vchoice[0], vchoice[1], lhs, 0, insout); //non-hook
		    agenda.push(a);
		    if (Globals::Get<int>("debuglevel") >= 3) 
		      fprintf(logfile, "Adding %s to the agenda (prob = %.5f, insout = %.5f)\n", a.data().data(), pd->prob, insout);

		  } // if max

	      
		  backpointer thebkptr;
		  thebkptr.lchild=neighbor_pd;         //(hk)?neighbor_pd:neighbor_pd->lchild;
		  thebkptr.rchild=popped_pd;
		  thebkptr.rule=rule;		

		  thebkptr.tri =  (ngram->N==3);
		  pd->bkptrs.push_back(thebkptr);


		} // e_rhs
	  } //end of enumerating neighbors
	}
      } // left side nonterms
   
      // right neighbors begin at position t:
      //    hash_map<int, neighbor_t>::iterator nr_itr;
      //    for (nr_itr = neighbors[0][hk?0:1][t][hk?u[0]:v[0]].begin(); nr_itr != neighbors[0][hk?0:1][t][hk?u[0]:v[0]].end(); 
      //         nr_itr++) {

      //      hash_map<int, hash_map<int, neighbor_t> >::iterator nll_itr;
      //      hash_map<int, neighbor_t >::iterator nl_itr;


      //nlll, first level iterator
      for (nll_itr = neighbors[0][0][t].begin(); nll_itr != neighbors[0][0][t].end();
	   nll_itr++) {
	for (nl_itr = (*nll_itr).second.begin(); nl_itr != (*nll_itr).second.end(); 
	     nl_itr++){


	  int nr = nl_itr->first;

	  neighbor_t * theneighbor = &(neighbors[0][0][t][nll_itr->first][nr]);
	  int nbr_neighbors = theneighbor->size();

	  for (int ni = 0; ni < nbr_neighbors; ni++){

	    // for each binary rule having specified children
	    lhs_inv_erhs_rule_hash *rnt = &grammar->rule_table[n][nr];
	    lhs_inv_erhs_rule_hash::iterator lhs_itr;
	    inv_erhs_rule_hash::iterator inv_itr;
	    erhs_rule_hash::iterator e_rhs_itr;
	 
	    for (lhs_itr = rnt->begin(); lhs_itr != rnt->end(); lhs_itr++)
	      for (inv_itr = lhs_itr->second.begin(); inv_itr != lhs_itr->second.end(); inv_itr++)
		for (e_rhs_itr = inv_itr->second.begin(); e_rhs_itr != inv_itr->second.end(); 
		     e_rhs_itr++) {

		  Rule *rule = e_rhs_itr->second;
		  int lhs = rule->lhs_int;


		  //		  if (! ( (rule->inverted && hk) || (!rule->inverted && !hk) ) )
		  //		    continue;


		  int S = (*theneighbor)[ni].S;

		  ITGDCell *neighbor_pd = dchart->cook(0, nr, t, S, (*theneighbor)[ni].u[0], (*theneighbor)[ni].u[1], 
						       (*theneighbor)[ni].v[1], (*theneighbor)[ni].v[0]); 
		  //ur[0], ur[1], vr[1], vr[0]);

		  //the right neighbor is not a hook
		  ul[0] = u[0];
		  ul[1] = u[1];
		  vl[0] = v[0];
		  vl[1] = v[1];
		  

		  sita4 = neighbor_pd->prob;

		  if (_prune_agenda) {
		    if (( bc[0][t][S].count(nr) && (sita4 < (bc[0][t][S][nr] - LOGRATIOBEAM) ) ) )  {
		    continue;
		    }
		  }

		  sita40 = sita4 + sita0;//log+

		  uchoice[0] = uchoice[1] = 0;
		  vchoice[0] = vchoice[1] = 0;
			
		  pd = dchart->cell(lhs, s, S, uchoice[0], uchoice[1], vchoice[1], vchoice[0]);
       
		  double newsita = //lm_prob + 
		    sita40; //lm prob already included
		  if (include_itgscore)
		    newsita += rule->prob;
        
		  double oldsita = pd->prob;

		  itemcount++;

		  if (newsita > oldsita){//maximizing
		    pd->N = lhs;
		    pd->s = s;
		    pd->t = S;
		    pd->prob = newsita;

		    DUEXASSIGN(pd->lword, uchoice);
		    DUEXASSIGN(pd->rword, vchoice);

		    pd->rule = rule;		// backpointers
		    pd->lchild = popped_pd;
		    pd->rchild = neighbor_pd; //neighbor_pd;

		    double insout = pd->prob + dchart->estimate(lhs, s, S, uchoice[0], uchoice[1], vchoice[1], vchoice[0]);
		    //put on agenda
		    agenda_node_t a(s, S, uchoice[0], uchoice[1], vchoice[0], vchoice[1], lhs, 0, insout); //non-hook
		    agenda.push(a);
		    if (Globals::Get<int>("debuglevel") >= 3) 
		      fprintf(logfile, "Adding %s to the agenda (prob = %.10e, insout = %.10e)\n", a.data().data(), pd->prob, insout);
		  }//end max

		  backpointer thebkptr;
		  thebkptr.lchild=popped_pd;
		  thebkptr.rchild=neighbor_pd; //(hk)?neighbor_pd:neighbor_pd->lchild;
		  thebkptr.rule=rule;		

		  thebkptr.tri =  (ngram->N==3);
		  pd->bkptrs.push_back(thebkptr);


		} // e_rhs
	  } //end of enumerating neighbors
	}
      } //end of right side nonterms
      
    }//end of epsilon translations


    
    //non-hook to hook
    if (!hk){
      hash_map <WORDTYPE, int >::iterator senv_r_itr;
      for (senv_r_itr=senvocab_r.begin(); senv_r_itr!=senvocab_r.end(); ++senv_r_itr){//all the right boundary words
	WORDTYPE hv = (*senv_r_itr).first;

	if (!hv)
	  continue;
	//do we have problems with (hv==0)?

	double trg = LMWEIGHT * ngram->bigram(hv, u[0]);
	pd = dchart->cook(1, n, s, t, hv, 0, v[1], v[0]); //hook

	double newsita = sita0 + trg;
	double oldsita = pd->prob;
         itemcount++;
	 hookitemcount++;

	//maximizing the hook and put onto the agenda
	if (newsita > oldsita){//maximizing
	  pd->N = n;
	  pd->s = s;
	  pd->t = t;
	  pd->prob = newsita;

	  //	  DUEXASSIGN(pd->lword, uchoice);
	  pd->lword[0]=hv;
	  pd->lword[1]=0;
	  DUEXASSIGN(pd->rword, v);

	  pd->rule = NULL;		// backpointers
	  pd->lchild = popped_pd; //like a unary rule
	  pd->rchild = popped_pd;  //special treatment //NULL; 

	  double insout = pd->prob + dchart->estimate(n, s, t, u[0], u[1], v[1], v[0]); //hook outside estimate needed
	  //put on agenda
	  agenda_node_t a(s, t, hv, 0, v[0], v[1], n, 1, insout);
	  agenda.push(a);
	  if (Globals::Get<int>("debuglevel") >= 3) 
	    fprintf(logfile, "Adding %s to the agenda (prob = %.10e, insout = %.10e)\n", a.data().data(), pd->prob, insout);

	}

	backpointer thebkptr;
	thebkptr.lchild=popped_pd;
	thebkptr.rchild=NULL;
	thebkptr.rule=NULL;

	thebkptr.tri =  (ngram->N==3);
	pd->bkptrs.push_back(thebkptr);

      }//end for hv 
    }


    /* unary rules */

    if (!hk){

      if (s==0 && t==T){
    lhs_inv_erhs_rule_hash *rnt = &grammar->rule_table[n][-1];
	
    lhs_inv_erhs_rule_hash::iterator lhs_itr;
    inv_erhs_rule_hash::iterator inv_itr;
    erhs_rule_hash::iterator e_rhs_itr;
	
    // look up all unary rules with n as rhs nonterminal
    for (lhs_itr = rnt->begin(); lhs_itr != rnt->end(); lhs_itr++)
      for (inv_itr = lhs_itr->second.begin(); inv_itr != lhs_itr->second.end(); inv_itr++)
        for (e_rhs_itr = inv_itr->second.begin(); e_rhs_itr != inv_itr->second.end(); e_rhs_itr++) {
          Rule *rule = e_rhs_itr->second;
          int lhs = rule->lhs_int;

          double lm_prob;
          uchoice[0] = uchoice[1] = 0;
          vchoice[0] = vchoice[1] = 0;
	   
          lm_prob = LMWEIGHT * compute_lm(u, NULL, v, NULL, rule, uchoice, vchoice);
	   
          double newsita = sita0 + lm_prob;
          if (include_itgscore)
            newsita += rule->prob;
          pd = dchart->cell(lhs, s, t, uchoice[0], uchoice[1], vchoice[1], vchoice[0]);
          double oldsita = pd->prob;
          itemcount++;

	  if (newsita > oldsita){//maximizing
            pd->prob = newsita;
            pd->N = lhs;
            pd->s = s;
            pd->t = t;
            DUEXASSIGN(pd->lword, uchoice);
            DUEXASSIGN(pd->rword, vchoice);

            pd->rule = rule;		// backpointers
	    pd->lchild = popped_pd;
	    pd->rchild = NULL;

            double insout = pd->prob + dchart->estimate(lhs, s, t, uchoice[0], uchoice[1], vchoice[1], vchoice[0]);
            //put on agenda
            agenda_node_t a(s, t, uchoice[0], uchoice[1], vchoice[0], vchoice[1], lhs, 0, insout);
            agenda.push(a);
            if (Globals::Get<int>("debuglevel") >= 3) 
              fprintf(logfile, "Adding %s to the agenda (prob = %.10e, insout = %.10e)\n", a.data().data(), pd->prob, insout);
          } //end of updating viterbi

	  backpointer thebkptr;
	  thebkptr.lchild=popped_pd;
	  thebkptr.rchild=NULL;
	  thebkptr.rule=rule;		
	  thebkptr.tri =  (ngram->N==3);
	  pd->bkptrs.push_back(thebkptr);


        } //end of matching rule
      } //end of top-level filtering
    }

    if (!hk){
    /* special case if the current cell is the root level */

    if ((s == 0) && (t == T) && (n == grammar->start_symbol)) {
      double lm_prob = 0.0;
      if (ngram->N == 2) {
        lm_prob = LMWEIGHT * (ngram->bigram(-1, u[0])+ngram->bigram(v[0], -2));
      } else if (ngram->N == 3) {
        lm_prob = LMWEIGHT * (ngram->trigram(-1, u[0], u[1])+ngram->trigram(v[1], v[0], -2));
        lm_prob += LMWEIGHT * (ngram->bigram(-1, u[0])); //adding the leading bigram probability
      }
       
      ITGDCell *cell = dchart->cell(grammar->start_symbol,0,T,u[0],u[1],v[1],v[0]);
      double newsita = cell->prob + lm_prob;
       
      //pd = dchart->cell(0, 0, T, 0, 0, 0, 0); //should use this to UNIQUELY represent the best translation 
      pd = dchart->cell(grammar->_nonterminal_cnt, 0, T, 0, 0, 0, 0); //should use this to UNIQUELY represent the best translation 
      double oldsita = pd->prob;
      itemcount++;

      if (newsita > oldsita){//maximizing
        pd->N = grammar->_nonterminal_cnt;
        pd->s = 0;
        pd->t = T;
        pd->prob = newsita;
	
	
        pd->lword[0] = -1;
        if (ngram->N == 2) {
          pd->lword[1] = 0;
          pd->rword[1] = 0;
        } else {
          pd->lword[1] = cell->lword[0];
          pd->rword[1] = cell->rword[0];
        }
        pd->rword[0] = -2;
	
	pd->rule = NULL;	// backpointers
	pd->lchild = cell;
	pd->rchild = NULL;

        double insout = pd->prob;
	agenda_node_t a(0, T, 0, 0, 0, 0, grammar->_nonterminal_cnt, 0, insout);
        agenda.push(a);

        if (Globals::Get<int>("debuglevel") >= 3) 
          fprintf(logfile, "Adding %s to the agenda (prob = %.10e, insout = %.10e)\n", a.data().data(), pd->prob, insout);
	 
      }//end of updating viterbi

      backpointer thebkptr;
      thebkptr.lchild=cell;
      thebkptr.rchild=NULL;
      thebkptr.rule=NULL;		
      thebkptr.tri =  (ngram->N==3);
      pd->bkptrs.push_back(thebkptr);


    }//end if top level
    }

  }//end of agenda
  

  /* get the best score from our unique best-cell representation */

  //  pd = dchart->cell(0, 0, T, 0, 0, 0, 0);
  pd = dchart->cell(grammar->_nonterminal_cnt, 0, T, 0, 0, 0, 0);
  double logscore = pd->prob;

  if (standalone){
    fprintf(logfile, "arcs popped %d\n", itemcount);
    fprintf(logfile, "hooks proposed %d\n", hookitemcount);
    fprintf(logfile, "agenda items explored %d\n", agenda_count);
    fprintf(logfile, "log prob %.20lf\n", logscore);
  }

  /* cleanup */

  for (int d=0; d<2; d++){
    for (int h=0; h<2; h++){
      delete[] neighbors[d][h];
    }
    delete[] neighbors[d];
  }    
  delete[] neighbors;  

  for (int h=0; h<2; h++){
    for (int iter=0; iter<=T; iter++){
      delete[] bc[h][iter];
    }
    delete[] bc[h];
  }
  delete[] bc;

  /* dump chart */

  if (Globals::Get<int>("debuglevel") >= 3) {
    dchart->dump(_sentno);
  }

  /* top-down reconstruction */

  if (standalone){
    pd->N = grammar->start_symbol;
    if (pd->prob > -INFINITY){
      return reconstruct(pd);
    }
    else
      return "";
  }
  else
    return "";
}

