/* itgdec-cyk-bitri.cpp
 *
 * Implementation file for the two pass agenda-based ITG decoder
 * 
 * $Id: itgdec-cyk-bitri.cpp,v 1.2 2011/10/16 22:36:43 gildea Exp $
 */


#include "itgdec-cyk-bitri.h"


/*
 *  Decoding with an agenda.
 */

string CYKDecoder::decode_cyk_bipass(){
  cyk_bottom_up();
  ITGDCell *pd=dchart->cell(grammar->_nonterminal_cnt, 0, T, 0 , 0, 0, 0);
  outsidepass(pd); //this can be improved to traverse all root level cells, oct 20

  return "";
}


string CYKDecoder::decode_cyk_hook_bipass(){
  cyk_hook_bottom_up();
  ITGDCell *pd=dchart->cell(grammar->_nonterminal_cnt, 0, T, 0 , 0, 0, 0);
  outsidepass(pd); //this can be improved to traverse all root level cells, oct 20

  return "";
}


string CYKDecoder::decode_hook_agenda_bipass(){
  decode_hook_agenda(false); 
  ITGDCell *pd=dchart->cell(grammar->_nonterminal_cnt, 0, T, 0 , 0, 0, 0);
  outsidepass(pd); 

  return "";
}

string CYKDecoder::decode_agenda_bitri(bool bilm) {

  //  FILE *outsidefile=fopen("/tmp/outsidefile", "a");

  int ul[2], vl[2], ur[2], vr[2], u[2], v[2];
  double sita0, sita3, sita4,  sita30, sita40;
  ITGDCell *pd = NULL; //pointer to a decoding chart cell
  int uchoice[2];
  int vchoice[2];
  int _prune_agenda = Globals::Get<bool>("prune_agenda");

  //for column-wise pruning
  //bc[left_boundary][right_boundary][nonterm_id]
  hash_map<int, double> **bc = new hash_map<int, double> *[T+1];
  for (int iter = 0; iter <= T; iter++){
    bc[iter] = new hash_map<int, double> [T+1];
  }

  //for agenda-based decoding
  agenda_t *agenda; 
  agenda=new agenda_t; 

  // neighbors' index
  // neighbors[left/right][boundary_location][nonterm_id]
  hash_map<int, neighbor_t> **neighbors;

  // neighbor index allocation
  neighbors = new hash_map<int, neighbor_t> *[2];
  for (int d = 0; d < 2; d++){
    neighbors[d] = new hash_map<int, neighbor_t> [T+1];
  }

  //first pass or second pass
  int passid=bilm?1:2;



  //bottom-up
  //initialize the agenda

  /* add phrase translations to the agenda */
  //issue: we are reenumerating the translation pairs
  //       but actually we have done dinit() and have
  //       the bottom level cells initialized in the
  //       chart all we need to do is to put them
  //       onto the agenda

  for (int s = 0; s <= T; s++){
    if (s < T) {
      for (int lspan = 1; lspan <= min(T - s, MAXPHR); lspan++) {
        int t = s + lspan;
        for (int uidx = 0; enclosure[s][lspan][uidx]; uidx++){
          WORDTYPE * ephr = enclosure[s][lspan][uidx];
          int u = *ephr;
          int v = last_word(ephr);
          int phrlen = phraselen(ephr);

          int u1 = 0;
          int v1 = 0;

          // update the inside words for 3+ grams
	  // support for bigram-based decoding?
	  if (ngram->N > 2 && phrlen > 1) {
	    u1 = ephr[1];
	    v1 = penultimate_word(ephr);
	  }

          //s, s+1, u, u
          pd = dchart->cell(grammar->T, s, t, u, u1, v1, v);

	  
	  if (bilm){//first pass, the bigram pass
	    pd->closed = 0;
	    double est =
	      dchart->estimate(grammar->T, s, t, u, u1, v1, v)
	      + borderest(u, u1, v, v1)
	      ;

	    double insout=pd->prob + est;//can have outside cost here

	    agenda_node_t a = agenda_node_t(s, t, u, u1, v, v1, grammar->T, insout);
	    agenda->push(a);

	  }
	  else{

	    //	    if (pd->closed==1){//only those visited in the first pass
	    //this is problematic in the phrase-based decoding
	    //because the bottom level cells do not share across the bi-pass and tri-pass

	    //best border cost plus the bigram outside cost
	    double est =
	      //dchart->estimate(grammar->T, s, t, u, u1, v1, v)+
	      dchart->outsideof(s, t, u, 0, v, 0, grammar->T) 
	      + borderest(u, u1, v, v1);

	    //	    printf ("border est: %.10e\n", borderest(u, u1, v, v1) );

	    
	    //	      fprintf(outsidefile, "%d\t%d\t%d\t%d\t%.10e\n", s, t, u, v, est);
	    
	    double insout = pd->prob + est;
	    
	    agenda_node_t a = agenda_node_t(s, t, u, u1, v, v1, grammar->T, insout);
	    agenda->push(a);


	    //	    }
	  }

	  //          fprintf(logfile, "Adding %s to the agenda (prob = %.10e, est = %.10e)\n", a.data().data(), pd->prob, est);

        }//end of enumerating the phrase pairs
      }//end of enumerating the spans (input side phrases)
    }//end if normal translations


    /* add insertions to the agenda */

    if (Globals::Get<bool>("allow_insertions")) {
      //epsilon to word
      for (int uidx = 0; enclosure[s][0][uidx]; uidx++){
	WORDTYPE * ephr = enclosure[s][0][uidx];
	int u = *ephr;
	int v = last_word(ephr);
	int phrlen = phraselen(ephr);

	int u1 = 0;
	int v1 = 0;

	// update the inside words for 3+ grams
	// again, support for bigrams?
	if (ngram->N > 2 && phrlen > 1) {
	  u1 = ephr[1];
	  v1 = penultimate_word(ephr);
	}

	//s, s, u, u
	pd = dchart->cell(grammar->T, s, s, u, u1, v1, v);

	if (bilm){//first pass
	  pd->closed = 0;

	  double est = 
	    dchart->estimate(grammar->T, s, s, u, u1, v1, v)
	    + borderest(u, u1, v, v1)
	    ;
	  
	  double insout=pd->prob + est;
	  agenda_node_t a=agenda_node_t(s, s, u, u1, v, v1, grammar->T, insout);
	  agenda->push(a);
	}
	else{
	  //	  if (pd->closed==1){//visited in the first pass
	  double est = 
	    //dchart->estimate(grammar->T, s, s, u, u1, v1, v)+
	    dchart->outsideof(s, s, u, 0, v, 0, grammar->T)
	    + borderest(u, u1, v, v1);

	  //	  printf ("border est: %.10e\n", borderest(u, u1, v, v1));
	  //	      fprintf(outsidefile, "%d\t%d\t%d\t%d\t%.10e\n", s, s, u, v, est);

	  double insout = pd->prob + est;
	  agenda_node_t a = agenda_node_t(s, s, u, u1, v, v1, grammar->T, insout);
	  agenda->push(a);

	  //	  }
	}

	//      fprintf(logfile, "Adding %s to the agenda (prob = %.10e, est = %.10e)\n", a.data().data(), pd->prob, est);
      }//end of enumerating all possible insertions
    }//end if allowing insertions
    
  }//end of enumerating all left splitting points
 
  bool include_itgscore = ! Globals::Get<bool>("suppress_itgscore");


  if (bilm){
    itemcount = 0;
  }

  if (!bilm){
    triitemcount = 0;
  }

  int agenda_count = 0;
  //  int curins; 

  int aftergoal =0;
  bool goalreached=false;

  //searching
  while (!agenda->empty()) {

    agenda_node_t anode = agenda->top();

    int s = anode.s;
    int t = anode.t;
    u[0] = anode.u[0];
    u[1] = anode.u[1];
    v[0] = anode.v[0];
    v[1] = anode.v[1];
    int n = anode.n;
    double score=anode.p;

    agenda->pop();



    agenda_count++;

    //    if (!bilm)
    //      fprintf(outsidefile, "popped  s: %d t: %d, lm: %d %d %d %d\n", 
    //    	      s, t, u[0], u[1], v[0], v[1]);

    if (n<grammar->_nonterminal_cnt && Globals::Get<int>("debuglevel") >= 3) {
      fprintf(logfile, "Agenda: popped  n: %s s: %d t: %d, lm: %d %d %d %d\n", 
              grammar->nonterminals[n].data(), s, t, u[0], u[1], v[0], v[1]);
    }

    //in bigram mode, u[1] and v[1] are zero
    ITGDCell *popped_pd = dchart->cell(n, s, t, u[0], u[1], v[1], v[0]);
    //    pd = dchart->cell(n, s, t, u[0], u[1], v[1], v[0]);

    //this is for future extension,
    //when we keep record of the inner words that 
    //yield the maximum score given the outer boundary words
    //    if (bilm){
    //      DUEXASSIGN(u, pd->u);
    //    DUEXASSIGN(v, pd->v);
    //  }

    sita0 = popped_pd->prob; //focus

    //    if (Globals::Get<int>("debuglevel") >= 3) 
    //      fprintf(logfile, "sita0 %.10e\n", sita0);

    //update the column best
    if (!bc[s][t].count(n) || sita0 > bc[s][t][n])
      bc[s][t][n] = sita0;

    /* GOAL 
     * The goal is distinguished by setting the nonterminal
     * spanning it to grammar->N.  We can't use S, because when
     * we see the chart entry (S,0,T), we need to add in the LM
     * probabilities for the end and beginning of the sentence.
     * When that is done, the entry is put back onto the agenda
     * with the special nonterminal grammar->N.
     */
    if ( (s == 0) && (t == T ) && (n == grammar->_nonterminal_cnt) ) {
      fprintf(logfile, "Agenda: popped goal %s\n", anode.data().data());
      
      if (bilm){
	//pop off more goals until falling out of the beam
        if (Globals::Get<int>("verbosity") >= 1) 
          printf("Goal reached %.10e\n", score);
	if (score<(bc[s][t][n]-LOGRATIOBEAM)){
	  break;
	}
      }
      else{
	//stop immediately when the goal is reached
	//	break;
	goalreached=true;

	//	if (score<(bc[s][t][n]-0.5)){
	//	break;
      //	}
      }
    }


    if (goalreached)
      aftergoal++;


    if (Globals::Get<int>("epass")){
      //            if (aftergoal>1000)
      if (aftergoal > 200*LOGRATIOBEAM)
      //      if (aftergoal >trivecsize)
	break;
    }
    else{
      if (goalreached)
	break;
    }



    //this assumes the first hyperpath reaching 
    //a node is the best
    //if explored already in this pass
    //    if (popped_pd->closed==passid){
    //          continue;
    //    }

    popped_pd->closed = passid;



    if (_prune_agenda){
      //different beam sizes for the two passes
      if (bilm){
	if (sita0 < (bc[s][t][n] - LOGRATIOBEAM) ) {
	  if (Globals::Get<int>("debuglevel") >= 3) 
	    fprintf(logfile, "sita0: %g, bc %g, logratiobeam %g\n",
		    sita0, bc[s][t][n], LOGRATIOBEAM);
	  continue;
	}
      }
      else{
	//now we use LOGRATIOBEAM+10 as the beam size for 
	//the second pass, which is essentially almost infinitely large
 	if (sita0 < (bc[s][t][n] - LOGRATIOBEAM - 010     ) ) {
 	  if (Globals::Get<int>("debuglevel") >= 3) 
 	    fprintf(logfile, "sita0: %g, bc %g, logratiobeam %g\n",
 		    sita0, bc[s][t][n], LOGRATIOBEAM);
 	  continue;
 	}
      }
    }


    neighbors[0][s][n].push_back(neighbor_node_t(t, u[0], u[1], v[0], v[1]));
    neighbors[1][t][n].push_back(neighbor_node_t(s, u[0], u[1], v[0], v[1]));

    // exploring, expanding the open set
    // left neighbors end at position s:
    hash_map<int, neighbor_t>::iterator nl_itr;
    for (nl_itr = neighbors[1][s].begin(); nl_itr != neighbors[1][s].end();
         nl_itr++) {

      int nl = nl_itr->first;

      neighbor_t * theneighbor = &(neighbors[1][s][nl]);
      int nbr_neighbors = theneighbor->size();

      for (int ni = 0; ni < nbr_neighbors; ni++){

	int S = (*theneighbor)[ni].S;
	      
	ul[0] = (*theneighbor)[ni].u[0];
	ul[1] = (*theneighbor)[ni].u[1];
	vl[0] = (*theneighbor)[ni].v[0];
	vl[1] = (*theneighbor)[ni].v[1];
	     
	ur[0] = u[0];
	ur[1] = u[1];
	vr[0] = v[0];
	vr[1] = v[1];
	     
	if (Globals::Get<int>("debuglevel") >= 3) 
	  fprintf(logfile, "left neighbor: %d lm %d %d %d %d\n", S, ul[0], ul[1], vl[1], vl[0]);


	ITGDCell *neighbor_pd = dchart->cell(nl, S, s, ul[0], ul[1], vl[1], vl[0]);
	//	sita3 = dchart->cell(nl, S, s, ul[0], ul[1], vl[1], vl[0])->prob;
	sita3 = neighbor_pd->prob;

	    	     
	if (_prune_agenda){
	  //different beams
	  if (bilm){
	    if (bc[S][s].count(nl) && sita3 < (bc[S][s][nl] - LOGRATIOBEAM))
	      continue;
	  }
	  else{
	    if (bc[S][s].count(nl) && sita3 < (bc[S][s][nl] - LOGRATIOBEAM  - 010   ))
	      continue;
	  }
	     
	}

	sita30 = sita3 + sita0;//log+



        // for each binary rule having specified children
        lhs_inv_erhs_rule_hash *rnt = &grammar->rule_table[nl][n];
        lhs_inv_erhs_rule_hash::iterator lhs_itr;
        inv_erhs_rule_hash::iterator inv_itr;
        erhs_rule_hash::iterator e_rhs_itr;

        for (lhs_itr = rnt->begin(); lhs_itr != rnt->end(); lhs_itr++)
          for (inv_itr = lhs_itr->second.begin(); inv_itr != lhs_itr->second.end(); inv_itr++)
            for (e_rhs_itr = inv_itr->second.begin(); e_rhs_itr != inv_itr->second.end(); 
                 e_rhs_itr++) {
	       
              Rule *rule = e_rhs_itr->second;
              int lhs = rule->lhs_int;

              if (Globals::Get<int>("debuglevel") >= 3)
                fprintf(logfile, "looking for left neighbor: 1 s: %d nl: %d\n", s, nl);
	     
              double lm_prob;
	     
              uchoice[0] = uchoice[1] = 0;
              vchoice[0] = vchoice[1] = 0;

	      //do bigram computation in the bilm mode
              if (rule->inverted) {
                lm_prob = LMWEIGHT * compute_lm(ur, ul, vr, vl, rule, uchoice, vchoice);
              } else {
                lm_prob = LMWEIGHT * compute_lm(ul, ur, vl, vr, rule, uchoice, vchoice);
              }
	     
	      if (bilm){
		uchoice[1]=vchoice[1]=0;
	      }

              pd = dchart->cell(lhs, S, t, uchoice[0], uchoice[1], vchoice[1], vchoice[0]);

	     
              double newsita = lm_prob + sita30;
              if (include_itgscore)
                newsita += rule->prob;
	     
              double oldsita = pd->prob;
	     
              itemcount++;
	      if (!bilm)
		triitemcount++;

              if (pd->inserted[(bilm)?0:1] != passid || newsita > oldsita){//maximizing
		if (pd->inserted[(bilm)?0:1] != passid){
		  pd->inserted[(bilm)?0:1]=passid;

		  //		  if (passid>1)
		  //		  fprintf(outsidefile, "pd: (%d\t%d\t%d\t%d\t%d\t%d\t%d)\n", S, t, uchoice[0], uchoice[1], vchoice[1], vchoice[0], lhs);
		}

                pd->prob = newsita;
		//	         	pd->S = s; //deprecated
		pd->s = S;
		pd->t = t;
		pd->N = lhs;
                pd->rule = rule;
		pd->lchild = neighbor_pd;
		pd->rchild = popped_pd;

		DUEXASSIGN(pd->lword, uchoice);
                DUEXASSIGN(pd->rword, vchoice);

		//deprecated
//                 DUEXASSIGN(pd->ul, ul);
//                 DUEXASSIGN(pd->vl, vl);
//                 DUEXASSIGN(pd->ur, ur);
//                 DUEXASSIGN(pd->vr, vr);
		

                double insout;
		if (bilm){
		  double est = dchart->estimate(lhs, S, t, uchoice[0], uchoice[1], vchoice[1], vchoice[0])
		    + borderest(uchoice[0], uchoice[1], vchoice[0], vchoice[1])
		    ;

		  insout=pd->prob + est;

		}
		else{
		  double est = 	//+ dchart->estimate(lhs, S, t, uchoice[0], uchoice[1], vchoice[1], vchoice[0])	 + 
		    dchart->outsideof(S, t, uchoice[0], 0, vchoice[0], 0, lhs)
		    + borderest(uchoice[0], uchoice[1], vchoice[0], vchoice[1]);

		  //		  printf ("border est: %.10e\n", borderest(uchoice[0], uchoice[1], vchoice[0], vchoice[1]));

		  //		  fprintf(outsidefile, "%d\t%d\t%d\t%d\t%d\t%c\t%d\n", S, t, uchoice[0], vchoice[0], lhs, dchart->cell(lhs, S, t, uchoice[0], 0, 0, vchoice[0])->inserted[0], dchart->cell(lhs, S, t, uchoice[0], 0, 0, vchoice[0])->inserted[0]);
		  //		  fprintf(outsidefile, "%d\t%d\t%d\t%d\t%d\t%.10e\n", S, t, uchoice[0], vchoice[0], lhs, est);
		  insout= pd->prob + est;


		}

                //put on agenda
                agenda_node_t a(S, t, uchoice[0], uchoice[1], vchoice[0], vchoice[1], lhs, insout);
                agenda->push(a);
                if (Globals::Get<int>("debuglevel") >= 3) 
                  fprintf(logfile, "Adding %s to the agenda (%.10e)\n", a.data().data(), insout);
		//                if (S == s)
		//                  pd->inserted = 1;
		
              } // if max

	      //store the backpointers...

	      //	      if (bilm){
		backpointer thebkptr;
		//		thebkptr.lchild=dchart->cell(nl, S, s, ul[0], 0, 0, vl[0]);
		//		thebkptr.rchild=dchart->cell(n, s, t, u[0], 0, 0, v[0]);

		thebkptr.lchild = neighbor_pd;
		thebkptr.rchild = popped_pd;

		//		thebkptr.lwidth=(s-S);
		//		thebkptr.rwidth=(t-s);
		thebkptr.rule=rule;

		thebkptr.tri= (ngram->N==3);

		pd->bkptrs.push_back(thebkptr);
		
		//	      }

            } // e_rhs
      } //end of enumerating neighbors
    } // left side nonterms
   


      // right neighbors begin at position t:
    hash_map<int, neighbor_t>::iterator nr_itr;
    for (nr_itr = neighbors[0][t].begin(); nr_itr != neighbors[0][t].end();
         nr_itr++) {

      int nr = nr_itr->first;

      neighbor_t * theneighbor = &(neighbors[0][t][nr]);
      int nbr_neighbors = theneighbor->size();
     
      for (int ni = 0; ni < nbr_neighbors; ni++){

	int S = (*theneighbor)[ni].S;

	ur[0] = (*theneighbor)[ni].u[0];
	ur[1] = (*theneighbor)[ni].u[1];
	vr[0] = (*theneighbor)[ni].v[0];
	vr[1] = (*theneighbor)[ni].v[1];
	       
	ul[0] = u[0];
	ul[1] = u[1];
	vl[0] = v[0];
	vl[1] = v[1];

	ITGDCell *neighbor_pd = dchart->cell(nr, t, S, ur[0], ur[1], vr[1], vr[0]);
	//	sita4 = dchart->cell(nr, t, S, ur[0], ur[1], vr[1], vr[0])->prob;
	sita4 = neighbor_pd->prob;
	

	if (_prune_agenda){
	  if (bilm){
	    if (bc[t][S].count(nr) && sita4 < (bc[t][S][nr] - LOGRATIOBEAM))
	      continue;
	  }
	  else{
	    //no beam necessary
	    if (bc[t][S].count(nr) && sita4 < (bc[t][S][nr] - LOGRATIOBEAM - 010    ))
	      continue;
	  }
	}

	sita40 = sita4 + sita0;//log+

        // for each binary rule having specified children
        lhs_inv_erhs_rule_hash *rnt = &grammar->rule_table[n][nr];
        lhs_inv_erhs_rule_hash::iterator lhs_itr;
        inv_erhs_rule_hash::iterator inv_itr;
        erhs_rule_hash::iterator e_rhs_itr;
	 
        for (lhs_itr = rnt->begin(); lhs_itr != rnt->end(); lhs_itr++)
          for (inv_itr = lhs_itr->second.begin(); inv_itr != lhs_itr->second.end(); inv_itr++)
            for (e_rhs_itr = inv_itr->second.begin(); e_rhs_itr != inv_itr->second.end(); 
                 e_rhs_itr++) {

              Rule *rule = e_rhs_itr->second;
              int lhs = rule->lhs_int;
	       
              double lm_prob;

              uchoice[0] = uchoice[1] = 0;
              vchoice[0] = vchoice[1] = 0;
			
              if (rule->inverted) {
                lm_prob = LMWEIGHT * compute_lm(ur, ul, vr, vl, rule, uchoice, vchoice);
              } else {
                lm_prob = LMWEIGHT * compute_lm(ul, ur, vl, vr, rule, uchoice, vchoice);
              }

	      if (bilm){
		uchoice[1]=vchoice[1]=0;
	      }

              pd = dchart->cell(lhs, s, S, uchoice[0], uchoice[1], vchoice[1], vchoice[0]);
       
              double newsita = lm_prob + sita40;
              if (include_itgscore)
                newsita += rule->prob;
        
              double oldsita = pd->prob;

              itemcount++;
	      if (!bilm)
		triitemcount++;

              if (pd->inserted[(bilm)?0:1]!=passid || newsita > oldsita){//maximizing
		if (pd->inserted[(bilm)?0:1]!=passid){
		  pd->inserted[(bilm)?0:1]=passid;

		  //		  if (passid>1)
		  //		  fprintf(outsidefile, "pd: (%d\t%d\t%d\t%d\t%d\t%d\t%d)\n", s, S, uchoice[0], uchoice[1], vchoice[1], vchoice[0], lhs);
		}

                pd->prob = newsita;
		//		                pd->S = t; //deprecated
		pd->s = s;
		pd->t = S;
		pd->N = lhs;
                pd->rule = rule;
		pd->lchild = popped_pd;
		pd->rchild = neighbor_pd;

		//deprecated
//                 DUEXASSIGN(pd->ul, ul);
//                 DUEXASSIGN(pd->vl, vl);
//                 DUEXASSIGN(pd->ur, ur);
//                 DUEXASSIGN(pd->vr, vr);
		

                DUEXASSIGN(pd->lword, uchoice);
                DUEXASSIGN(pd->rword, vchoice);

		double insout;
		if (bilm){
		  double est = 	dchart->estimate(lhs, s, S, uchoice[0], uchoice[1], vchoice[1], vchoice[0]) 
		    + borderest(uchoice[0], uchoice[1], vchoice[0], vchoice[1])
		    ;
		  insout=pd->prob + est;
		}
		else{
		  double est = 		    //+ dchart->estimate(lhs, s, S, uchoice[0], uchoice[1], vchoice[1], vchoice[0]) + 
		    dchart->outsideof(s, S, uchoice[0], 0, vchoice[0], 0, lhs)
		    + borderest(uchoice[0], uchoice[1], vchoice[0], vchoice[1]);

		  //		  printf ("border est: %.10e\n", borderest(uchoice[0], uchoice[1], vchoice[0], vchoice[1]));

		  //		  fprintf(outsidefile, "%d\t%d\t%d\t%d\t%d\t%c\t%d\n", s, S, uchoice[0], vchoice[0], lhs, dchart->cell(lhs, s, S, uchoice[0], 0, 0, vchoice[0])->inserted[0], dchart->cell(lhs, s, S, uchoice[0], 0, 0, vchoice[0])->inserted[0]);
		  //		  fprintf(outsidefile, "%d\t%d\t%d\t%d\t%d\t%.10e\n", s, S, uchoice[0], vchoice[0], lhs, est);
		  insout= pd->prob + est;

		}

                //put on agenda
                agenda_node_t a(s, S, uchoice[0], uchoice[1], vchoice[0], vchoice[1], lhs, insout);
                agenda->push(a);
                if (Globals::Get<int>("debuglevel") >= 3) 
                  fprintf(logfile, "Adding %s to the agenda (prob = %.10e, insout = %.10e)\n", a.data().data(), pd->prob, insout);
		//              if (S == t)
		//                pd->inserted = 1;
              }//if max

	      //store the backpointers...
	      //	      if (bilm){
		backpointer thebkptr;
		//		thebkptr.lchild=dchart->cell(n, s, t, u[0], 0, 0, v[0]);
		//		thebkptr.rchild=dchart->cell(nr, t, S, ur[0], 0, 0, vr[0]);

		thebkptr.lchild = popped_pd;
		thebkptr.rchild = neighbor_pd;

		//		thebkptr.lwidth=(t-s);
		//		thebkptr.rwidth=(S-t);
		thebkptr.rule=rule;

		//	      thebkptr.lptr->deps++;
		//	      thebkptr.rptr->deps++;

		thebkptr.tri= (ngram->N==3);;

		pd->bkptrs.push_back(thebkptr);

		//	      }
            } // e_rhs
      } //end of enumerating neighbors
    } //end of right side nonterms


      /* unary rules */

    if (s==0 && t==T){ //added on oct 8, to filter, unary rules only appear on the top in the current grammar
    lhs_inv_erhs_rule_hash *rnt = &grammar->rule_table[n][-1];
	
    lhs_inv_erhs_rule_hash::iterator lhs_itr;
    inv_erhs_rule_hash::iterator inv_itr;
    erhs_rule_hash::iterator e_rhs_itr;
	
    // look up all unary rules with n as rhs nonterminal
    for (lhs_itr = rnt->begin(); lhs_itr != rnt->end(); lhs_itr++)
      for (inv_itr = lhs_itr->second.begin(); inv_itr != lhs_itr->second.end(); inv_itr++)
        for (e_rhs_itr = inv_itr->second.begin(); e_rhs_itr != inv_itr->second.end(); e_rhs_itr++) {
          Rule *rule = e_rhs_itr->second;
          int lhs = rule->lhs_int;

          double lm_prob;
          uchoice[0] = uchoice[1] = 0;
          vchoice[0] = vchoice[1] = 0;
	   
          lm_prob = LMWEIGHT * compute_lm(u, NULL, v, NULL, rule, uchoice, vchoice);

	  if (bilm){
	    uchoice[1]=vchoice[1]=0;
	  }

	   
          double newsita = sita0 + lm_prob;
          if (include_itgscore)
            newsita += rule->prob;
          pd = dchart->cell(lhs, s, t, uchoice[0], uchoice[1], vchoice[1], vchoice[0]);
          double oldsita = pd->prob;
          itemcount++;
	  if (!bilm)
	    triitemcount++;
          if (pd->inserted[(bilm)?0:1]!=passid || newsita > oldsita){//maximizing
	    if (pd->inserted[(bilm)?0:1]!=passid){
	      pd->inserted[(bilm)?0:1]=passid;

	      //	      if (passid>1)
	      //	      fprintf(outsidefile, "pd: (%d\t%d\t%d\t%d\t%d\t%d\t%d)\n", s, t, uchoice[0], uchoice[1], vchoice[1], vchoice[0], lhs);
      	    }

            pd->prob = newsita;
	    //	                pd->S = -1; //deprecated
	    pd->s = s;
	    pd->t = t;
	    pd->N = lhs;
            pd->rule = rule;
	    pd->lchild = popped_pd;
	    pd->rchild = NULL;
	    
            DUEXASSIGN(pd->lword, uchoice);
            DUEXASSIGN(pd->rword, vchoice);

	    //deprecated
//             DUEXASSIGN(pd->ul, u);
//             DUEXASSIGN(pd->vl, v);
//             DUEXASSIGN(pd->ur, u);
//             DUEXASSIGN(pd->vr, v);
	    

            double insout;
	    if (bilm){
	      double est =  dchart->estimate(lhs, s, t, uchoice[0], uchoice[1], vchoice[1], vchoice[0])
		+ borderest(uchoice[0], uchoice[1], vchoice[0], vchoice[1])
		;

	      insout = pd->prob + est;
	    }
	    else{
	      double est = 		//+ dchart->estimate(lhs, s, t, uchoice[0], uchoice[1], vchoice[1], vchoice[0])  + 
		dchart->outsideof(s, t, uchoice[0], 0, vchoice[0], 0, lhs)
		+ borderest(uchoice[0], uchoice[1], vchoice[0], vchoice[1]);

	      //	      printf ("border est: %.10e\n", borderest(uchoice[0], uchoice[1], vchoice[0], vchoice[1]));

	      //	      if (lhs==grammar->start_symbol)
	      //		est=1000; //added oct 8

	      //	      fprintf(outsidefile, "%d\t%d\t%d\t%d\t%d\t%c\t%d\n", s, t, uchoice[0], vchoice[0], lhs, dchart->cell(lhs, s, t, uchoice[0], 0, 0, vchoice[0])->inserted[0], dchart->cell(lhs, s, t, uchoice[0], 0, 0, vchoice[0])->inserted[0]);
	      //	      fprintf(outsidefile, "%d\t%d\t%d\t%d\t%d\t%.10e\n", s, t, uchoice[0], vchoice[0], lhs, est);

	      insout = pd->prob + est;

	    }

            //put on agenda
            agenda_node_t a(s, t, uchoice[0], uchoice[1], vchoice[0], vchoice[1], lhs, insout);
            agenda->push(a);
            if (Globals::Get<int>("debuglevel") >= 3) 
              fprintf(logfile, "Adding %s to the agenda (prob = %.10e, insout = %.10e)\n", a.data().data(), pd->prob, insout);
          } //end of updating viterbi

	  //store the backpointers...
	  
	  //	  if (bilm){
	    backpointer thebkptr;
	    //	    thebkptr.lchild=dchart->cell(n, s, t, u[0], 0, 0, v[0]);

	    thebkptr.lchild = popped_pd;

	    thebkptr.rchild=NULL;
	    //	    thebkptr.rptr=dcell(s, t, u[0], u[1], v[0], v[1], n);
	    thebkptr.rule=rule;

	    //	      thebkptr.lptr->deps++;
	    thebkptr.tri= (ngram->N==3);;

	    pd->bkptrs.push_back(thebkptr);
	    //            if (Globals::Get<int>("debuglevel") >= 3) 
	    //              fprintf(logfile, "backpointing\n");

	    //	  }
	  
        } //end of matching rule
  
      }

    /* special case if the current cell is the root level */

    if ((s == 0) && (t == T) && (n == grammar->start_symbol)) {

      double lm_prob = 0.0;
      if (ngram->N == 2) {
        lm_prob = LMWEIGHT * (ngram->bigram(-1, u[0])+ngram->bigram(v[0], -2));
      } else if (ngram->N == 3) {
        lm_prob = LMWEIGHT * (ngram->trigram(-1, u[0], u[1])+ngram->trigram(v[1], v[0], -2));
        lm_prob += LMWEIGHT * (ngram->bigram(-1, u[0])); //adding the leading bigram probability
      }

      ITGDCell *cell = dchart->cell(grammar->start_symbol,0,T,u[0],u[1],v[1],v[0]);
      //      double newsita = dchart->cell(grammar->start_symbol,0,T,u[0],u[1],v[1],v[0])->prob + lm_prob;
      double newsita = cell->prob + lm_prob;
       
      pd = dchart->cell(grammar->_nonterminal_cnt, 0, T, 0, 0, 0, 0); //should use this to UNIQUELY represent the best translation 
      double oldsita = pd->prob;
      itemcount++;
      if (!bilm)
	triitemcount++;
      if (pd->inserted[(bilm)?0:1]!=passid || newsita > oldsita){
	if (pd->inserted[(bilm)?0:1]!=passid){
	  pd->inserted[(bilm)?0:1]=passid;
	  
	  //	  if (passid>1)
	  //	  fprintf(outsidefile, "pd: (%d\t%d\t%d\t%d\t%d\t%d\t%d)\n", 0, T, 0, 0, 0, 0, grammar->_nonterminal_cnt);
	}

        pd->prob = newsita;
	//		pd->S = -1; //deprecated
	pd->s = 0;
	pd->t = T;
	//	pd->N = grammar->_nonterminal_cnt;
	//to agree with the change in cyk
	pd->N = grammar->start_symbol;

	pd->rule = NULL;
	pd->lchild = cell;
	pd->rchild = NULL;

        double insout = pd->prob;
        //    agenda.push(agenda_node_t(0, T, u[0], u[1], v[0], v[1], grammar->_nonterminal_cnt, insout));


	//	DUEXASSIGN(pd->lword, u);
	//        DUEXASSIGN(pd->rword, v);
	//to agree with the change in cyk
	pd->lword[0] = -1;
        if (ngram->N == 2) {
          pd->lword[1] = 0;
          pd->rword[1] = 0;
        } else {
          pd->lword[1] = cell->lword[0];
          pd->rword[1] = cell->rword[0];
        }
        pd->rword[0] = -2;
	//deprecated
//         DUEXASSIGN(pd->ul, u);
//         DUEXASSIGN(pd->vl, v);
//         DUEXASSIGN(pd->ur, u);
//         DUEXASSIGN(pd->vr, v);
	

        agenda_node_t a(0, T, 0, 0, 0, 0, grammar->_nonterminal_cnt, insout);
        agenda->push(a);
        if (Globals::Get<int>("debuglevel") >= 3) 
          fprintf(logfile, "Adding %s to the agenda (prob = %.10e, insout = %.10e)\n", a.data().data(), pd->prob, insout);
	 
      }//end of updating viterbi

      //backpointers 
      //      if (bilm){
	backpointer thebkptr;
	//	thebkptr.lchild = dchart->cell(n, 0, T, u[0], 0, 0, v[0]);

	thebkptr.lchild = cell;

	thebkptr.rchild = NULL;
	 
	thebkptr.tri= (ngram->N==3);;
	pd->bkptrs.push_back(thebkptr);
	//      }
    }//end if top level
  
  }//end of agenda






  //then top-down
  if (bilm){
    pd=dchart->cell(grammar->_nonterminal_cnt, 0, T, 0 , 0, 0, 0);
    outsidepass(pd); //compute the outside costs
  }//end if bigram decoding




  /* get the best score from our unique best-cell representation */

  pd = dchart->cell(grammar->_nonterminal_cnt, 0, T, 0, 0, 0, 0);
  double logscore = pd->prob;
  //  DUEXASSIGN(u, pd->ul); // ??
  //  DUEXASSIGN(v, pd->vl); // ??

  //  fprintf(logfile, "arcs popped %d\n", itemcount);
  fprintf(logfile, "agenda items explored %d\n", agenda_count);
  //  fprintf(logfile, "log prob %.20lf\n", logscore);

  /* cleanup */

  for (int d=0; d<2; d++){
    delete[] neighbors[d];
  }
  delete[] neighbors;  

  for (int iter=0; iter<=T; iter++){
    delete[] bc[iter];
  }
  delete[] bc;

  delete agenda;
  /* dump chart */

  
  if (Globals::Get<int>("debuglevel") >= 3) {
    dchart->dump(_sentno);
  }
  
  //  fclose(  outsidefile);

  /* top-down reconstruction */

  //  pd = dchart->cell(grammar->start_symbol, 0, T, u[0], u[1], v[1], v[0]);
  if (!bilm){

    if (Globals::Get<int>("epass")){
      
      
      printf ("epass begins\n");
      trioutsidepass(pd);
      maxEpass(pd);    
      printf ("epass ends\n");
      
      pd = dchart->cell(grammar->_nonterminal_cnt, 0, T, 0, 0, 0, 0);
      logscore = pd->prob;
    }

    fprintf(logfile, "arcs popped %d\n", itemcount);
    fprintf(logfile, "hooks proposed %d\n", hookitemcount);
    fprintf(logfile, "triedges popped %d\n", triitemcount);
    fprintf(logfile, "log prob %.20lf\n", logscore);

    if (pd->prob > -INFINITY ){
      //      return reconstruct(pd->lchild);
      return reconstruct(pd);
    }
    else
      return "";
  }
  else
    return "";
}

#define maxEcons 1

#ifdef maxEcons
/* traverse the forest, maximizing the expected total counts
 *
 */
double CYKDecoder::maxEpass(ITGDCell *pd){
  bkptr_v::iterator bkptr_itr;
  ITGDCell *lpd, *rpd;
  double maxE;

  maxE = -INFINITY;
  int uchoice[2];
  int vchoice[2];


  if (pd->inserted[1]=='i')
    return pd->outside;

  pd->inserted[1]='i'; //color it

  if (pd->bkptrs.size() > 0)
    for (bkptr_itr=pd->bkptrs.begin(); bkptr_itr!=pd->bkptrs.end(); ++bkptr_itr){
      //skip non-trigram backpointers
      if (!bkptr_itr->tri)
	continue;

      lpd=bkptr_itr->lchild;
      rpd=bkptr_itr->rchild;
      double lmaxE, rmaxE;

      
      if (lpd){
	lmaxE = maxEpass(lpd);
      }
      else
	lmaxE = 0;

      if (rpd){
	rmaxE = maxEpass(rpd);
      }
      else
	rmaxE = 0;

      uchoice[0] = uchoice[1] = 0;
      vchoice[0] = vchoice[1] = 0;

      double totalE = lmaxE + rmaxE ;//+ lmc;  //summing up. (should be in the real domain)

      if (totalE > maxE){
	maxE = totalE;

	//	printf("updated\n");
	//update the viterbi backpointers
	pd->lchild = lpd;
	pd->rchild = rpd;

      }

    }//end of enumerating the children
  else
    maxE = 0;
  
  double nodeE = pd->prob  + pd->outside;

  pd->outside = exp(nodeE) + maxE;

  return pd->outside;
  // return nodeE + maxE;
}
#else
/* traverse the forest, maximizing the expected total counts
 *
 */
double CYKDecoder::maxEpass(ITGDCell *pd){
  bkptr_v::iterator bkptr_itr;
  ITGDCell *lpd, *rpd;
  double maxE;

  maxE = -INFINITY;
  int uchoice[2];
  int vchoice[2];


  if (pd->inserted[1]=='i')
    return pd->outside;

  pd->inserted[1]='i'; //color it

  if (pd->bkptrs.size() > 0)
    for (bkptr_itr=pd->bkptrs.begin(); bkptr_itr!=pd->bkptrs.end(); ++bkptr_itr){
      //skip non-trigram backpointers
      if (!bkptr_itr->tri)
	continue;

      lpd=bkptr_itr->lchild;
      rpd=bkptr_itr->rchild;
      double lmaxE, rmaxE;
      double lprob, rprob;
      
      if (lpd){
	lmaxE = maxEpass(lpd);
	lprob = lpd->prob;
      }
      else{
	lmaxE = 0;
	lprob = 0;
      }
      if (rpd){
	rmaxE = maxEpass(rpd);
	rprob = rpd->prob;
      }
      else{
	rmaxE = 0;
	rprob = 0;
      }

      uchoice[0] = uchoice[1] = 0;
      vchoice[0] = vchoice[1] = 0;

      double lmc=0;

      if (lpd && rpd){
	if (bkptr_itr->rule->inverted) {
	  lmc = LMWEIGHT * compute_lm(rpd->lword, lpd->lword, rpd->rword, lpd->rword, bkptr_itr->rule, uchoice, vchoice);
	} else {
	  lmc = LMWEIGHT * compute_lm(lpd->lword, rpd->lword, lpd->rword, rpd->rword, bkptr_itr->rule, uchoice, vchoice);
	}
      }


      double totalE =  exp(lprob + rprob + pd->outside + lmc) + lmaxE + rmaxE ;//+ lmc;  //summing up. (should be in the real domain)

      if (totalE > maxE){
	maxE = totalE;

	//	printf("updated\n");
	//update the viterbi backpointers
	pd->lchild = lpd;
	pd->rchild = rpd;

      }

    }//end of enumerating the children
  else
    maxE = 0;
  
  //  double nodeE = pd->prob  + pd->outside;

  if (maxE == 0)
    maxE = exp(pd->prob + pd->outside);

  //  pd->outside = exp(nodeE) + maxE;
  pd->outside = maxE;

  return pd->outside;
  // return nodeE + maxE;
}
#endif

//we need a trigram outside pass

void CYKDecoder::trioutsidepass(ITGDCell *pd){

  int uchoice[2];
  int vchoice[2];

  pd_vec  toporder; //order the nodes topologically in the vector

  //  pd=dchart->cell(grammar->_nonterminal_cnt, 0, T, 0 , 0, 0, 0);

  depthfirst(pd, toporder, 1);

  //initialize from the root

  ITGDCell *thepd;
  bkptr_v::iterator bkptr_itr, bkptrptr_itr;

  //reach from the root to the top level parents of binary chilren
  //this is making assumptions on the grammar structure
  for (bkptr_itr=pd->bkptrs.begin(); bkptr_itr!=pd->bkptrs.end(); ++bkptr_itr){

    //skip non-trigram backpointers
    if (!bkptr_itr->tri)
      continue;

    bkptr_itr->lchild->outside =
      LMWEIGHT * (ngram->bigram(-1, bkptr_itr->lchild->lword[0])
		  + ngram->trigram(-1, bkptr_itr->lchild->lword[0], bkptr_itr->lchild->lword[1])
		  + ngram->trigram(bkptr_itr->lchild->rword[1], bkptr_itr->lchild->rword[0], -2) )
      ;


    for (bkptrptr_itr=bkptr_itr->lchild->bkptrs.begin(); bkptrptr_itr!=bkptr_itr->lchild->bkptrs.end(); ++bkptrptr_itr){
      //skip non-trigram backpointers
      if (!bkptrptr_itr->tri)
	continue;

      thepd=bkptrptr_itr->lchild; //root->S->A/

      thepd->outside =
	LMWEIGHT * (ngram->bigram(-1, bkptr_itr->lchild->lword[0])
		    + ngram->trigram(-1, bkptr_itr->lchild->lword[0], bkptr_itr->lchild->lword[1])
		    + ngram->trigram(bkptr_itr->lchild->rword[1], bkptr_itr->lchild->rword[0], -2) )
	+bkptrptr_itr->rule->prob;// + bkptr_itr->rule->prob;

      if (Globals::Get<int>("debuglevel") >= 3) 
	fprintf(logfile, "root level outside %.10e\n", thepd->outside);
	
    }
  }

  //problem:: insertions, fixed if we use the topological sort

  int vecsize=toporder.size();

  if (Globals::Get<int>("verbosity") >= 1)
    printf ("vecsize %d\n", vecsize);

  if (Globals::Get<int>("debuglevel") >= 3) 
    fprintf(logfile, "Vec size %d\n", vecsize);

  for (int idx=vecsize-1; idx>=0; idx--){ //backwards
    toporder[idx]->inserted[1]='v';

    pd=toporder[idx];

    if (pd->rchild==NULL){
      continue;
    }

      
    ITGDCell *lpd, *rpd;

    // compiler warned that this isn't used
//     double summingtotalinside=0;

    //iterate over all back pointers
    for (bkptr_itr=pd->bkptrs.begin(); bkptr_itr!=pd->bkptrs.end(); ++bkptr_itr){
      //skip non-trigram backpointers
      if (!bkptr_itr->tri)
	continue;

      lpd=bkptr_itr->lchild;
      rpd=bkptr_itr->rchild;

      bool lunseen, runseen;


      lunseen=(lpd->inserted[1]!='v');
      runseen=(rpd->inserted[1]!='v');

      if (lunseen)
	lpd->inserted[1]='v';
      if (runseen)
	rpd->inserted[1]='v';

      double lmc;


      uchoice[0] = uchoice[1] = 0;
      vchoice[0] = vchoice[1] = 0;

      //for non-hook versions
	
      if (bkptr_itr->rule->inverted) {
	lmc = LMWEIGHT * compute_lm(rpd->lword, lpd->lword, rpd->rword, lpd->rword, bkptr_itr->rule, uchoice, vchoice);
      } else {
	lmc = LMWEIGHT * compute_lm(lpd->lword, rpd->lword, lpd->rword, rpd->rword, bkptr_itr->rule, uchoice, vchoice);
      }
     

      double newlout = lmc +
	pd->outside + rpd->prob + bkptr_itr->rule->prob;

      double newrout = lmc + 
	pd->outside + lpd->prob + bkptr_itr->rule->prob;


      // compiler warned that this isn't used
//       double totalinside = lmc +
// 	lpd->prob + rpd->prob + bkptr_itr->rule->prob;

    
        
      if (lunseen || newlout>lpd->outside)
	lpd->outside = newlout;
      

      /*      
      if (lunseen)
	lpd->outside = newlout;
      else{
	lpd->outside = log (exp(lpd->outside) + exp(newlout) );
      }
      */

      
      
      if (runseen || newrout>rpd->outside)
	rpd->outside = newrout;
	
      

      /*
      if (runseen)
	rpd->outside = newrout;
      else{
	rpd->outside = log (exp(rpd->outside) + exp(newrout));
      }
      */
	
      //      summingtotalinside=log(exp(summingtotalinside) + exp(totalinside) );

    }//end of iterating over all backpointers

    //    pd->prob=summingtotalinside;

  }//end of vising all the nodes in topological order
  
}

/* compute the outside costs following the
 * back pointers in topological order
 */

void CYKDecoder::outsidepass(ITGDCell *pd){
  //  ITGDCell *pd = NULL; //pointer to a decoding chart cell
  int uchoice[2];
  int vchoice[2];

  bool bilm=true;

  pd_vec  toporder; //order the nodes topologically in the vector

  //  pd=dchart->cell(grammar->_nonterminal_cnt, 0, T, 0 , 0, 0, 0);

  depthfirst(pd, toporder, 0);

  //initialize from the root

  ITGDCell *thepd;
  bkptr_v::iterator bkptr_itr, bkptrptr_itr;

  //reach from the root to the top level parents of binary chilren
  //this is making assumptions on the grammar structure
  for (bkptr_itr=pd->bkptrs.begin(); bkptr_itr!=pd->bkptrs.end(); ++bkptr_itr){

    bkptr_itr->lchild->outside =
      LMWEIGHT * (ngram->bigram(-1, bkptr_itr->lchild->lword[0])+ngram->bigram(bkptr_itr->lchild->rword[0], -2))
      ;
      //      +bkptr_itr->rule->prob;
    //    printf("bkptr_itr ->rule->prob: %.10e\n", bkptr_itr->rule->prob);

    for (bkptrptr_itr=bkptr_itr->lchild->bkptrs.begin(); bkptrptr_itr!=bkptr_itr->lchild->bkptrs.end(); ++bkptrptr_itr){
      thepd=bkptrptr_itr->lchild; //root->S->A/

      thepd->outside =
	LMWEIGHT * (ngram->bigram(-1, bkptrptr_itr->lchild->lword[0])+ngram->bigram(bkptrptr_itr->lchild->rword[0], -2))
	+bkptrptr_itr->rule->prob;// + bkptr_itr->rule->prob;

      if (Globals::Get<int>("debuglevel") >= 3) 
	fprintf(logfile, "root level outside %.10e\n", thepd->outside);
	
    }
  }

  //problem:: insertions, fixed if we use the topological sort

  int vecsize=toporder.size();

  trivecsize = vecsize;

  if (Globals::Get<int>("verbosity") >= 1)
    printf ("vecsize %d\n", vecsize);

  if (Globals::Get<int>("debuglevel") >= 3) 
    fprintf(logfile, "Vec size %d\n", vecsize);

  for (int idx=vecsize-1; idx>=0; idx--){ //backwards
    toporder[idx]->inserted[(bilm)?0:1]='v';
    //    if (toporder[idx]->S==-1)
    //      continue; //we've done everything for the top level nodes

    pd=toporder[idx];

    if (pd->rchild==NULL){
      continue;
    }

    if (pd->lchild == pd->rchild){ //hook 

      ITGDCell *lpd;
      
      //iterate over all back pointers
      for (bkptr_itr=pd->bkptrs.begin(); bkptr_itr!=pd->bkptrs.end(); ++bkptr_itr){
	lpd=bkptr_itr->lchild;

	bool lunseen;

	lunseen=(lpd->inserted[(bilm)?0:1]!='v');

	if (lunseen)
	  lpd->inserted[(bilm)?0:1]='v';
	

	double lmc;


	lmc = LMWEIGHT * ngram->bigram(pd->lword[0], lpd->lword[0]);
          

	double newlout = lmc + pd->outside ;


	if (lunseen || newlout>lpd->outside){
	  lpd->outside=newlout;
	}      
	
      }//end of all backpointers
    }//end of hook nodes
    else{
      
    ITGDCell *lpd, *rpd;
      
    //iterate over all back pointers
    for (bkptr_itr=pd->bkptrs.begin(); bkptr_itr!=pd->bkptrs.end(); ++bkptr_itr){
      lpd=bkptr_itr->lchild;
      rpd=bkptr_itr->rchild;

      bool lunseen, runseen;


      lunseen=(lpd->inserted[(bilm)?0:1]!='v');
      runseen=(rpd->inserted[(bilm)?0:1]!='v');

      if (lunseen)
	lpd->inserted[(bilm)?0:1]='v';
      if (runseen)
	rpd->inserted[(bilm)?0:1]='v';

      double lmc;

      if (Globals::Get<int>("use_hooks"))
	lmc=0;
      else{
	uchoice[0] = uchoice[1] = 0;
	vchoice[0] = vchoice[1] = 0;

	//for non-hook versions
	
	if (bkptr_itr->rule->inverted) {
	  lmc = LMWEIGHT * compute_lm(rpd->lword, lpd->lword, rpd->rword, lpd->rword, bkptr_itr->rule, uchoice, vchoice);
	} else {
	  lmc = LMWEIGHT * compute_lm(lpd->lword, rpd->lword, lpd->rword, rpd->rword, bkptr_itr->rule, uchoice, vchoice);
	}
     
      }

      double newlout = lmc +
	pd->outside + rpd->prob + bkptr_itr->rule->prob;

      double newrout = lmc + 
	pd->outside + lpd->prob + bkptr_itr->rule->prob;


      if (lunseen || newlout>lpd->outside){
	lpd->outside=newlout;
      }
		
      if (runseen || newrout>rpd->outside){
	rpd->outside=newrout;
      }

      //	fprintf(outsidefile, "lpd->outside: %.10e\t rpd->outside: %.10e\n", lpd->outside, rpd->outside);
      //	fflush(outsidefile);

      //        if (Globals::Get<int>("debuglevel") >= 3) 
      //          fprintf(logfile, "lpd->outside: %.10e\t rpd->outside: %.10e\n", lpd->outside, rpd->outside);

	
    }//end of iterating over all backpointers
    }//end of non-hook nodes
  }//end of vising all the nodes in topological order

}

/* Deep-first traversal of the forest, following the
   backpointers
*/


void CYKDecoder::depthfirst(ITGDCell *pd, pd_vec &thevec, int biortri){
  bkptr_v::iterator bkptr_itr;
  ITGDCell *lpd, *rpd;

  pd->inserted[biortri]='i'; //color it

  for (bkptr_itr=pd->bkptrs.begin(); bkptr_itr!=pd->bkptrs.end(); ++bkptr_itr){
    //    if (bkptr_itr->tri != (biortri==1) )
    if (biortri == 1){
      if (!bkptr_itr->tri)
	continue;
    }

    lpd=bkptr_itr->lchild;
    rpd=bkptr_itr->rchild;
	
    if (lpd && (lpd->inserted[biortri]!='i')){
      depthfirst(lpd, thevec, biortri);
    }

    if (rpd && (rpd->inserted[biortri]!='i')){
      depthfirst(rpd, thevec, biortri);
    }
  }
  
  //parent is put after the children
  thevec.push_back(pd);

  return;
}


