/* itgdec-cyk.cpp
 *
 * Implementation file for basic bottom-up CYK-style decoder, as
 * described in Wu (ACL 1996)
 *
 * $Id: itgdec-cyk.cpp,v 1.2 2011/10/16 22:36:43 gildea Exp $
 */

#include <sys/stat.h>
#include <cmath>
#include <vector>
#include <iomanip>

#include "itgdec-cyk.h"
// #include "itgchart.h"

using namespace std;

ITGGrammar* grammar;

//#define DUEXASSIGN(x, y) {x[0] = y[0]; x[1] = y[1];}

// const unsigned int TOPK_BEAM_1 = 5; // for others
// const unsigned int TOPK_BEAM_2 = 200; // for C

int ITGDCell::NUM_BACKPOINTERS = 1;

double LENGTH_BONUS;

string dumpdir;

/* The chart is organized into <key,ITGDCell> pairs.
 * <key> is actually an encoding of the two (bigram case) or
 * four (trigram) case border words of the cell.  These
 * macros take a key and produces the indices into the
 * candidates array, which is a vector mapping from the
 * English bag of words to the actual English IDs from the
 * vocabulary.
 */

CYKDecoder::CYKDecoder(ITGGrammar* ig, ITGCorp* ic) : Decoder(ig, ic) {

  dchart = new ITGDChart(MAXT); // maximum lengths

  LENGTH_BONUS = G_DOUBLE("length_bonus");

  ::grammar = ig;

  int t;
  enclosure = new WORDTYPE ***[MAXT+1];
  for (t = 0; t <= MAXT; t++) { 
    enclosure[t] = new WORDTYPE **[MAXT+1]; 
    enclosure[t][0] = new WORDTYPE * [MAXINS + 1]; //allows different number of insertions than normal translations
    for (int lspan = 1; lspan <= MAXT; lspan++)
      enclosure[t][lspan] = new WORDTYPE * [MAXTRS + 1];
  }
}

CYKDecoder::~CYKDecoder() {

  for (int t = 0; t <= MAXT; t++){
    for (int to = 0; to <= MAXT; to++){ 
      delete[] enclosure[t][to];
    } 
    delete[] enclosure[t];
  }
  delete[] enclosure;

  delete dchart;
}

bool operator>(const backpointer& a, const backpointer& b) {
  backpointer_lt blt;
  return ! blt(a, b);
}

bool operator==(const backpointer& a, const backpointer& b) {
  return false;
}

ostream & operator<<(ostream &out, agenda_node_t a) {
  out << a.data();
  return out;
}

ostream& operator<<(ostream& out, const ITGDCell& cell) {
  //  if (cell.N < 0) // uninitialized cell
  //    return out;

  out << "[" << grammar->nonterminals[cell.N] << " " << cell.s << ".." << cell.t << "]";

  if (ngram->loaded) {
    if (ngram->N == 3)
      out << " <" << l1vocab[cell.lword[0]] << "," << l1vocab[cell.lword[1]] 
          << "," << l1vocab[cell.rword[1]] << "," << l1vocab[cell.rword[0]] << "> ";
    else if (ngram->N == 2)
      out << " <" << l1vocab[cell.lword[0]] << "," << l1vocab[cell.rword[0]] << "> ";
  }

  if (cell.lchild == NULL && cell.rchild == NULL) { // leaf
    out << " " << grammar->nonterminals[cell.N] << " -> ["
        << transliterate(cell.ephr,&l1vocab) << "/" 
        << transliterate(cell.cphr,&l2vocab) << "]"; 
  }

  if (cell.rule != NULL) // NULL for top-level rule (with <s>...</.s>) and terminals
    out << " " << *cell.rule;

  out << " in=" << cell.prob << " out=" << cell.outside;
  
  return out;
}


/** CYKDecoder::fillin(t, lspan, & kbests, candidates)
 *
 *  This function takes a kbest list of candidate translations for a
 *  Chinese phrase and puts them on the chart for the span (t, t+lspan).
 *  It also updates the candidates set, which is a mapping between
 *  ... [figure this out and then update this comment].
 */
inline void CYKDecoder::fillin(int t, int lspan, list <WORDTYPE*> &kbest) {

  // index into the translation list for a given span in the target lang
  int uidx = 0;

  // a phrase_entry is a (WORDTYPE *, double) pair (phrase, probability)
  list <WORDTYPE*>::iterator kitr;
  for (kitr = kbest.begin(); kitr != kbest.end(); kitr++) {
    WORDTYPE *phrase = *kitr;  // the english candidate phrase 

    // enclosure[t][lspan] holds the list of possible phrase translations
    enclosure[t][lspan][uidx] = phrase;
    uidx++;
  }

  enclosure[t][lspan][uidx] = 0;//zero-ending list
}

/** CYKDecoder::dinit()
 *
 * Seeds the chart with phrasal translations.
 */
void CYKDecoder::dinit(bool firstpass) {
  //  FILE *dinitfile=fopen("/tmp/dinitfile", "a");

  bool kbest_init = Globals::Get<int>("lazy_kbest"); 

  if (firstpass){
    senvocab_l.clear();
    senvocab_r.clear();
    senvocab.clear();
    senvocab[0]=1; //0 must be in the vocab
    dchart->clean();
  }

  int passid=(firstpass)?0:1;


  if (Globals::Get<int>("debuglevel") >= 2) {
    fprintf(logfile, "\n\n-- Decoding sentence #%d (len %d)\n", _sentno, T);
    fprint_array(logfile, cn);
    fprintf(logfile, "\n");
    fflush(logfile);
  }

  int inited = 0;

  for (int lspan = 1; lspan <= T; lspan++) {
    for (int s = 0; s <= T - lspan; s++) {
      for (int uidx = 0; enclosure[s][lspan][uidx]; uidx++) {
        WORDTYPE *ephr = enclosure[s][lspan][uidx];
        int phrlen = phraselen(ephr);

        WORDTYPE u[2], v[2];
        u[0] = *ephr;
        v[0] = last_word(ephr);
        u[1] = v[1] = 0;
        if (ngram->N > 2 && phrlen > 1) {
          u[1] = ephr[1];
          v[1] = penultimate_word(ephr);
        }


	if (firstpass){
	  senvocab_l[u[0]]=1;
	  senvocab_r[v[0]]=1;
	  WORDTYPE *pephr=ephr;
	  while (*pephr){
	    senvocab[*pephr]=1;
	    pephr++;
	  }
	}

        WORDTYPE* cphr_t = splice(cn, s, s + lspan);
        WORDTYPE* cphr = grammar->phrase_symbol_table_entry(cphr_t);
        delete[] cphr_t;

        if (lspan == 1 && !grammar->probs_table.count(cphr))
          cphr = grammar->unkphr;

        lhs_to_prob_table yitr = grammar->probs_table[cphr][ephr];
        lhs_to_prob_table::iterator l_iter;
        for (l_iter = yitr.begin(); l_iter != yitr.end(); l_iter++) {
          // l_iter->first is preterminal id
          // l_iter->second is rule prob
	  int preT = l_iter->first;
          // the old/existing cell
          ITGDCell *pd = dchart->cell(preT, s, s + lspan, u[0], u[1], v[1], v[0]);
          // create the new cell
          ITGDCell new_cell(preT, s, s + lspan, u[0], u[1], v[1], v[0]);
          new_cell.ephr = ephr;
          new_cell.cphr = cphr;

          double phr_prob = l_iter->second;
          if (phrlen == 0)
            phr_prob *= Globals::Get<double>("delpenalty");
          
          double lm_prob = ngram->prob(ephr);
          double newsita = 0.0;
          if (weights.loaded) { // discriminative
            new_cell.features = feature_init(new_cell,phr_prob,lm_prob);
            newsita = new_cell.features * weights;
          } else {
            newsita = phr_prob + LMWEIGHT * lm_prob + LENGTH_BONUS * phrlen;
          }
	  double oldsita = pd->prob;

	  //	  if (newsita < oldsita)
	  //	    cerr<<s<<" "<<s+lspan<<" "<<u[0]<<" "<<u[1]<<" "<<v[1]<<" "<<v[0]<<" "<<l_iter->second<<" "<<oldsita<<" "<<newsita<<endl;

	  
	  /*
	   *  dynamic programming maximization
	   */
	  if (pd->inserted[passid] != passid || newsita > oldsita){
	    if (pd->inserted[passid] != passid)
	      pd->inserted[passid] = passid;

            pd->N = preT; // grammar->T;
            pd->s = s;
            pd->t = s + lspan;
            pd->rule = NULL;
            pd->lchild = NULL;
            pd->rchild = NULL;
            DUEXASSIGN(pd->lword, u);
            DUEXASSIGN(pd->rword, v);
            pd->features = new_cell.features;

            pd->prob = newsita;
            pd->ephr = ephr;
            pd->cphr = cphr;

	    if (kbest_init)
	      KBESTS(pd->s, pd->t, preT)->KV.push_back(pd);

	    //	    fprintf(dinitfile, "%d\t%d\t%.10e\n", s, s+lspan, pd->prob);
	    //	    fflush(dinitfile);

	    if (Globals::Get<int>("debuglevel") >= 2) {
	      fprintf(logfile, "Leaf [%d,%d] = ", s, s + lspan);
	      fprint_array(logfile, ephr);
	      if (phrlen > 1)
		fprintf(logfile, " [sec = %d, pen = %d]", ephr[1], penultimate_word(ephr));
	      fprintf(logfile, " (%d, %d, %d, %d)", pd->lword[0], pd->lword[1], pd->rword[1], pd->rword[0]);
	      fprintf(logfile, " (pr = %.5e)\n", pd->prob);
	      fflush(logfile);
	    }
	  } //end of maximization
          inited++;
        } // loop over preterminal symbols (l_iter)
      }
    }
  }

  /* INSERTIONS */
  if (Globals::Get<bool>("allow_insertions")) {
    for (int s = 0; s <= T; s++) {
      for (int uidx = 0; enclosure[s][0][uidx]; uidx++){
        WORDTYPE* ephr = enclosure[s][0][uidx];
        int phrlen = phraselen(ephr);


        //s, s, u, v
        WORDTYPE u[2], v[2];
        u[0] = *ephr;
        v[0] = last_word(ephr);
        u[1] = v[1] = 0;
        if (ngram->N > 2 && phrlen > 1) {
          u[1] = ephr[1];
          v[1] = penultimate_word(ephr);
        }


	if (firstpass){
	  senvocab_l[u[0]]=1;
	  senvocab_r[v[0]]=1;
	  WORDTYPE *pephr=ephr;
	  while (*pephr){
	    senvocab[*pephr]=1;
	    pephr++;
	  }
	}

        ITGDCell *pd = dchart->cell(grammar->T, s, s, u[0], u[1], v[1], v[0]);

        // create the new cell
        ITGDCell new_cell(grammar->T, s, s, u[0], u[1], v[1], v[0]);
        new_cell.ephr = ephr;

        double newsita = 0.0;
        double ins_prob = grammar->phr2phr_prob(ephr, grammar->nullphr) * Globals::Get<double>("inscredit");
        double lm_prob = ngram->prob(ephr);
        if (weights.loaded) { // discriminative
          new_cell.features = feature_init(new_cell, ins_prob, lm_prob);
          newsita = new_cell.features * weights;
        } else
          newsita = ins_prob + lm_prob * LMWEIGHT + LENGTH_BONUS * phrlen;

	double oldsita = pd->prob;
	if (pd->inserted[passid] != passid || newsita > oldsita){
	  if (pd->inserted[passid] != passid)
	    pd->inserted[passid] = passid;

          pd->N = grammar->T;
          pd->s = s;
          pd->t = s;
 	  pd->rule = NULL;
          pd->lchild = NULL;
          pd->rchild = NULL;
          DUEXASSIGN(pd->lword, u);
          DUEXASSIGN(pd->rword, v);
 	  pd->prob = newsita;
 	  pd->ephr = ephr;

	  //	  KBESTS(pd->s, pd->t, preT)->KV.push_back(pd);

	  //	  fprintf(dinitfile, "%d\t%d\t%.10e\n", s, s, pd->prob);
	  //	  fflush(dinitfile);

	  if (Globals::Get<int>("debuglevel") >= 2) {
	    fprintf(logfile, "Leaf [%d,%d] = ", s, s);
	    fprint_array(logfile, ephr);
	    fprintf(logfile, " (%d, %d, %d, %d)", pd->lword[0], pd->lword[1], pd->rword[1], pd->rword[0]);
	    fprintf(logfile, " (pr = %.5e)\n", pd->prob);
	  }
	}
        inited++;
      }

    }
  }

  if (Globals::Get<int>("debuglevel") >= 1) 
    fprintf(logfile, "* Initialized %d chart items\n", inited);

  //  fprintf(dinitfile, "\n");
  //  fclose(dinitfile);
}

/* CYKDecoder::compute_estimate()
 *
 */
void CYKDecoder::compute_estimate(bool bilm) {
  hash_map <WORDTYPE, int >::iterator senv_itr, senv_iitr, senv_l_itr, senv_r_itr;

  dchart->bestborder.clear();

  if (bilm){
    //iterate over all english words
    for (senv_itr=senvocab.begin(); senv_itr!=senvocab.end(); ++senv_itr){
      WORDTYPE u=(*senv_itr).first;

      double curbest=ngram->bigram(-1, u);

      for (senv_r_itr=senvocab_r.begin(); senv_r_itr!=senvocab_r.end(); ++senv_r_itr){
	WORDTYPE w=(*senv_r_itr).first;
	double curbi;
	
	if (!w)
	  continue;

	curbi=ngram->bigram(w, u);

	if (curbi>curbest)
	  curbest=curbi;
      }//end of iter over w

      dchart->bestborder[u][0].left=curbest*LMWEIGHT;


      curbest=ngram->bigram(u, -2);

      for (senv_l_itr=senvocab_l.begin(); senv_l_itr!=senvocab_l.end(); ++senv_l_itr){
	WORDTYPE w=(*senv_l_itr).first;
	double curbi;
	
	if (!w)
	  continue;

	curbi=ngram->bigram(u, w);

	if (curbi>curbest)
	  curbest=curbi;
      }//end of iter over w

      dchart->bestborder[0][u].right=curbest*LMWEIGHT;
	
    }//end of iter over u

  }
  else{

    //iterate over all english words
    for (senv_itr=senvocab.begin(); senv_itr!=senvocab.end(); ++senv_itr){
      WORDTYPE u=(*senv_itr).first;
      for (senv_iitr=senvocab.begin(); senv_iitr!=senvocab.end(); ++senv_iitr){
	WORDTYPE v=(*senv_iitr).first;
      
	double curbest=ngram->trigram(-1, u, v);

	for (senv_r_itr=senvocab_r.begin(); senv_r_itr!=senvocab_r.end(); ++senv_r_itr){
	  WORDTYPE w=(*senv_r_itr).first;
	  double curtri;

	  //otherwise, the deletion word will get the highest trigram score
	  if (!w)
	    continue;

	  curtri=ngram->trigram(w, u, v);

	  if (curtri>curbest)
	    curbest=curtri;
	}//end of iter over w

	dchart->bestborder[u][v].left=curbest*LMWEIGHT;

	//      printf("bestborder[%d][%d].left: %.10e\n", u, v, curbest*LMWEIGHT);

      }//end of iter over v

      //then the right side

      for (senv_iitr=senvocab.begin(); senv_iitr!=senvocab.end(); ++senv_iitr){
	WORDTYPE v=(*senv_iitr).first;
      
	double curbest=ngram->trigram(u, v, -2);

	for (senv_l_itr=senvocab_l.begin(); senv_l_itr!=senvocab_l.end(); ++senv_l_itr){
	  WORDTYPE w=(*senv_l_itr).first;
	  double curtri;

	  if (!w)
	    continue;
	
	  curtri=ngram->trigram(u, v, w);

	  if (curtri>curbest)
	    curbest=curtri;
	}//end of iter over w

	dchart->bestborder[u][v].right=curbest*LMWEIGHT;

	//      printf("bestborder[%d][%d].right: %.10e\n", u, v, curbest*LMWEIGHT);

      }//end of iter over v

    }//end of iter over u

  }

}

/** CYKDecoder::decode()
 *
 *  Translate a sentence and return the result as a char*.
 */
string CYKDecoder::decode() {
  if (Globals::Get<int>("use_agenda")) {
    if (Globals::Get<int>("two_pass")){
      ngram->N=2;
      this->dinit(true);
      if (Globals::Get<int>("cyk_first_pass")){

	if (Globals::Get<int>("cyk_hooks")){
	  this->decode_cyk_hook_bipass();
	}
	else
	  this->decode_cyk_bipass();
      }
      else if (Globals::Get<int>("use_hooks")){
      	this->decode_hook_agenda_bipass();
      }
      else{
	this->compute_estimate(true);
	this->decode_agenda_bitri(true);
      }
      ngram->N=3;
      this->dinit(false);
      this->compute_estimate(false);
      return this->decode_agenda_bitri(false);
    }
    else if (Globals::Get<int>("use_hooks")){
      ngram->N=2;
      this->dinit(true);
      return this->decode_hook_agenda(true);
    }
    else{
      this->dinit(true);
      return this->decode_agenda();
    }
  }
  else
    if (Globals::Get<int>("lazy_kbest")){
      this->dinit(true);
      return this->topdowndecode();
    }
    else{
      this->dinit(true);
      return this->decode_cyk();
    }
}

/** CYKDecoder::decode_cyk()
 *
 *  Decoding, bottom-up style CYK algorithm.
 */
string CYKDecoder::decode_cyk() {

  cyk_bottom_up();

  /*  recurse to reconstruct it */
  ITGDCell* top = dchart->cell(grammar->_nonterminal_cnt, 0, T, 0, 0, 0, 0); 

  fprintf(logfile, "arcs popped %d\n", itemcount);
  fprintf(logfile, "log prob %.20lf\n", top->prob);

  if (top->bkptrs.size()) {
    if (G_INT("verbosity") >= 1)
      cout << build_leaves(top, true) << endl;

    return reconstruct(top);
  }

  return "<NONE>";
}

void CYKDecoder::cyk_bottom_up() {
  double LOGRATIOBEAM_PER_NTERM;
  if(Globals::Has_key("logratiobeam_per_nterm")) {
    LOGRATIOBEAM_PER_NTERM = Globals::Get<double>("logratiobeam_per_nterm");
  } else {    
    LOGRATIOBEAM_PER_NTERM = Globals::Get<double>("logratiobeam");
  }
  edgecount = 0;
  if (Globals::Get<int>("verbosity") >= 1) {
    cout << "* running CYK bottom up...";
    fflush(stdout);
  }
  string prune_method = Globals::Get<string>("prune_method");

  hash_map <LMSTATE, ITGDCell> *lcol, *rcol;
  hash_map <LMSTATE, ITGDCell>::iterator litr, ritr;

  // stext holds the dp result for the extended span (s, t) - with insertions
  // stext is indexed by stage, nonterm, lm state
  hash_map<int, hash_map <LMSTATE, ITGDCell> > stext[3];

  /* There are three stages: 
   *
   * - stage 1 handles the normal case when two adjacent nonzero
   *   blocks are combined
   * - stage 2 handles the case when the left block is an insertion (0
   *   span in source lang) 
   * - stage 3 handles the case when the right block is an insertion
   *
   * The reason Hao broke this into three stages is because insertions
   * create a new block over the exact same span -- in other words,
   * you update what you're looking at. Thus a separate record has to
   * be kept of it, since you can't alter the data structure you're
   * enumerating.  After all three stages have been gone through, a
   * single pass over everything is made in order to handle the
   * pruning.
   */

  itemcount = 0;
  hookitemcount = 0;

  for (int span = 1; span <= T; span++) {
    
    // cerr << "span: " << span << endl;

    for (int s = 0; s <= T - span; s++) {
      int t = s + span;

      // keep track of max item created over span; allows some pruning at creation
      // time rather than just as an afterstep
      double maxscore = -INFINITY;

      hash_map<int, double> nterm_maxsita;

      //dp for (s, t)
      //for binary rules
      for (int stage = 0; stage < 3; stage++) {

        // clear the map for each nonterminal for the current stage
        stext[stage].clear();//prepare for holding new results

        // stage 0: deltas = 1..span-1 (combine nonzero blocks)
        // stage 1: deltas = 0 (insertion on left side)
        // stage 2: deltas = 0 (insertion on right side)
        int from = 0, to = 0;
        switch(stage) {
        case 0: // combine two nonempty blocks
          from = 1;
          to = span;
          break;
        case 1: // insertion on left side
          from = 0;
          to = 1;
          break;
        case 2: // insertion on right side
          from = 0;
          to = 1;
          break;
        }

        for (int deltas = from; deltas < to; deltas++) {
          int S;
          if (stage == 0) {
            S = s + deltas; 
          }
          else if (stage == 1) {
            S = s; 
            // right column same as
            // whole span, so must
            // update parallel data
            // structure instead
          } else { //stage==2
            S = t; 
            // left column same as
            // whole span, so must
            // update parallel data
            // structure instead
          }
          if (Globals::Get<int>("debuglevel") >= 2) {
            fprintf(logfile, "stage = %d, span = %d-%d-%d\n", stage, s, S, t);
          }
          if (Globals::Get<int>("verbosity") >= 3) {
            printf("SPAN: (%d,%d)  split=%d  stage=%d\n", s, t, S, stage);
            fflush(stdout);
          }
	  
          hash_map<int, hash_map <LMSTATE, ITGDCell> > *lspan, *rspan;
          hash_map<int, hash_map <LMSTATE, ITGDCell> >::iterator lnt_itr, rnt_itr;
          lspan = dchart->span(s, S);
          rspan = dchart->span(S, t);

          // see which nonterm pairs exist in chart
          for (lnt_itr = lspan->begin(); lnt_itr != lspan->end(); lnt_itr++) {
	    int nl = lnt_itr->first;
            for (rnt_itr = rspan->begin(); rnt_itr != rspan->end(); rnt_itr++) {
	      
              int nr = rnt_itr->first;

              // lookahead, avoid enumerating one column while the other is empty
              if (!dchart->has_column(nl, s, S) || !dchart->has_column(nr, S, t))
                continue;
	      
              if (!grammar->rule_table[nl].count(nr))
                continue;

              lcol = dchart->column(nl, s, S);
              rcol = dchart->column(nr, S, t);

	      // cerr << "n: " << nl << "\t" << nr << endl;
	      
              // for each binary rule having specified children
              lhs_inv_erhs_rule_hash *rnt = &grammar->rule_table[nl][nr];
	      
              lhs_inv_erhs_rule_hash::iterator lhs_itr;
              inv_erhs_rule_hash::iterator inv_itr;
              erhs_rule_hash::iterator e_rhs_itr;

              for (lhs_itr = rnt->begin(); lhs_itr != rnt->end(); lhs_itr++)
                for (inv_itr = lhs_itr->second.begin(); inv_itr != lhs_itr->second.end(); inv_itr++)
                  for (e_rhs_itr = inv_itr->second.begin(); e_rhs_itr != inv_itr->second.end(); e_rhs_itr++) {
                    Rule* rule = e_rhs_itr->second;

                    int n = rule->lhs_int;

                    if (Globals::Get<int>("verbosity") >= 3) {
                      printf(" rule %s  ", rule->str().c_str());
                      if (rule->e_rhs)
                        fprint_array(stdout, rule->e_rhs);
                      printf("\n");
                    }

                    for (litr = lcol->begin(); litr != lcol->end(); litr++) {
                      int ul[2], vl[2];

                      ITGDCell* lcell = &litr->second;

                      // the frontiers of the left constituent
                      ul[0] = dchart->u0of(litr->first);
                      ul[1] = dchart->u1of(litr->first);
                      vl[0] = dchart->v0of(litr->first);
                      vl[1] = dchart->v1of(litr->first);
		      
                      double sita3 = lcell->prob;
		      
                      for (ritr = rcol->begin(); ritr != rcol->end(); ritr++){
                        int ur[2], vr[2], uchoice[2], vchoice[2];

                        ITGDCell* rcell = &ritr->second;

                        // the frontiers of the right constituent
                        ur[0] = dchart->u0of(ritr->first);
                        ur[1] = dchart->u1of(ritr->first);
                        vr[0] = dchart->v0of(ritr->first);
                        vr[1] = dchart->v1of(ritr->first);
			
                        double sita4 = rcell->prob;
			
                        double sita34 = sita3 + sita4; //log +
			
                        double lm_prob = 0.0;
			
                        uchoice[0] = uchoice[1] = 0;
                        vchoice[0] = vchoice[1] = 0;

                        if (rule->inverted) {
                          lm_prob = compute_lm(ur, ul, vr, vl, rule, uchoice, vchoice);
                        } else {
                          lm_prob = compute_lm(ul, ur, vl, vr, rule, uchoice, vchoice);
                        }

                        // create new cell
                        ITGDCell new_cell(n, s, t, uchoice[0], uchoice[1], vchoice[1], vchoice[0]);
                        
                        double newsita = 0.0;
                        if (weights.loaded) { // discriminative
                          new_cell.features = feature_merge(&new_cell, lcell, rcell, rule, lm_prob);
                          newsita = new_cell.features * weights;
                        } else {
                          newsita = rule->prob + sita34 + lm_prob * LMWEIGHT;
                        }

                        // this saves a ton of memory because we don't
                        // waste space populating stext with entries
                        // that are going to get pruned anyway. it can
                        // be the difference between out-of-memory
                        // errors and successful runs

			if ((((nterm_maxsita[rule->lhs_int] == 0) || 
			      (nterm_maxsita[rule->lhs_int] - LOGRATIOBEAM_PER_NTERM) < newsita))
			    && (newsita >= maxscore - LOGRATIOBEAM)) {

			  if ((nterm_maxsita[rule->lhs_int] == 0) || (nterm_maxsita[rule->lhs_int] < newsita))
			    nterm_maxsita[rule->lhs_int] = newsita;

                          // update new high score for span
                          maxscore = max(newsita, maxscore);

                          ITGDCell* pd = &(stext[stage][n][dchart->u01v01(uchoice[0], uchoice[1], vchoice[1], vchoice[0])]);
                          double oldsita = pd->prob;
                          itemcount++;
                          //                         pd->backptrs.insert(backpointer(rule, lcell, rcell, newsita));
			  
                          //add the backpointer
                          backpointer thebkptr(rule, lcell, rcell, newsita);
			  thebkptr.tri =  (ngram->N==3);
                          pd->bkptrs.push_back(thebkptr);
			  
                          if (newsita > oldsita) { //maximizing
                            pd->N = rule->lhs_int;
                            pd->s = s;
                            pd->t = t;
                            DUEXASSIGN(pd->lword, uchoice);
                            DUEXASSIGN(pd->rword, vchoice);
                            pd->prob = newsita;
                            // best backpointers
                            pd->lchild = lcell;
                            pd->rchild = rcell;
                            pd->rule = rule;

                            pd->features = new_cell.features;

                            if (Globals::Get<int>("debuglevel") >= 3) {
                              fprintf(logfile, "chart ");
                              fprintf(logfile, " %d %d %s[%s %s ... %s %s]  %g   lm: %g\n",
                                      s, t, rule->lhs.data(), 
                                      l1vocab[uchoice[0]].data(), l1vocab[uchoice[1]].data(), 
                                      l1vocab[vchoice[1]].data(), l1vocab[vchoice[0]].data(),
                                      newsita, lm_prob * LMWEIGHT);
                            }
                          }
                        }
                      }//end of rcol
                    }//end of lcol
                  }//end for all binary rules 
            } // end for right child nonterminal
          } // end for left child nonerminal
	  
        }//end for deltas, splitting S

	hash_map<int, hash_map <LMSTATE, ITGDCell> >::iterator nt_itr;

        // copy from stext to chart
        for (nt_itr = stext[stage].begin(); nt_itr != stext[stage].end(); nt_itr++){
          int n = nt_itr->first;
          lcol = &(nt_itr->second);

          for (litr = lcol->begin(); litr != lcol->end(); litr++){
            LMSTATE index = litr->first;
            itemcount++;
            if ( ((*dchart->column(n, s, t))[index].prob == 0.0) 
                 || ((*dchart->column(n, s, t))[index].prob < litr->second.prob) ) {
              (*dchart->column(n, s, t))[index] = litr->second;
            }
          }
        }
      }	// end for stage

      hash_map<int, hash_map <LMSTATE, ITGDCell> > *span = dchart->span(s, t);
      hash_map<int, hash_map <LMSTATE, ITGDCell> >::iterator nt_itr;
      
      double maxsita = -INFINITY;
      // with-ratio pruning
      for (nt_itr = span->begin(); nt_itr != span->end(); nt_itr++) {
        lcol = &(*nt_itr).second;
        for (litr = lcol->begin(); litr != lcol->end(); litr++){
          double thesita = litr->second.prob;
          if (thesita < 0 && thesita > maxsita) {
            maxsita = thesita;
          }
        }
      }

      if (Globals::Get<int>("verbosity") >= 3) {
	int entries = 0;
	for (nt_itr = span->begin(); nt_itr != span->end(); nt_itr++){
	  lcol = &(nt_itr->second);
	  entries += lcol->size();
	}
	cout << "* Entries over (" << s << "," << t <<
	  ") BEFORE PRUNING: " << entries << "\n";
      }


      hash_map <LMSTATE, ITGDCell>::iterator bklitr;
      if (prune_method == "topk") {
	for (nt_itr = span->begin(); nt_itr != span->end(); nt_itr++) {
          int N = nt_itr->first;
          lcol = &(nt_itr->second);

          // preparing for topk pruning
          priority_queue<double> topk_pq_1;
          priority_queue<double> topk_pq_2;
          int count1 = 0;
          int count2 = 0;
          double mymax = -INFINITY;
          if (prune_method == "topk") {
            for (litr = lcol->begin(); litr != lcol->end();litr++){
              double thesita = litr->second.prob;
              ++count1;
              if (thesita > mymax) {
                mymax = thesita;
              }
              topk_pq_1.push(-thesita);
              if (topk_pq_1.size() > Globals::Get<double>("prune_topk_other_nonterm")) {
                topk_pq_1.pop();
              }
              topk_pq_2.push(-thesita);
              if (topk_pq_2.size() > Globals::Get<double>("prune_topk_universal_nonterm")) {
                topk_pq_2.pop();
              }
            }
          }
         
          for (litr = lcol->begin(); litr != lcol->end();){
            double thesita = litr->second.prob;
            if (N != 3) {
              if (-thesita > topk_pq_1.top()) {
                bklitr=litr;
                litr++;
                lcol->erase(bklitr);
                ++count2;
              } else {
                litr++;
              }
            } else {
              if (-thesita > topk_pq_2.top()) {
                bklitr=litr;
                litr++;
                lcol->erase(bklitr);
                ++count2;
              } else {
                litr++;
              }
            }
          }
	}
      } else {
	    // ratio
  	    // use the highest prob to prune everything that falls outside the ratio beam
	    // logratio beam pruning

            for (nt_itr = span->begin(); nt_itr != span->end(); nt_itr++) {
              lcol = &(nt_itr->second);
  	      int nterm = nt_itr->first;

	      hash_map <LMSTATE, ITGDCell>::iterator bklitr;
	      for (litr = lcol->begin(); litr != lcol->end();){
	        double thesita = litr->second.prob;
	        if ((thesita < (maxsita - LOGRATIOBEAM))
		    || ((nterm_maxsita[nterm] != 0)
		        && ((thesita < nterm_maxsita[nterm] - LOGRATIOBEAM_PER_NTERM)))) { //log +
	          bklitr=litr;
	          litr++;
	          lcol->erase(bklitr);
	        } else {
	          litr++;
	        }
	      }
            }
      }

      if (Globals::Get<int>("verbosity") >= 3) {
	int entries = 0;
	for (nt_itr = span->begin(); nt_itr != span->end(); nt_itr++){
	  lcol = &(nt_itr->second);
	  entries += lcol->size();
	}
	cout << "* Entries over (" << s << "," << t <<
	  ") AFTER PRUNING: " << entries << "\n";
      }


      // unary rules
      // (no split point)

      // for unary rules, there is a order related issue.
      // We can not control the order because the rules are stored in a hash table, 
      // even worse, it will be altered inside the loop
      // So, we repeately apply the unary rules until no more changes.
      // Practically, we stop after 99 iterations to avoid infinite loop.
      // Another option is to preprocess the grammar to get a topological order.
      // -Zhao

      int LOOP_COUNT = 99;
      bool changed = true;
      for (int loop_count = 0; loop_count <= LOOP_COUNT; ++loop_count) {
	if (loop_count >= LOOP_COUNT) {
	  cerr << "WARNING: Your grammar may have loop for unary rules!" << endl;
	  break;
	}
	if (!changed) {
	  break;
	}
	changed = false;
	// cerr << endl << "loop... " << endl;

	for (nt_itr = span->begin(); nt_itr != span->end(); nt_itr++) {
	  int rhs = nt_itr->first;
	  lhs_inv_erhs_rule_hash *rnt = &grammar->rule_table[rhs][-1];
	  
	  lhs_inv_erhs_rule_hash::iterator lhs_itr;
	  inv_erhs_rule_hash::iterator inv_itr;
	  erhs_rule_hash::iterator e_rhs_itr;
	  
	  for (lhs_itr = rnt->begin(); lhs_itr != rnt->end(); lhs_itr++)
	    for (inv_itr = lhs_itr->second.begin(); inv_itr != lhs_itr->second.end(); inv_itr++)
	      for (e_rhs_itr = inv_itr->second.begin(); e_rhs_itr != inv_itr->second.end(); e_rhs_itr++) {
		Rule* rule = e_rhs_itr->second;
		
		int lhs = rule->lhs_int;
		
		//filtering non-root S-es except at the end
		if ((lhs == grammar->start_symbol) && (s > 0 || t < T))
		  break;
		
		if (!dchart->has_column(rhs, s, t))
		  continue;
		
		lcol = dchart->column(rhs, s, t);
		for (litr = lcol->begin(); litr != lcol->end(); litr++){
		  int u[2], v[2], uchoice[2], vchoice[2];
		  
		  ITGDCell* child = &litr->second;
		  
		  u[0] = dchart->u0of(litr->first);
		  u[1] = dchart->u1of(litr->first);
		  v[0] = dchart->v0of(litr->first);
		  v[1] = dchart->v1of(litr->first);
		  
		  double lm_prob;
		  uchoice[0] = uchoice[1] = 0;
		  vchoice[0] = vchoice[1] = 0;
		  
		  lm_prob = compute_lm(u, NULL, v, NULL, rule, uchoice, vchoice);

                  //		  if ((((nterm_maxsita[rule->lhs_int] - LOGRATIOBEAM_PER_NTERM) < newsita)
                  //		       || (nterm_maxsita[rule->lhs_int] == 0))
                  //		      && (newsita >= maxscore - LOGRATIOBEAM)) {
                  {

                    ITGDCell new_cell(lhs,s,t,uchoice[0], uchoice[1], vchoice[1], vchoice[0]);

                    double newsita = 0.0;
                    if (weights.loaded) { // discriminative
                      new_cell.features = feature_merge(&new_cell,child,NULL,rule,lm_prob);
                      newsita = new_cell.features * weights;
                    } else {
                      newsita = rule->prob + lm_prob * LMWEIGHT + child->prob;
                    }

                    // uchoice and vchoice include any target language terminals

		    if ((nterm_maxsita[rule->lhs_int] < newsita) || (nterm_maxsita[rule->lhs_int] == 0))
		      nterm_maxsita[rule->lhs_int] = newsita;
		   
		    maxscore = max(newsita, maxscore);

		    // uchoice and vchoice include any target language terminals
		    // that were added by the unary rule.
		    ITGDCell* pd = dchart->cell(lhs, s, t, uchoice[0], uchoice[1], vchoice[1], vchoice[0]);

                    double oldsita = pd->prob;
                    itemcount++;
		  
                    // 		  pd->backptrs.insert(backpointer(rule, child, NULL, newsita));
		  
		    // add the backpointer
		    backpointer thebkptr(rule, child, NULL, newsita);
		    thebkptr.tri =  (ngram->N==3);
		    pd->bkptrs.push_back(thebkptr);
	  
		  if (newsita > oldsita){ //maximizing
		    // cerr << lhs << "  ->  " << rhs << "\t";
		    // cerr << "(" << newsita << " > " << oldsita << ")" << endl;
		    changed = true;

		    pd->N = rule->lhs_int;
		    pd->s = s;
		    pd->t = t;
		    DUEXASSIGN(pd->lword, uchoice);
		    DUEXASSIGN(pd->rword, vchoice);
		    pd->prob = newsita;

		    // best backpointers
		    pd->lchild = child;
		    pd->rchild = NULL;
		    pd->rule = rule;

                    pd->features = new_cell.features;

		      if (Globals::Get<int>("debuglevel") >= 3) {
			fprintf(logfile, "u chart ");
			fprintf(logfile, " %d %d %d[%s %s ... %s %s]  %g   \n", s, t, lhs, 
				l1vocab[uchoice[0]].data(), l1vocab[uchoice[1]].data(), 
				l1vocab[vchoice[1]].data(), l1vocab[vchoice[0]].data(), newsita);
		      }
		    }//end of updating viterbi
		  } // end of beam
		}//end for lcol   
	      }//end for unary rules with specified rhs
	}//end of for nt_itr over rhs
      } // loop until no change

      for (nt_itr = span->begin(); nt_itr != span->end(); nt_itr++) {
	int N = nt_itr->first;
	// speed up for generalized ITG grammar
	// do not create empty entries in the hash table, that wastes a huge mount of time!
	if (dchart->has_column(N,s,t)) {
	  edgecount += dchart->column(N,s,t)->size();
	}
      }

      /* dump chart */
      if (Globals::Get<int>("debug_dump_charts") >= 2)
        dchart->dump(_sentno, s, t);

    }//end for s, t
  }//end for span

  /* find the best cell */
  /* 1. find the best entry that covers the entire span */
  lcol = dchart->column(grammar->start_symbol, 0, T);
  for (litr = lcol->begin(); litr != lcol->end(); litr++){
    ITGDCell* cell = &litr->second;

    int u[2], v[2];
    u[0] = dchart->u0of(litr->first);
    u[1] = dchart->u1of(litr->first);
    v[0] = dchart->v0of(litr->first);
    v[1] = dchart->v1of(litr->first);

    //  if (n>0)
    //    continue;//only looking at the "S"
    if ( (u[0] > 0 && v[0] > 0) || ((u[0] + v[0]) == 0)) {
      double lm_prob = 0.0;
      if (ngram->N == 2) {
        lm_prob = (ngram->bigram(-1, u[0]) + ngram->bigram(v[0], -2));
      } else { // trigram
        lm_prob = (ngram->trigram(-1, u[0], u[1]) + ngram->trigram(v[1], v[0], -2));
        lm_prob += (ngram->bigram(-1, u[0])); //adding the leading bigram probability
      }

      ITGDCell new_cell(grammar->start_symbol, 0, T, -1,
                        ngram->N == 2 ? 0 : cell->lword[0],
                        ngram->N == 2 ? 0 : cell->rword[0], -2);
                        

      double prob = 0.0;
      if (weights.loaded) { // discriminative
        new_cell.features = feature_merge(&new_cell,cell,NULL,NULL,lm_prob);
        prob = new_cell.features * weights;
      } else {
        prob = cell->prob + lm_prob * LMWEIGHT;
      }

      ITGDCell* top = dchart->cell(grammar->_nonterminal_cnt, 0, T, 0, 0, 0, 0); 

      top->bkptrs.push_back(backpointer(NULL, cell, NULL, prob));
//       top->backptrs.insert(backpointer(NULL, cell, NULL, prob));

      if (prob > top->prob) {
        top->N = grammar->start_symbol;
        top->s = 0;
        top->t = T;
        top->prob = prob;
        top->lword[0] = -1;
        if (ngram->N == 2) {
          top->lword[1] = 0;
          top->rword[1] = 0;
        } else {
          top->lword[1] = cell->lword[0];
          top->rword[1] = cell->rword[0];
        }
        top->rword[0] = -2;

        // cache best backpointers
        top->rule = NULL;
        top->lchild = cell;
        top->rchild = NULL;

        top->features = new_cell.features;
      }
    }
  }

  //  fprintf(logfile, "arcs popped %d\n", itemcount);

  if (Globals::Get<int>("verbosity") == 1) {
    cout << "produced forest of " << edgecount << " edges\n";
    fflush(stdout);
  }
}

struct cell_lt {
  bool operator()(const ITGDCell* a, const ITGDCell* b) {
    return (a->t - a->s) < (b->t - b->s);
  }
};

// NOTE: correctness not guaranteed when insertions are allowed
// (since no topological sort is done to straighten out the resulting unary chains)
void CYKDecoder::cyk_top_down() {
  edgecount = 0; // number of outside probs computed

  string msg = "* running CYK top down...";
  if (Globals::Get<int>("verbosity") >= 1) {
    cout << msg;
    fflush(stdout);
  }

  priority_queue<ITGDCell*,vector<ITGDCell*>,cell_lt> cells;

//   hash_map <LMSTATE, ITGDCell>* col = dchart->column(grammar->start_symbol, 0, T);
//   hash_map <LMSTATE, ITGDCell>::iterator itr;
//   for (itr = col->begin(); itr != col->end(); itr++) {
//     ITGDCell* pcell = &itr->second;
//     if (Globals::Get<int>("debuglevel") >= 3)
//       cout << "* Adding " << *pcell << " to the priority queue\n";
//     cells.push(pcell);
//   }

  ITGDCell* top = dchart->cell(grammar->_nonterminal_cnt, 0, T, 0, 0, 0, 0);
  top->outside = 0.0;
  cells.push(top);

  ITGDCell* parent;
  while (! cells.empty()) {
    parent = cells.top();
    cells.pop();

    if (parent->closed == 2)
      continue;

    parent->closed = 2;
    edgecount++;

    if (Globals::Get<int>("debuglevel") >= 3) {
//       cout << parent->N << endl;
//       cout << parent->s << endl;
//       cout << parent->t << endl;
//       cout << parent->lword[0] << endl;
//       cout << parent->rword[0] << endl;
//       cout << (void*)parent->rule << endl;
//       fflush(stdout);
      cout << "POPPED: " << *parent << " at address " << (void*)parent << endl;
      fflush(stdout);
    }

    // for all the back pointers
    bkptr_v::iterator bit;
    for (bit = parent->bkptrs.begin(); bit != parent->bkptrs.end(); bit++) {

      ITGDCell* lchild = bit->lchild;
      ITGDCell* rchild = bit->rchild;

      Rule* rule = bit->rule;

      // compute the outside probability of all (0, 1, or 2) children
      if (lchild && rchild) {
        int uchoice[2];  uchoice[0] = uchoice[1] = 0;
        int vchoice[2];  vchoice[0] = vchoice[1] = 0;

        int ul[2];  DUEXASSIGN(ul, lchild->lword);
        int vl[2];  DUEXASSIGN(vl, lchild->rword);
        int ur[2];  DUEXASSIGN(ur, rchild->lword);
        int vr[2];  DUEXASSIGN(vr, rchild->rword);

        double lm_prob;
        if (rule->inverted)
          lm_prob = LMWEIGHT * compute_lm(ur, ul, vr, vl, rule, uchoice, vchoice);
        else
          lm_prob = LMWEIGHT * compute_lm(ul, ur, vl, vr, rule, uchoice, vchoice);

        // outside prob for left child
        double outside = lm_prob + rule->prob + rchild->prob + parent->outside;
        lchild->outside = max(lchild->outside, outside);
        if (lchild->closed != 2)  cells.push(lchild);

        // right child
        outside = lm_prob + rule->prob + lchild->prob + parent->outside;
        rchild->outside = max(rchild->outside, outside);
        if (rchild->closed != 2)  cells.push(rchild);

      } else if (lchild && !rule) { // TOP-level unary rule
        double lm_prob = LMWEIGHT * (ngram->bigram(-1, lchild->lword[0]) + ngram->bigram(lchild->rword[0], -2));
        double outside = lm_prob;
        lchild->outside = max(lchild->outside, outside);
        if (lchild->closed != 2)  cells.push(lchild);
      } else if (lchild) { // ordinary unary rule
        double outside = rule->prob + parent->outside;
        lchild->outside = max(lchild->outside, outside);
        if (lchild->closed != 2)  cells.push(lchild);
      }
    }
  }        

  if (Globals::Get<int>("verbosity") >= 1) {
    cout << "processed " << edgecount << " edges\n";
    fflush(stdout);
  }

  if (Globals::Get<int>("debug_dump_charts") >= 2)
    dchart->dump(_sentno);
}


int CYKDecoder::readin(ITGCorp* corp) {
  if (corp->retrieve(&cn, &T, MAXT)) {
    init_phrases(); // loads phrasal translations
    return 1;
  }
  return 0;
}

void CYKDecoder::init_phrases() {
  list<WORDTYPE*> kbest;

  //0: initialize the translation candidates for the gaps between
  //the words in the target language (insertions)

  if (Globals::Get<bool>("allow_insertions")) {
    findkbest(grammar->nullphr, MAXINS, kbest);
  }
  for (int t = 0; t <= T; t++)
    fillin(t, 0, kbest);
      
  //1: initialize the translation candidates for all the phrases
  //seen in the source sentence 

  for (int t = 0; t < T; t++){
    for (int lspan = 1; lspan <= T - t; lspan++) {
      WORDTYPE *phrase = splice(cn, t, t + lspan);
      int numfound = findkbest(phrase, MAXTRS, kbest);

      //           printf("** [%d,%d] Found %d translations\n", t, t+lspan, numfound);
      // if no translation candidates were found (including deletion, which
      // is modeled as a translation candidate), then at least allow the word
      // to get deleted.
      if (1 == lspan && 0 == numfound) {
        fprintf(stderr, "* WARNING: no candidates found for sentence %d/word %d (%d)\n",
                _sentno + 1, t + 1, cn[t]);

      }
      fillin(t, lspan, kbest);
      delete[] phrase;
    }
  }  
}

void CYKDecoder::readin(WORDTYPE* icn, int iT, int iMAXT) {
  list<WORDTYPE*> kbest;

  int i = 0;
  for (i = 0; *icn; icn++, i++)
    cn[i] = *icn;
  cn[i] = 0;
  T = iT;
  MAXT = iMAXT;

  init_phrases();
}

/* reconstructs the best parse top-down */
string CYKDecoder::reconstruct(ITGDCell *cell, int indent) {
  cout << setw(indent) << " " << *cell << endl;

  /* leaves */
  if (cell->lchild == NULL && cell->rchild == NULL)
    return transliterate(cell->ephr, &l1vocab) + " ";

  /* top-level parent */
  if (cell->rule == NULL) {
    return reconstruct(cell->lchild, indent + 2);
  }
  
  /* unary rules */
  if (cell->lchild && cell->rchild) {
    Rule* rule = cell->rule;
    int inv = rule->inverted;

    string lstr = reconstruct(cell->lchild, indent + 2);
    string rstr = reconstruct(cell->rchild, indent + 2);

    string combined;

    WORDTYPE *p = NULL;
    if (rule->e_rhs)
      for (p = rule->e_rhs; *p > 0; p++)	// terminals in rule
        combined += l1vocab[*p] + " ";

    combined += (inv ? rstr : lstr );		// child chart item

    if (rule->e_rhs)
      for (p++; *p > 0; p++) {
        combined += l1vocab[*p] + " ";
      }

    combined += (inv ? lstr : rstr );

    if (rule->e_rhs)
      for (p++; *p > 0; p++) 
        combined += l1vocab[*p] + " ";

    return combined;

  } else { /* unary rules */
    assert(cell->lchild);
    Rule* rule = cell->rule;
    ITGDCell* child = cell->lchild;

    string combined;
    WORDTYPE *p = NULL;
    if (rule->e_rhs)
      for (p = rule->e_rhs; *p > 0; p++)		// terminals in rule
        combined += l1vocab[*p] + " ";

    combined += reconstruct(child, indent + 2);	// child chart item
    
    if (rule->e_rhs)
      for (p++; *p > 0; p++) 
        combined += l1vocab[*p] + " ";
    
    return combined;
  }
}

string CYKDecoder::build_leaves(const ITGDCell* cell, bool delim) {
  stringstream ss;

  if (cell->lchild == NULL && cell->rchild == NULL) {
    if (delim) ss << "(";
    ss << transliterate(cell->ephr, &l1vocab);
    if (delim) ss << ")";
  } else {
    if (cell->lchild)
      ss << build_leaves(cell->lchild, delim);
    if (cell->rchild)
      ss << " " << build_leaves(cell->rchild, delim);
  }

  return ss.str();
}

string outfile(string root) {
  string file = root + "." + Globals::Get<string>("runid");
  int from = Globals::Get<int>("sentence_from");
  int to = Globals::Get<int>("sentence_to");

  if (from != 0 || to != 0)
    file += "." + Xtos(from) + "-" + Xtos(to);

  return file;
}

void CYKDecoder::run() {
  if (Globals::Get<int>("verbosity") >= 1)
    print_header("Translating with CYK decoder");

  int sent1 = Globals::Get<int>("sentence_from");
  int sentN = Globals::Get<int>("sentence_to");

  ofstream out;
  out.open(outfile("out").data(), ios::out);
  if (!out.is_open()) {
    cerr << "* FATAL: couldn't write to file '" << outfile("out") << "'\n";
    exit(1);
  }

  while (readin(corpus)) {
    _sentno++;

    if (sent1 && _sentno < sent1)
      continue;
    if (sentN && _sentno > sentN) {
      cout << endl;
      print_header("");
      cout << "Quitting after processing sentences " << sent1 << " to " << sentN << endl;
      print_header("");
      break;
    }

    if (G_INT("debug_dump_charts") >= 1) {
      dumpdir = G_STRING("rundir") + "/debug/" + Xtos(_sentno);
      int status = mkdir(dumpdir.data(), S_IRWXU | S_IRWXG | S_IROTH | S_IXOTH);
      if (status == 0)
        cout << "* created debug directory " << dumpdir << endl;
    }

    if (Globals::Get<int>("verbosity") >= 1)
      cout << "[" << _sentno << "] Translating '" << cn << "' (len = " << phraselen(cn) << ")\n";

    time_t start_time = time(NULL);
    //    dinit(); //initialization of the decoding mode chart

    string result = decode();

    cout << result << endl;
    out << result << endl;

    fprintf(corpus->ef, "%s\n", result.data());
    fflush(corpus->ef); 

    fprintf(logfile, "pair %d: <%d>\t %ld Secs\n", _sentno, T, 
            time(NULL) - start_time);
    fflush(logfile);

    //     printf("\rFinished translating sentence %d", _sentno);
    //     fflush(stdout);
  } // end of sentence loop

  out.close();

  cout << endl;
} // end of decoding mode


void ITGDChart::dump(int sentno, int s, int t) {
  for (int N = 0; N < grammar->_nonterminal_cnt; N++) {
    dumpdir = G_STRING("rundir") + "/debug/" + Xtos(sentno);
    string filename = "chart." + grammar->nonterminals[N] + "." + Xtos(s) + "-" + Xtos(t);
    string path = dumpdir + "/" + filename;

    ofstream out(path.data());

    hash_map<LMSTATE, ITGDCell>* col = column(N, s, t);

    for (hash_map<LMSTATE, ITGDCell>::iterator it = col->begin(); it != col->end(); it++) {
      ITGDCell& cell = it->second;
      out << cell << endl;
    }
    out.close();
  }
}

void ITGDChart::dump(int sentno) {
  string filename = "debug.chart." + Xtos(sentno);
  string path = Globals::Get<string>("rundir") + "/" + filename;

  ofstream out(path.data());

  for (int span = 1; span <= T; span++) {
    for (int s = 0; s <= T - span; s++) {
      int t = s + span;
      for (int N = 0; N < grammar->_nonterminal_cnt; N++) {
        hash_map<LMSTATE, ITGDCell>* col = column(N, s, t);
        for (hash_map<LMSTATE, ITGDCell>::iterator it = col->begin(); it != col->end(); it++) {
          ITGDCell& cell = it->second;
          out << cell << endl;
        }
      }
    }
  }

  out.close();
}
