/* itgparse.cpp
 *
 * Contains implementation for viterbi parsing.
 *
 * $Id: itgparse.cpp,v 1.2 2011/10/16 21:42:44 gildea Exp $
 */

#include "itgparse.h"
#include <sys/time.h>

namespace viterbi {

  using namespace std;

  char* get_current_dir_name() {
    char* buf = NULL;
    return getcwd(buf,0);
  }

  Parser::Parser(ITGGrammar* ig, ITGCorp* ic) : ITG(ig, ic)
  {
    vchart = new ITGVChart(MAXT, MAXV);

    viterbi_file = runid + ".viterbi";
    afile_file = runid + ".afile";
    info_file = runid + ".info";

    pfviterbi = fopen(viterbi_file.data(), "w");
    if (NULL == pfviterbi) {
      cerr << "FATAL ERROR: can't open viterbi file '" << viterbi_file << endl;
      exit(1);
    } else {
      char* dir = get_current_dir_name();
      cout << "Will write viterbi parses to '" << dir << "/" << viterbi_file << endl;
      free(dir);
    }
    /*
    // for writing the .afile file
    pfafile = fopen(afile_file.data(), "w");
    if (NULL == pfafile) {
      cerr << "FATAL ERROR: can't open alignment file '" << afile_file << "'\n";
      exit(1);
    } else {
      char* dir = get_current_dir_name();
      cout << "Will write viterbi alignments to '" << dir << "/" << afile_file << endl;
      delete[] dir;
    }

    // for writing the .info file
    pfinfo = fopen(info_file.data(), "w");
    if (NULL == pfinfo) {
      cerr << "FATAL ERROR: can't open info file '" << info_file << "'\n";
      exit(1);
    } else {
      char* dir = get_current_dir_name();
      cout << "Will write viterbi parse information to '" << dir << "/" << info_file << "'\n";
      delete[] dir;
    }
    */
    en = new WORDTYPE[MAXT];
    cn = new WORDTYPE[MAXV];

    // CLONING
    CLONING = Globals::Get<bool>("cloning");
    if (CLONING)  {
      clonechart = new ITGViterbiCloneChart(MAXT,MAXV);
    }
    viterbi_clone_cnt = 0; //used to count num clones used in viterbi parses
    viterbi_phrase_count = 0;
  }


  Parser::~Parser()
  {
    delete vchart;
    fclose(pfviterbi);
    /*fclose(pfafile);
      fclose(pfinfo);*/

    delete[] en;
    delete[] cn;
  }

  /** Parser::vinit()
   *
   *  Initializes the viterbi chart for phrase-based Viterbi parsing.
   *  This requires an O(n^4) pass over the sentence pair, instead of
   *  the O(n^2) require for word-based initialization.
   */
  void Parser::vinit() {
    pi = 0.0;   // reset the sentence probability
    _sentno++;  // counts which sentence we're on

    vchart->clean(T, V); // zeroes out the chart
    if (CLONING)
      clonechart->clean();

    // initialize viterbi backtrace table with all possibile phrase
    // translations

    int MAXPHRLEN = Globals::Get<int>("maxphrlen");

    // initialize chart with phrase pairs found in sentence pair
    for (int deltas = 0; deltas <= MAXPHRLEN; deltas++)  {
      for (int s = 0; s <= T - deltas; s++)  {
        int t = s + deltas;
        WORDTYPE *en_part = splice(en,s,t);
        for (int deltau = 0; deltau <= MAXPHRLEN; deltau++)  {
          if (deltau + deltas == 0)  continue;
          for (int u = 0; u <= V - deltau; u++)  {
            int v = u + deltau;
            WORDTYPE *cn_part = splice(cn,u,v);

            double phr_prob = grammar->phr2phr_prob(en_part, cn_part);
            if (phr_prob > 0.0) {
              ITGVCell *cell = vchart->cell(grammar->T,s,t,u,v);

              double prob = 0.0;
//               ITGVCell new_cell(grammar->T, s, t, u, v);
              prob = phr_prob;

              cell->prob = prob;
              cell->rule = new PhrasePair(grammar->nonterminals[grammar->T], grammar->T, en_part, cn_part, phr_prob);
//               cell->features = new_cell.features;

//               cell->uses_clone = 0;
//               cell->clone_start = -1;
//               cell->clone_stop = -1;
            }

#ifdef DEBUG
            printf("viterbi: prob[%d,%d,%d,%d,%s] = %.20e\n",
                   s,t,u,v,grammar->nonterminals[lhs],pr);
            printf("  en = ");
            fprint_array(stdout,en_part);
            printf("\n");
            printf("  cn = ");
            fprint_array(stdout,cn_part);
            printf("\n");
            fflush(stdout);
#endif
            delete[] cn_part;
          } //end of u loop
        } //end of deltau loop
        delete[] en_part;
      } //end of s loop
    } //end of deltas loop

    //   dump_chart("/tmp/chart", _sentno);
  }




  /** Parser::viterbi
   *  This implementation of the Viterbi algorithm reorders the loops to
   *  allow for cloning.  Whereas Hao's version guaranteed that smaller
   *  blocks will have been built when we consider larger blocks, this
   *  version guarantees that smaller intervals in L1 will have been
   *  built.  
   */

  void Parser::viterbi() {
    vinit();//initialization of the viterbi mode chart

    //_sentno counts which sentence we're on
    /*
    fprintf(pfafile, "begin %d\n", _sentno);
    fprintf(pfinfo, "begin %d\n", _sentno);
    */
#ifdef DEBUG
    printf("Processing sentence #%d (T=%d,V=%d)\n", _sentno, T, V);
#endif

    //bottom-up first
    for (int deltas = 0; deltas <= T; deltas++)  {

      for (int s = 0; s <= T - deltas; s++)  {
        int t = s + deltas;

        for (int deltau = 0; deltau <= V; deltau++)  {

          for (int u = 0; u <= V - deltau; u++)  {
            int v = u + deltau;

            /* CASE 1: best parse uses binary rule here */
            for (int ruleno = 0; ruleno < grammar->B; ruleno++)  {
              Rule *rule = grammar->binary_rules[ruleno];

              for (int S = s; S <= t; S++) {

                for (int U = u; U <= v; U++) {

                  ITGVCell* left = NULL, *right = NULL;

                  bool do_clone = CLONING; //default

                  if (rule->inverted) {
                    left = vchart->cell(rule->rhsl_int, s, S, U, v);
                    if (left->prob <= 0.0) continue;
                    right = vchart->cell(rule->rhsr_int, S, t, u, U);
                    do_clone = CLONING && S != s && S != t && u == U;
                  } else {
                    left = vchart->cell(rule->rhsl_int, s, S, u, U);
                    if (left->prob <= 0.0)  continue;
                    right = vchart->cell(rule->rhsr_int, S, t, U, v);
                    do_clone = CLONING && S != s && S != t && v == U;
                  }

                  /* regular */

                  double newgama = rule->prob * right->prob * left->prob;
                  if (do_clone)  newgama *= 1 - grammar->pins;
                  ITGVCell *cell = vchart->cell(rule->lhs_int, s, t, u, v);

                  if (newgama > cell->prob) {
                    cell->prob = newgama;
                    cell->uses_clone = 0;
                    cell->clone_start = -1;
                    cell->clone_stop = -1;
                    // backpointers
                    cell->rule = rule;
                    cell->lchild = left;
                    cell->rchild = right;
                  }

                  /* cloning */

                  if (do_clone) {
                    double gama_c = clonechart->prob(S,t);
                    if (gama_c > 0.0) {
                      newgama = left->prob * gama_c * grammar->pins;
                      if (newgama > cell->prob) {
                        ITGViterbiCloneCell *clone = clonechart->get(S,t);
                        cell->prob = newgama;
                        cell->uses_clone = 1;
                        cell->clone_start = clone->u;
                        cell->clone_stop = clone->v;
                        // backpointers
                        cell->rule = rule;
                        cell->lchild = left;
                        cell->rchild = left;
                      }
                    }
                  }
                } // end U split
              } // end S split

              /* update best clone for span */

              if (CLONING && v > u && t > s)  {
                //we don't need to consider unary rules, because it will
                //always be more probable to clone the bottom node of a
                //unary production, and we always have our choice
                ITGVCell *cell = vchart->cell(rule->lhs_int, s, t, u, v);
                double newgama = cell->prob;
                double oldgama = clonechart->prob(s,t);
                if (newgama > oldgama)  {
                  ITGViterbiCloneCell clonecell(u, v, rule->lhs_int, newgama);
                  clonechart->put(s,t,clonecell);
                  //                 printf("  Recording (%d,%d,%d,%d) as best clone for (%d,%d)\n",
                  //                        s,t,u,v,s,t);
                  //                 printf("    rule = %s, S = %d, U = %d, inv = %d, logprob = %.5f\n", rule->lhs.data(), cell->S, cell->U, cell->inv, log(cell->prob));

                }//end up updating best clone
              }//end of best clone check
            }

            /* CASE 2: best parse uses unary rule here */

            for (int ruleno = 0; ruleno < grammar->U; ruleno++) {
              Rule *rule = grammar->unary_rules[ruleno];
              int lhs = rule->lhs_int;
              int rhs = rule->rhsl_int;
              ITGVCell* child = vchart->cell(rhs, s, t, u, v);
              double newgama = rule->prob * child->prob;
              ITGVCell *cell = vchart->cell(lhs, s, t, u, v);
              double oldgama = cell->prob;
              if (newgama > oldgama){//maximizing
                cell->prob = newgama;
                cell->uses_clone = 0;
                cell->clone_start = -1;
                cell->clone_stop = -1;
                // backpointers
                cell->rule = rule;
                cell->lchild = child;
                cell->rchild = NULL;
              }//end of updating viterbi
            }//end of for unary rules

            /* CASE 3: best parse makes this spot a phrase leaf */
            // this is handled with base cases in vinit()
          }
        }
      }
    }

    /*
      if (CLONING)
      for (int s = 0; s < MAXT; s++)  {
      for (int t = s + 1; t <= MAXT; t++)  {
      ITGViterbiCloneCell *clone = clonechart->get(s,t);
      if (clone->N != 0)  {
      printf("Sentence %d: best clone for (%d,%d) = ", _sentno, s, t);
      printf("(%s,%d,%d,%d,%d), logprob = %.5f\n", 
      grammar->nonterminals[clone->N].data(), 
      s, t, clone->u, clone->v, 
      log(clone->prob));
      ITGVCell *cell = vchart->cell(clone->N, s,t,clone->u,clone->v);
      printf("  S = %d, U = %d, inv = %d, logprob = %.5f\n",
      cell->S, cell->U, cell->inv, log(cell->prob));
      }
      }                 
      }
    */

    /* top-down reconstruction */

    //   cout << "Dumping chart for sentence " << _sentno << endl;
    //   dump_chart("/tmp/chart", _sentno);

    pi = vchart->prob(grammar->start_symbol, 0, T, 0, V);
    if (pi <= 0.0) {
      fprintf(pfviterbi, "* No parse\n");
      /*
      fprintf(pfafile, "0\t0\nend\n"); 
      */
      printf("* WARNING: no parse for sentence %d. Dumping chart to /tmp/chart.%d\n", _sentno, _sentno);
      dump_chart("/tmp/chart", _sentno);
      return;
    } else {

      topcell_ = vchart->cell(grammar->start_symbol, 0, T, 0, V);

      fprintf(pfviterbi, "%s\n",
	      reconstruct(vchart->cell(grammar->start_symbol, 0, T, 0, V)).c_str());
    }
  }

  string Parser::reconstruct(ITGVCell* cell) {
    stringstream sstr;
    if (cell->lchild && cell->rchild) { // binary rule
      string open = cell->rule->inverted ? "<" : "[";
      string close = cell->rule->inverted ? ">" : "]";
      sstr << open << cell->rule->lhs << " " << reconstruct(cell->lchild)
           << " " << reconstruct(cell->rchild) << close << " ";
    } else if (cell->lchild) { // unary rule
      sstr << "[" << cell->rule->lhs << " " << reconstruct(cell->lchild) << "] ";
    } else { // leaf
      PhrasePair* pair = static_cast<PhrasePair*>(cell->rule);
      sstr << "[" << pair->lhs;
//       sstr << " { " << transliterate(pair->l1phr,&l1vocab) << " / " << transliterate(pair->l2phr, &l2vocab) << " } ] ";
      sstr << " { " << pair->l1phrstr << " / " << pair->l2phrstr << " } ] ";
    }

    return sstr.str();
  }

  /*
  string reconstruct_old() {
    string viterbi = "";
    stack<ViterbiNode> stack;
    stack.push(ViterbiNode(grammar->start_symbol, 0, T, 0, V));
    int clone_cnt = 0;
    int phrase_count = 0;

    while (! stack.empty()) {
      ViterbiNode node = stack.top();
      stack.pop();

      // look for closing brackets
      if (node.lhs == '>' or node.lhs == ']') {
        stringstream s;
        s << (char)node.lhs << " ";
        viterbi += s.str();
        continue;
      }

      fprintf(pfinfo, "%d %d %d %d", node.s, node.t, node.u, node.v);


      ITGVCell *best = vchart->cell(node.lhs, node.s, node.t, node.u, node.v);
      if (node.lhs == grammar->T) { //leaf
        // add to the viterbi string
        PhrasePair *pair = (PhrasePair *)(best->rule);
        string pairstr = "{ " + pair->l1phr + " / " + pair->l2phr + " }";
        viterbi += pairstr + " ";

        // write out the alignments
        //       if (node.u != node.v)   // only if not l1 singleton
        if (node.s < node.t && node.u < node.v) {
          for (int l1 = node.s; l1 < node.t; l1++) {
            fprintf(pfafile, "%d\t", l1+1);
            for (int l2 = node.u; l2 < node.v; l2++)
              fprintf(pfafile, "%d ", l2+1);
            fprintf(pfafile, "\n");
          }

          if (node.t > node.s + 1 || node.v > node.u + 1)
            phrase_count++;
        }
        fprintf(pfinfo, " %s %s %.10e\n", 
                best->rule->lhs.data(), pairstr.data(), pair->prob);

      } else if (best->rule->binary) {
        char ruletype = 'b';      // [b]inary, [c]lone
        char ruledir = 's';       // [s]traight or [i]nverted
        int rhsr = best->rule->rhsr_int;
        int rhsl = best->rule->rhsl_int;
        int u, v, ul, vl;
        if (best->rule->inverted) {
          ruledir = 'i';
          viterbi += "<" + grammar->nonterminals[node.lhs] + " ";
          stack.push(ViterbiNode('>'));
          u = node.u;
          v = best->U;
          if (best->uses_clone) {
            ITGViterbiCloneCell *clone = clonechart->get(node.s, node.t);
            rhsr = clone->N;
            u = clone->u;
            v = clone->v;
            clone_cnt++;
            ruletype = 'c';
          }
          stack.push(ViterbiNode(rhsr, best->S, node.t, u, v));
          stack.push(ViterbiNode(rhsl, node.s, best->S, best->U, node.v));

          ul = best->U;
          vl = node.v;

        } else {
          viterbi += "[" + grammar->nonterminals[node.lhs] + " ";
          stack.push(ViterbiNode(']'));
          u = best->U;
          v = node.v;
          if (best->uses_clone) {
            ITGViterbiCloneCell *clone = clonechart->get(node.s, node.t);
            rhsr = clone->N;
            u = clone->u;
            v = clone->v;
            clone_cnt++;
            ruletype = 'c';
          }
          stack.push(ViterbiNode(rhsr, best->S, node.t, u, v));
          stack.push(ViterbiNode(rhsl, node.s, best->S, node.u, best->U));

          ul = node.u;
          vl = best->U;
        }

        fprintf(pfinfo, " %s %c %c",
                best->rule->lhs.data(), ruletype, ruledir);
        fprintf(pfinfo, " %s %d %d %d %d",
                best->rule->rhsl.data(), node.s, best->S, ul, vl);
        fprintf(pfinfo, " %s %d %d %d %d",
                best->rule->rhsr.data(), best->S, node.t, u, v);
        fprintf(pfinfo, " %.10e\n",              best->rule->prob);

      } else { // unary
        int rhsl = best->rule->rhsl_int;
        viterbi += "[" + grammar->nonterminals[node.lhs] + " ";
        stack.push(ViterbiNode((int)']'));
        stack.push(ViterbiNode(rhsl, node.s, node.t, node.u, node.v));

        fprintf(pfinfo, " %s u %s %d %d %d %d %.10e\n",
                best->rule->lhs.data(), best->rule->rhsl.data(), 
                node.s, node.t, node.u, node.v, best->rule->prob);
      }
    }
    viterbi_clone_cnt += clone_cnt;
    viterbi_phrase_count += phrase_count;

    // viterbi
    fprintf(pfviterbi, "%s", viterbi.data());
    if (CLONING)
      fprintf(pfviterbi, " (%d clones)\n", clone_cnt);
    else
      fprintf(pfviterbi, "\n");
    fflush(pfviterbi);

    // alignment
    fprintf(pfafile, "end\n");
    fflush(pfafile);

    fprintf(pfinfo, "end\n");
    fflush(pfinfo);

#ifdef DEBUG
    printf("Done with sentence %d\n", _sentno);
#endif
  }
  */


  /** Parser::dump_chart()
   *
   *  Dumps the Viterbi chart to disk.
   */
  void Parser::dump_chart(const char *file, int sentno)
  {
    char outfile[100];
    sprintf(outfile, "%s.%d", file, sentno);

    FILE *out = fopen(outfile, "w");
    if (NULL == out) {
      fprintf(stderr, "Can't dump chart to %s\n", file);
      return;
    }

    fprintf(out, "Sentence pair %d\n", sentno);
    fprintf(out, "Length(l1) = %d\nLength(l2) = %d\n", phraselen(en), phraselen(cn));
    fprintf(out, "l1: ");
    fprint_array(out, en);
    fprintf(out, "\nl2: ");
    fprint_array(out, cn);
    fprintf(out, "\n");

    for (int deltas = 0; deltas <= T; deltas++) {
      for (int s = 0; s <= T - deltas; s++)  {
        int t = s + deltas;
        for (int deltau = 0; deltau <= V; deltau++)  {
          if (deltau + deltas == 0)  continue;
          for (int u = 0; u <= V - deltau; u++)  {
            int v = u + deltau;
            for (int lhs = 0; lhs < grammar->_nonterminal_cnt; lhs++)  {
              double pr = vchart->prob(lhs, s, t, u, v);
              fprintf(out, "%d,%d,%d,%d %s %.20e\n", 
                      s, t, u, v, grammar->nonterminals[lhs].data(), pr);
            }

            for (int ruleno = 0; ruleno < grammar->U; ruleno++) {

            }
          }
        }
      }
    }

    fclose(out);
  }


  void Parser::run()
  {
    if (Globals::Get<int>("verbosity") >= 1)
      print_header("Computing viterbi parses");

    int sens = 0;
    while (readin(corpus)){
      gettimeofday(&start, 0);
      viterbi();
      double pi_max = pi;
      gettimeofday(&finish, 0);
      long compTime = (finish.tv_sec - start.tv_sec);
      fprintf(logfile, "pair %d: <%d, %d>\t%ld seconds, pi_max = %.20e\n", 
              ++sens, T, V, compTime, pi_max);
      fflush(logfile);

//       printf("\rFinished parsing sentence %d", _sentno);
//       fflush(stdout);
    }

    // number of phrases
    printf("\nPhrases / sentence = %d / %d = %.2f\n",
           viterbi_phrase_count, sens,
           (double) viterbi_phrase_count / (double) sens);

    if (CLONING)
      printf("Clones / sentence = %d / %d = %.2f\n", 
             viterbi_clone_cnt, sens, 
             (double) viterbi_clone_cnt / (double) sens);

  }

  void ITGVChart::clean(int T, int V){
//     cout << "ITGVChart::clean(): zeroing " << sizeof(ITGVCell)*MAXN*totalDimen 
//          << " bytes (sizeof(ITGVCell)=" << sizeof(ITGVCell) 
//          << ", MAXN=" << MAXN << ", totalDimen=" << totalDimen << ")\n";
    memset(cells, 0, sizeof(ITGVCell)*MAXN*totalDimen);
  }

}


