/* itggrmr.cpp
 *
 * $Id: itggrmr.cpp,v 1.2 2011/10/16 22:33:38 gildea Exp $
 */

#include "itggrmr.h"
#include "array.h"
#include "globals.h"
#include "digamma.h"
#include <cstring>
#include <cmath>
#include <iostream>
#include <sstream>
#include <fstream>
#include <cassert>

using namespace std;

//mixed binary rules and unary rules supported
//but the unary rules can not form a link
//also, unary rules are only allowed at higher than preterm level

#define STRAITCONNECTOR "->"
#define INVERSIONCONNECTOR "-<"
#define FANOUT 2
#define MAXLINELEN 100
#define SEP " \n\r\t"

double fact(int arg) {
  double val = 2.0;
  while (arg > 2)
    val *= arg--;
  return val;
}


// a phrase pair's probability is checked:
// -- according to its entry in the t-table
// -- defaultprob if both phrases are <= MAXPHRLEN_SMOOTH
// -- 0

/* returns the phrase translation probability.  one has to use find()
 * when searching hash tables for the presence of an entry because []
 * creates entries that are not there already
 */

/* July 2006 -- only give probabilty to phrases with fewer words
 * than MAXPHR
 */
double ITGGrammar::phr2phr_prob(WORDTYPE* l1, WORDTYPE* l2){
  phrase_table::iterator l2itr;
  phrase_entry::iterator l1itr;

  /* try to find the phrase pair in the t-table */

  l2itr = probs_table.find(l2);
  if (l2itr != probs_table.end()){
    l1itr = l2itr->second.find(l1);
    if (l1itr != l2itr->second.end()) {
      double p = l1itr->second[T];
      return p;
    }
  }

  int len1 = phraselen(l1);
  int len2 = phraselen(l2);
  int len = len1 > len2 ? len1 : len2;

  /* if control reaches here we know we haven't found the phrase
   * in the ttable, so check if it should get a default prob
   */
  static int max = Globals::Get<int>("maxphrlen_smooth");
  if (len <= max) {
    double p = defaultprob;

    // put the phrase in the ttable if it wasn't found
    WORDTYPE *syml1 = phrase_symbol_table_entry(l1);
    WORDTYPE *syml2 = phrase_symbol_table_entry(l2);
    probs_table[syml2][syml1][T] = p;
    counts_table[syml2][syml1][T] = 0.0;  // counts table (for training only)

    return p;
  }

  /* phrase pair not found, and too long for default prob */
  if (mode == 'd' || mode == 'e') 
    return -INFINITY;	// log prob
  else 
    return 0;		// prob
}


/* updates the count of a pair of phrases, and returns the new value;
   NOTE: we depend on this function to only update counts for phrases
   that exist.  in other words, this function can be called with phrase
   pairs that are not in the lexicon, in which case nothing is returned.
   this means that we can't use [][]s, because those create entries
   that are not present. */
double ITGGrammar::phr2phr_cntsum(WORDTYPE *l1, WORDTYPE *l2, double val) {
  phrase_table::iterator l2itr;
  phrase_entry::iterator l1itr;

  l2itr = counts_table.find(l2);
  if (l2itr != counts_table.end()){
    l1itr = (*l2itr).second.find(l1);
    if (l1itr != (*l2itr).second.end()) {

      //do this here for phrases; make sure not to do it twice
      N_cnt[T] += val;

      //       printf("N_cnt[%d] = %.5f + %.5f = ", T, N_cnt[T], val);
      //       printf("%.5f\n\n", N_cnt[T]);

#ifdef DEBUG
      printf("* phr2phr_cntsum(");
      fprint_array(stdout,l1);
      printf(", ");
      fprint_array(stdout,l2);
      printf(") += %.5f\n", val);
      fflush(stdout);
#endif
      (*l1itr).second[T] += val;
      return (*l1itr).second[T];
    }
  }

  return 0.0;
}

void ITGGrammar::combinecnt(const char *flist){
  ifstream list(flist);
  if (! list.is_open())  {
    printf("* FATAL: can't read list of files '%s'\n", flist);
    exit(-1);
  }

  string filename;
  if (Globals::Get<int>("verbosity") >= 1) {
    cout << "Reading files...";
    fflush(stdout);
  }
  while (getline(list, filename)) {
    cout << filename << "...";
    fflush(stdout);
    ifstream in(filename.c_str());
    if (! in.is_open()) {
      printf("* FATAL: can't read '%s' listed in %s\n", filename.c_str(), flist);
      exit(-1);
    }
 
    /* read in nonterminal masses */
    for (unsigned i = 0; i < nonterminals.size(); i++) {
      string nt, line;
      double x;

      getline(in, line);
      stringstream str(line);

      str >> x >> nt; //read in order
      N_cnt[indices[nt]] += x;
    }

    /* read in rule counts */
    for (unsigned i = 0; i < rules.size(); i++) {
      Rule *rule = rules[i];
      double x;
      string line;
      getline(in, line);
      stringstream str(line);
      str >> x;
      rule->count += x;
    }

    //CLONING: in order to be backward compatible with previous
    //versions, the insertion probability used for cloning is hidden
    //under a phrase rule of the following form
    //     2^31-1 0 2^31-1 0 <pins>
    //     2^31-2 0 2^31-2 0 <mass>
    //this should cause no harm to previous versions, so long as no
    //one ever tries to run it over a vocabulary with 2 billion items.
    if (CLONING) {
      WORDTYPE e[MAXPHR];
      WORDTYPE c[MAXPHR];
      double cnt;

      //get the clone mass
      string line;
      getline(in, line);
      read_phrase(line, e, c, &cnt);

      //if this rule didn't appear first, complain, and default to
      //the pins value specified in the config file
      if (e[0] != CLONERULE)  {
        cerr << "* Warning: cloning requested, but no clone count found in '"
             << probsf.data() << "'\n";
        abort();
      } else {
        clone_mass += cnt;
      }

      //get the total mass
      e[0] = 0;
      c[0] = 0;
      getline(in, line);
      read_phrase(line, e, c, &cnt);

      if (e[0] != CLONERULE - 1)  {
        cerr << "* Warning: cloning requested, but no total count found in '"
             << probsf << "'\n";
        abort();
      } else {
        total_mass += cnt;
      }
    }

    //phrase version
    while (! in.eof())  {
      WORDTYPE e[MAXPHR], c[MAXPHR];
      double cnt;
      string line;

      // returns false when it fails to read a phrase
      getline(in, line);
      read_phrase(line, e, c, &cnt);

      WORDTYPE *syme = phrase_symbol_table_entry(e);
      WORDTYPE *symc = phrase_symbol_table_entry(c);
      counts_table[symc][syme][T] += cnt;  // counts table
      in >> ws;
    }

    in.close();
  }
  cout << endl;

  list.close();
}


void ITGGrammar::dumpprb(string r_dpfile, string t_dpfile) {
  FILE *r_dumpf = fopen(r_dpfile.data(), "w");
  FILE *t_dumpf = fopen(t_dpfile.data(), "w");

  if (NULL == r_dumpf )  {
    cerr << "FATAL ERROR: can't write to probability file '" << r_dpfile << "'\n";
    exit(1);
  }

  if (NULL == t_dumpf )  {
    cerr << "FATAL ERROR: can't write to probability file '" << t_dpfile << "'\n";
    exit(1);
  }

  /* write rule probs */
  // write them in the same order they were read in

  //write the nonterminals
  for (unsigned i = 0; i < nonterminals.size(); i++) {
    fprintf(r_dumpf, "%s ", nonterminals[i].c_str());
  }
  fprintf(r_dumpf, "\n");


  //write the preterminals


  fprintf(r_dumpf, "%s ", nonterminals[T].c_str());
  fprintf(r_dumpf, "%s ", nonterminals[TL1].c_str());
  fprintf(r_dumpf, "%s ", nonterminals[TL2].c_str());
  fprintf(r_dumpf, "\n");

  
  //write the rules
  for (unsigned i = 0; i < rules.size(); i++) {
    Rule *rule = rules[i];
    fprintf(r_dumpf, "%s %.80e\n", rule->rulestr_inputformat().c_str(), rule->prob);
  }


  /*
  for (unsigned i = 0; i < binary_rules.size(); i++) {
    Rule *rule = binary_rules[i];
    fprintf(r_dumpf, "%.80e\n", rule->prob);
    printf("Writing %s rule %s\n", 
           rule->binary ? "binary" : "unary", rule->str().data());
  }
  for (unsigned i = 0; i < unary_rules.size(); i++) {
    Rule *rule = unary_rules[i];
    fprintf(r_dumpf, "%.80e\n", rule->prob);
    printf("Writing %s rule %s\n", 
           rule->binary ? "binary" : "unary", rule->str().data());
  }
  */

  //if cloning is enabled, dump the insertion probability and
  //mass counts
  if (CLONING)  {
    fprintf(t_dumpf, "%d 0 %d 0 %.80e\n", CLONERULE, CLONERULE, pins);
  }

  bool drop = Globals::Get<bool>("drop_lowcount_phrases");

  //phrases
  phrase_entry::iterator l1itr;
  phrase_table::iterator l2itr;
  for (l2itr = probs_table.begin(); l2itr != probs_table.end(); l2itr++) {
    WORDTYPE *l2 = l2itr->first;
    phrase_entry l1_tab = l2itr->second;
    for (l1itr = l1_tab.begin(); l1itr != l1_tab.end(); l1itr++) {
      WORDTYPE *l1 = l1itr->first;
      double pr = l1itr->second[T];
      if ( (pr > defaultprob && drop) || (! drop) ) {
        fprint_array(t_dumpf,l1);
        fprintf(t_dumpf, " ");
        fprint_array(t_dumpf,l2);
        fprintf(t_dumpf, " %e\n", pr);
      }
    }
  }

  fclose(r_dumpf);
  fclose(t_dumpf);
}


/** ITGGrammar::dumpcnt()
 *  Dumps the partial counts to a file for later recombination.  This
 *  is so that the training corpus can be split up across many computers.
 *  Note that the format differs slightly from the probability files,
 *  because we dump counts for the nonterminals (e.g., "A") in addition
 *  to counts for the rules (e.g., A -> B C)
 */
void ITGGrammar::dumpcnt(string dgfile) {
  FILE *dumpf  = fopen(dgfile.data(), "w");
  if (NULL == dumpf) {
    cerr << "* FATAL: Can't write to file " << dgfile << endl;
    exit(1);
  }

  /* dump nonterminal counts with labels */
  for (int i = 0; i < _nonterminal_cnt; i++) {
    fprintf(dumpf, "%.80e %s\n", N_cnt[i], nonterminals[i].data());
  }

  /* dump rule counts, in the same order as they were read in */
  for (unsigned i = 0; i < rules.size(); i++) {
    Rule *rule = rules[i];
    fprintf(dumpf, "%.80e %s\n", rule->count, rule->rulestr().data());
  }

  /* dump cloning counts */
  if (CLONING)  {
    fprintf(dumpf, "%d 0 %d 0 %.80e\n", CLONERULE, CLONERULE, clone_mass);
    fprintf(dumpf, "%d 0 %d 0 %.80e\n", CLONERULE-1, CLONERULE-1, total_mass);
  }

  double cutoff = Globals::Get<double>("phrase_count_cutoff");
  bool drop = Globals::Get<bool>("drop_lowcount_phrases");

  /* dump phrases */
  phrase_table::iterator l2itr;
  phrase_entry::iterator l1itr;
  for (l2itr = counts_table.begin(); l2itr != counts_table.end(); l2itr++) {
    WORDTYPE *l2 = (*l2itr).first;
    phrase_entry l1_tab = (*l2itr).second;
    for (l1itr = l1_tab.begin(); l1itr != l1_tab.end(); l1itr++) {
      WORDTYPE *l1 = (*l1itr).first;
      double cnt = (*l1itr).second[T];
      if ( (cnt > cutoff && drop) || (! drop) )  {
        fprint_array(dumpf,l1);
        fprintf(dumpf, " ");
        fprint_array(dumpf,l2);
        fprintf(dumpf, " %.80e\n", cnt);
        //         printf("COUNT: ");
        //         fprint_array(stdout,l1);
        //         printf(" ");
        //         fprint_array(stdout,l2);
        //         printf(" %.80e\n", cnt);
        fflush(dumpf);
      }
    }
  }

  fclose(dumpf);
}

/* either 
 * (a) loads the traditional ITG grammar rules or 
 * (b) loads nothing, in which case Chiang-style rules are expected in the phrase table
 *     (loaded next)
 */
void ITGGrammar::load_grammar() {
  string line;
  stringstream tokens;

  ifstream grammar(rulesf.data());
  if (! grammar.is_open()) {
    cout << "* FATAL: can't read grammar file " << rulesf << endl;
    exit(-1);
  }    

  /* line 1: list of nonterminals */
  getline(grammar, line);
  tokens << line;
  string token;
  while (tokens >> token) {
    nonterminals.push_back(token);
    indices[token] = nonterminals.size() - 1;
  }
  _nonterminal_cnt = nonterminals.size();

  start_symbol = indices["S"];


  /* line 2: list of three preterminals for l1/l2, l1/e, e/l2 */

  vector<string> preterminals;
  getline(grammar, line);
  stringstream tokens2; // can't figure out how to reuse tokens
  tokens2 << line;
  while (tokens2 >> token) {
    preterminals.push_back(token);
  }

  T = indices[preterminals[0]];
  TL1 = indices[preterminals[1]];
  TL2 = indices[preterminals[2]];

  /* lines 3..N: the rules */    

  while (getline(grammar,line)) {
    Rule *rule = new Rule(nonterminals, line);
    rule->lhs_int = indices[rule->lhs];
    rule->rhsl_int = indices[rule->rhsl];
    if (rule->binary) {
      rule->rhsr_int = indices[rule->rhsr];
      binary_rules.push_back(rule);
    } else {
      unary_rules.push_back(rule);
    }
    rules.push_back(rule);

    int lhs = indices[rule->lhs];
    int inv = rule->inverted ? 1 : 0;
    int rhsl = indices[rule->rhsl];
    int rhsr = rule->binary ? indices[rule->rhsr] : -1;
    rule_table[rhsl][rhsr][lhs][inv][nullphr] = rule;

    lhs_rule_table[lhs].push_back(rule); //added by Hao


    if (Globals::Get<int>("verbosity") >= 2) {
      string type = (rule->binary) ? "binary" : "unary";
      printf("Read %s rule %s\n", type.data(), rule->str().data());
    }
  }

  B = binary_rules.size();
  U = unary_rules.size();

  grammar.close();

  max_binary_rule_prob = -9999999;
  for (int b = 0; b < B; b++) {
    if (binary_rules[b]->prob > max_binary_rule_prob)
      max_binary_rule_prob = binary_rules[b]->prob;
  }

  if (G_INT("verbosity") >= 1 && binary_rules.size() && unary_rules.size())
    cout << "Read " << binary_rules.size() << " binary rules and " 
         << unary_rules.size() << " unary rules.\n";
}


/*  Loads the grammar (ITG-style or Chiang-style)
 */
void ITGGrammar::load() {
  if (loaded)
    return;

  loaded = true;

  string line;
  stringstream tokens;

  // set the null phrase
  WORDTYPE nullphr_t[1] = { 0 };
  nullphr = phrase_symbol_table_entry(nullphr_t);

  /* read the grammar rules and probabilities */
  load_grammar();

  /* read the phrase pairs and probabilities */
  if (B || U) {
    // grammar rules were read by load_grammar 
    load_phrase_table();
  } else {
    // grammar rules are part of phrase table (i.e., hiero-style rules)
    load_binarized_grammar();
  }

  // check whether there was an entry for pins
  if (CLONING) {
    WORDTYPE *phr = new WORDTYPE[2];
    phr[0] = CLONERULE;
    phr[1] = 0;
    phrase_table::iterator found = probs_table.find(phr);
    if (found != probs_table.end())  {
      phrase_entry::iterator guy = found->second.find(phr);
      if (guy != found->second.end()) {
        cout << "Changing pins from default " << pins << " to ";
        pins = guy->second[T];
        cout << pins << endl;
      }
      found->second.erase(phr);
    }
    probs_table.erase(phr);
  }

  // set the unk phrase for l2
  WORDTYPE unkphr_t[2];
  unkphr_t[0] = l2vocab["<unk>"]; unkphr_t[1] = 0;
  unkphr = phrase_symbol_table_entry(unkphr_t);

  if (! *unkphr)
    cerr << "* WARNING: l2 <unk> id = " << *unkphr << endl;
}


void ITGGrammar::normalize() {
  for (int i = 0; i < _nonterminal_cnt; i++){
    if (N_cnt[i] < threshold)
      N_cnt[i] = threshold;
  }
  //normalize for the nonterminal rules
  for (unsigned i = 0; i < rules.size(); i++) {
    Rule *rule = rules[i];
    int lhs = rule->lhs_int;

    //normalize the rule probability

    if (Globals::Get<int>("verbosity") >= 1)
      cout << "Normalize: " << rule->rulestr() 
          << " " << rule->count << " / N_cnt[" << lhs << "] = " 
          << N_cnt[lhs] << " -> ";
    double prob = rule->count / N_cnt[lhs];
    prob = (prob > threshold) ? prob : threshold;
    rule->prob = prob;
    rule->count = 0.0;
//    cout << rule->prob << endl;
  }

  //cloning
  if (CLONING) {
    pins = clone_mass / total_mass;
    //     printf("* Computing new pins: %.5f / %.5f = %.5f\n", 
    //            clone_mass, total_mass, pins);
  }

  //phrases
  int num_phrases_seen = 0;
  phrase_table::iterator l2itr;
  if (Globals::Get<bool>("variational_bayes")) {
    double alpha = Globals::Get<double>("variational_alpha");
    double total_mass = 0.0;
    for (l2itr = counts_table.begin(); l2itr != counts_table.end(); l2itr++) {

      phrase_entry &l1_tab = (*l2itr).second;	// reference avoids deep copy
      phrase_entry::iterator l1itr;
      for (l1itr = l1_tab.begin(); l1itr != l1_tab.end(); l1itr++) {

	double cnt = (*l1itr).second[T];
	if (cnt + alpha > 0.0)			
	  l1itr->second[T] = exp(digamma(cnt + alpha));
	else 
	  l1itr->second[T] = 0.0;
	total_mass += l1itr->second[T];
	//	num_phrases_seen++;
      }
    }
    N_cnt[T] = exp(digamma(N_cnt[T] + num_phrases_seen*alpha));
  } // end of variational bayes count adjustment

  // normalize phrase table
  for (l2itr = counts_table.begin(); l2itr != counts_table.end(); l2itr++) {

    WORDTYPE *l2 = (*l2itr).first;
    phrase_entry &l1_tab = (*l2itr).second;
    phrase_entry::iterator l1itr;
    for (l1itr = l1_tab.begin(); l1itr != l1_tab.end(); l1itr++) {

      WORDTYPE *l1 = (*l1itr).first;
      double cnt = (*l1itr).second[T];
      double total = N_cnt[T];
      double pnew = cnt / total;

      num_phrases_seen++;
      
      l1itr->second[T] = 0.0;  // reset the count

      //             printf("  counts_table[l2][l1] = %.5f (should be 0)\n",
      //                    counts_table[l2][l1]);
      probs_table[l2][l1][T] = pnew > threshold ? pnew : defaultprob;
    }
  }
  //   printf("  N_cnt[%d] = %.5f, mass = %.5f\n", T, N_cnt[T], mass);

  printf("Normalization: saw %d distinct phrase pairs\n", num_phrases_seen);

  // reset counter
  memset(N_cnt, 0, sizeof(N_cnt));
}

/* m-step in the mixture of dp model
 */
void ITGGrammar::mdpnormalize(){
  for (int i = 0; i < _nonterminal_cnt; i++){
    if (N_cnt[i] < threshold)
      N_cnt[i] = threshold;
  }
  //normalize for the nonterminal rules
  for (unsigned i = 0; i < rules.size(); i++) {
    Rule *rule = rules[i];
    int lhs = rule->lhs_int;

    //normalize the rule probability
    if (Globals::Get<int>("verbosity") >= 1)
      cout << "Normalize: " << rule->rulestr() 
           << " " << rule->count << " / N_cnt[" << lhs << "] = " 
           << N_cnt[lhs] << " -> ";
    double prob = rule->count / N_cnt[lhs];
    prob = (prob > threshold) ? prob : threshold;
    rule->prob = prob;
    rule->count = 0.0;
 //   cout << rule->prob << endl;
  }

  // reset counter
  memset(N_cnt, 0, sizeof(N_cnt));



  //vectorize the set of phrase pairs
  phrlist.clear();
  phrase_table::iterator l2itr;
  int num_phrases_seen = 0;
  for (l2itr = counts_table.begin(); l2itr != counts_table.end(); l2itr++) {
    WORDTYPE *l2 = (*l2itr).first;

    phrase_entry &l1_tab = (*l2itr).second;	// reference avoids deep copy
    phrase_entry::iterator l1itr;

    for (l1itr = l1_tab.begin(); l1itr != l1_tab.end(); l1itr++) {
      WORDTYPE *l1 = (*l1itr).first;
      double cnt = (*l1itr).second[T];

      phrinfo_t ph;
      ph.count = cnt;
      //fill in other fields
      ph.residual_count = 0;
      ph.l1 = l1;
      ph.l2 = l2;
      
      phrlist.push_back(ph);

      l1itr->second[T] = 0.0;  // reset the count

      num_phrases_seen++;
    }
  }


  printf("Normalization: saw %d distinct phrase pairs\n", num_phrases_seen);

  //maybe we need to sort the list of phrase pairs according to their expected counts
  //  sort(phrlist.begin(), phrlist.end(), greater_ec());
  //  phrlist.resize(num_phrases_seen);

  //iterate over the phrase list backward to compute \gamma_{t, 2}
  double residual_count = 0; 
  for (int idx = num_phrases_seen-1; idx >=0; idx--){
    phrlist[idx].residual_count = residual_count;
    residual_count += phrlist[idx].count;
  }

  double alpha = Globals::Get<double>("variational_alpha");

  //iterate over the phrase list forward to compute S_{t}
  double accumulant_val = 0;
  for (int idx = 0; idx < num_phrases_seen; idx++){
    double gammasum = digamma(1 + phrlist[idx].count + alpha + phrlist[idx].residual_count);
    phrlist[idx].S = digamma(1 + phrlist[idx].count) - gammasum + accumulant_val;  //maybe need to add P_0
    accumulant_val += digamma(alpha + phrlist[idx].residual_count) - gammasum;
  }


  //fill in the table of rule probs
  double Z=0;
  for (int idx =0; idx < num_phrases_seen; idx++){
    Z += exp(phrlist[idx].S);
  }

  //fill in the table of rule probs
  for (int idx =0; idx < num_phrases_seen; idx++){
    double pnew = exp(phrlist[idx].S); ///Z;

    WORDTYPE *l1 = phrlist[idx].l1;
    WORDTYPE *l2 = phrlist[idx].l2;

    probs_table[l2][l1][T] = pnew > threshold? pnew: defaultprob;
  }
  
  //release the vector
  phrlist.clear();

}


/** ITGGrammar::read_phrase()
 *
 * Takes an input line read from the phrase table, and splits it into the
 * phrase pairs and probability.
 *
 * Return value: true if the phrase pair was read without error.
 */
bool ITGGrammar::read_phrase(string line, WORDTYPE *e, WORDTYPE *c, double *p)
{
  stringstream tokens(line);

  int num = 0;
  int i = 0;
  int saw = 0;
  while (tokens >> num)  {
    saw++;
    if (num > MAXWORDID) {
      fprintf(stderr, "* FATAL: word id %d is out of range (max %d)", num, MAXWORDID);
      exit(1);
    }
    e[i] = num;  i++;
    if (num == 0)  break;
  }
  i = 0;
  while (tokens >> num) {
    saw++;
    if (num > MAXWORDID) {
      fprintf(stderr, "* FATAL: word id %d is out of range (max %d)", num, MAXWORDID);
      exit(1);
    }
    c[i] = num;  i++;
    if (num == 0)  break;
  }

  /* if there was only one token on the line, it probably means the 
   * probfile format is the old style where the rule probs were on
   * top.  warn the user!
   */
  if (saw == 0) {
    cerr << "* FATAL: it looks like you're using the deprecated probfile\n"
         << "   format with the rule probs at the top.\n";
    exit(1);
  }
 
  double prob;
  tokens >> prob;

  *p = prob;

  return true;
}

/*	Reads phrase table format where all phrases have same preterminal
 *
 */
void ITGGrammar::load_phrase_table() {
  ifstream probs(probsf.data());
  if (! probs.is_open()) {
    cerr << "* FATAL: can't read t-table " << probsf << endl;
    exit(-1);
  }

  int skipped = 0;
  int pairno = 0;
  while (! probs.eof()) {
    //phrase rules
    WORDTYPE e[200];
    WORDTYPE c[200];
    double prob;

    string line;
    getline(probs, line);
    if (line == "")
      continue;
    read_phrase(line, e, c, &prob);

    // for decoding, maxphrlen applies only to the target side phrase
    if (mode == 'd' || mode == 'e') {
      if (mode == 'd' && phraselen(e) > maxphrlen) {
        skipped++;
        continue;
      } else if (mode == 'e' && phraselen(c) > maxphrlen) {
        skipped++;
        continue;
      }
    } else if (phraselen(c) > maxphrlen || phraselen(e) > maxphrlen) {
      skipped++;
      continue;
    }

    // use log probs for decoding
    if (mode == 'd' || mode == 'e') {
      prob = log(prob);
    }

    pairno++;
    if (Globals::Get<int>("verbosity") >= 1)
      if (! (pairno % 10000)) {
        printf("\rRead phrase pair %d", pairno);
        fflush(stdout);
      }

    WORDTYPE* syme = phrase_symbol_table_entry(e);
    WORDTYPE* symc = phrase_symbol_table_entry(c);
    probs_table[symc][syme][T] = prob;
    counts_table[symc][syme][T] = 0.0;  // counts table
  }
  probs.close();

  if (Globals::Get<int>("verbosity") >= 1)
    printf("\rRead %d phrases from t-table. Skipped another %d which were too long.\n", pairno, skipped);

}

/* loads output of binarizer: rules with mixed terminals
 * and nonterminals (hiero-style rules).
 *
 * Lines have the following format:
 * <prob> <rule_type> <lhs_id> <rule>
 *
 * rule_type is one of 0 or 1 (terminal phrase pair),
 * 2 (binary or unary rule), or 4 (inverted binary rule)
 *
 * <lhs_id> is the nonterminal ID of the rule.  There are
 * many nonterminals used to handle binarization.
 *
 * <rule> is of the form 
 *    <cid>{0,2} 0 <eid>* 0
 *
 * where <cid> is a sequence of zero, one, or two chinese terminals
 * and nonterminals and eid is a sequence of any number of
 * english-side terminals and nonterminals.  There will always be only
 * zero, one, or two items on the Chinese side because we binarize the
 * source side for efficient parsing.
 */
void ITGGrammar::load_binarized_grammar() {
  int count = 0; // how many rules have we read?

  ifstream probs(probsf.data());
  if (! probs.is_open()) {
    cerr << "* FATAL: can't read t-table " << probsf << endl;
    exit(-1);
  }

  while (! probs.eof()) {
    string line;
    getline(probs, line);
    if (is_comment_or_empty_line(line)) {
      continue;
    }

    ++count;
    stringstream tokens(line);
    double prob;
    int rule_type;
    int lhs_int;
    
    tokens >> prob;
    tokens >> rule_type;
    tokens >> lhs_int;

    if (mode == 'd' || mode == 'e')
      prob = log(prob);

    int num = 0;
    int i = 0;
    WORDTYPE e[200];
    WORDTYPE c[200];
    while (tokens >> num)  { // read in chinese side up to the first 0
      if (num > MAXWORDID) {
        fprintf(stderr, "* FATAL: word id %d is out of range (max %d)", 
                num, MAXWORDID);
        exit(1);
      }
      c[i] = num;  i++;
      if (num == 0)  break;
    }
    i = 0;
    while (tokens >> num) { // read in english side up to the second 0
      if (num > MAXWORDID) {
        fprintf(stderr, "* FATAL: word id %d is out of range (max %d)", 
                num, MAXWORDID);
        exit(1);
      }
      e[i] = num;  i++;
      if (num == 0)  break;
    }

    /* rule types:
     *   0,1: terminal phrase pair
     *   2:   binary or unary rule
     *   4:   inverted binary rule
     *
     *   nonterminals are represented by negative numbers
     */
    if (rule_type == 0 || rule_type == 1) { // preterminal rule
      // chinese side consists of terminals
      WORDTYPE* syme = phrase_symbol_table_entry(e);
      WORDTYPE* symc = phrase_symbol_table_entry(c);
      probs_table[symc][syme][-lhs_int] = prob;
      //counts_table[symc][syme][-lhs_int] = 0.0;  // counts table
    } else { // unary or binary rule
      // chinese side consists of nonterminals
      Rule *rule = new Rule;
      rule->prob = prob;		
      rule->lhs_int = -lhs_int;		rule->lhs = nonterminal_name(rule->lhs_int);
      rule->rhsl_int = -c[0];		rule->rhsl = nonterminal_name(rule->rhsl_int);
      if (c[1] == 0) {
        // unary rule
        rule->binary = false;
        rule->rhsr_int = -1;
        unary_rules.push_back(rule);
      } else {
        // binary rule      
        rule->binary = true;
        rule->rhsr_int = -c[1];		rule->rhsr = nonterminal_name(rule->rhsr_int);
        binary_rules.push_back(rule);
      }
      rule->e_rhs = phrase_symbol_table_entry(e);
      rule->inverted = (rule_type == 4);
      rule_table[rule->rhsl_int][rule->rhsr_int][rule->lhs_int][rule->inverted][rule->e_rhs] = rule;

      lhs_rule_table[rule->lhs_int].push_back(rule); //added by Hao

      rules.push_back(rule);

      if (G_INT("verbosity") >= 2) {
        string type = (rule->binary) ? "binary" : "unary";
        printf("Read %s rule %s\n", type.data(), rule->str().data());
      }
      
    }
    probs >> ws;	// eat whitespace to reach eof after last rule
  }
  probs.close();
  B = binary_rules.size();
  U = unary_rules.size();

  max_binary_rule_prob = -9999999;
  for (int b = 0; b < B; b++) {
    if (binary_rules[b]->prob > max_binary_rule_prob)
      max_binary_rule_prob = binary_rules[b]->prob;
  }

  if (G_INT("verbosity") >= 1) {
    printf("Read %d binary rule(s), %d unary rule(s), and %d lexical rule(s). [%d]\n\n", B, U, count-B-U, count);
  }
}

ostream& operator<< (ostream& out, Rule& rule) {
  out << rule.str();
  return out;
};

Rule::Rule(vector<string> nonterminals, string line, double prob) {
  stringstream tokens(line);
  if (! (tokens >> lhs >> sign >> rhsl)) {
    cerr << "* WARNING: rule '" << line << "' is invalid\n";
  }

  // if there is another token, this is a binary rule and next is prob
  // if there isn't another token, this is a unary rule and this is prob
  tokens >> rhsr;
  string pr;
  if (tokens >> pr) {
    binary = true;
  } else {
    pr = rhsr;
    binary = false;
    rhsr = "";
  }
  stringstream t(pr);
  t >> prob;

  if (prob == 0) {
    cerr << "* FATAL: does your rules file contain rule probabilities?\n";
    exit(1);
  }

  if (binary && sign == "-<") {
    inverted = true;
  } else {
    inverted = false;
  }

  char mode = Globals::Get<char>("runmode");
  if (mode == 'd' || mode == 'e')
    prob = log(prob);
  this->prob = prob;

  count = 0.0;
  e_rhs = NULL;
  lhs_int = rhsl_int = rhsr_int = -1;
}
