/* itggrmr.h
 *
 * $Id: itggrmr.h,v 1.3 2011/10/17 13:17:08 gildea Exp $
 */

#ifndef __ITGGRMR_H__
#define __ITGGRMR_H__

#include <stdlib.h>
#include <stdio.h>
#include <string.h>
#include <vector>
#include <string>
#include <iomanip>
#include <sstream>

#include "globals.h"
#include "array.h"

#define MAXN 5
#define MAXR 20

#define MAXPHR 10 //maximum phrase length

//we hide cloning params as an improbable phrase rule 
//with the following id, in order to maintain backward
//file compatibility with other itg versions floating around
//since words are stored in unsigned shorts, there's less space...
#define MAXWORDID (((int)pow(2.0,(double)sizeof(WORDTYPE)*8))-1)  // 2^16-1
#define CLONERULE MAXWORDID

// definitions for phrase table
// deep compare of zero-terminated strings of ints
struct eqphrase {
  bool operator()(const WORDTYPE* a, const WORDTYPE* b) const {
    while (*a && *b) {
      if (*a != *b) 
        return 0;
      a++;
      b++;
    }
    return (*a == *b);
  }
};
// deep hash of zero-terminated string of ints
struct hashphrase {
  size_t operator()(const WORDTYPE *s) const {	// hash 0-terminated string of shorts
    const WORDTYPE * p;
    unsigned int h;	// assumes 32-bit int
    h = 0;
    for (p = s; *p; p++) {
      // treat it as an integer
      int t = *p;
      h = (h << 4) + t;
      h = (h ^ ((h & 0xf0000000) >> 24)) & 0x0fffffff;
    }

    return h;
  }
};

struct lhs_to_prob_table : hash_map<int, double, hash<int>, eqint> {
  // small initial bucket size to conserve memory
  lhs_to_prob_table() : hash_map<int, double, hash<int>, eqint>(1) {}
};
struct phrase_entry : hash_map<WORDTYPE*, lhs_to_prob_table, hashphrase, eqphrase> {
  phrase_entry() : hash_map<WORDTYPE*, lhs_to_prob_table, hashphrase, eqphrase>(1) {}
};
struct phrase_table : hash_map<WORDTYPE*, phrase_entry, hashphrase, eqphrase> {};

typedef phrase_entry::iterator phrase_entry_iter;


/* typedef hash_map<WORDTYPE*, double, hashphrase, eqphrase> phrase_entry; */
/* typedef hash_map<WORDTYPE*, phrase_entry, hashphrase, eqphrase> phrase_table; */

class Rule {
public:
  double prob, count;
  string lhs, sign, rhsl, rhsr;
  bool inverted, binary;

  int lhs_int, rhsl_int, rhsr_int;

  WORDTYPE *e_rhs;	// for mixed terms and nonterms

  Rule() { }

  Rule(vector<string>, string, double prob = 0.0);

  virtual ~Rule() { }

  string rulestr() const {
    string s;
    if (binary && inverted)
      s = lhs + " -> <" + rhsl + " " + rhsr + ">";
    else if (binary)
      s = lhs + " -> [" + rhsl + " " + rhsr + "]";
    else
      s = lhs + " -> " + rhsl;
    return s;
  }

  string rulestr_inputformat() const {
    string s;
    if (binary && inverted)
      s = lhs + " -< " + rhsl + " " + rhsr ;
    else if (binary)
      s = lhs + " -> " + rhsl + " " + rhsr ;
    else
      s = lhs + " -> " + rhsl;
    return s;
  }

  string str() const {
    stringstream ss;
    ss << " (" << setprecision(5) << prob << ")";
    return rulestr() + ss.str();
  }

};

ostream& operator<< (ostream& out, Rule& rule);

// type definitions for rule table: multilevel hash
// indexed by : rhsl rhsr lhs inv e_rhs
typedef hash_map<WORDTYPE*,  Rule *, hashphrase, eqphrase> erhs_rule_hash;
typedef hash_map<int, erhs_rule_hash> inv_erhs_rule_hash;
typedef hash_map<int, inv_erhs_rule_hash> lhs_inv_erhs_rule_hash;
typedef hash_map<int, lhs_inv_erhs_rule_hash> rhsr_lhs_inv_erhs_rule_hash;
typedef hash_map<int, rhsr_lhs_inv_erhs_rule_hash> rhsl_rhsr_lhs_inv_erhs_rule_hash;


typedef hash_map<int, vector<Rule *> > lhs_rule_hash;


typedef struct phrinfo_t{
  double count;
  double residual_count;
  double S;

  WORDTYPE* l1;
  WORDTYPE* l2;
} phrinfo_t;


struct greater_ec : public binary_function<phrinfo_t, phrinfo_t, bool> {
  bool operator()(phrinfo_t x, phrinfo_t y) { return x.count > y.count; }
};



class ITGGrammar {
 public:
  int _nonterminal_cnt;//number of nonterminals;
  int T;//preterm for e\c
  int TL1;//preterm for e\cpsilon
  int TL2;//preterm for epsilon\c
  int R;//number of rules;
  int U;//number of unary rules;
  int B;//number of binary rules;
  string rulesf;//grammar file name
  string probsf;//input probabilities file name 
  double N_cnt[MAXN];//nonterminal expected counts

  // indexed by : rhsl rhsr lhs inv e_rhs
  rhsl_rhsr_lhs_inv_erhs_rule_hash rule_table;

  lhs_rule_hash lhs_rule_table; //indexed by lhs

  bool loaded;

  char mode;

  int start_symbol;

  phrase_table probs_table;   //holds phrase probabilities
  phrase_table counts_table;  //holds phrase soft counts

  vector<phrinfo_t> phrlist; //vector of phrase pairs;

  double defaultprob, threshold;

  int drop_lowcount_phrases; //whether to not write phrases with counts below the threshold

  // maps from integers to strings
  vector<string> nonterminals;
  vector<Rule *> rules, binary_rules, unary_rules;
  map<string,int> indices;

  WORDTYPE *unkphr;
  WORDTYPE *nullphr;

  int maxphrlen;


  /* CLONING */
  bool CLONING;     //to clone or not to clone (sorry...)
  double pins;      //insertion probability for cloning
  double clone_mass; //for training
  double total_mass;//for training


  hash_map<WORDTYPE *, WORDTYPE *, hashphrase, eqphrase> phrase_symbol_tbl;
  /* global symbol table - key and value in hash table must be identical */
  WORDTYPE *phrase_symbol_table_entry(WORDTYPE *ai) {
    if (phrase_symbol_tbl.count(ai)) {
      return phrase_symbol_tbl[ai];
    } else {
      WORDTYPE *np = new WORDTYPE[phraselen(ai)+1];
      int j;
      for (j = 0; ai[j]; j++) {
        np[j] = ai[j];
      }
      np[j] = 0;
      phrase_symbol_tbl[np] = np;
      return np;
    }
  }

  double max_binary_rule_prob;  // (log prob) used for outside prob for decoding

  // private constructor prevents new()
  ITGGrammar (string rules, string probs) {
    if (Globals::Get<int>("verbosity"))
      print_header("Loading ITG grammar");

    rulesf = rules;
    probsf = probs;
    _nonterminal_cnt = 0;
    T = 0;
    R = 0;
    U = 0;
    B = 0;

    //CLONING
    clone_mass = 0.0;
    total_mass = 0.0;
    pins = 0.0;

    Globals* globals = Globals::Instance();

    // set defaults
    defaultprob = globals->get<double>("defaultprob");
    threshold = globals->get<double>("threshold");
    mode = globals->get<char>("runmode");

    memset(N_cnt, 0, sizeof(N_cnt));

    //read info from the config file
    CLONING = globals->get<bool>("cloning");
    pins = globals->get<double>("pins");
    maxphrlen = globals->get<int>("maxphrlen");
    drop_lowcount_phrases = globals->get<bool>("drop_lowcount_phrases");

    loaded = false;
  }

  ~ITGGrammar() {
  }

  //unary rules
  inline double rulprob(int lhs, int rhs){
    Rule *rule = rule_table[rhs][-1][lhs][0][nullphr];
    if (NULL == rule)
      return 0.0;

    return rule->prob;
  }

  inline double rulcntsum(int lhs, int rhs, double val){
/*     N_cnt[lhs] += val; */
    Rule *rule = rule_table[rhs][-1][lhs][0][nullphr];
    if (NULL == rule)
      return 0.0;

    return rule->count += val;
  }

  //binary rules
  inline double rulprob(int lhs, int rhsl, int rhsr, int inv){
    Rule *rule = rule_table[rhsl][rhsr][lhs][inv][nullphr];
    if (NULL == rule)
      return 0.0;

    return rule->prob;
  }

  inline double rulcntsum(int lhs, int rhsl, int rhsr, int inv, double val){
/*     N_cnt[lhs] += val; */
    Rule *rule = rule_table[rhsl][rhsr][lhs][inv][nullphr];
    if (NULL == rule)
      return 0.0;
      
    return rule->count += val;
  }

  string nonterminal_name(unsigned int i) {
    while (i >= nonterminals.size()) {
      stringstream ss;
      ss << "V" << nonterminals.size();
      string token = ss.str();
      nonterminals.push_back(token);
      indices[token] = nonterminals.size() - 1;
      _nonterminal_cnt = nonterminals.size();
    }
    
    return nonterminals[i];
  }

  //phrases at the leaves
  double phr2phr_cntsum(WORDTYPE *, WORDTYPE *, double);
  double phr2phr_prob(WORDTYPE *, WORDTYPE *);

  //network version
  double phrase_prob(WORDTYPE *, WORDTYPE *);

  void normalize();
  void mdpnormalize();
  void load();
  void load_grammar();
  void load_phrase_table();  	/* phrase pairs with preterm specified by grammar */
  void load_binarized_grammar();  /* rules with mixed terminals and nonterminals */
  int uninonz(int n);
  int binnonz(int m, int n); //boolean function, returns true when the bigram (m,n) exits, otherwise returns false
  int biwnonz(int m, int n);
  int trinonz(int l, int m, int n);//trigram nonzero
  void dumpcnt(string);
  void dumpprb(string, string);
  void combinecnt(const char *flist);

  // reads a phrase rule into the arguments
  bool read_phrase(string, WORDTYPE *, WORDTYPE *, double *);
};

class PhrasePair : public Rule {
 public:
  WORDTYPE *l1phr, *l2phr;
  string l1phrstr, l2phrstr;
  bool leaf;

  PhrasePair(string lhs, int lhs_int, WORDTYPE *l1, WORDTYPE *l2, double prob) {
    this->lhs = lhs;
    this->lhs_int = lhs_int;
    this->prob = prob;
    l1phr = l1;
    l2phr = l2;
    leaf = true;
    binary = false;

    for (int i = 0; ; i++) {
      stringstream s;
      if (l1[i] == 0)  break;
      s << l1[i] << " ";
      l1phrstr += s.str();
    }
    l1phrstr += "0";

    for (int i = 0; ; i++) {
      stringstream s;
      if (l2[i] == 0)  break;
      s << l2[i] << " ";
      l2phrstr += s.str();
    }
    l2phrstr += "0";
  }
};

#endif



