/* itgdec-cyk.h
 *
 * Implements a basic bottom-up CYK-style decoder.
 *
 * $Id: itgdec-cyk.h,v 1.41 2008/05/08 15:12:53 post Exp $
 */

#ifndef __ITGDECCYK_H__
#define __ITGDECCYK_H__

#include <vector>
#include <set>
#include "ngram.h"
#include "itgdec.h"
#include "vocab.h"
#include "cell.h"
using namespace std;

#define DUEXASSIGN(x, y) {x[0] = y[0]; x[1] = y[1];}


/** DECODING ****************************************************/

// DINT is the type used for each boundary word in the language model state.
// It used to be a char that was mapped to English words on a per-sentence basis.
// Now it is just the English word, represetned as a WORDTYPE.
typedef WORDTYPE DINT;

struct LMSTATE {
  DINT u0, u1, v0, v1;
  bool operator==(const LMSTATE x) const { 
    return u0 == x.u0 && u1 == x.u1 && v0 == x.v0 && v1 == x.v1; 
  }
};

BEGIN_HASH_NAMESPACE
  template<>
  struct hash<LMSTATE>
    { 
      size_t
      operator()(LMSTATE __x) const
      { return __x.u0+__x.u1+__x.v0+__x.v1; }
    };
END_HASH_NAMESPACE

struct ITGDCell;

struct backpointer {
  Rule* rule;
  ITGDCell* lchild;
  ITGDCell* rchild;
  double prob;
  bool tri; // trigram states' backpointers
  backpointer(){};
  backpointer(Rule* irule, ITGDCell* ilchild, ITGDCell* irchild, double iprob) {
    rule = irule;
    lchild = ilchild;
    rchild = irchild;
    prob = iprob;
  }
};

struct backpointer_lt { 
  bool operator()(const backpointer& a, const backpointer& b) {
    return a.prob <= b.prob;
  }
};

bool operator>(const backpointer&, const backpointer&);
bool operator==(const backpointer&, const backpointer&);

/* typedef TopK<backpointer> backptrs_t; */
typedef vector<backpointer> bkptr_v;

struct ITGDCell : public Cell {
  static int NUM_BACKPOINTERS;

  // dp state
  int N, s, t;
  WORDTYPE lword[2], rword[2]; 

  double prob; // accumulated probability 
  double outside; // outside probability

  //  double maxe; //expected value

  // backpointers
  bkptr_v bkptrs;

  // backpointers tracking the best way to construct this cell
  Rule* rule;
  ITGDCell* lchild, *rchild;

  // for terminal cells
  WORDTYPE* ephr;
  WORDTYPE* cphr;

  char inserted[2]; //for both passes

  char closed; //flag

  ITGDCell(int Ni, int si, int ti, WORDTYPE lwordi, WORDTYPE liwordi, WORDTYPE riwordi, WORDTYPE rwordi) {
    N = Ni;
    s = si;
    t = ti;
    lword[0] = lwordi;
    lword[1] = liwordi;
    rword[1] = riwordi;
    rword[0] = rwordi;
    lchild = rchild = NULL;
    rule = NULL;
    ephr = cphr = NULL;
    inserted[0] = inserted[1] = closed = 0;
    prob = -INFINITY;
/*     outside = -INFINITY; */
  }    

  ITGDCell() {
    // this *should* determine if the CYK decoder is being used as a
    // coarse-level decoder and set k high only in that case.  for now
    // it assumes that bigrams = coarse
/*     backptrs.k = NUM_BACKPOINTERS; */
    prob = outside = -INFINITY;
    N = s = t = -1;
    rule = NULL;
    lchild = rchild = NULL;
    ephr = cphr = NULL;
    inserted[0] = inserted[1] = closed = 0;
  }
}; 

struct border_t {
  double left;
  WORDTYPE lword;
  double right;
  WORDTYPE rword;
};

typedef vector<ITGDCell *> pd_vec;



///////////////////////////////////////////
typedef struct kb_item_struct{
  unsigned int l;
  unsigned int r; //indices 
  int S; //split point
  Rule *rul; //applied rule 
  DINT ul[2];
  DINT vl[2];
  DINT ur[2];
  DINT vr[2];
  double c; //cost
}kb_item_t;


struct kbless: public binary_function<kb_item_t, kb_item_t, bool>{
  bool operator()(const kb_item_t& kb1, const kb_item_t& kb2) const{
    return kb1.c<kb2.c;
  }
};


typedef priority_queue<kb_item_t, vector<kb_item_t>, kbless> kb_cand_t;


struct pdless:public binary_function<ITGDCell*, ITGDCell*, bool>{
  bool operator()(const ITGDCell* pd1, const ITGDCell* pd2) const{
    return pd1->prob<pd2->prob;
  }
};


//the priority queue implementation
typedef priority_queue<ITGDCell *, vector<ITGDCell*>, pdless> kv_buf_t;



typedef struct ITGKbest_struct{
  vector <ITGDCell *> KV; //vector of kbests, after re-ranking
  //  vector <ITGDCell *> KV_buf; //before re-ranking
  kv_buf_t KV_buf;
  //  hash_map <long, int> KV_buf_map;
  kb_cand_t Kand;
  double bound; //lower bound on the cost, upper bound on the prob
} ITGKb;

/////////////////////////////////////////////////////

/** class ITGDChart
 *
 *  This is the chart used for bottom-up decoding.  The most important
 *  data structure is cells, which is an array of array of hashes.
 *  The outer level array is a unique position for every (s,t) span,
 *  computed based on the maximum width of the input sentence.  The
 *  next level of arrays is the nonterminal of the span.  The hash map
 *  maps from integers to the actual cells.  The integer keys are
 *  computed from (t_1,t_2,t_3,t_4) target word tuples, where each t_i
 *  is drawn from a finite set of translation candidates.
 */
struct ITGDChart {
  int T; // number of words in the source sentence
  int dimenT;

public:

  hash_map <int, hash_map <LMSTATE, ITGDCell > > *cells;
  hash_map <int, hash_map <LMSTATE, ITGDCell > > *hooks;
  
  hash_map <DINT, hash_map <int, hash_map <LMSTATE, ITGDCell > > > *indexedhooks;

  hash_map <int, ITGKb > *kbests; 

  // outside estimators
  //  hash_map <WORDTYPE,border_t>** bestborder;
  hash_map <WORDTYPE, hash_map <WORDTYPE, border_t > > bestborder;

  hash_map <int, hash_map <int, double> > bestpossible;
  hash_map <int, hash_map <int,  hash_map <int, hash_map <int, border_t > > > > consbestborder;

  /** Constructor
   *  Creates the list of cells.
   *  mt = maximum words in (translation) source sentence
   *  mv = maximum words in target sentence
   *  nc = number of nonterminals
   */
  ITGDChart(DINT mt) {
    T = mt;

    dimenT = T * (T + 3) / 2 + 1;

    cells = new hash_map <int, hash_map<LMSTATE, ITGDCell> > [dimenT];
    
    hooks = new hash_map <int, hash_map<LMSTATE, ITGDCell> > [dimenT];

    indexedhooks = new hash_map <DINT, hash_map <int, hash_map <LMSTATE, ITGDCell > > > [dimenT];


    kbests = new hash_map <int, ITGKb > [dimenT]; 

  }



#define ESTWEIGHT 1

  /* Returns the estimate for the outside probability computed
   * from one of dinit_{bigram|trigram}_estimate
   */
  double estimate(int n, int s, int t, int u0, int u1, int v1, int v0) {

    /*    double a = bestpossible[s][t]; */
    /*    double left = consbestborder[u0][u1][s][t].left; */
    /*    double right = consbestborder[v1][v0][s][t].right; */
    double ruleprb = grammar->max_binary_rule_prob;
    int outside_span = T - t + s; // T - (t - s) -- assume n words
    // would be covered with n-1 rules

    //   double est = a + ((left + right) * LMWEIGHT) + ruleprb * outside_span;
    double est = ruleprb * outside_span;
    return est*ESTWEIGHT;
  }

  ~ITGDChart() { 
    delete[] cells;

    delete[] hooks;

    delete[] indexedhooks;

    delete[] kbests; 
  }

  void dump(int sentno, int s, int t);
  void dump(int sentno);

  /** ITGDChart::cell()
   *  Returns the specified decoding chart cell.
   *  n = index of nonterminal
   *  s,t = starting,ending pos in L2
   *  {lr}_{io} = indices of words, l = left, r = right, i = inner, o = outer
   */
  ITGDCell *cell(int n, int s, int t, int l_o, int l_i, int r_i, int r_o) {
    // TODO: bounds checking

    // offset(s,t) maps {(0,0) -> 0, (0,1) -> 1, (0,T) -> T, (1,1) -> T+1, (1,2) -> T+2} ...
    int offset = s * (T + 1) - s * (s + 1) / 2 + t;
    LMSTATE wordpos = candidate_offset(l_o, l_i, r_i, r_o);

    return &cells[offset][n][wordpos];
  }

  ITGDCell *hook(int n, int s, int t, int l_o, int l_i, int r_i, int r_o) {
    int offset = s * (T + 1) - s * (s + 1) / 2 + t;
    LMSTATE wordpos = candidate_offset(l_o, l_i, r_i, r_o);

    return &hooks[offset][n][wordpos];
  }

  ITGDCell *indexedhook(int n, int s, int t, int l_o, int l_i, int r_i, int r_o) {
    int offset = s * (T + 1) - s * (s + 1) / 2 + t;
    LMSTATE wordpos = candidate_offset(l_o, l_i, r_i, r_o);

    return &indexedhooks[offset][l_o][n][wordpos];
  }


  ITGDCell *cook(char hk, int n, int s, int t, int l_o, int l_i, int r_i, int r_o) {
    if (hk)
      return hook(n, s, t, l_o, l_i, r_i, r_o);
    else
      return cell(n, s, t, l_o, l_i, r_i, r_o);
  }

  
  ITGKb *KBESTS(int s, int t, int n){
    int offset = s * (T + 1) - s * (s + 1) / 2 + t;    

    return &kbests[offset][n];
  }

  /** ITGDChart::candidate_offset
   *
   *  Given a set of four word translation candidates that are the
   *  borders of a cell, compute their offset into an array that holds
   *  them.
   */
  inline LMSTATE candidate_offset(int l_o, int l_i, int r_i, int r_o)
  {
    LMSTATE lm;
    lm.u0 = l_o; lm.u1 = l_i; lm.v1 = r_i; lm.v0 = r_o;
    return lm;
  }

  LMSTATE u01v01(int u0, int u1, int v1, int v0) {
    LMSTATE lm;
    lm.u0 = u0; lm.u1 = u1; lm.v1 = v1; lm.v0 = v0;
    return lm;
  }

  int u1of(LMSTATE x) {    
    return x.u1;
  }
  int v1of(LMSTATE x) {    
    return x.v1;
  }
  int u0of(LMSTATE x) {     
    return x.u0;
  }
  int v0of(LMSTATE x) {    
    return x.v0;
  }


  /** ITGDChart::column()
   *
   *  Returns the column of entries for a given (nonterminal, input
   *  range).  This is a number of chart entries differing in (a)
   *  their outside english words and (b) in their composition
   *  (potentially) from subconstituents.
   *
   *  s,t are the start and end positions over the input
   *  n is the nonterminal
   */
  hash_map <LMSTATE, ITGDCell> * column(int n, int s, int t) {
    int offset = s * (T + 1) - s * (s + 1) / 2 + t;
    return &cells[offset][n];
  }

  hash_map <LMSTATE, ITGDCell> *hookcolumn(int s, int t, int n, DINT v){
    int offset = s * (T + 1) - s * (s + 1) / 2 + t;
    return &indexedhooks[offset][v][n];
  }


  
  int has_column(int n, int s, int t) {
    int offset = s * (T + 1) - s * (s + 1) / 2 + t;
    return cells[offset].count(n);
  }


  hash_map<int, hash_map <LMSTATE, ITGDCell> > * span(int s, int t) {
    int offset = s * (T + 1) - s * (s + 1) / 2 + t;
    return &cells[offset];
    
  }




  void clean(){
    for (int t = 0; t < dimenT; t++){
      cells[t].clear();
      hooks[t].clear();

      indexedhooks[t].clear();
    }

    for (int t = 0; t < dimenT; t++)
      kbests[t].clear();
    
  }

  //  inline double outsideof(int s, int t, int u, int u1, int v, int v1, int n){
  //    return 0;
  //  }
  
  inline double outsideof(int s, int t, int u, int u1, int v, int v1, int n){
    int offset = s * (T + 1) - s * (s + 1) / 2 + t;
    LMSTATE wordpos = candidate_offset(u, u1, v1, v);

    if (cells[offset].count(n)){
      if (cells[offset][n].count(wordpos)){
	{
	  return cell(n, s, t, u, u1, v1, v)->inserted[0]=='v'? cell(n, s, t, u, u1, v1, v)->outside:-1000;
	}
      }
    }
    return -1000;
  }
  
};


/** ITGHChart
 *
 * used for hooks
 */
class ITGHChart {
 public:
  //   ITGDCell *cells;
  // hash_map <int, hash_map <int, hash_map <int, ITGDCell> > > *cells;
  hash_map <int, ITGDCell> **cells;
  double **bestpossible;

  border_t **bestborder;

  //  int totalDimen;
  DINT MAXT;
  DINT MAXV;//max number of translation word candicates for a sentence in lang 2

  int dimenT;
  //int dimenV;

  ITGHChart(DINT mt, DINT mv, int nc){
    MAXT = mt;
    MAXV = mv;
    
    dimenT = MAXT * (MAXT + 3) / 2 + 1;

    cells = new hash_map <int, ITGDCell > *[dimenT];
    for (int t = 0; t < dimenT; t++){
      cells[t] = new hash_map <int , ITGDCell>[MAXN * MAXN];
    }

    bestpossible = new double *[MAXT+1];
    for (int t = 0; t <= MAXT; t++){
      bestpossible[t] = new double[MAXT+1];
    }

    /*     printf("MAXV is %d\n", MAXV); */
    bestborder = new border_t *[((int)MAXV)+1];
    for (int s=0; s<=MAXV; s++){
      bestborder[s] = new border_t[((int)MAXV)+1];
    }
  }

  ~ITGHChart(){ 
    for (int t=0; t<dimenT; t++){
      delete[] cells[t];
    }
    delete[] cells;

    
    for (int t=0; t<=MAXT; t++){
      delete[] bestpossible[t];
    }
    delete[] bestpossible;


    for (int s=0; s<=MAXV; s++){
      delete[] bestborder[s];
    }
    delete[] bestborder;
  }
  
  void clean(int T, int V, int N) {
    //  memset(cells, 0, sizeof(ITGDCell)*MAXN*totalDimen);
    for (int t=0; t<dimenT; t++)
      for (int n=0; n<MAXN*MAXN; n++)
        cells[t][n].clear();
  }
};


/** agenda_node_t
 *
 * This class represents an entry in the agenda.
 */
class agenda_node_t {
 public:
  agenda_node_t(char s_arg, char t_arg, 
                WORDTYPE u0_arg, WORDTYPE u1_arg, WORDTYPE v0_arg, WORDTYPE v1_arg, char n_arg, double p_arg) {
    s = s_arg;
    t = t_arg;
    u[0] = u0_arg;
    u[1] = u1_arg;
    v[0] = v0_arg;
    v[1] = v1_arg;
    n = n_arg;
    p = p_arg;
  }

  agenda_node_t(char s_arg, char t_arg, 
                WORDTYPE u0_arg, WORDTYPE u1_arg, WORDTYPE v0_arg, WORDTYPE v1_arg, char n_arg, char hk_arg, double p_arg) {
    s = s_arg;
    t = t_arg;
    u[0] = u0_arg;
    u[1] = u1_arg;
    v[0] = v0_arg;
    v[1] = v1_arg;
    n = n_arg;
    p = p_arg;

    h=hk_arg;
  }

  string data() {
    char str[100];
    char sym = (n == grammar->_nonterminal_cnt) ? '*' : grammar->nonterminals[n][0]; // use special * for goal
    sprintf(str, "[%c(%d,%d) %.10e]", sym, s, t, p);
    return string(str);
  }

  double p;
  char s;
  char t;
  WORDTYPE u[2];
  WORDTYPE v[2];
  int n;
  char h;
};


struct prless: public binary_function<agenda_node_t, agenda_node_t, bool>{
  bool operator()(const agenda_node_t& an1, const agenda_node_t& an2) const{
    return an1.p < an2.p;
  }
};

typedef priority_queue<agenda_node_t, vector<agenda_node_t>, prless> agenda_t;

class neighbor_node_t {
 public:
  neighbor_node_t(char S_arg, WORDTYPE u0_arg, WORDTYPE u1_arg, WORDTYPE v0_arg, WORDTYPE v1_arg){
    S = S_arg;
    u[0] = u0_arg;
    u[1] = u1_arg;
    v[0] = v0_arg;
    v[1] = v1_arg;
  }

  char S;
  WORDTYPE u[2];
  WORDTYPE v[2];
};


//vector imple
typedef vector<neighbor_node_t> neighbor_t;

class CYKDecoder : public Decoder {
 public:
  CYKDecoder(ITGGrammar*, ITGCorp* ic = NULL);
  ~CYKDecoder();

  int trivecsize;

  int itemcount; //the counter for the explored hyperedges
  int hookitemcount; 
  int triitemcount;

  void run();

  int readin(ITGCorp *corp);  // overloaded
  void readin(WORDTYPE*, int, int);  // overloaded

  void init_phrases(); // loads phrase translations

  void dinit(bool firstpass); //initialize sita, before decoding
  void compute_estimate(bool bilm); //compute outside estimate
  inline double borderest(int u, int u1, int v, int v1){
    //    return 0;
    return dchart->bestborder[u][u1].left + dchart->bestborder[v1][v].right; 
  }
  string decode();//get the best translation of the chinese(l2) sentence
  string decode_cyk();
  string decode_agenda();
  string decode_agenda_bitri(bool bilm);//bi-tri two pass
  string decode_hook_agenda(bool standalone);

  string decode_cyk_bipass(); //cyk bigram pass
  string decode_cyk_hook_bipass();
  string decode_hook_agenda_bipass(); //hook-agenda bi-pass

  string topdowndecode(); 
  void lazyjthbest(int s, int t, int n, unsigned int j);


  ITGKb *KBESTS(int s, int t, int n){return dchart->KBESTS(s, t, n);};
  void cyk_bottom_up();
  void cyk_hook_bottom_up();
  hash_map<int, hash_map <LMSTATE, ITGDCell> > * matchinghookspan(int s, int t, int n, DINT v, bool left);

  void cyk_top_down();
  void compute_outside_prob(int,int,int,int);

  double rootlmcost(int *u, int *v); 

  string reconstruct(int, int, ITGDCell*, int indent = 0);
  string reconstruct(ITGDCell*, int indent = 0);

  string build_leaves(const ITGDCell*, bool delim = false);

  void depthfirst(ITGDCell *pd, pd_vec &thevec, int biortri);

  void outsidepass(ITGDCell *pd);

  void trioutsidepass(ITGDCell *pd);

  double maxEpass(ITGDCell *pd);

  // enters the kbest translations onto the chart
 void fillin(int, int, list <WORDTYPE*> &);

  // decoding chart
  ITGDChart *dchart;
  // contains translation hypotheses for each source span
  WORDTYPE ****enclosure;

  int edgecount;

  //  vector<WORDTYPE> localvocab_l, localvocab_r; //list of boundary words in the english phrases
  hash_map <WORDTYPE, int > senvocab_l, senvocab_r, senvocab; //list of boundary words / words in the english phrases
};

#endif
