/********************************************************************************************************
 * QRNA - Comparative analysis of biological sequences 
 *         with pair hidden Markov models, pair stochastic context-free
 *        grammars, and probabilistic evolutionary  models.
 *       
 * Version 2.0.0 (JUN 2003)
 *
 * Copyright (C) 2000-2003 Howard Hughes Medical Institute/Washington University School of Medicine
 * All Rights Reserved
 * 
 *     This source code is distributed under the terms of the
 *     GNU General Public License. See the files COPYING and LICENSE
 *     for details.
 ***********************************************************************************************************/

/* othdp.c
 *
 * ER, Mon Jul 26 10:06:21 CDT 1999 [St. Louis]
 * 
 * dynamic programming (viterbi and forward) with the othermodel
 *
 * calculates:
 *                       P(seqX,seqY \pi^* | othemodel)  [viterbi algorithm; \pi^* = best path ]
 *              \sum_\pi P(seqX,seqY \pi   | othemodel)  [forward algorithm ]
 * 
 */

#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <ctype.h>
#include <limits.h>

#include "funcs.h"
#include "globals.h"
#include "squid.h"
#include "structs.h"

#ifdef MEMDEBUG
#include "dbmalloc.h"
#endif

static double forwB(int i, int j, int Lx, int Ly, struct othdpf_s *dp, struct othmodel_s *oth);

static double forwBdiag(int i, struct othdpd_s *dp, struct othmodel_s *oth);
static double forwMdiag(int i, int L, int cur_x, int cur_y, struct othdpd_s *dp, struct othmodel_s *oth);
static double forwXdiag(int i, int L, int cur_x, int cur_y, struct othdpd_s *dp, struct othmodel_s *oth);
static double forwYdiag(int i, int L, int cur_x, int cur_y, struct othdpd_s *dp, struct othmodel_s *oth);
static double forwEdiag(int i, int cur_x, int cur_y, struct othdpd_s *dp, struct othmodel_s *oth);

static void tracebackOTH_L2(FILE *ofp, SQINFO sqinfoX, int *seqX, SQINFO sqinfoY, int *seqY, 
			    int startX, int startY, int Lx, int Ly, int *ret_len_ali,
			    double score, struct othdpf_s *dp, struct othmodel_s *oth, 
			    struct ali_s *ali, int alignment, int traceback, int voutput, struct  end_s *othends);
static void tracebackOTHdiag_L(FILE *ofp, SQINFO sqinfoX, int *seqX, SQINFO sqinfoY, int *seqY, 
			       int start, int L, double score, struct othdpd_s *dp,
			       struct othmodel_s *oth, struct ali_s *ali, 
			       int alignment, int traceback, int voutput, struct  end_s *othends);

/* Function: AllocDpDiagOTH()
 * Date:     ER, Mon Jul 26 14:56:57 CDT 1999 [St. Louis]
 *
 * Purpose:  Allocates memory for the dp matrices of the OTH model- diagonal DP
 *
 * Returns:  othdp_s are allocated
 */
struct othdpd_s *
AllocDpDiagOTH(int L)
{
  struct othdpd_s *othdp;       /* structure with dp matrices   */
  int i, d;

  othdp = (struct othdpd_s *) MallocOrDie (sizeof(struct othdpd_s));

  othdp->flmx    = (double *)  MallocOrDie (sizeof(double)   * (L+1)            );
  othdp->frmx    = (double *)  MallocOrDie (sizeof(double)   * (L+1)            );
  othdp->fjmx    = (double **) MallocOrDie (sizeof(double *) *  L               );
  othdp->fjmx[0] = (double *)  MallocOrDie (sizeof(double)   *  L    * (L+3) / 2);

  othdp->bmx = (double *) MallocOrDie (sizeof(double) * L);
  othdp->mmx = (double *) MallocOrDie (sizeof(double) * L);
  othdp->xmx = (double *) MallocOrDie (sizeof(double) * L);
  othdp->ymx = (double *) MallocOrDie (sizeof(double) * L);
  othdp->emx = (double *) MallocOrDie (sizeof(double) * L);
  
  for (i = 0; i < L; i++) 
    othdp->fjmx[i] = othdp->fjmx[0] + i*(i+3)/2;

  for (i = 0; i < L; i++) {
    othdp->flmx[i] = -BIGFLOAT;
    othdp->frmx[i] = -BIGFLOAT;
    
    for (d = 0; d <= i+1; d++) 
      othdp->fjmx[i][d] = -BIGFLOAT;
    
    othdp->bmx[i] = -BIGFLOAT;
    othdp->mmx[i] = -BIGFLOAT;
    othdp->xmx[i] = -BIGFLOAT;
    othdp->ymx[i] = -BIGFLOAT;
    othdp->emx[i] = -BIGFLOAT;
  }
  othdp->flmx[L] = -BIGFLOAT;
  othdp->frmx[L] = -BIGFLOAT;
  
  return othdp;
}

/* Function: AllocDpFullOTH()
 * Date:     ER, Sun Nov 28 17:59:34 CST 1999 [St. Louis]
 *
 * Purpose:  Allocates memory for the dp matrices of the OTH model- full DP
 *
 * Returns:  othdp_s are allocated
 */
struct othdpf_s *
AllocDpFullOTH(int Lx, int Ly)
{
  struct othdpf_s *othdp;       /* structure with dp matrices   */
  int i, j;
  int dx, dy;
  int ij;

  othdp = (struct othdpf_s *) MallocOrDie (sizeof(struct othdpf_s));

  othdp->flxmx    = (double *)  MallocOrDie (sizeof(double)   * (Lx+1)              );
  othdp->frxmx    = (double *)  MallocOrDie (sizeof(double)   * (Lx+1)              );
  othdp->fjxmx    = (double **) MallocOrDie (sizeof(double *) * (Lx+1)              );
  othdp->fjxmx[0] = (double *)  MallocOrDie (sizeof(double)   *  Lx * (Lx+3) / 2 + 1);

  othdp->flymx    = (double *)  MallocOrDie (sizeof(double)   * (Ly+1)              );
  othdp->frymx    = (double *)  MallocOrDie (sizeof(double)   * (Ly+1)              );
  othdp->fjymx    = (double **) MallocOrDie (sizeof(double *) * (Ly+1)              );
  othdp->fjymx[0] = (double *)  MallocOrDie (sizeof(double)   *  Ly * (Ly+3) / 2 + 1);

  othdp->bmx = (double *) MallocOrDie (sizeof(double) * (Lx+1) * (Ly+1));
  othdp->mmx = (double *) MallocOrDie (sizeof(double) * (Lx+1) * (Ly+1));
  othdp->xmx = (double *) MallocOrDie (sizeof(double) * (Lx+1) * (Ly+1));
  othdp->ymx = (double *) MallocOrDie (sizeof(double) * (Lx+1) * (Ly+1));
  othdp->emx = (double *) MallocOrDie (sizeof(double) * (Lx+1) * (Ly+1));
  
  for (i = 0; i < Lx; i++) {
    othdp->fjxmx[i] = othdp->fjxmx[0] + i*(i+3)/2;

    for (dx = 0; dx <= i+1; dx++) 
      othdp->fjxmx[i][dx] = -BIGFLOAT;
  }
  
  for (j = 0; j < Ly; j++) {
    othdp->fjymx[j] = othdp->fjymx[0] + j*(j+3)/2;

    for (dy = 0; dy <= j+1; dy++) 
      othdp->fjymx[j][dy] = -BIGFLOAT;
  }
  othdp->fjxmx[Lx] = othdp->fjxmx[0] + Lx*(Lx+3)/2;
  othdp->fjymx[Ly] = othdp->fjymx[0] + Ly*(Ly+3)/2;

  othdp->fjxmx[Lx][0] = -BIGFLOAT;
  othdp->fjymx[Ly][0] = -BIGFLOAT;
  
  for (i = 0; i <= Lx; i++) 
    othdp->flxmx[i] = -BIGFLOAT;

  for (j = 0; j <= Ly; j++) 
    othdp->flxmx[j] = -BIGFLOAT;

  for (i = 0; i <= Lx; i++) 
    for (j = 0; j <= Ly; j++) {
      ij = i*Ly + j;
      othdp->bmx[ij] = -BIGFLOAT;
      othdp->mmx[ij] = -BIGFLOAT;
      othdp->xmx[ij] = -BIGFLOAT;
      othdp->ymx[ij] = -BIGFLOAT;
      othdp->emx[ij] = -BIGFLOAT;
    }
  
  return othdp;
}

/* Function: AllocMatrixDiag()
 * Date:     ER, Tue Jan 23 09:22:23 CST 2001 [St. Louis]
 *
 * Purpose:  Allocates memory for a matrix that contains all the scores of 
 *           a diagonal alignment.
 *
 *           m[i][l] 
 *
 *             with       i = 0, L-1 --- first position in alignmente
 *                        l = 0, i+1 --- number of nucleotides in alignment.
 *
 *                 aligned position from i to (i+l-1)
 *
 * Returns:  m is allocated
 */
double **
AllocMatrixDiag(int L)
{
  int      i, d;
  double **m;

  m    = (double **) MallocOrDie (sizeof(double *) * L);
  m[0] = (double  *) MallocOrDie (sizeof(double  ) * L * (L+3) / 2);

  for (i = 1; i < L; i++) 
    m[i] = m[0] + i*(i+3)/2;
  
  for (i = 0; i < L; i++)  
    for (d = 0; d <= i+1; d++) 
      m[i][d] = -BIGFLOAT;
  
  return m;
}

/* Function: AllocMatrixFull()
 * Date:     ER, Tue Jan 23 09:30:13  CST 2001 [St. Louis]
 *
 * Purpose:  Allocates memory for a matrix that contains all the scores of 
 *           a non-diagonal alignment.
 *
 *           m[i(Ly+1)+j][dx(j+1)+dy]  
 *
 *             with       i  = 0, Lx-1 
 *                        dx = 0, i   
 *                        j  = 0, Ly-1 
 *                        dy = 0, j  
 *
 *                   aligns fragments: (i-dx,i) to (j-dy,j)
 *
 *
 *           m[i(Ly+1)+Ly][dx] 
 *
 *             with       i  = 0, Lx-1 
 *                        dx = 0, i   
 *
 *                   aligns fragments: (i-dx,i) to nothing
 *
 *
 *           m[Lx(Ly+1)+j][dy]
 *
 *             with       j  = 0, Ly-1 
 *                        dy = 0, y   
 *
 *                   aligns fragments: nothing to (j-dy,j)
 *
 *
 *           m[Lx(Ly+1)+Ly][0] aligns nothing to noting
 *
 * Returns:  m is allocated
 */
double **
AllocMatrixFull(int Lx,  int Ly)
{
  int      i, dx;
  int      j, dy;
  int      ij;
  int      prv_ij;
  int      add;
  double **m;

  m    = (double **) MallocOrDie (sizeof(double *) * (Lx+1) * (Ly+1));
  m[0] = (double *)  MallocOrDie (sizeof(double)   * (Lx*(Lx+1)/2 + 1) * (Ly*(Ly+1)/2 + 1));

  add =  0;
  prv_ij = 0;
  for (i = 0; i < Lx; i++) 
    for (j = 0; j < Ly; j++) {
      ij = i*(Ly+1) + j;

      if (ij > 0) add += (prv_ij/(Ly+1) + 1 )  * (prv_ij%(Ly+1) + 1 );

      m[ij] = m[0] + add;
      prv_ij = ij;
    }

  for (i = 0; i < Lx; i++) 
    m[i*(Ly+1)+Ly] = m[0] + (Lx*(Lx+1)/2)*(Ly*(Ly+1)/2) + (i*(i+1)/2);
  
  for (j = 0; j < Ly; j++) 
    m[Lx*(Ly+1)+j] = m[0] + (Lx*(Lx+1)/2)*(Ly*(Ly+1)/2) + (Lx*(Lx+1)/2) + (j*(j+1)/2);
  
  m[Lx*(Ly+1)+Ly] =  m[0] + (Lx*(Lx+1)/2)*(Ly*(Ly+1)/2) + (Lx*(Lx+1)/2) + (Ly*(Ly+1)/2);
  
  for (i = 0; i < Lx; i++) 
    for (j = 0; j < Ly; j++) 
      for (dx = 0; dx <= i; dx++) 
	for (dy = 0; dy <= j; dy++) 
	  m[i*(Ly+1)+j][dx*(j+1)+dy] = -BIGFLOAT;

  for (i = 0; i < Lx; i++) 
    for (dx = 0; dx <= i; dx++) 
      m[i*(Ly+1)+Ly][dx] = -BIGFLOAT;
  
  for (j = 0; j < Ly; j++)  
    for (dy = 0; dy <= j; dy++) 
      m[Lx*(Ly+1)+j][dy] = -BIGFLOAT;
  
  m[Lx*(Ly+1)+Ly][0] = -BIGFLOAT;
  
  return m;
}

/* Function: CompareOTH()
 * Date:     ER, Fri Dec 31 06:27:11 CST 1999 [Panticosa]
 *
 * Purpose:  Compare different functions created with the OTH model
 *          
 *           1.- ScoreWithOTHMatrix()
 *           2.- ViterbiOTHDiagMatrix()         
 *           3.- ForwardOTHDiagMatrix()         
 *
 * Args:     seqX, seqY   -- equal length sequences, ACGT and gaps (-) 
 *           start        -- starting position
 *           L            -- lengths of sX,sY 
 *           oth          -- oth_model structure
 *           dp           -- matrices for dynamic programming
 *           ali          -- array with alignment
 *           traceback    -- if true due traceback
 *           alignment    -- if true print alignment
 *
 * Returns:  void.
 */
void
CompareOTH(FILE *ofp, SQINFO sqinfoX, int *seqX, SQINFO sqinfoY, int *seqY, int start, int L, 
	   struct othmodel_s *oth, struct othdpd_s *othdp, double *sc, struct ali_s *ali, 
	   int alignment, int traceback)
{
  int      i, d;
  int      iabs;
  double **m1, **m2, **m3;
  
  m1 = AllocMatrixDiag(L);
  m2 = AllocMatrixDiag(L);
  m3 = AllocMatrixDiag(L);
 
  ViterbiOTHDiagMatrix(ofp, m2, sqinfoX, seqX, sqinfoY, seqY, start, L, oth, othdp);
  ForwardOTHDiagMatrix(ofp, m3, seqX, seqY, start, L, oth, othdp, sc);
  
  for (i = 0; i < L; i++) { 
    iabs = i + start;
    
    m1[i][0] = ScoreWithOTH(ofp, seqX, seqY, iabs, 0, oth, FALSE);
    
    for (d = 1; d <= i+1; d++)       
      if (i == 0 || d == 1) 
	m1[i][d] = ScoreWithOTH(ofp, seqX, seqY, iabs, d, oth, FALSE);
      else 
	m1[i][d] = m1[i-1][d-1] + ScoreWithOTHMatrix(seqX, seqY, iabs, oth); 
  }
  
  for (i = 0; i < L; i++)
    for (d = 0; d <= i+1; d++) 
      printf ("check %d %d  | sc = %6.4f vi = %6.4f fr = %6.4f \n", i, d, m1[i][d], m2[i][d], m3[i][d]);
  
  free(m1[0]); 
  free(m2[0]); 
  free(m3[0]);
  free(m1); 
  free(m2); 
  free(m3);
}

/* Function: CompareOTHFull()
 * Date:     ER, Fri Dec 31 06:27:11 CST 1999 [Panticosa]
 *
 * Purpose:  Compare different functions created with the OTH model
 *          
 *           1.- chech -- ViterbiOTH_L2(()
 *           2.- m[]   -- ViterbiOTHFullVector()       
 *           3.- v[]   -- ForwardOTHFullMatrix()         
 *
 * Args:     seqX, seqY   -- equal length sequences, ACGT and gaps (-) 
 *           start        -- starting position
 *           L            -- lengths of sX,sY 
 *           oth          -- oth_model structure
 *           dp           -- matrices for dynamic programming
 *           ali          -- array with alignment
 *           traceback    -- if true due traceback
 *           alignment    -- if true print alignment
 *
 * Returns:  void.
 */
void
CompareOTHFull(FILE *ofp, SQINFO sqinfoX, int *seqX, SQINFO sqinfoY, int *seqY, 
	       int startX, int Lx, int startY, int Ly, int *ret_leg, int Lw,
	       struct othmodel_s *oth, struct othdpf_s *othdp, struct ali_s *ali, struct windowends_s *windowends)
{
  struct end_s *othends;
  int      i, dx;
  int      j, dy;
  int      iabs, jabs;
  int      ij, dxy;
  int      leg;
  double   check, acc;
  double **m, *v;

  othends = windowends->fwd->oth;

  leg = *ret_leg;

  acc = 0.00001;

  m = AllocMatrixFull(Lx, Ly);
  v = (double *)  MallocOrDie (sizeof(double) * (Lx+1) * (Ly+1));

  for (i = 0; i <= Lx; i++) 
    for (j = 0; j <= Ly; j++) 
      v[i*(Ly+1)+j] = -BIGFLOAT;
    
  ViterbiOTHFullMatrix(ofp, m, sqinfoX, seqX, sqinfoY, seqY, startX, startY, Lx, Ly, Lw, oth, othdp);
  ViterbiOTHFullVector(ofp, v, sqinfoX, seqX, sqinfoY, seqY, startX, startY, Lx, Ly, oth, othdp);
  
  for (i = 0; i <= Lx; i++) { 
    iabs = i + startX;
    for (j = 0; j <= Ly; j++) { 
      jabs = j + startY;
      
      ij = i*(Ly+1) + j;

      if (i == Lx && j == Ly) {
	check = ViterbiOTH_L2(stdout, sqinfoX, seqX, sqinfoY, seqY, startX+Lx, startY+Ly, 0, 0, &leg, Lw,
			      oth, othdp, ali, FALSE, FALSE, FALSE, FALSE, othends);
	if (m[ij][0] > check+acc || m[ij][0] < check-acc)
	  printf ("check matrix -- -- -- --  | m = %6.4f vi = %6.4f \n", m[ij][0], check);
	
	if (v[ij] > check+acc || v[ij] < check-acc)
	  printf ("-->check vector -- -- | v = %6.4f vi = %6.4f \n", v[ij], check);
      }
      else if (j == Ly) {
	for (dx = 0; dx <= i; dx++) {
	  
	  check = ViterbiOTH_L2(stdout, sqinfoX, seqX, sqinfoY, seqY, iabs-dx, startY+Ly, dx+1, 0, &leg, Lw,
				oth, othdp, ali, FALSE, FALSE, FALSE, FALSE, othends);
	  
	  if (m[ij][dx] > check+acc || m[ij][dx] < check-acc)
	    printf ("check matrix %d -- %d --  | m = %6.4f vi = %6.4f \n", i, dx, m[ij][dx], check);
	  
	  if (dx == i)  
	    if (v[ij] > check+acc || v[ij] < check-acc)
	      printf ("-->check vector %d -- | v = %6.4f vi = %6.4f \n", i, v[ij], check);
	}
      }
      else if (i == Lx) {
	for (dy = 0; dy <= j; dy++) {
	  
	  check = ViterbiOTH_L2(stdout, sqinfoX, seqX, sqinfoY, seqY, startX+Lx, jabs-dy, 0, dy+1, &leg, Lw,
				oth, othdp, ali, FALSE, FALSE, FALSE, FALSE, othends);
	  

	  if (m[ij][dy] > check+acc || m[ij][dy] < check-acc)
	    printf ("check matrix -- %d -- %d  | m = %6.4f vi = %6.4f \n", j, dy, m[ij][dy], check);
	  
	  
	  if (dy == j)  
	    if (v[ij] > check+acc || v[ij] < check-acc)
	      printf ("-->check vector -- %d | v = %6.4f vi = %6.4f \n", j, v[ij], check);
	}
      }
      else {
      for (dx = 0; dx <= i; dx++) 
	for (dy = 0; dy <= j; dy++) {

	  dxy = dx*(j+1) + dy;

	  check = ViterbiOTH_L2(stdout, sqinfoX, seqX, sqinfoY, seqY, iabs-dx, jabs-dy, dx+1, dy+1, &leg, Lw,
				oth, othdp, ali, FALSE, FALSE, FALSE, FALSE, othends);
	  
	  if (m[ij][dxy] > check+acc || m[ij][dxy] < check-acc)
	    printf ("check matrix %d %d %d %d | m = %6.4f vi = %6.4f \n", i, j, dx, dy, m[ij][dxy], check);

	  if (dx == i && dy == j) 
	    if (v[ij] > check+acc || v[ij] < check-acc)
	      printf ("-->check vector %d %d | v = %6.4f vi = %6.4f \n", i, j, v[ij], check);
	  
	}
      }
    }
  }
  
  free(m[0]); 
  free(m); 
  free(v); 

  *ret_leg = leg;
}

/* Function: ForwardOTHFull()
 * Date:     ER, Tue Oct  9 20:00:12 CDT 2001 [St. Louis]
 * Purpose:  Calculates \sum_{align} P(X,Y align| OTH model)
 *                       in this case we do not keep the alignment, so the two sequences
 *                       can have different length.
 *
 * Strategy: The algorith is O(L1*L2) in time, because we "remove" the original gaps
 *               and let the model put their own gaps.
 *           The algorithm is O(L1*L2) in memory
 *
 * Args:     ofp          -- output file
 *           seqX, seqY   -- equal length sequences, ACGT and gaps (-) 
 *           start        -- start position
 *           d            -- length of sX,sY 
 *           othmodel     -- oth_model structure
 *
 * Returns:  log likelihood, log P(seqX,seqY,\pi_{blast} | Othmodel)
 */
void
ForwardOTHFull(FILE *ofp, int *seqX, int *seqY, int startX, int startY, int Lx, int Ly, int Lw,
	      struct othmodel_s *oth, struct othdpf_s *othdp, struct sc2_s *othsc, double *sc, int ones)
{
  int *irX, *irY;     /* reverse complements */

  othsc->pl = ForwardOTH_L2(ofp, seqX, seqY, startX, startY, Lx, Ly, Lw, oth, othdp, sc);

  if (!ones) {
    /* revcomp */
    irX = MallocOrDie(sizeof(int) * (Lx+1));
    irY = MallocOrDie(sizeof(int) * (Ly+1));
    RevComp(irX, startX+seqX, Lx);
    RevComp(irY, startY+seqY, Ly);
    
    othsc->mn =  ForwardOTH_L2(ofp, irX, irY, 0, 0, Lx, Ly, Lw, oth, othdp, sc);
    
    free(irX);
    free(irY);
  }
  else othsc->mn = 0.0;
  
}

/* Function: ForwardOTH_L2()
 * Date:     ER, Tue Nov  2 14:29:01 CST 1999 [St. Louis]
 *
 * Purpose:  Calculates \sum_{align} P(X,Y align| OTH model)
 *                       in this case we do not keep the alignment, so the two sequences
 *                       can have different length.
 *
 * Strategy: The algorith is O(L1*L2) in time, because we "remove" the original gaps
 *               and let the model put their own gaps.
 *           The algorithm is O(L1*L2) in memory
 *
 * Args:     ofp          -- output file
 *           seqX, seqY   -- equal length sequences, ACGT and gaps (-) 
 *           start        -- start position
 *           d            -- length of sX,sY 
 *           othmodel     -- oth_model structure
 *
 * Returns:  log likelihood, log P(seqX,seqY,\pi_{blast} | Othmodel)
 */
double
ForwardOTH_L2(FILE *ofp, int *seqX, int *seqY, int startX, int startY, int Lx, int Ly, int Lw,
	      struct othmodel_s *oth, struct othdpf_s *othdp, double *sc)
{
  int      dimX, dimY;
  int      i,j;		             /* relative positions in seqX, seqY                       */
  int      iabs, jabs;		     /* absolute positions in seqX, seqY                       */
  int      dx, dy;
  int      pos, posi, posj, posij;   /* pos=(i,j), posi=(i-1,j), posj=(i,j-1), posij=(i-1,j-1) */
  int      cur_x, cur_y;             /* nucleotides at those positions                         */
  int      idx = 0;
  double   sco[4];

  dimX = Lx + 1;
  dimY = Ly + 1;

  PatternDpFullOTH(Lx, Ly, othdp);

  othdp->flxmx[Lx]    = ScoreWithNullX(seqX, startX,    0, oth->FLN);
  othdp->flymx[Ly]    = ScoreWithNullY(seqY, startY,    0, oth->FLN);
  othdp->frxmx[Lx]    = ScoreWithNullX(seqX, startX+Lx, 0, oth->FRN);
  othdp->frymx[Ly]    = ScoreWithNullY(seqY, startY+Ly, 0, oth->FRN);
  othdp->fjxmx[Lx][0] = ScoreWithNullX(seqX, startX,    0, oth->FJN);
  othdp->fjymx[Ly][0] = ScoreWithNullY(seqY, startY,    0, oth->FJN);

  /* calculate  flanking model matrices
   */
  for (i = 0; i < Lx; i++) {
    
    iabs = i + startX;
    cur_x = seqX[iabs];

    if (i == 0) {
      othdp->flxmx[i]      = ScoreWithNullX(seqX, iabs,        1, oth->FLN);
      othdp->frxmx[Lx-i-1] = ScoreWithNullX(seqX, Lx+startX-1, 1, oth->FRN);
    }
    else {
      othdp->flxmx[i]      = othdp->flxmx[i-1]  + NullAddOneX(cur_x,               oth->FLN);
      othdp->frxmx[Lx-i-1] = othdp->frxmx[Lx-i] + NullAddOneX(seqX[Lx+startX-i-1], oth->FRN);
    }
    
    othdp->fjxmx[i][0] = ScoreWithNullX(seqX, iabs, 0, oth->FJN);
    for (dx = 1; dx <= i+1; dx++) {
      if (i == 0)
	othdp->fjxmx[i][dx] = ScoreWithNullX(seqX, iabs-dx+1, dx, oth->FJN);
      else
	othdp->fjxmx[i][dx] = othdp->fjxmx[i-1][dx-1] + NullAddOneX(cur_x, oth->FJN);
    }
  }

  for (j = 0; j < Ly; j++) {
    jabs = j + startY;
    cur_y = seqY[jabs];

    if (j == 0) {
      othdp->flymx[j]      = ScoreWithNullY(seqY, jabs,        1, oth->FLN);
      othdp->frymx[Ly-j-1] = ScoreWithNullY(seqY, Ly+startY-1, 1, oth->FRN);
    }
    else {
      othdp->flymx[j]      = othdp->flymx[j-1]  + NullAddOneY(cur_y,               oth->FLN);
      othdp->frymx[Ly-j-1] = othdp->frymx[Ly-j] + NullAddOneY(seqY[Ly+startY-j-1], oth->FRN);
    }
    
    othdp->fjymx[j][0] = ScoreWithNullY(seqY, jabs, 0, oth->FJN);
    for (dy = 1; dy <= j+1; dy++) {
      if (j == 0)
	othdp->fjymx[j][dy] = ScoreWithNullY(seqY, jabs-dy+1, dy, oth->FJN);
      else
	othdp->fjymx[j][dy] = othdp->fjymx[j-1][dy-1] + NullAddOneY(cur_y, oth->FJN);
    }
  } 
 
  /* SPECIAL CASE: m(-,-)
   */
  pos = Lx*dimY + Ly;
  othdp->bmx[pos] = -BIGFLOAT;
  othdp->mmx[pos] = -BIGFLOAT;
  othdp->xmx[pos] = -BIGFLOAT;
  othdp->ymx[pos] = -BIGFLOAT;
  othdp->emx[pos] = -BIGFLOAT;

  if (Lx == 0 && Ly == 0) 
    return othdp->frxmx[Lx] + othdp->frymx[Ly] + oth->t[TFLFR] + othdp->frxmx[Lx] + othdp->frymx[Ly];

  sc[idx++] = othdp->frxmx[Lx] + othdp->frymx[Ly] + oth->t[TFLFR] + othdp->frxmx[0] + othdp->frymx[0];

  
  /* SPECIAL CASE: m(i,-)
   */
  for (i = 0; i < Lx; i++) {
    
    iabs = i + startX;
    cur_x = seqX[iabs]; 
    
    pos  = i    *dimY + Ly;
    posi = (i-1)*dimY + Ly;


    /* state B(i,-) 
     */
    othdp->bmx[pos] = forwB(i, Ly, Lx, Ly, othdp, oth);
    
    /* state M(i,-) 
     */
    othdp->mmx[pos] = -BIGFLOAT;
    
    /* state X(i,-) 
     */
      if (i == 0)
	othdp->xmx[pos] = othdp->flxmx[Lx] + othdp->flymx[Ly] + 
	  oth->t[TBX] + oth->t[TFLB] + oth->xem[cur_x];
      else {
	sco[0] = oth->t[TBX] + othdp->bmx[posi];
	sco[1] = oth->t[TMX] + othdp->mmx[posi];
	sco[2] = oth->t[TXX] + othdp->xmx[posi];
	sco[3] = oth->t[TYX] + othdp->ymx[posi];
	othdp->xmx[pos] = oth->xem[cur_x] + DLog2Sum(sco, 4);
      }

    /* state Y(i,-) 
     */
      othdp->ymx[pos] = -BIGFLOAT;

    /* state E(i,-) 
     */
    sco[0] = oth->t[TME] + othdp->mmx[pos];
    sco[1] = oth->t[TXE] + othdp->xmx[pos];
    sco[2] = oth->t[TYE] + othdp->ymx[pos];
    othdp->emx[pos] = DLog2Sum(sco, 3);
    
    /* state T(i,-) 
     */
    sc[idx++] = othdp->emx[pos]                    + oth->t[TEFR]  + othdp->frxmx[i+1] + othdp->frymx[Ly];
    sc[idx++] = othdp->flxmx[i] + othdp->flymx[Ly] + oth->t[TFLFR] + othdp->frxmx[i+1] + othdp->frymx[Ly];
  }
  if (Ly == 0) return DLog2Sum(sc, idx);
  
  /* SPECIAL CASE:  m(-,j)
   */
  for (j = 0; j < Ly; j++) {
    jabs = j + startY;
    cur_y = seqY[jabs];
    
    pos  = Lx*dimY + j;
    posj = Lx*dimY + j-1;

    /* state B(-,j) 
     */
    othdp->bmx[pos] = forwB(Lx, j, Lx, Ly, othdp, oth);
    
    /* state M(-,j) 
     */
    othdp->mmx[pos] = -BIGFLOAT;

    /* state X(-,j) 
     */
    othdp->xmx[pos] = -BIGFLOAT;
    

    /* state Y(-,j) 
     */
    if (j == 0)
      othdp->ymx[pos] = othdp->flxmx[Lx] + othdp->flymx[Ly] + 
	oth->t[TBY] + oth->t[TFLB] + oth->yem[cur_y];
    else {
      sco[0] = oth->t[TBY] + othdp->bmx[posj];
      sco[1] = oth->t[TMY] + othdp->mmx[posj];
      sco[2] = oth->t[TXY] + othdp->xmx[posj];
      sco[3] = oth->t[TYY] + othdp->ymx[posj];
      othdp->ymx[pos] = oth->yem[cur_y] + DLog2Sum(sco, 4);
    }

    /* state E(-,j) 
     */
    sco[0] = oth->t[TME] + othdp->mmx[pos];
    sco[1] = oth->t[TXE] + othdp->xmx[pos];
    sco[2] = oth->t[TYE] + othdp->ymx[pos];
    othdp->emx[pos] = DLog2Sum(sco, 3);

    /* state T(-,j) 
     */
    sc[idx++] = othdp->emx[pos]                    + oth->t[TEFR]  + othdp->frxmx[Lx] + othdp->frymx[j+1];
    sc[idx++] = othdp->flxmx[Lx] + othdp->flymx[j] + oth->t[TFLFR] + othdp->frxmx[Lx] + othdp->frymx[j+1];
  }
  if (Lx == 0) return DLog2Sum(sc, idx);
  
  /* GENERAL CASE:  i \in (0,Lx-1),  j\ in (0,Ly-1)
   */
  for (j = 0; j < Ly; j++) {
    jabs = j + startY;
    cur_y = seqY[jabs];
    
    for (i = 0; i < Lx; i++) {
      iabs = i + startX;
      cur_x = seqX[iabs];
      
      pos = i*dimY + j;
      if (i == 0 && j == 0) 
	{
	  posi  = Lx*dimY + j;
 	  posj  = i *dimY + Ly;
	  posij = Lx*dimY + Ly;
	}
      else if (i == 0)
	{
	  posi  = Lx*dimY + j;
	  posj  = i *dimY + j-1;
	  posij = Lx*dimY + j-1;
	}
      else if (j == 0)
	{
	  posi  = (i-1)*dimY + j;
	  posj  = i    *dimY + Ly;
	  posij = (i-1)*dimY + Ly;
	}
      else
	{
	  posi  = (i-1)*dimY + j;
 	  posj  = i    *dimY + j-1;
	  posij = (i-1)*dimY + j-1;
	}
      
      /* B(i,j) 
       */
      othdp->bmx[pos] = forwB(i, j, Lx, Ly, othdp, oth);
      
      /* state M(i,j)
       */
      sco[0] = oth->t[TBM] + othdp->bmx[posij];
      sco[1] = oth->t[TMM] + othdp->mmx[posij];
      sco[2] = oth->t[TXM] + othdp->xmx[posij];
      sco[3] = oth->t[TYM] + othdp->ymx[posij];
      othdp->mmx[pos] = oth->mem[idx(cur_x,cur_y)] + DLog2Sum(sco, 4);
      
      /* state X(i,j)
       */
      sco[0] = oth->t[TBX] + othdp->bmx[posi];
      sco[1] = oth->t[TMX] + othdp->mmx[posi];
      sco[2] = oth->t[TXX] + othdp->xmx[posi];
      sco[3] = oth->t[TYX] + othdp->ymx[posi];
      othdp->xmx[pos] = oth->xem[cur_x] + DLog2Sum(sco, 4);
      
       /* state Y(i,j)
       */
      sco[0] = oth->t[TBY] + othdp->bmx[posj];
      sco[1] = oth->t[TMY] + othdp->mmx[posj];
      sco[2] = oth->t[TXY] + othdp->xmx[posj];
      sco[3] = oth->t[TYY] + othdp->ymx[posj];
      othdp->ymx[pos] = oth->yem[cur_y] + DLog2Sum(sco, 4);
      
      /* state E(i,j)
       */
      sco[0] = oth->t[TME] + othdp->mmx[pos];
      sco[1] = oth->t[TXE] + othdp->xmx[pos];
      sco[2] = oth->t[TYE] + othdp->ymx[pos];
      othdp->emx[pos] = DLog2Sum(sco, 3);

      /* T(Lx-1,Ly-1) = \max_{i=0}^{i=Lx-1} \max_{j=0}^{i=Ly-1} 
       *                   E(i,j)*FRN(iabs+1 to Lx-1,jabs+1 to Ly-1)
       */
      sc[idx++] = othdp->emx[pos] + oth->t[TEFR] + othdp->frxmx[i+1] + othdp->frymx[j+1];
      sc[idx++] = othdp->flxmx[i] + othdp->flymx[j] + oth->t[TFLFR] + 
	othdp->frxmx[i+1] + othdp->frymx[j+1];

    } /* while i */
  } /* while j */
  
  return DLog2Sum(sc, idx);
}



/* Function: ForwardOTHDiag()
 * Date:     ER, Tue Oct  9 20:09:08 CDT 2001  [St. Louis]
 *
 * Purpose:  Calculates \sum_{align} P(X,Y, align | OTH model)
 *
 * Strategy: The algorith is O(L) in time because we respect the  gaps
 *               so there is only one "length".
 *           The algorithm is O(L) in memory, but it can be implemented keeping 
 *               only the previous value.
 *
 * Args:     ofp          -- output file
 *           seqX, seqY   -- equal length sequences, ACGT and gaps (-) 
 *           start        -- start position
 *           d            -- length of sX,sY 
 *           othmodel     -- oth_model structure
 *           sc           -- vector of scores
 *
 * Returns:  log likelihood, log P(seqX,seqY,\pi_{blast} | Othmodel)
 */
void
ForwardOTHDiag(FILE *ofp, int *seqX, int *seqY, int start, int L, struct othmodel_s *oth, 
		 struct othdpd_s *othdp, struct sc2_s *othsc, double *sc, int ones)
{
  int *irX, *irY;     /* reverse complements */

  othsc->pl = ForwardOTHDiag_L(ofp, seqX, seqY, start, L, oth, othdp, sc);

  if (!ones) {
    /* revcomp */
    irX = MallocOrDie(sizeof(int) * (L+1));
    irY = MallocOrDie(sizeof(int) * (L+1));
    RevComp(irX, start+seqX, L);
    RevComp(irY, start+seqY, L);
    
    othsc->mn =  ForwardOTHDiag_L(ofp, irX, irY, 0, L, oth, othdp, sc);
    
    free(irX);
    free(irY);
  }
  else othsc->mn = 0.0;

}

/* Function: ForwardOTHDiag_L()
 * Date:     ER, Tue Nov  2 11:41:58 CST 1999 [St. Louis]
 *
 * Purpose:  Calculates \sum_{align} P(X,Y, align | OTH model)
 *
 * Strategy: The algorith is O(L) in time because we respect the  gaps
 *               so there is only one "length".
 *           The algorithm is O(L) in memory, but it can be implemented keeping 
 *               only the previous value.
 *
 * Args:     ofp          -- output file
 *           seqX, seqY   -- equal length sequences, ACGT and gaps (-) 
 *           start        -- start position
 *           d            -- length of sX,sY 
 *           othmodel     -- oth_model structure
 *           sc           -- vector of scores
 *
 * Returns:  log likelihood, log P(seqX,seqY,\pi_{blast} | Othmodel)
 */
double
ForwardOTHDiag_L(FILE *ofp, int *seqX, int *seqY, int start, int L, struct othmodel_s *oth, 
		 struct othdpd_s *othdp, double *sc)
{
  int     i, d;		            /* relative positions in seqX, seqY   */
  int     iabs;		            /* absolute positions in seqX, seqY   */
  int     cur_x, cur_y;             /* nucleotides at those positions     */
  int     idx = 0;

  PatternDpDiagOTH(L, othdp);

  othdp->flmx[L] = ScoreWithNullDiag(seqX, seqY, start,   0, oth->FLN);
  othdp->frmx[L] = ScoreWithNullDiag(seqX, seqY, start+L, 0, oth->FRN);

  if (L == 0) 
    return oth->t[TFLFR] + othdp->flmx[L] + othdp->frmx[L];

  for (i = 0; i < L; i++) {
    
    iabs = i + start;
    cur_x = seqX[iabs];
    cur_y = seqY[iabs];

    if (i == 0) {
      othdp->flmx[i]     = ScoreWithNullDiag(seqX, seqY, iabs,      1, oth->FLN);
      othdp->frmx[L-i-1] = ScoreWithNullDiag(seqX, seqY, L+start-1, 1, oth->FRN);
    }
    else {
      othdp->flmx[i]     = othdp->flmx[i-1] + NullAddOnePair(cur_x,             cur_y,             oth->FLN);
      othdp->frmx[L-i-1] = othdp->frmx[L-i] + NullAddOnePair(seqX[L+start-i-1], seqY[L+start-i-1], oth->FRN);
    }
    
    othdp->fjmx[i][0] = ScoreWithNullDiag(seqX, seqY, iabs, 0, oth->FJN);

    for (d = 1; d <= i+1; d++) {
      if (i == 0)
	othdp->fjmx[i][d] = ScoreWithNullDiag(seqX, seqY, iabs-d+1, d, oth->FJN);
      else
	othdp->fjmx[i][d] = othdp->fjmx[i-1][d-1] + NullAddOnePair(cur_x, cur_y, oth->FJN);
    }
    
  }

  sc[idx++] = othdp->flmx[L] + oth->t[TFLFR] + othdp->frmx[0];
  
  for (i = 0; i < L; i++) {
    
    iabs = i + start;
    cur_x = seqX[iabs];
    cur_y = seqY[iabs];

    othdp->bmx[i] = forwBdiag(i, othdp, oth);
    othdp->mmx[i] = forwMdiag(i, L, cur_x, cur_y, othdp, oth);
    othdp->xmx[i] = forwXdiag(i, L, cur_x, cur_y, othdp, oth);
    othdp->ymx[i] = forwYdiag(i, L, cur_x, cur_y, othdp, oth);
    othdp->emx[i] = forwEdiag(i, cur_x, cur_y, othdp, oth);
     
    /* T(L-1) = \sum_{i=0}^{i=L-1} E(i)*eta*FRNull(i+1 to L-1)  
     */
    sc[idx++] = othdp->emx[i]  + oth->t[TEFR]  + othdp->frmx[i+1];
    sc[idx++] = othdp->flmx[i] + oth->t[TFLFR] + othdp->frmx[i+1];
  }

  return DLog2Sum(sc, idx);
}

/* Function: ForwardOTHDiagMatrix() 
 * Date:     ER, Thu Dec 30 03:11:19 CST 1999 [Zaragoza] 
 * 
 * Purpose:  Calculates, i \in [start, start+L-1] --- l \in [1, i+1]
 *           mt[i][l] = \sum_{align} P(X,Y, begin = start+i-l+1, end = start+i, align | OTH model) 
 * 
 * Args:     ofp          -- output file 
 *           seqX, seqY   -- equal length sequences, ACGT and gaps (-)  
 *           start        -- start position 
 *           L            -- length of sX,sY  
 *           oth          -- oth_model structure 
 *           dp           -- matrices for dynamic programming
 *           sc           -- vector of scores
 * 
 * Returns:  void. fill mt[][], allocated and freed by caller
 */
void
ForwardOTHDiagMatrix(FILE *ofp, double **mt, int *seqX, int *seqY, int start, int L, 
		     struct othmodel_s *oth, struct othdpd_s *othdp, double *sc)
{
  int     i, st;	      /* relative positions in seqX, seqY    */
  int     iabs, stabs;	      /* absolute positions in seqX, seqY    */
  int     Lmax,  l; 
  int     d; 
  int     cur_x, cur_y;       /* nucleotides at those positions      */
  int     idx = 0;
  double  fr;
  
  PatternDpDiagOTH(L, othdp);

  othdp->flmx[L] = ScoreWithNullDiag(seqX, seqY, start,     0, oth->FLN);
  othdp->frmx[L] = ScoreWithNullDiag(seqX, seqY, start+L-1, 0, oth->FRN);
  
  mt[0][0] = oth->t[TFLFR] + othdp->flmx[L] + othdp->frmx[L];
  
  for (st = 0; st < L; st++) {
    stabs = st + start;
    Lmax = L - st;
    
    othdp->flmx[L] = ScoreWithNullDiag(seqX, seqY, stabs,        0, oth->FLN);
    othdp->frmx[L] = ScoreWithNullDiag(seqX, seqY, stabs+Lmax-1, 0, oth->FRN);

    mt[st][0] = mt[0][0];

    for (l = 1; l <= Lmax; l++) {
      i = st + l - 1;
      iabs = i + start;
      cur_x = seqX[iabs];
      cur_y = seqY[iabs];
      
      if (i == st) {
	othdp->flmx[i]   = ScoreWithNullDiag(seqX, seqY, iabs,      l, oth->FLN);
	othdp->frmx[L-l] = ScoreWithNullDiag(seqX, seqY, L+start-1, l, oth->FRN);
      }
      else {
	othdp->flmx[i]   = othdp->flmx[i-1]   + NullAddOnePair(cur_x,           cur_y,           oth->FLN);
	othdp->frmx[L-l] = othdp->frmx[L-l+1] + NullAddOnePair(seqX[L+start-l], seqY[L+start-l], oth->FRN);
      }
      
      othdp->fjmx[i][0] = ScoreWithNullDiag(seqX, seqY, iabs, 0, oth->FJN);
      for (d = 1; d <= l; d++) {
	if (i == st)
	  othdp->fjmx[i][d] = ScoreWithNullDiag(seqX, seqY, iabs-d+1, d, oth->FJN);
	else
	  othdp->fjmx[i][d] = othdp->fjmx[i-1][d-1] + NullAddOnePair(cur_x, cur_y, oth->FJN);
      }
     
      /* B(i) 
       */
      idx = 0;
      sc[idx++] = oth->t[TFLB] + othdp->flmx[i];
      for (d = 1; d < l; d++) 
	sc[idx++] = othdp->emx[st+d-1] + oth->t[TEFJ] + oth->t[TFJB] + othdp->fjmx[i][l-d];
      
      othdp->bmx[i] = DLog2Sum(sc, idx);
      
      /* state M(i)
       */
      idx = 0;
      if (cur_x < 4 && cur_y < 4) {
	if (i == st) 
	  sc[idx++] = oth->t[TBM] + oth->t[TFLB] + othdp->flmx[L];
	else {
	  sc[idx++] = oth->t[TBM] + othdp->bmx[i-1];
	  sc[idx++] = oth->t[TMM] + othdp->mmx[i-1];
	  sc[idx++] = oth->t[TXM] + othdp->xmx[i-1];
	  sc[idx++] = oth->t[TYM] + othdp->ymx[i-1];
	}
      }
      othdp->mmx[i] = DLog2Sum(sc, idx) + oth->mem[idx(cur_x,cur_y)];
      
      /* state X(i)
       */
      idx = 0;
      if (cur_x < 4 && cur_y == 4) {
	if (i == st) 
	  sc[idx++] = oth->t[TBX] + oth->t[TFLB] + othdp->flmx[L];
	else {
	  sc[idx++] = oth->t[TBX] + othdp->bmx[i-1];
	  sc[idx++] = oth->t[TMX] + othdp->mmx[i-1];
	  sc[idx++] = oth->t[TXX] + othdp->xmx[i-1];
	  sc[idx++] = oth->t[TYX] + othdp->ymx[i-1];
	}
      }
      othdp->xmx[i] = DLog2Sum(sc, idx) + oth->xem[cur_x];
      
      /* state Y(i)
       */
      idx = 0;
      if (cur_x == 4 && cur_y < 4) { 
	if (i == st) 
	  sc[idx++] = oth->t[TBY] + oth->t[TFLB] + othdp->flmx[L];
	else {
	  sc[idx++] = oth->t[TBY] + othdp->bmx[i-1];
	  sc[idx++] = oth->t[TMY] + othdp->mmx[i-1];
	  sc[idx++] = oth->t[TXY] + othdp->xmx[i-1];
	  sc[idx++] = oth->t[TYY] + othdp->ymx[i-1];
	}
      }
      othdp->ymx[i] = DLog2Sum(sc, idx) + oth->yem[cur_y];
      
      /* state E(i)
       */
      idx = 0;
      sc[idx++] = oth->t[TME] + othdp->mmx[i];
      sc[idx++] = oth->t[TXE] + othdp->xmx[i];
      sc[idx++] = oth->t[TYE] + othdp->ymx[i];
      othdp->emx[i] = DLog2Sum(sc, idx);

      /* T[i][l] = \max_{d=st}^{d=i} E(d)*FRN(d+1 to i)
       *
       * test: mt[i][l] = ViterbiOTHDiag_L(start=iabs-l-1, L=l)
       */
      idx = 0;
      for (d = 1; d <= l; d++) {
	fr = ScoreWithNullDiag(seqX, seqY, start+st+d, l-d, oth->FRN);

	sc[idx++] = othdp->flmx[st+d-1] + oth->t[TFLFR] + fr;
	sc[idx++] = othdp->emx[st+d-1]  + oth->t[TEFR]  + fr;
      }
      
      mt[i][l] = DLog2Sum(sc, idx);
      
    } /* while l */
  } /* while start */
}

/* Function: ForwardOTHDiagVector() 
 * Date:     ER, Thu Dec 30 03:11:43 CST 1999 [Zaragoza] 
 * 
 * Purpose:  Calculates, i \in [start, start+L-1] 
 *           mt[i] = \sum_{align} P(X,Y, begin = start, end = start+i, align | OTH model) 
 * 
 * Args:     ofp          -- output file 
 *           seqX, seqY   -- equal length sequences, ACGT and gaps (-)  
 *           start        -- start position 
 *           L            -- length of sX,sY  
 *           oth          -- oth_model structure 
 *           dp           -- matrices for dynamic programming
 *           ali          -- array with alignment
 *           traceback    -- if true due traceback
 *           alignment    -- if true print alignment
 *           sc           -- vector of scores
 * 
 * Returns:  fills v[i], allocated and freed by caller
 */
void
ForwardOTHDiagVector(FILE *ofp, double *v, int *seqX, int *seqY, int start, int L, 
		     struct othmodel_s *oth, struct othdpd_s *othdp, double *sc)
{
  int     i;	              /* relative positions in seqX, seqY    */
  int     iabs;	              /* absolute positions in seqX, seqY    */
  int     l; 
  int     d; 
  int     cur_x, cur_y;       /* nucleotides at those positions      */
  int     idx = 0;
  double  fr;
  
  PatternDpDiagOTH(L, othdp);

  othdp->flmx[L] = ScoreWithNullDiag(seqX, seqY, start,     0, oth->FLN);
  othdp->frmx[L] = ScoreWithNullDiag(seqX, seqY, start+L-1, 0, oth->FRN);
  
  for (l = 1; l <= L; l++) {
    i = l - 1;
    iabs = i + start;
    cur_x = seqX[iabs];
    cur_y = seqY[iabs];
    
    if (i == 0) {
      othdp->flmx[i]   = ScoreWithNullDiag(seqX, seqY, iabs,      l, oth->FLN);
      othdp->frmx[L-l] = ScoreWithNullDiag(seqX, seqY, L+start-1, l, oth->FRN);
    }
    else {
      othdp->flmx[i]   = othdp->flmx[i-1]   + NullAddOnePair(cur_x,           cur_y,           oth->FLN);
      othdp->frmx[L-l] = othdp->frmx[L-l+1] + NullAddOnePair(seqX[L+start-l], seqY[L+start-l], oth->FRN);
    }
    
    othdp->fjmx[i][0] = ScoreWithNullDiag(seqX, seqY, iabs, 0, oth->FJN);
    for (d = 1; d <= l; d++) {
      if (i == 0)
	othdp->fjmx[i][d] = ScoreWithNullDiag(seqX, seqY, iabs-d+1, d, oth->FJN);
      else
	othdp->fjmx[i][d] = othdp->fjmx[i-1][d-1] + NullAddOnePair(cur_x, cur_y, oth->FJN);
    }
    
    /* B(i) 
     */
    idx = 0;
    sc[idx++] = oth->t[TFLB] + othdp->flmx[i];
    for (d = 1; d < l; d++) 
      sc[idx++] = othdp->emx[d-1] + oth->t[TEFJ] + oth->t[TFJB] + 
	   othdp->fjmx[i][l-d];
    
    othdp->bmx[i] = DLog2Sum(sc, idx);
    
    /* state M(i)
     */
    idx = 0;
    if (cur_x < 4 && cur_y < 4) {
      if (i == 0) 
	sc[idx++] = oth->t[TBM] + oth->t[TFLB] + othdp->flmx[L];
      else {
	sc[idx++] = oth->t[TBM] + othdp->bmx[i-1];
	sc[idx++] = oth->t[TMM] + othdp->mmx[i-1];
	sc[idx++] = oth->t[TXM] + othdp->xmx[i-1];
	sc[idx++] = oth->t[TYM] + othdp->ymx[i-1];
      }
    }
    othdp->mmx[i] = DLog2Sum(sc, idx) + oth->mem[idx(cur_x,cur_y)];
    
    /* state X(i)
     */
    idx = 0;
    if (cur_x < 4 && cur_y == 4) {
      if (i == 0) 
	sc[idx++] = oth->t[TBX] + oth->t[TFLB] + othdp->flmx[L];
      else {
	sc[idx++] = oth->t[TBX] + othdp->bmx[i-1];
	sc[idx++] = oth->t[TMX] + othdp->mmx[i-1];
	sc[idx++] = oth->t[TXX] + othdp->xmx[i-1];
	sc[idx++] = oth->t[TYX] + othdp->ymx[i-1];
      }
    }
    othdp->xmx[i] = DLog2Sum(sc, idx) + oth->xem[cur_x];
    
    /* state Y(i)
     */
    idx = 0;
    if (cur_x == 4 && cur_y < 4) { 
      if (i == 0) 
	sc[idx++] = oth->t[TBY] + oth->t[TFLB] + othdp->flmx[L];
      else {
	sc[idx++] = oth->t[TBY] + othdp->bmx[i-1];
	sc[idx++] = oth->t[TMY] + othdp->mmx[i-1];
	sc[idx++] = oth->t[TXY] + othdp->xmx[i-1];
	sc[idx++] = oth->t[TYY] + othdp->ymx[i-1];
      }
    }
    othdp->ymx[i] = DLog2Sum(sc, idx) + oth->yem[cur_y];
    
    /* state E(i)
     */
    idx = 0;
    sc[idx++] = oth->t[TME] + othdp->mmx[i];
    sc[idx++] = oth->t[TXE] + othdp->xmx[i];
    sc[idx++] = oth->t[TYE] + othdp->ymx[i];
    othdp->emx[i] = DLog2Sum(sc, idx);
    
    /* T[i][l] = \max_{d=0}^{d=i} E(d)*FRN(d+1 to i)
     *
     * test: v[l] = ViterbiOTHDiag_L(start, L=l)
     */
    idx = 0;
    for (d = 1; d <= l; d++) {
      fr = ScoreWithNullDiag(seqX, seqY, start+d, l-d, oth->FRN);
      sc[idx++] = othdp->flmx[d-1] + oth->t[TFLFR] + fr;
      sc[idx++] = othdp->emx[d-1]  + oth->t[TEFR]  + fr;  
    }

    v[i] = DLog2Sum(sc, idx);
    
  } /* while i */
}


/* Function: FreeDpOTH()
 * Date:     ER, Tue Nov  2 14:29:24 CST 1999 [St. Louis]
 *
 * Purpose:  frees memory for the dp matrices of the OTH model
 *
 * Returns:  dpmatrixoth are allocated
 */
void
FreeDpDiagOTH(struct othdpd_s *dp)
{
  free(dp->flmx);
  free(dp->frmx);
  free(dp->fjmx[0]);
  free(dp->fjmx);

  free(dp->bmx);
  free(dp->mmx);
  free(dp->xmx);
  free(dp->ymx);
  free(dp->emx);

  free(dp);
}
void
FreeDpFullOTH(struct othdpf_s *dp)
{
  free(dp->flxmx);
  free(dp->frxmx);
  free(dp->fjxmx[0]);
  free(dp->fjxmx);

  free(dp->flymx);
  free(dp->frymx);
  free(dp->fjymx[0]);
  free(dp->fjymx);

  free(dp->bmx);
  free(dp->mmx);
  free(dp->xmx);
  free(dp->ymx);
  free(dp->emx);

  free(dp);
}

/* Function: NullOJ()
 * Date:     ER, Mon Jan 29 09:43:56 CST 2001 [St. Louis]
 *
 * Purpose:  score 2 fragments as if passing thorugh an OTH model but
 *           scoring only though the flanking states.
 *
 * Args:     iseqX, iseqY -- sequences (in integers 0-4)
 *           iabs         -- start position iseqX
 *           jabs         -- start position iseqY
 *           dx           -- fragment of iseqX scored
 *           dy           -- fragment of iseqY scored
 *           othmodel     -- oth_model structure
 *
 * Returns:  
 */
double 
NullOJ(int iabs, int jabs, int dx, int dy, int *seqX, int *seqY, struct othmodel_s *oth)
{
  double nulloj;

  nulloj = oth->t[TFLFR]   
    + ScoreWithNullX(seqX, iabs-dx+1, dx, oth->FLN) 
    + ScoreWithNullY(seqY, jabs-dy+1, dy, oth->FLN) 
    + ScoreWithNullX(seqX, iabs,       0, oth->FRN)      
    + ScoreWithNullY(seqY, jabs,       0, oth->FRN);

  return nulloj;
}


void
PatternDpDiagOTH(int L, struct othdpd_s *dp)
{
  int i, d;

 for (i = 0; i < L; i++) {
    dp->flmx[i] = -BIGFLOAT;
    dp->frmx[i] = -BIGFLOAT;

    for (d = 0; d <= i+1; d++) 
      dp->fjmx[i][d] = -BIGFLOAT;
    
    dp->bmx[i] = -BIGFLOAT;
    dp->mmx[i] = -BIGFLOAT;
    dp->xmx[i] = -BIGFLOAT;
    dp->ymx[i] = -BIGFLOAT;
    dp->emx[i] = -BIGFLOAT;
  }
    dp->flmx[L] = -BIGFLOAT;
    dp->frmx[L] = -BIGFLOAT;
}

void
PatternDpFullOTH(int Lx,int Ly, struct othdpf_s *dp)
{
  int i, j, ij;
  int dx, dy;

  for (i = 0; i < Lx; i++) {
    dp->flxmx[i] = -BIGFLOAT;
    dp->frxmx[i] = -BIGFLOAT;
    
    for (dx = 0; dx <= i+1; dx++) {
      dp->fjxmx[i][dx] = -BIGFLOAT;
      dp->fjymx[i][dx] = -BIGFLOAT;
    }
  }
  
  for (j = 0; j < Ly; j++) {
    dp->flymx[j] = -BIGFLOAT;
    dp->frymx[j] = -BIGFLOAT;

    for (dy = 0; dy <= j+1; dy++) {
      dp->fjxmx[j][dy] = -BIGFLOAT;
      dp->fjymx[j][dy] = -BIGFLOAT;
    }
  }


  for (i = 0; i < Lx; i++) {
    for (j = 0; j < Ly; j++) {
      ij = i*Ly + j;
      dp->bmx[ij] = -BIGFLOAT;
      dp->mmx[ij] = -BIGFLOAT;
      dp->xmx[ij] = -BIGFLOAT;
      dp->ymx[ij] = -BIGFLOAT;
      dp->emx[ij] = -BIGFLOAT;
    }
  }
  dp->flxmx[Lx] = -BIGFLOAT;
  dp->frxmx[Lx] = -BIGFLOAT;
  dp->flymx[Ly] = -BIGFLOAT;
  dp->frymx[Ly] = -BIGFLOAT;
}

/* Function: TestForwardOTH()
 * Date:     ER, Thu Dec 30 03:28:39 CST 1999 [Zaragoza]
 *
 * Purpose:  Compare different functions created with the OTH model
 *          
 *           1.- ForwardOTHDiagMatrix()         
 *           2.- ForwardOTHDiag_L()
 *
 *           3.- ForwardOTHDiagVector()
 *           4.- ForwardOTHDiag_L()
 *
 *           (1) and (2) should give the same answer
 *           (3) and (4) should give the same answer
 *
 * Args:     seqX, seqY   -- equal length sequences, ACGT and gaps (-) 
 *           start        -- starting position
 *           L            -- lengths of sX,sY 
 *           oth          -- oth_model structure
 *           dp           -- matrices for dynamic programming
 *           ali          -- array with alignment
 *           traceback    -- if true due traceback
 *           alignment    -- if true print alignment
 *
 * Returns:  void.
 */
void
TestForwardOTH(FILE *ofp, int *seqX, int *seqY, int start, int L, struct othmodel_s *oth, 
	       struct othdpd_s *othdp, double *sc)
{
  int      i, d;
  int      iabs;
  double  *v1, *v2;
  double **m1, **m2;
  
  m1 = (double **) MallocOrDie (sizeof(double *) * L);
  m2 = (double **) MallocOrDie (sizeof(double *) * L);

  v1 = (double *)  MallocOrDie (sizeof(double) * L);
  v2 = (double *)  MallocOrDie (sizeof(double) * L);

  m1[0] = (double *)  MallocOrDie (sizeof(double) * L * (L+3) / 2);
  m2[0] = (double *)  MallocOrDie (sizeof(double) * L * (L+3) / 2);

  for (i = 1; i < L; i++) {
    m1[i] = m1[0] + i*(i+3)/2;
    m2[i] = m2[0] + i*(i+3)/2;
  }
  
  for (i = 0; i < L; i++) { 
    v1[i] = -BIGFLOAT;
    v2[i] = -BIGFLOAT;
    for (d = 0; d <= i+1; d++) {
      m1[i][d] = -BIGFLOAT;
      m2[i][d] = -BIGFLOAT;
    }
  }

  ForwardOTHDiagVector(ofp, v1, seqX, seqY, start, L, oth, othdp, sc); 
  ForwardOTHDiagMatrix(ofp, m1, seqX, seqY, start, L, oth, othdp, sc);
  
  for (i = 0; i < L; i++) { 
    
    iabs = i + start;

    v2[i]    = ForwardOTHDiag_L(ofp, seqX, seqY, start, i+1, oth, othdp, sc); 
    m2[i][0] = ForwardOTHDiag_L(ofp, seqX, seqY, iabs, 0, oth, othdp, sc);
    
   for (d = 1; d <= i+1; d++)     
      m2[i][d] = ForwardOTHDiag_L(ofp, seqX, seqY, iabs-d+1, d, oth, othdp, sc);    
  }
  
  for (i = 0; i < L; i++) 
    if (v1[i] == v2[i]) 
      printf ("OK    %d | %f %f \n", i, v1[i], v2[i]);
    else 
      printf ("check %d | %f %f \n", i, v1[i], v2[i]);
  
  for (i = 0; i < L; i++) 
    for (d = 0; d <= i+1; d++) {
      if (m1[i][d] == m2[i][d]) 
	printf ("OK    %d %d  | %f %f \n", i, d, m1[i][d], m2[i][d]);
      else 
	printf ("check %d %d  | %f %f \n", i, d, m1[i][d], m2[i][d]);
    }

  free(v1); 
  free(v2); 

  free(m1[0]); 
  free(m2[0]); 
  free(m1); 
  free(m2); 
}

/* Function: TestViterbiOTH()
 * Date:     ER, Tue Dec 28 10:10:24 CST 1999 [Zaragoza]
 *
 * Purpose:  Compare different functions created with the OTH model
 *          
 *           1.- ViterbiOTHDiagMatrix()         
 *           2.- ViterbiOTHDiag_L()
 *
 *           3.- ScoreWithOTHMatrix()
 *           4.- ScoreWithOTH()
 *
 *           (1) and (2) should always give the same answer
 *           (3) and (4) should always give the same answer
 *           [ the two groups should give the same answer if the flanking models do not generate
 *             any base]
 *
 * Args:     seqX, seqY   -- equal length sequences, ACGT and gaps (-) 
 *           start        -- starting position
 *           L            -- lengths of sX,sY 
 *           oth          -- oth_model structure
 *           dp           -- matrices for dynamic programming
 *           ali          -- array with alignment
 *           traceback    -- if true due traceback
 *           alignment    -- if true print alignment
 *
 * Returns:  void.
 */
void
TestViterbiOTH(FILE *ofp, SQINFO sqinfoX, int *seqX, SQINFO sqinfoY, int *seqY, int start, int L, 
	       struct othmodel_s *oth, struct othdpd_s *othdp, struct ali_s *ali, 
	       int alignment, int traceback, struct end_s *othends)
{
  int      i, d;
  int      iabs;
  double  *v1, *v2;
  double **m1, **m2, **m3, **m4;
  
  m1 = AllocMatrixDiag(L);
  m2 = AllocMatrixDiag(L);
  m3 = AllocMatrixDiag(L);
  m4 = AllocMatrixDiag(L);

  v1 = (double *)  MallocOrDie (sizeof(double) * L);
  v2 = (double *)  MallocOrDie (sizeof(double) * L);

  
  for (i = 0; i < L; i++) { 
    v1[i] = -BIGFLOAT;
    v2[i] = -BIGFLOAT;
  }
  
  ViterbiOTHDiagVector(ofp, v1, sqinfoX, seqX, sqinfoY, seqY, start, L, oth, othdp); 
  tracebackOTHdiag_L(ofp, sqinfoX, seqX, sqinfoY, seqY, start, L, v1[L-1], othdp, oth, ali, 
		     alignment, traceback, FALSE, othends);
  ViterbiOTHDiagMatrix(ofp, m1, sqinfoX, seqX, sqinfoY, seqY, start, L, oth, othdp);
  
  for (i = 0; i < L; i++) { 
    
    iabs = i + start;
    
    v2[i] = ViterbiOTHDiag_L(ofp, sqinfoX, seqX, sqinfoY, seqY, start, i+1, oth, othdp, 
			     ali, FALSE, FALSE, FALSE, FALSE, othends); 
   
    m2[i][0] = ViterbiOTHDiag_L(ofp, sqinfoX, seqX, sqinfoY, seqY, iabs, 0, oth, 
				othdp, ali, FALSE, FALSE, FALSE, FALSE, othends);
    m3[i][0] = ScoreWithOTH(ofp, seqX, seqY, iabs, 0, oth, FALSE);
    m4[i][0] = m3[i][0];
    
    for (d = 1; d <= i+1; d++) {      
      m2[i][d] = ViterbiOTHDiag_L(ofp, sqinfoX, seqX, sqinfoY, seqY, iabs-d+1, d, 
				  oth, othdp, ali, FALSE, FALSE, FALSE, FALSE, othends);    
      
      if (i == 0 || d == 1) 
	m3[i][d] = ScoreWithOTH(ofp, seqX, seqY, iabs, d, oth, FALSE);
      else 
	m3[i][d] = m3[i-1][d-1] + ScoreWithOTHMatrix(seqX, seqY, iabs, oth); 

      m4[i][d] = ScoreWithOTH(ofp, seqX, seqY, iabs-d+1, d, oth, FALSE);
    }
  }
  
  for (i = 0; i < L; i++) 
    if (v1[i] == v2[i]) 
      printf ("OK vector   %d | %f %f \n", i, v1[i], v2[i]);
    else 
      printf ("check vector %d | %f %f \n", i, v1[i], v2[i]);
  
  for (i = 0; i < L; i++) 
    for (d = 0; d <= i+1; d++) {
      if (m1[i][d] == m2[i][d] && m3[i][d] == m4[i][d] && m3[i][d] == m2[i][d]) 
	printf ("OK matrix   %d %d \n", i, d);
      else 
	printf ("check matrix %d %d  | %f %f %f %f \n", i, d, m1[i][d], m2[i][d], m3[i][d], m4[i][d]);
    }
 
  free(v1); 
  free(v2); 

  free(m1[0]); 
  free(m2[0]); 
  free(m3[0]);
  free(m4[0]);
  free(m1); 
  free(m2); 
  free(m3);
  free(m4);
}


/* Function: ViterbiOTHFull()
 * Date:     ER,  Tue Oct  9 20:20:09 CDT 2001 [St. Louis]
 *
 * Purpose:  Calculates P(X,Y, best align | OTH model) and traceback
 *
 * Args:     ofp          -- output file
 *           seqX, seqY   -- equal length sequences, ACGT and gaps (-) 
 *           start        -- start position
 *           d            -- length of sX,sY 
 *           othmodel     -- oth_model structure
 *
 * Returns:  log likelihood, log P(seqX,seqY,best align | OTHmodel). prints traceback
 */
void
ViterbiOTHFull(FILE *ofp, SQINFO sqinfoX, int *seqX, SQINFO sqinfoY, int *seqY, int startX, int startY,
	       int Lx, int Ly, int *ret_len_ali, int Lw, struct othmodel_s *oth, struct othdpf_s *othdp, struct sc2_s *othsc,
	       struct ali_s *ali, int alignment, int ones, int sweep, int traceback, int doends, struct windowends_s *windowends)
{
  int          *irX, *irY;     /* reverse complements */

  othsc->pl = ViterbiOTH_L2(ofp, sqinfoX, seqX, sqinfoY, seqY, startX, startY, Lx, Ly, ret_len_ali, Lw, 
			    oth, othdp, ali, alignment, sweep, traceback, doends, windowends->fwd->oth);

  if (!ones) {
    /* revcomp */
    irX = MallocOrDie(sizeof(int) * (Lx+1));
    irY = MallocOrDie(sizeof(int) * (Ly+1));
    RevComp(irX, startX+seqX, Lx);
    RevComp(irY, startY+seqY, Ly);
    
    othsc->mn =  ViterbiOTH_L2(ofp, sqinfoX, seqX, sqinfoY, seqY, 0, 0, Lx, Ly, ret_len_ali, Lw, 
			       oth, othdp, ali, FALSE, sweep, traceback, doends, windowends->rev->oth);
    
    free(irX);
    free(irY);
  }
  else 
    othsc->mn = 0.0;

}

/* Function: ViterbiOTH_L2()
 * Date:     ER, Tue Nov  2 14:29:39 CST 1999 [St. Louis]
 *
 * Purpose:  Calculates P(X,Y, best align | OTH model) and traceback
 *
 * Args:     ofp          -- output file
 *           seqX, seqY   -- equal length sequences, ACGT and gaps (-) 
 *           start        -- start position
 *           d            -- length of sX,sY 
 *           othmodel     -- oth_model structure
 *
 * Returns:  log likelihood, log P(seqX,seqY,best align | OTHmodel). prints traceback
 */
double
ViterbiOTH_L2(FILE *ofp, SQINFO sqinfoX, int *seqX, SQINFO sqinfoY, int *seqY, int startX, int startY,
	      int Lx, int Ly, int *ret_len_ali, int Lw, struct othmodel_s *oth, struct othdpf_s *othdp, 
	      struct ali_s *ali, int alignment, int sweep, int traceback, int doends, struct end_s *othends)
{
  int     dimY;
  int     i, j;		           /* relative positions in seqX, seqY                       */
  int     iabs, jabs;              /* absolute positions in seqX, seqY                       */
  int     dx, dy;
  int     pos, posi, posj, posij;  /* pos=(i,j), posi=(i-1,j), posj=(i,j-1), posij=(i-1,j-1) */
  int     cur_x, cur_y;            /* nucleotides at those positions                         */
  double  score;                   /* viterbi score                                          */
  double  sc, bestsc;

  PatternDpFullOTH(Lx, Ly, othdp);

  dimY = Ly + 1;

  othdp->flxmx[Lx]    = ScoreWithNullX(seqX, startX,    0, oth->FLN);
  othdp->flymx[Ly]    = ScoreWithNullY(seqY, startY,    0, oth->FLN);
  othdp->frxmx[Lx]    = ScoreWithNullX(seqX, startX+Lx, 0, oth->FRN);
  othdp->frymx[Ly]    = ScoreWithNullY(seqY, startY+Ly, 0, oth->FRN);
  othdp->fjxmx[Lx][0] = ScoreWithNullX(seqX, startX,    0, oth->FJN);
  othdp->fjymx[Ly][0] = ScoreWithNullY(seqY, startY,    0, oth->FJN);

  
  if (Lx == 0 && Ly == 0) 
    return oth->t[TFLFR] + othdp->flxmx[Lx] + othdp->flymx[Ly] + othdp->frxmx[Lx] + othdp->frymx[Ly];

  /* calculate flanking model matrices
   */
  for (i = 0; i < Lx; i++) {
    iabs = i + startX;
    cur_x = seqX[iabs];

    if (i == 0) {
      othdp->flxmx[i]      = ScoreWithNullX(seqX, iabs,        1, oth->FLN);
      othdp->frxmx[Lx-i-1] = ScoreWithNullX(seqX, Lx+startX-1, 1, oth->FRN);
    }
    else {
      othdp->flxmx[i]      = othdp->flxmx[i-1]  + NullAddOneX(cur_x,               oth->FLN);
      othdp->frxmx[Lx-i-1] = othdp->frxmx[Lx-i] + NullAddOneX(seqX[Lx+startX-i-1], oth->FRN);
    }
    
    othdp->fjxmx[i][0] = ScoreWithNullX(seqX, iabs, 0, oth->FJN);
    for (dx = 1; dx <= i+1; dx++) {
      if (i == 0)
	othdp->fjxmx[i][dx] = ScoreWithNullX(seqX, iabs-dx+1, dx, oth->FJN);
      else
	othdp->fjxmx[i][dx] = othdp->fjxmx[i-1][dx-1] + NullAddOneX(cur_x, oth->FJN);
    }
  }

  for (j = 0; j < Ly; j++) {
    jabs = j + startY;
    cur_y = seqY[jabs];

    if (j == 0) {
      othdp->flymx[j]      = ScoreWithNullY(seqY, jabs,        1, oth->FLN);
      othdp->frymx[Ly-j-1] = ScoreWithNullY(seqY, Ly+startY-1, 1, oth->FRN);
    }
    else {
      othdp->flymx[j]      = othdp->flymx[j-1]  + NullAddOneY(cur_y,               oth->FLN);
      othdp->frymx[Ly-j-1] = othdp->frymx[Ly-j] + NullAddOneY(seqY[Ly+startY-j-1], oth->FRN);
    }
    
    othdp->fjymx[j][0] = ScoreWithNullY(seqY, jabs, 0, oth->FJN);
    for (dy = 1; dy <= j+1; dy++) {
      if (j == 0)
	othdp->fjymx[j][dy] = ScoreWithNullY(seqY, jabs-dy+1, dy, oth->FJN);
      else
	othdp->fjymx[j][dy] = othdp->fjymx[j-1][dy-1] + NullAddOneY(cur_y, oth->FJN);
    }
  } 

  score = -BIGFLOAT;

  /* SPECIAL CASE: m(-,-)
   */
  pos = Lx*dimY + Ly;
  othdp->bmx[pos] = -BIGFLOAT;
  othdp->mmx[pos] = -BIGFLOAT;  
  othdp->xmx[pos] = -BIGFLOAT;
  othdp->ymx[pos] = -BIGFLOAT;
  othdp->emx[pos] = -BIGFLOAT;
  score = oth->t[TFLFR] + othdp->flxmx[Lx] + othdp->flymx[Ly] + othdp->frxmx[0] + othdp->frymx[0];

  
 
  /* SPECIAL CASE: m(i,-)
   */
  for (i = 0; i < Lx; i++) {
    
    iabs = i + startX;
    cur_x = seqX[iabs];
    
    pos  = i    *dimY + Ly;
    posi = (i-1)*dimY + Ly;

    /* state B(i,-) 
     */
    bestsc = oth->t[TFLB] + othdp->flxmx[i] + othdp->flymx[Ly];
    for (dx = 1; dx <= i; dx++)
      if ((sc = othdp->emx[(dx-1)*dimY+Ly] + oth->t[TEFJ] + oth->t[TFJB] + 
	   othdp->fjxmx[i][i-dx+1] + othdp->fjymx[Ly][0]) > bestsc) bestsc = sc;
    othdp->bmx[pos] = bestsc;
    
    /* state M(i,-) 
     */
    othdp->mmx[pos] = -BIGFLOAT;
    
    /* state X(i,-) 
     */
    if (i == 0)
      othdp->xmx[pos] = othdp->flxmx[Lx] + othdp->flymx[Ly] + oth->t[TFLB] + oth->t[TBX] + oth->xem[cur_x];
    else
      othdp->xmx[pos] = oth->t[TXX] + oth->xem[cur_x] + othdp->xmx[posi];
 
    /* state Y(i,-) 
     */
    othdp->ymx[pos] = -BIGFLOAT;
    
    /* state E(i,-) 
     */
    othdp->emx[pos] = oth->t[TXE] + othdp->xmx[pos];
    
    /* state T(i,-) 
     */
    if ((sc = othdp->emx[pos] + oth->t[TEFR] + othdp->frxmx[i+1] + othdp->frymx[0]) > score) 
	score = sc;
    if ((sc = othdp->flxmx[i] + othdp->flymx[Ly] + oth->t[TFLFR] + othdp->frxmx[i+1] + othdp->frymx[0]) > score) 
      score = sc;
  }
  if (Ly == 0) return score;
  
  /* SPECIAL CASE:  m(-,j)
   */
  for (j = 0; j < Ly; j++) {
    jabs = j + startY;
    cur_y = seqY[jabs];
    
    pos  = Lx*dimY + j;
    posj = Lx*dimY + j-1;

    /* state B(-,j) 
     */
    bestsc = oth->t[TFLB] + othdp->flymx[j] + othdp->flxmx[Lx];
    for (dy = 1; dy <= j; dy++)
      if ((sc = othdp->emx[Lx*dimY+dy-1] + oth->t[TEFJ] + oth->t[TFJB] + 
	   othdp->fjymx[j][j-dy+1] + othdp->fjxmx[Lx][0]) > bestsc) bestsc = sc;
    othdp->bmx[pos] = bestsc;
    
    /* state M(-,j) 
     */
    othdp->mmx[pos] = -BIGFLOAT;

    /* state X(-,j) 
     */
    othdp->xmx[pos] = -BIGFLOAT;
    
    /* state Y(-,j) 
     */
    if (j == 0)
      othdp->ymx[pos] = othdp->flxmx[Lx] + othdp->flymx[Ly] + oth->t[TFLB] + oth->t[TBY] + oth->yem[cur_y];
    else
      othdp->ymx[pos] = oth->t[TYY] + oth->yem[cur_y] + othdp->ymx[posj];
    
    /* state E(-,j) 
     */
    othdp->emx[pos] = oth->t[TYE] + othdp->ymx[pos];
    
    /* state T(-,j) 
     */
    if ((sc = othdp->emx[pos] + oth->t[TEFR] + othdp->frymx[j+1] + othdp->frxmx[0]) > score) 
      score = sc; 
    if ((sc = othdp->flxmx[Lx] + othdp->flymx[j] + oth->t[TFLFR] + othdp->frxmx[0] + othdp->frymx[j+1]) > score) 
      score = sc;
  }
  if (Lx == 0) return score;
  
  /* GENERAL CASE:  i \in (0,Lx-1),  j\ in (0,Ly-1)
   */

  for (j = 0; j < Ly; j++) {
    jabs = j + startY;
    cur_y = seqY[jabs];
 
    for (i = 0; i < Lx; i++) {
      iabs = i + startX;
      cur_x = seqX[iabs];
      
      pos = i*dimY + j;

      if (i == 0 && j == 0) 
	{
	  posi  = Lx*dimY + j;
 	  posj  = i *dimY + Ly;
	  posij = Lx*dimY + Ly;
	}
      else if (i == 0)
	{
	  posi  = Lx*dimY + j;
	  posj  = i *dimY + j-1;
	  posij = Lx*dimY + j-1;
	}
      else if (j == 0)
	{
	  posi  = (i-1)*dimY + j;
	  posj  = i    *dimY + Ly;
	  posij = (i-1)*dimY + Ly;
	}
      else
	{
	  posi  = (i-1)*dimY + j;
 	  posj  = i    *dimY + j-1;
	  posij = (i-1)*dimY + j-1;
	}
     

      /* state B(i,j) 
       */
      bestsc = oth->t[TFLB] + othdp->flxmx[i] + othdp->flymx[j];
      for (dx = 1; dx <= i; dx++)
	for (dy = 1; dy <= j; dy++)
	  if ((sc = othdp->emx[(dx-1)*dimY+(dy-1)] + oth->t[TEFJ] + oth->t[TFJB] + 
	       othdp->fjxmx[i][i-dx+1] + othdp->fjymx[j][j-dy+1]) > bestsc) bestsc = sc;
      othdp->bmx[pos] = bestsc;
    
      /* state M(i,j)
       */
      if (i == 0 && j == 0) 
	bestsc = oth->t[TBM] + oth->t[TFLB] + othdp->flxmx[Lx] + othdp->flymx[Ly];
      else {
	bestsc = -BIGFLOAT;
	if ((sc = oth->t[TBM] + othdp->bmx[posij]) > bestsc) bestsc = sc;
	if ((sc = oth->t[TMM] + othdp->mmx[posij]) > bestsc) bestsc = sc;
	if ((sc = oth->t[TXM] + othdp->xmx[posij]) > bestsc) bestsc = sc;
	if ((sc = oth->t[TYM] + othdp->ymx[posij]) > bestsc) bestsc = sc;
      }
      othdp->mmx[pos] = bestsc + oth->mem[idx(cur_x,cur_y)];
      
      /* state X(i,j)
       */
      bestsc = -BIGFLOAT;
      if ((sc = oth->t[TBX] + othdp->bmx[posi]) > bestsc) bestsc = sc;
      if ((sc = oth->t[TMX] + othdp->mmx[posi]) > bestsc) bestsc = sc;
      if ((sc = oth->t[TXX] + othdp->xmx[posi]) > bestsc) bestsc = sc;
      if ((sc = oth->t[TYX] + othdp->ymx[posi]) > bestsc) bestsc = sc;
      othdp->xmx[pos] = bestsc + oth->xem[cur_x];
      
      /* state Y(i,j)
       */
      bestsc = -BIGFLOAT;
      if ((sc = oth->t[TBY] + othdp->bmx[posj]) > bestsc) bestsc = sc;
      if ((sc = oth->t[TMY] + othdp->mmx[posj]) > bestsc) bestsc = sc;
      if ((sc = oth->t[TXY] + othdp->xmx[posj]) > bestsc) bestsc = sc;
      if ((sc = oth->t[TYY] + othdp->ymx[posj]) > bestsc) bestsc = sc;
      othdp->ymx[pos] = bestsc + oth->yem[cur_y];
      
      /* state E(i,j)
       */
      bestsc = -BIGFLOAT;
      if ((sc = oth->t[TME] + othdp->mmx[pos]) > bestsc) bestsc = sc;
      if ((sc = oth->t[TXE] + othdp->xmx[pos]) > bestsc) bestsc = sc;
      if ((sc = oth->t[TYE] + othdp->ymx[pos]) > bestsc) bestsc = sc;
      othdp->emx[pos] = bestsc;
      
      /* T(Lx-1,Ly-1) = 
       *         \max_{i=0}^{i=Lx-1} \max_{j=0}^{i=Ly-1} E(i,j)*FRN(iabs+1 to Lx-1,jabs+1 to Ly-1)
       */
      if ((sc = othdp->emx[pos] + oth->t[TEFR] + othdp->frxmx[i+1] + othdp->frymx[j+1]) > score)
	score = sc;
      if ((sc = othdp->flxmx[i] + othdp->flymx[j] + oth->t[TFLFR] + 
	   othdp->frxmx[i+1] + othdp->frymx[j+1]) > score)
	score = sc;
   
    } /* while i */ 
  } /* while j */ 
  
  /* calculate and print the traceback 
   */
  if ((traceback || alignment || doends) && (!sweep || score > 0.0)) 
  tracebackOTH_L2(ofp, sqinfoX, seqX, sqinfoY, seqY, startX, startY, Lx, Ly, ret_len_ali, 
		  score, othdp, oth, ali, alignment, traceback, doends, othends); 
  
  return score; 
} 

/* Function: ViterbiOTHDiag() 
 * Date:     ER, Tue Oct  9 20:11:09 CDT 2001  [St. Louis] 
 * 
 * Purpose:  Calculates P(X,Y, best align | OTH model) and traceback 
 * 
 * Args:     ofp          -- output file 
 *           seqX, seqY   -- equal length sequences, ACGT and gaps (-)  
 *           start        -- start position 
 *           d            -- length of sX,sY  
 *           othmodel     -- oth_model structure 
 * 
 * Returns:  log likelihood, log P(seqX,seqY,best align | OTHmodel). prints traceback 
 */
void
ViterbiOTHDiag(FILE *ofp, SQINFO sqinfoX, int *seqX, SQINFO sqinfoY, int *seqY, int start, int L,  
	       struct othmodel_s *oth, struct othdpd_s *othdp, struct sc2_s *othsc, struct ali_s *ali, 
	       int alignment, int ones, int sweep, int traceback, int doends, struct windowends_s *windowends) 
{
  int          *irX, *irY;     /* reverse complements */

  othsc->pl = ViterbiOTHDiag_L(ofp, sqinfoX, seqX, sqinfoY, seqY, start, L, oth, othdp, ali, 
			       alignment, sweep, traceback, doends, windowends->fwd->oth);

  if (!ones) {
			/* revcomp */
    irX = MallocOrDie(sizeof(int) * (L+1));
    irY = MallocOrDie(sizeof(int) * (L+1));
    RevComp(irX, start+seqX, L);
    RevComp(irY, start+seqY, L);
    
    othsc->mn = ViterbiOTHDiag_L(ofp, sqinfoX, irX, sqinfoY, irY, 0, L, oth, othdp, ali, 
				 FALSE, sweep, traceback, doends, windowends->rev->oth);

    free(irX);
    free(irY);
  }
  else 
    othsc->mn = 0.0;

}

/* Function: ViterbiOTHDiag_L() 
 * Date:     ER, Tue Nov  2 11:44:24 CST 1999 [St. Louis] 
 * 
 * Purpose:  Calculates P(X,Y, best align | OTH model) and traceback 
 * 
 * Args:     ofp          -- output file 
 *           seqX, seqY   -- equal length sequences, ACGT and gaps (-)  
 *           start        -- start position 
 *           d            -- length of sX,sY  
 *           othmodel     -- oth_model structure 
 * 
 * Returns:  log likelihood, log P(seqX,seqY,best align | OTHmodel). prints traceback 
 */
double 
ViterbiOTHDiag_L(FILE *ofp, SQINFO sqinfoX, int *seqX, SQINFO sqinfoY, int *seqY, int start, int L,  
  		 struct othmodel_s *oth, struct othdpd_s *othdp, struct ali_s *ali, 
		 int alignment, int sweep, int traceback, int doends, struct end_s *othends) 
{ 
  int     i;		      /* relative positions in seqX, seqY    */
  int     iabs;		      /* absolute positions in seqX, seqY    */
  int     d; 
  int     cur_x, cur_y;       /* nucleotides at those positions      */
  double  score;              /* viterbi score                       */
  double  sc, bestsc; 
  
  PatternDpDiagOTH(L, othdp);

  othdp->flmx[L] = ScoreWithNullDiag(seqX, seqY, start,     0, oth->FLN);
  othdp->frmx[L] = ScoreWithNullDiag(seqX, seqY, start+L-1, 0, oth->FRN);
  
  if (L == 0) {
    sc = oth->t[TFLFR] + othdp->flmx[L] + othdp->frmx[L];
    return sc;
  }
  
  score = -BIGFLOAT;
  
  for (i = 0; i < L; i++) {
    
    iabs = i + start;
    cur_x = seqX[iabs];
    cur_y = seqY[iabs];
    
    if (i == 0) {
      othdp->flmx[i]     = ScoreWithNullDiag(seqX, seqY, iabs,      1, oth->FLN);
      othdp->frmx[L-i-1] = ScoreWithNullDiag(seqX, seqY, L+start-1, 1, oth->FRN);
    }
    else {
      othdp->flmx[i] = othdp->flmx[i-1] + 
	NullAddOnePair(cur_x, cur_y, oth->FLN);
      othdp->frmx[L-i-1] = othdp->frmx[L-i] + 
	NullAddOnePair(seqX[L+start-i-1], seqY[L+start-i-1], oth->FRN);
    }
    
    othdp->fjmx[i][0] = ScoreWithNullDiag(seqX, seqY, iabs, 0, oth->FJN);
    for (d = 1; d <= i+1; d++) {
      if (i == 0)
	othdp->fjmx[i][d] = ScoreWithNullDiag(seqX, seqY, iabs-d+1, d, oth->FJN);
      else
	othdp->fjmx[i][d] = othdp->fjmx[i-1][d-1] + NullAddOnePair(cur_x, cur_y, oth->FJN);
    }
    
    /* B(i) 
     */
    bestsc = oth->t[TFLB] + othdp->flmx[i];
    for (d = 1; d <= i; d++) 
      if ((sc = othdp->emx[d-1] + oth->t[TEFJ] + oth->t[TFJB] + 
	   othdp->fjmx[i][i+1-d]) > bestsc) bestsc = sc;
    othdp->bmx[i] = bestsc;
    
    /* state M(i)
     */
    bestsc = -BIGFLOAT;
    if (cur_x < 4 && cur_y < 4) {
      if (i == 0) 
	bestsc = oth->t[TBM] + oth->t[TFLB] + othdp->flmx[L];
      else {
	if ((sc = oth->t[TBM] + othdp->bmx[i-1]) > bestsc) bestsc = sc;
	if ((sc = oth->t[TMM] + othdp->mmx[i-1]) > bestsc) bestsc = sc;
	if ((sc = oth->t[TXM] + othdp->xmx[i-1]) > bestsc) bestsc = sc;
	if ((sc = oth->t[TYM] + othdp->ymx[i-1]) > bestsc) bestsc = sc;
      }
      bestsc += oth->mem[idx(cur_x,cur_y)];
    }
    othdp->mmx[i] = bestsc;
    
    /* state X(i)
     */
    bestsc = -BIGFLOAT;
    if (cur_x < 4 && cur_y == 4) {
      if (i == 0) 
	bestsc = oth->t[TBX] + oth->t[TFLB] + othdp->flmx[L];
      else {
	if ((sc = oth->t[TBX] + othdp->bmx[i-1]) > bestsc) bestsc = sc;
	if ((sc = oth->t[TMX] + othdp->mmx[i-1]) > bestsc) bestsc = sc;
	if ((sc = oth->t[TXX] + othdp->xmx[i-1]) > bestsc) bestsc = sc;
	if ((sc = oth->t[TYX] + othdp->ymx[i-1]) > bestsc) bestsc = sc;
      }
      bestsc += oth->xem[cur_x];
    }
    othdp->xmx[i] = bestsc;
    
    /* state Y(i)
     */
    bestsc = -BIGFLOAT;
    if (cur_x == 4 && cur_y < 4) { 
      if (i == 0) 
	bestsc = oth->t[TBY] + oth->t[TFLB] + othdp->flmx[L];
      else {
	if ((sc = oth->t[TBY] + othdp->bmx[i-1]) > bestsc) bestsc = sc;
	if ((sc = oth->t[TMY] + othdp->mmx[i-1]) > bestsc) bestsc = sc;
	if ((sc = oth->t[TXY] + othdp->xmx[i-1]) > bestsc) bestsc = sc;
	if ((sc = oth->t[TYY] + othdp->ymx[i-1]) > bestsc) bestsc = sc;
      }
      bestsc += oth->yem[cur_y];
    }
    othdp->ymx[i] = bestsc;
    
    /* state E(i)
     */
    bestsc = -BIGFLOAT;
    if ((sc = oth->t[TME] + othdp->mmx[i]) > bestsc) bestsc = sc;
    if ((sc = oth->t[TXE] + othdp->xmx[i]) > bestsc) bestsc = sc;
    if ((sc = oth->t[TYE] + othdp->ymx[i]) > bestsc) bestsc = sc;
    othdp->emx[i] = bestsc;
   
  } /* while i */
  
  /* T(L-1) = \max_{i=0}^{i=L-1} E(i)*FRN(iabs+1 to L-1)
   */
  if ((sc = othdp->flmx[L]   + oth->t[TFLFR] + othdp->frmx[0]) > score)  
    score = sc;

  for (i = 0; i < L; i++) {
    if ((sc = othdp->emx[i]  + oth->t[TEFR]  + othdp->frmx[i+1]) > score) 
      score = sc; 
    if ((sc = othdp->flmx[i] + oth->t[TFLFR] + othdp->frmx[i+1]) > score) 
      score = sc; 
  }
 

 /* calculate and print the traceback
   */
  if ((traceback || doends) && (!sweep || score > 0.0)) 
    tracebackOTHdiag_L(ofp, sqinfoX, seqX, sqinfoY, seqY, start, L, score, othdp, oth, ali, 
		       alignment, traceback, doends, othends);
  
  return score;
}



/* Function: ViterbiOTHDiagMatrix() 
 * Date:     ER, Wed Dec 29 11:28:15 CST 1999 [Zaragoza] 
 * 
 * Purpose:  Calculates, i \in [start, start+L-1] --- l \in [1, i+1]
 *           mt[i][l] = P(X,Y, begin = start+i-l+1, end = start+i, best align | OTH model) 
 * 
 * Args:     ofp          -- output file 
 *           seqX, seqY   -- equal length sequences, ACGT and gaps (-)  
 *           start        -- start position 
 *           L            -- length of sX,sY  
 *           oth          -- oth_model structure 
 *           dp           -- matrices for dynamic programming
 *           ali          -- array with alignment
 *           traceback    -- if true due traceback
 *           alignment    -- if true print alignment
 * 
 * Returns:  void. fill mt[][], allocated and freed by caller
 */
void
ViterbiOTHDiagMatrix(FILE *ofp, double **mt, SQINFO sqinfoX, int *seqX, SQINFO sqinfoY, 
		     int *seqY, int start, int L, struct othmodel_s *oth, struct othdpd_s *othdp)
{
  int     i, st;	      /* relative positions in seqX, seqY    */
  int     iabs, stabs;	      /* absolute positions in seqX, seqY    */
  int     Lmax,  l; 
  int     d; 
  int     cur_x, cur_y;       /* nucleotides at those positions      */
  double  fr;
  double  sc, bestsc; 
  
  PatternDpDiagOTH(L, othdp);

  othdp->flmx[L] = ScoreWithNullDiag(seqX, seqY, start,     0, oth->FLN);
  othdp->frmx[L] = ScoreWithNullDiag(seqX, seqY, start+L-1, 0, oth->FRN);
  
  mt[0][0] = oth->t[TFLFR] + othdp->flmx[L] + othdp->frmx[L];
  
  for (st = 0; st < L; st++) {
    stabs = st + start;
    Lmax = L - st;
    
    mt[st][0] = mt[0][0];
    
    for (l = 1; l <= Lmax; l++) {
      i = st + l - 1;
      iabs = i + start;
      cur_x = seqX[iabs];
      cur_y = seqY[iabs];
      
      if (i == st) {
	othdp->flmx[i]   = ScoreWithNullDiag(seqX, seqY, iabs,      l, oth->FLN);
	othdp->frmx[L-l] = ScoreWithNullDiag(seqX, seqY, L+start-1, l, oth->FRN);
      }
      else {
	othdp->flmx[i] = othdp->flmx[i-1] + 
	  NullAddOnePair(cur_x, cur_y, oth->FLN);

	othdp->frmx[L-l] = othdp->frmx[L-l+1] + 
	  NullAddOnePair(seqX[L+start-l], seqY[L+start-l], oth->FRN);
      }
      
      othdp->fjmx[i][0] = ScoreWithNullDiag(seqX, seqY, iabs, 0, oth->FJN);
      for (d = 1; d <= l; d++) {
	if (i == st)
	  othdp->fjmx[i][d] = ScoreWithNullDiag(seqX, seqY, iabs-d+1, d, oth->FJN);
	else
	  othdp->fjmx[i][d] = othdp->fjmx[i-1][d-1] + NullAddOnePair(cur_x, cur_y, oth->FJN);
      }
      
      /* state B(i) 
       */
      bestsc = oth->t[TFLB] + othdp->flmx[i];
      for (d = 1; d < l; d++) 
	if ((sc = othdp->emx[st+d-1] + oth->t[TEFJ] + oth->t[TFJB] + 
	     othdp->fjmx[i][l-d]) > bestsc) bestsc = sc;
      
      othdp->bmx[i] = bestsc;
      
      /* state M(i)
       */
      if (cur_x < 4 && cur_y < 4) {
	if (i == st) 
	  bestsc = oth->t[TBM] + oth->t[TFLB] + othdp->flmx[L];
	else {
	  bestsc = -BIGFLOAT;
	  if ((sc = oth->t[TBM] + othdp->bmx[i-1]) > bestsc) bestsc = sc;
	  if ((sc = oth->t[TMM] + othdp->mmx[i-1]) > bestsc) bestsc = sc;
	  if ((sc = oth->t[TXM] + othdp->xmx[i-1]) > bestsc) bestsc = sc;
	  if ((sc = oth->t[TYM] + othdp->ymx[i-1]) > bestsc) bestsc = sc;
	}
	
	bestsc += oth->mem[idx(cur_x,cur_y)];
      }
      othdp->mmx[i] = bestsc;
      
      /* state X(i)
       */
      if (cur_x < 4 && cur_y == 4) {
	if (i == st) 
	  bestsc = oth->t[TBX] + oth->t[TFLB] + othdp->flmx[L];
	else {
	  bestsc = -BIGFLOAT;
	  if ((sc = oth->t[TBX] + othdp->bmx[i-1]) > bestsc) bestsc = sc;
	  if ((sc = oth->t[TMX] + othdp->mmx[i-1]) > bestsc) bestsc = sc;
	  if ((sc = oth->t[TXX] + othdp->xmx[i-1]) > bestsc) bestsc = sc;
	  if ((sc = oth->t[TYX] + othdp->ymx[i-1]) > bestsc) bestsc = sc;
	}
	bestsc += oth->xem[cur_x];
      }
      othdp->xmx[i] = bestsc;
      
      /* state Y(i)
       */
      if (cur_x == 4 && cur_y < 4) { 
	if (i == st) 
	  bestsc = oth->t[TBY] + oth->t[TFLB] + othdp->flmx[L];
	else {
	  bestsc = -BIGFLOAT;
	  if ((sc = oth->t[TBY] + othdp->bmx[i-1]) > bestsc) bestsc = sc;
	  if ((sc = oth->t[TMY] + othdp->mmx[i-1]) > bestsc) bestsc = sc;
	  if ((sc = oth->t[TXY] + othdp->xmx[i-1]) > bestsc) bestsc = sc;
	  if ((sc = oth->t[TYY] + othdp->ymx[i-1]) > bestsc) bestsc = sc;
	}
	bestsc += oth->yem[cur_y];
      }
      othdp->ymx[i] = bestsc;
      
      /* state E(i)
       */
      bestsc = -BIGFLOAT;
      if ((sc = oth->t[TME] + othdp->mmx[i]) > bestsc) bestsc = sc;
      if ((sc = oth->t[TXE] + othdp->xmx[i]) > bestsc) bestsc = sc;
      if ((sc = oth->t[TYE] + othdp->ymx[i]) > bestsc) bestsc = sc;
      othdp->emx[i] = bestsc;

      /* mt[i][l] = \max_{d=st}^{d=i} E(d)*FRN(d+1 to i)
       *
       * test: mt[i][l] = ViterbiOTHDiag_L(start=iabs-l-1, L=l)
       */
      bestsc = -BIGFLOAT;
      for (d = 1; d <= l; d++) {
	fr = ScoreWithNullDiag(seqX, seqY, start+st+d, l-d, oth->FRN);

	if ((sc = othdp->emx[st+d-1]  + oth->t[TEFR]  + fr) > bestsc) bestsc = sc;
	if ((sc = othdp->flmx[st+d-1] + oth->t[TFLFR] + fr) > bestsc) bestsc = sc; 
      }
      mt[i][l] = bestsc;

    } /* while i */
  } /* while start */
}

/* Function: ViterbiOTHFullMatrix() 
 * Date:     ER, Mon Oct 23 13:43:39 CDT 2000 [St. Louis] 
 * 
 * Purpose:  Calculates, sti \in [start, startX+Lx-1] --- lx \in [1, Lx] -- i = sti + lx - 1
 *                       stj \in [start, startY+Ly-1] --- ly \in [1, Ly] -- j = stj + lj - 1
 *
 *           mt[i*dimY+j][lx*(j+1)+ly] = 
 *               P(X,Y, begin = (startX+i-lx+1,startY+j-l+1) end = (startX+i,startY+j) , best align | OTH model) 
 * 
 * Args:     ofp          -- output file 
 *           seqX, seqY   -- equal length sequences, ACGT and gaps (-)  
 *           start        -- start position 
 *           Lx           -- length of sX  
 *           Ly           -- length of sY  
 *           oth          -- oth_model structure 
 *           dp           -- matrices for dynamic programming
 *           ali          -- array with alignment
 *           traceback    -- if true due traceback
 *           alignment    -- if true print alignment
 * 
 * Returns:  void. fill mt[][], allocated and freed by caller
 */
void
ViterbiOTHFullMatrix(FILE *ofp, double **mt, SQINFO sqinfoX, int *seqX, SQINFO sqinfoY, 
		     int *seqY, int startX, int startY, int Lx, int Ly, int Lw,
		     struct othmodel_s *oth, struct othdpf_s *othdp)
{
  int     dimY;
  int     i, sti;	      /* relative positions in seqX    */
  int     j, stj;	      /* relative positions in seqY    */
  int     ij;
  int     iabs, stiabs;	      /* absolute positions in seqX, seqY    */
  int     jabs, stjabs;	      /* absolute positions in seqX, seqY    */
  int     Lxmax,  lx; 
  int     Lymax,  ly; 
  int     dx, dy; 
  int     cur_x, cur_y;       /* nucleotides at those positions      */
  int     prv_i, prv_j, prv_ij;
  double  fr;
  double  sc, bestsc; 
  
  PatternDpFullOTH(Lx, Ly, othdp);

  dimY = Ly + 1;

  othdp->flxmx[Lx]    = ScoreWithNullX(seqX, startX,    0, oth->FLN);
  othdp->flymx[Ly]    = ScoreWithNullY(seqY, startY,    0, oth->FLN);
  othdp->frxmx[Lx]    = ScoreWithNullX(seqX, startX+Lx, 0, oth->FRN);
  othdp->frymx[Ly]    = ScoreWithNullY(seqY, startY+Ly, 0, oth->FRN);
  othdp->fjxmx[Lx][0] = ScoreWithNullX(seqX, startX,    0, oth->FJN);
  othdp->fjymx[Ly][0] = ScoreWithNullY(seqY, startY,    0, oth->FJN);

  /* START filling the mt matrix
   */
  
  /* SPECIAL CASE:
   *  
   *  M(-,-)
   *
   */
  mt[Lx*dimY+Ly][0] = oth->t[TFLFR] 
    + othdp->flxmx[Lx] + othdp->flymx[Ly] 
    + othdp->frxmx[Lx] + othdp->frymx[Ly];
  

  for (sti = 0; sti < Lx; sti++) {
    stiabs = sti + startX;
    Lxmax = Lx - sti;
    
    for (stj = 0; stj < Ly; stj++) {
      stjabs = stj + startY;
      Lymax = Ly - stj;
      
      /* FLX states
       */
      for (lx = 1; lx <= Lxmax; lx++) {
	i = sti + lx - 1;
	iabs = i + startX;
	
	cur_x = seqX[iabs];

	if (i == sti) {
	  othdp->flxmx[i]     = ScoreWithNullX(seqX, iabs,        lx, oth->FLN);
	  othdp->frxmx[Lx-lx] = ScoreWithNullX(seqX, Lx+startX-1, lx, oth->FRN);
	}
	else {
	  othdp->flxmx[i]     = othdp->flxmx[i-1]     + NullAddOneX(cur_x,              oth->FLN);
	  othdp->frxmx[Lx-lx] = othdp->frxmx[Lx-lx+1] + NullAddOneX(seqX[Lx+startX-lx], oth->FRN);
	}
	
	
	othdp->fjxmx[i][0] = ScoreWithNullX(seqX, iabs, 0, oth->FJN);
	for (dx = 1; dx <= lx; dx++) {
	  if (i == sti)
	    othdp->fjxmx[i][dx] = ScoreWithNullX(seqX, iabs-dx+1, dx, oth->FJN);
	  else
	    othdp->fjxmx[i][dx] = othdp->fjxmx[i-1][dx-1] + NullAddOneX(cur_x, oth->FJN);
	}
      }
      
      /* FLY states
       */
      for (ly = 1; ly <= Lymax; ly++) {
	j = stj + ly - 1;
	jabs = j + startY;
	cur_y = seqY[jabs];
	
	if (j == stj) {
	  othdp->flymx[j]     = ScoreWithNullY(seqY, jabs,        ly, oth->FLN);
	  othdp->frymx[Ly-ly] = ScoreWithNullY(seqY, Ly+startY-1, ly, oth->FRN);
	}
	else {
	  
	  othdp->flymx[j]     = othdp->flymx[j-1]     + NullAddOneY(cur_y,              oth->FLN);
	  othdp->frymx[Ly-ly] = othdp->frymx[Ly-ly+1] + NullAddOneY(seqY[Ly+startY-ly], oth->FRN);
	}
	
	othdp->fjymx[j][0] = ScoreWithNullY(seqY, jabs, 0, oth->FJN);
	for (dy = 1; dy <= ly; dy++) {
	  if (j == stj)
	    othdp->fjymx[j][dy] = ScoreWithNullY(seqY, jabs-dy+1, dy, oth->FJN);
	  else
	    othdp->fjymx[j][dy] = othdp->fjymx[j-1][dy-1] + NullAddOneY(cur_y, oth->FJN);
	}
      }
      
      /* SPECIAL CASE:  sti \in [0,Lx-1]  **  lx \in [1, Lx - sti]   **   i = sti+lx-1  
       *  
       *  M(i,-)
       *
       */
      for (lx = 1; lx <= Lxmax; lx++) {
	i = sti + lx - 1;
	iabs = i + startX;
	cur_x = seqX[iabs];
	ij = i*dimY + Ly;
	
	/* B(i,-) 
	 */
	bestsc = oth->t[TFLB] + othdp->flxmx[i] + othdp->flymx[Ly];
	for (dx = 1; dx < lx; dx++) 
	  if ((sc = othdp->emx[(sti+dx-1)*dimY+Ly] + oth->t[TEFJ] + oth->t[TFJB] 
	       + othdp->fjxmx[i][lx-dx] + othdp->fjymx[0][0]) 
	      > bestsc) bestsc = sc;
	
	othdp->bmx[ij] = bestsc;
	
	/* state M(i,-)
	 */
	othdp->mmx[ij] = -BIGFLOAT;
	
	/* state X(i,-)
	 */
	if (i == sti) 
	  othdp->xmx[ij] = oth->t[TBX] + oth->t[TFLB] + othdp->flxmx[Lx] + othdp->flymx[Ly] + oth->xem[cur_x];
	else
	  othdp->xmx[ij] = oth->t[TXX] + othdp->xmx[(i-1)*dimY+Ly] + oth->xem[cur_x];
	
	/* state Y(i,-)
	 */
	othdp->ymx[ij] = -BIGFLOAT;
	
	/* state E(i,-)
	 */
	othdp->emx[ij] =  oth->t[TXE] + othdp->xmx[ij];
	
	/* mt[i*dimY+Ly][lx] = 
	 *                  \max_{dx=sti}^{dx=i} E(dx,-)*FRN(dx+1 to i)
	 *
	 */
	bestsc = -BIGFLOAT;
	for (dx = 1; dx <= lx; dx++) {
	  fr = ScoreWithNullX(seqX, startX+sti+dx, lx-dx, oth->FRN) 
	    +  ScoreWithNullY(seqY, startY+stj,        0, oth->FRN);
	  
	  if ((sc = othdp->emx[ (sti+dx-1)*dimY + Ly ]        + oth->t[TEFR]  + fr) > bestsc) bestsc = sc;
	  if ((sc = othdp->flxmx[sti+dx-1] + othdp->flymx[Ly] + oth->t[TFLFR] + fr) > bestsc) bestsc = sc; 
	}
	
	mt[ij][lx-1] = bestsc;   
      }
      
      /* SPECIAL CASE:  stj \in [0,Ly-1]  **  ly \in [1, Ly - stj] ** j = stj+ly-1 
       *  
       *  M(-,j)
       *
       */
      for (ly = 1; ly <= Lymax; ly++) {
	j = stj + ly - 1;
	jabs = j + startY;
	cur_y = seqY[jabs];
	
	ij = Lx*dimY + j;
	
	/* B(-,j) 
	 */
	bestsc = oth->t[TFLB] + othdp->flxmx[Lx] + othdp->flymx[j];
	for (dy = 1; dy < ly; dy++) 
	  if ((sc = othdp->emx[Lx*dimY+(stj+dy-1)] + oth->t[TEFJ] + oth->t[TFJB] 
	       + othdp->fjxmx[0][0] + othdp->fjymx[j][ly-dy]) 
	      > bestsc) bestsc = sc;
	
	othdp->bmx[ij] = bestsc;
	
	/* state M(-,j)
	 */
	othdp->mmx[ij] = -BIGFLOAT;
	
	/* state X(-,j)
	 */
	othdp->xmx[ij] = -BIGFLOAT;
	
	/* state Y(-,j)
	 */
	if (j == stj) 
	  othdp->ymx[ij] = oth->t[TBY] + oth->t[TFLB] + othdp->flxmx[Lx] + othdp->flymx[Ly] + oth->yem[cur_y];
	else
	  othdp->ymx[ij] = oth->t[TYY] + othdp->ymx[Lx*dimY+(j-1)] + oth->yem[cur_y];
	
	/* state E(-,j)
	 */
	othdp->emx[ij] = oth->t[TYE] + othdp->ymx[ij];
	
	/* mt[Lx*dimY+j][ly] = 
	 *                  \max_{dy=stj}^{dy=j} E(-,dy)*FRN(dy+1 to j)
	 *
	 */
	bestsc = -BIGFLOAT;
	for (dy = 1; dy <= ly; dy++) {
	  fr = ScoreWithNullX(seqX, startX+sti,        0, oth->FRN) 
	    +  ScoreWithNullY(seqY, startY+stj+dy, ly-dy, oth->FRN);
	  
	  if ((sc = othdp->emx[ Lx*dimY + (stj+dy-1) ]        + oth->t[TEFR]  + fr) > bestsc) bestsc = sc;
	  if ((sc = othdp->flxmx[Lx] + othdp->flymx[stj+dy-1] + oth->t[TFLFR] + fr) > bestsc) bestsc = sc; 
	}
	
	mt[ij][ly-1] = bestsc;
      }
      
      /* GENERAL CASE:  sti \in [0,Lx-1]  **  lx \in [1, Lx - sti]   ** i = sti+lx-1   
       *                stj \in [0,Ly-1]  **  ly \in [1, Ly - stj]   ** j = stj+ly-1   
       *
       *  
       *  M(i,j)
       *
       */

      for (lx = 1; lx <= Lxmax; lx++) {
	i = sti + lx - 1;
	iabs = i + startX;
	
	for (ly = 1; ly <= Lymax; ly++) {
	  j = stj + ly - 1;
	  jabs = j + startY;
	  
	  ij = i*dimY + j;
	  cur_x = seqX[iabs];
	  cur_y = seqY[jabs];
	  
	  if (i == sti && j == stj) 
	    {
	      prv_i  = Lx*dimY + j;
	      prv_j  = i *dimY + Ly;
	      prv_ij = Lx*dimY + Ly;
	    }
	  else if (i == sti)
	    {
	      prv_i  = Lx*dimY + j;
	      prv_j  = i *dimY + j-1;
	      prv_ij = Lx*dimY + j-1;
	    }
	  else if (j == stj)
	    {
	      prv_i  = (i-1)*dimY + j;
	      prv_j  = i    *dimY + Ly;
	      prv_ij = (i-1)*dimY + Ly;
	    }
	  else
	    {
	      prv_i  = (i-1)*dimY + j;
	      prv_j  = i    *dimY + j-1;
	      prv_ij = (i-1)*dimY + j-1;
	    }
	  
	  /* B(ij) 
	   */
	  bestsc = oth->t[TFLB] + othdp->flxmx[i] + othdp->flymx[j];
	  for (dx = 1; dx < lx; dx++) 
	    for (dy = 1; dy < ly; dy++) 
	      if ((sc = othdp->emx[(sti+dx-1)*dimY+(stj+dy-1)] + oth->t[TEFJ] + oth->t[TFJB] 
		   + othdp->fjxmx[i][lx-dx] + othdp->fjymx[j][ly-dy]) 
		  > bestsc) bestsc = sc;
	  othdp->bmx[ij] = bestsc;
	  
	  /* state M(ij)
	   */
	  if (i == sti && j == stj) 
	    bestsc = oth->t[TBM] + oth->t[TFLB] + othdp->flxmx[Lx] + othdp->flymx[Ly];
	  else {
	    bestsc = -BIGFLOAT;
	    if ((sc = oth->t[TBM] + othdp->bmx[prv_ij]) > bestsc) bestsc = sc;
	    if ((sc = oth->t[TMM] + othdp->mmx[prv_ij]) > bestsc) bestsc = sc;
	    if ((sc = oth->t[TXM] + othdp->xmx[prv_ij]) > bestsc) bestsc = sc;
	    if ((sc = oth->t[TYM] + othdp->ymx[prv_ij]) > bestsc) bestsc = sc;
	  }
	  othdp->mmx[ij] = bestsc + oth->mem[idx(cur_x,cur_y)];
	  
	  /* state X(ij)
	   */
	  bestsc = -BIGFLOAT;
	  if ((sc = oth->t[TBX] + othdp->bmx[prv_i]) > bestsc) bestsc = sc;
	  if ((sc = oth->t[TMX] + othdp->mmx[prv_i]) > bestsc) bestsc = sc;
	  if ((sc = oth->t[TXX] + othdp->xmx[prv_i]) > bestsc) bestsc = sc;
	  if ((sc = oth->t[TYX] + othdp->ymx[prv_i]) > bestsc) bestsc = sc;
	  othdp->xmx[ij] = bestsc + oth->xem[cur_x];
	  
	  /* state Y(ij)
	   */
	  bestsc = -BIGFLOAT;
	  if ((sc = oth->t[TBY] + othdp->bmx[prv_j]) > bestsc) bestsc = sc;
	  if ((sc = oth->t[TMY] + othdp->mmx[prv_j]) > bestsc) bestsc = sc;
	  if ((sc = oth->t[TXY] + othdp->xmx[prv_j]) > bestsc) bestsc = sc;
	  if ((sc = oth->t[TYY] + othdp->ymx[prv_j]) > bestsc) bestsc = sc;
	  othdp->ymx[ij] = bestsc + oth->yem[cur_y];
	  
	  /* state E(ij)
	   */
	  bestsc = -BIGFLOAT;
	  if ((sc = oth->t[TME] + othdp->mmx[ij]) > bestsc) bestsc = sc;
	  if ((sc = oth->t[TXE] + othdp->xmx[ij]) > bestsc) bestsc = sc;
	  if ((sc = oth->t[TYE] + othdp->ymx[ij]) > bestsc) bestsc = sc;
	  othdp->emx[ij] = bestsc;
	  
      
	  /* mt[i*dimY+j][(lx-1)*(j+1)+ly-1] = 
	   *                  \max_{dx=sti,dy=stj}^{dx=i,dy=j} E(dx,dy)*FRN(dx+1 to i,dy+1 to j)
	   *
	   * test: mt[ij][lx*(j+2)+ly] = ViterbiOTHDiag_L(startX=iabs-lx+1, startY=jabs-ly+1, Lx=lx, Ly=ly)
	   */
	  bestsc = -BIGFLOAT;
	  for (dx = 1; dx <= lx; dx++) 
	    for (dy = 1; dy <= ly; dy++) {
	      fr = ScoreWithNullX(seqX, startX+sti+dx, lx-dx, oth->FRN) 
		+  ScoreWithNullY(seqY, startY+stj+dy, ly-dy, oth->FRN);
	      
	      if ((sc = othdp->emx[(sti+dx-1)*dimY+(stj+dy-1)]          + oth->t[TEFR]  + fr) > bestsc) bestsc = sc;
	      if ((sc = othdp->flxmx[sti+dx-1] + othdp->flymx[stj+dy-1] + oth->t[TFLFR] + fr) > bestsc) bestsc = sc; 
	    }

	  mt[ij][(lx-1)*(j+1)+ly-1] = bestsc;
	} /* while ly */
      } /* while lx */
    } /* while stj */
  } /* while sti */
}

/* Function: ViterbiOTHFullVector() 
 * Date:     ER, Mon Oct 30 16:46:14 CST 2000 [St. Louis] 
 * 
 * Purpose:  Calculates, lx \in [1, Lx] -- i = lx - 1 
 *                       ly \in [1, Ly] -- j = ly - 1 
 *
 *           v[lx*(j+1)+ly] = P(X,Y, begin = (startX,startY) end = (startX+i,startY+j) , best align | OTH model) 
 * 
 * Args:     ofp          -- output file 
 *           seqX, seqY   -- equal length sequences, ACGT and gaps (-)  
 *           start        -- start position 
 *           Lx           -- length of sX  
 *           Ly           -- length of sY  
 *           oth          -- oth_model structure 
 *           dp           -- matrices for dynamic programming
 *           ali          -- array with alignment
 *           traceback    -- if true due traceback
 *           alignment    -- if true print alignment
 * 
 * Returns:  void. fill mt[][], allocated and freed by caller
 */
void
ViterbiOTHFullVector(FILE *ofp, double *v, SQINFO sqinfoX, int *seqX, SQINFO sqinfoY, 
		     int *seqY, int startX, int startY, int Lx, int Ly, 
		     struct othmodel_s *oth, struct othdpf_s *othdp)
{
  int     dimY;
  int     i;	      /* relative positions in seqX    */
  int     j;	      /* relative positions in seqY    */
  int     ij;
  int     iabs;	      /* absolute positions in seqX, seqY    */
  int     jabs;	      /* absolute positions in seqX, seqY    */
  int     lx; 
  int     ly; 
  int     dx, dy; 
  int     cur_x, cur_y;       /* nucleotides at those positions      */
  int     prv_i, prv_j, prv_ij;
  double  fr;
  double  sc, bestsc; 
  
  PatternDpFullOTH(Lx, Ly, othdp);

  dimY = Ly + 1;

  othdp->flxmx[Lx]    = ScoreWithNullX(seqX, startX,    0, oth->FLN);
  othdp->flymx[Ly]    = ScoreWithNullY(seqY, startY,    0, oth->FLN);
  othdp->frxmx[Lx]    = ScoreWithNullX(seqX, startX+Lx, 0, oth->FRN);
  othdp->frymx[Ly]    = ScoreWithNullY(seqY, startY+Ly, 0, oth->FRN);
  othdp->fjxmx[Lx][0] = ScoreWithNullX(seqX, startX,    0, oth->FJN);
  othdp->fjymx[Ly][0] = ScoreWithNullY(seqY, startY,    0, oth->FJN);

  /* START filling the mt matrix
   */
  v[0] = oth->t[TFLFR] 
    + othdp->flxmx[Lx] + othdp->flymx[Ly] 
    + othdp->frxmx[Lx] + othdp->frymx[Ly];
  
  
  for (lx = 1; lx <= Lx; lx++) {
    i = lx - 1;
    iabs = i + startX;
    cur_x = seqX[iabs];
    
    /* FLX state
     */
    if (i == 0) {
      othdp->flxmx[i]     = ScoreWithNullX(seqX, iabs,        lx, oth->FLN);
      othdp->frxmx[Lx-lx] = ScoreWithNullX(seqX, Lx+startX-1, lx, oth->FRN);
    }
    else {
      othdp->flxmx[i]     = othdp->flxmx[i-1]     + NullAddOneX(cur_x,              oth->FLN);
      othdp->frxmx[Lx-lx] = othdp->frxmx[Lx-lx+1] + NullAddOneX(seqX[Lx+startX-lx], oth->FRN);
    }
    
    
    othdp->fjxmx[i][0] = ScoreWithNullX(seqX, iabs, 0, oth->FJN);
    for (dx = 1; dx <= lx; dx++) {
      if (i == 0)
	othdp->fjxmx[i][dx] = ScoreWithNullX(seqX, iabs-dx+1, dx, oth->FJN);
      else
	othdp->fjxmx[i][dx] = othdp->fjxmx[i-1][dx-1] + NullAddOneX(cur_x, oth->FJN);
    }

    /* SPECIAL CASE:
     *  
     *  M(-,-)
     *
     */
    v[Lx*dimY+Ly] = v[0];

    /* SPECIAL CASE:  lx \in [1, Lx]   **   i =lx-1  
     *  
     *  M(i,-)
     *
     */
    ij = i*dimY + Ly;

    /* B(i,-) 
     */
    bestsc = oth->t[TFLB] + othdp->flxmx[i] + othdp->flymx[Ly];
    for (dx = 1; dx < lx; dx++) 
      if ((sc = othdp->emx[(dx-1)*dimY + Ly] + oth->t[TEFJ] + oth->t[TFJB] 
	   + othdp->fjxmx[i][lx-dx] + othdp->fjymx[0][0]) 
	  > bestsc) bestsc = sc;
    
    othdp->bmx[ij] = bestsc;
    
    /* state M(i,-)
     */
    othdp->mmx[ij] = -BIGFLOAT;
    
    /* state X(i,-)
     */
    if (i == 0) 
      othdp->xmx[ij] = oth->t[TBX] + oth->t[TFLB] + othdp->flxmx[Lx] + othdp->flymx[Ly] + oth->xem[cur_x];
    else 
      othdp->xmx[ij] = oth->t[TXX] + othdp->xmx[(i-1)*dimY+Ly] + oth->xem[cur_x];
    
    /* state Y(i,-)
     */
    othdp->ymx[ij] = -BIGFLOAT;
    
    /* state E(i,-)
     */
    othdp->emx[ij] =  oth->t[TXE] + othdp->xmx[ij];
    
    /* v[i] = \max_{dx=0}^{dx=i} E(dx,-)*FRN(dx+1 to i)
     *
     */
    bestsc = -BIGFLOAT;
    for (dx = 1; dx <= lx; dx++) {
      fr = ScoreWithNullX(seqX, startX+dx, lx-dx, oth->FRN) 
	+  ScoreWithNullY(seqY, startY,        0, oth->FRN);
      
      if ((sc = othdp->emx[ (dx-1)*dimY + Ly ]        + oth->t[TEFR]  + fr) > bestsc) bestsc = sc;
      if ((sc = othdp->flxmx[dx-1] + othdp->flymx[Ly] + oth->t[TFLFR] + fr) > bestsc) bestsc = sc; 
    }
    
    v[ij] = bestsc;
    
  }
  
  for (ly = 1; ly <= Ly; ly++) {
    j = ly - 1;
    jabs = j + startY;
    cur_y = seqY[jabs];
    
    /* FLY state
     */
    if (j == 0) {
      othdp->flymx[j]     = ScoreWithNullY(seqY, jabs,        ly, oth->FLN);
      othdp->frymx[Ly-ly] = ScoreWithNullY(seqY, Ly+startY-1, ly, oth->FRN);
    }
    else {
      othdp->flymx[j]     = othdp->flymx[j-1]     + NullAddOneY(cur_y,              oth->FLN);
      othdp->frymx[Ly-ly] = othdp->frymx[Ly-ly+1] + NullAddOneY(seqY[Ly+startY-ly], oth->FRN);
    }
    
    othdp->fjymx[j][0] = ScoreWithNullY(seqY, jabs, 0, oth->FJN);
    for (dy = 1; dy <= ly; dy++) {
      if (j == 0)
	othdp->fjymx[j][dy] = ScoreWithNullY(seqY, jabs-dy+1, dy, oth->FJN);
      else
	othdp->fjymx[j][dy] = othdp->fjymx[j-1][dy-1] + NullAddOneY(cur_y, oth->FJN);
    }
    
    
    /* SPECIAL CASE:  ly \in [1, Ly] ** j = ly-1 
     *  
     *  M(-,j)
     *
     */
    ij = Lx*dimY + j;
    
    /* B(-,j) 
     */
    bestsc = oth->t[TFLB] + othdp->flxmx[Lx] + othdp->flymx[j];
    for (dy = 1; dy < ly; dy++) 
      if ((sc = othdp->emx[Lx*dimY + (dy-1)] + oth->t[TEFJ] + oth->t[TFJB] 
	   + othdp->fjxmx[0][0] + othdp->fjymx[j][ly-dy]) 
	  > bestsc) bestsc = sc;
    
    othdp->bmx[ij] = bestsc;
    
    /* state M(-,j)
     */
    othdp->mmx[ij] = -BIGFLOAT;
    
    /* state X(-,j)
     */
    othdp->xmx[ij] = -BIGFLOAT;
    
    /* state Y(-,j)
     */
    if (j == 0) 
      othdp->ymx[ij] = oth->t[TBY] + oth->t[TFLB] + othdp->flxmx[Lx] + othdp->flymx[Ly] + oth->yem[cur_y];
    else 
      othdp->ymx[ij] = oth->t[TYY] + othdp->ymx[Lx*dimY+(j-1)] + oth->yem[cur_y];
    
    /* state E(-,j)
     */
    othdp->emx[ij] = oth->t[TYE] + othdp->ymx[ij];
    
    /* v[j] = \max_{dy=0}^{dy=j} E(-,dy)*FRN(dy+1 to j)
     *
     */
    bestsc = -BIGFLOAT;
    for (dy = 1; dy <= ly; dy++) {
      fr = ScoreWithNullX(seqX, startX,        0, oth->FRN) 
	+  ScoreWithNullY(seqY, startY+dy, ly-dy, oth->FRN);
      
      if ((sc = othdp->emx[ Lx*dimY + (dy-1) ]        + oth->t[TEFR]  + fr) > bestsc) bestsc = sc;
      if ((sc = othdp->flxmx[Lx] + othdp->flymx[dy-1] + oth->t[TFLFR] + fr) > bestsc) bestsc = sc; 
    }
    
    v[ij] = bestsc;
  }
  
  /* GENERAL CASE:  lx \in [1, Lx]   ** i = lx-1   
   *                ly \in [1, Ly]   ** j = ly-1   
   *
   *  
   *  M(i,j)
   *
   */
  for (lx = 1; lx <= Lx; lx++) {
    i = lx - 1;
    iabs = i + startX;

    cur_x = seqX[iabs];
    
    for (ly = 1; ly <= Ly; ly++) {
      j = ly - 1;
      jabs = j + startY;
      
      cur_y = seqY[jabs];
      
      ij = i*dimY + j;
      
      if (i == 0 && j == 0) 
	{
	  prv_i  = Lx*dimY + j;
	  prv_j  = i *dimY + Ly;
	  prv_ij = Lx*dimY + Ly;
	}
      else if (i == 0)
	{
	  prv_i  = Lx*dimY + j;
	  prv_j  = i *dimY + j-1;
	  prv_ij = Lx*dimY + j-1;
	}
      else if (j == 0)
	{
	  prv_i  = (i-1)*dimY + j;
	  prv_j  = i    *dimY + Ly;
	  prv_ij = (i-1)*dimY + Ly;
	}
      else
	{
	  prv_i  = (i-1)*dimY + j;
	  prv_j  = i    *dimY + j-1;
	  prv_ij = (i-1)*dimY + j-1;
	}
      
      /* B(ij) 
       */
      bestsc = oth->t[TFLB] + othdp->flxmx[i] + othdp->flymx[j];
      for (dx = 1; dx < lx; dx++) 
	for (dy = 1; dy < ly; dy++) 
	  if ((sc = othdp->emx[(dx-1)*dimY + (dy-1)] + oth->t[TEFJ] + oth->t[TFJB] 
	       + othdp->fjxmx[i][lx-dx] + othdp->fjymx[j][ly-dy]) 
	      > bestsc) bestsc = sc;
      
      othdp->bmx[ij] = bestsc;
      
      /* state M(ij)
       */
      if (i == 0 && j == 0) 
	bestsc = oth->t[TBM] + oth->t[TFLB] + othdp->flxmx[Lx] + othdp->flymx[Ly];
      else {
	bestsc = -BIGFLOAT;
	if ((sc = oth->t[TBM] + othdp->bmx[prv_ij]) > bestsc) bestsc = sc;
	if ((sc = oth->t[TMM] + othdp->mmx[prv_ij]) > bestsc) bestsc = sc;
	if ((sc = oth->t[TXM] + othdp->xmx[prv_ij]) > bestsc) bestsc = sc;
	if ((sc = oth->t[TYM] + othdp->ymx[prv_ij]) > bestsc) bestsc = sc;
      }
      othdp->mmx[ij] = bestsc + oth->mem[idx(cur_x,cur_y)];

      /* state X(ij)
       */
      bestsc = -BIGFLOAT;
      if ((sc = oth->t[TBX] + othdp->bmx[prv_i]) > bestsc) bestsc = sc;
      if ((sc = oth->t[TMX] + othdp->mmx[prv_i]) > bestsc) bestsc = sc;
      if ((sc = oth->t[TXX] + othdp->xmx[prv_i]) > bestsc) bestsc = sc;
      if ((sc = oth->t[TYX] + othdp->ymx[prv_i]) > bestsc) bestsc = sc;
      othdp->xmx[ij] = bestsc + oth->xem[cur_x];
      
      /* state Y(ij)
       */
      bestsc = -BIGFLOAT;
      if ((sc = oth->t[TBY] + othdp->bmx[prv_j]) > bestsc) bestsc = sc;
      if ((sc = oth->t[TMY] + othdp->mmx[prv_j]) > bestsc) bestsc = sc;
      if ((sc = oth->t[TXY] + othdp->xmx[prv_j]) > bestsc) bestsc = sc;
      if ((sc = oth->t[TYY] + othdp->ymx[prv_j]) > bestsc) bestsc = sc;
      othdp->ymx[ij] = bestsc + oth->yem[cur_y];
      
      /* state E(ij)
       */
      bestsc = -BIGFLOAT;
      if ((sc = oth->t[TME] + othdp->mmx[ij]) > bestsc) bestsc = sc;
      if ((sc = oth->t[TXE] + othdp->xmx[ij]) > bestsc) bestsc = sc;
      if ((sc = oth->t[TYE] + othdp->ymx[ij]) > bestsc) bestsc = sc;
      othdp->emx[ij] = bestsc;
      
      
      /* mt[i*dimY+j] = \max_{dx=0,dy=0}^{dx=i,dy=j} E(dx,dy)*FRN(dx+1 to i,dy+1 to j)
       *
       */
      bestsc = -BIGFLOAT;
      for (dx = 1; dx <= lx; dx++) 
	for (dy = 1; dy <= ly; dy++) {
	  fr = ScoreWithNullX(seqX, startX+dx, lx-dx, oth->FRN) 
	    +  ScoreWithNullY(seqY, startY+dy, ly-dy, oth->FRN);
	  
	  if ((sc = othdp->emx[ (dx-1)*dimY + (dy-1) ]      + oth->t[TEFR]  + fr) > bestsc) bestsc = sc;
	  if ((sc = othdp->flxmx[dx-1] + othdp->flymx[dy-1] + oth->t[TFLFR] + fr) > bestsc) bestsc = sc; 
	}
      
      v[ij] = bestsc;
      
    } /* while lx */
  } /* while ly */
}

/* Function: ViterbiOTHDiagVector() 
 * Date:     ER, Wed Dec 29 11:28:15 CST 1999 [Zaragoza] 
 * 
 * Purpose:  Calculates, i \in [start, start+L-1] 
 *           mt[i] = P(X,Y, begin = start, end = start+i, best align | OTH model) 
 * 
 * Args:     ofp          -- output file 
 *           seqX, seqY   -- equal length sequences, ACGT and gaps (-)  
 *           start        -- start position 
 *           L            -- length of sX,sY  
 *           oth          -- oth_model structure 
 *           dp           -- matrices for dynamic programming
 *           ali          -- array with alignment
 *           traceback    -- if true due traceback
 *           alignment    -- if true print alignment
 * 
 * Returns:  fills v[i], allocated and freed by caller
 */
void
ViterbiOTHDiagVector(FILE *ofp, double *v, SQINFO sqinfoX, int *seqX, SQINFO sqinfoY, 
		     int *seqY, int start, int L, struct othmodel_s *oth, struct othdpd_s *othdp)
{
  int     i;	              /* relative positions in seqX, seqY    */
  int     iabs;	              /* absolute positions in seqX, seqY    */
  int     l; 
  int     d; 
  int     cur_x, cur_y;       /* nucleotides at those positions      */
  double  fr;
  double  sc, bestsc; 
  
  PatternDpDiagOTH(L, othdp);

  othdp->flmx[L] = ScoreWithNullDiag(seqX, seqY, start,     0, oth->FLN);
  othdp->frmx[L] = ScoreWithNullDiag(seqX, seqY, start+L-1, 0, oth->FRN);
  
  for (l = 1; l <= L; l++) {
    i = l - 1;
    iabs = i + start;
    cur_x = seqX[iabs];
    cur_y = seqY[iabs];
    
    if (i == 0) {
      othdp->flmx[i]   = ScoreWithNullDiag(seqX, seqY, iabs,      l, oth->FLN);
      othdp->frmx[L-l] = ScoreWithNullDiag(seqX, seqY, L+start-l, l, oth->FRN);
    }
    else {
      othdp->flmx[i] = othdp->flmx[i-1] + 
	NullAddOnePair(cur_x, cur_y, oth->FLN);
      othdp->frmx[L-l] = othdp->frmx[L-l+1] + 
	NullAddOnePair(seqX[L+start-l], seqY[L+start-l], oth->FRN);
    }
    
    othdp->fjmx[i][0] = ScoreWithNullDiag(seqX, seqY, iabs, 0, oth->FJN);
    for (d = 1; d <= l; d++) {
      if (i == 0)
	othdp->fjmx[i][d] = ScoreWithNullDiag(seqX, seqY, iabs-d+1, d, oth->FJN);
      else
	othdp->fjmx[i][d] = othdp->fjmx[i-1][d-1] + NullAddOnePair(cur_x, cur_y, oth->FJN);
    }

    /* B(i) 
     */
    bestsc = oth->t[TFLB] + othdp->flmx[i];
    for (d = 1; d < l; d++) 
      if ((sc = othdp->emx[d-1] + oth->t[TEFJ] + oth->t[TFJB] + 
	   othdp->fjmx[i][l-d]) > bestsc) bestsc = sc;
    
    othdp->bmx[i] = bestsc;
    
    /* state M(i)
     */
    bestsc = -BIGFLOAT;
    if (cur_x < 4 && cur_y < 4) {
      if (i == 0) 
	bestsc = oth->t[TBM] + oth->t[TFLB] + othdp->flmx[L];
      else {
	if ((sc = oth->t[TBM] + othdp->bmx[i-1]) > bestsc) bestsc = sc;
	if ((sc = oth->t[TMM] + othdp->mmx[i-1]) > bestsc) bestsc = sc;
	if ((sc = oth->t[TXM] + othdp->xmx[i-1]) > bestsc) bestsc = sc;
	if ((sc = oth->t[TYM] + othdp->ymx[i-1]) > bestsc) bestsc = sc;
      }
      
      bestsc += oth->mem[idx(cur_x,cur_y)];
    }
    othdp->mmx[i] = bestsc;
    
    /* state X(i)
     */
    bestsc = -BIGFLOAT;
    if (cur_x < 4 && cur_y == 4) {
      if (i == 0) 
	bestsc = oth->t[TBX] + oth->t[TFLB] + othdp->flmx[L];
      else {
	if ((sc = oth->t[TBX] + othdp->bmx[i-1]) > bestsc) bestsc = sc;
	if ((sc = oth->t[TMX] + othdp->mmx[i-1]) > bestsc) bestsc = sc;
	if ((sc = oth->t[TXX] + othdp->xmx[i-1]) > bestsc) bestsc = sc;
	if ((sc = oth->t[TYX] + othdp->ymx[i-1]) > bestsc) bestsc = sc;
      }
      bestsc += oth->xem[cur_x];
    }
    othdp->xmx[i] = bestsc;
    
    /* state Y(i)
     */
    bestsc = -BIGFLOAT;
    if (cur_x == 4 && cur_y < 4) { 
      if (i == 0) 
	bestsc = oth->t[TBY] + oth->t[TFLB] + othdp->flmx[L];
      else {
	if ((sc = oth->t[TBY] + othdp->bmx[i-1]) > bestsc) bestsc = sc;
	if ((sc = oth->t[TMY] + othdp->mmx[i-1]) > bestsc) bestsc = sc;
	if ((sc = oth->t[TXY] + othdp->xmx[i-1]) > bestsc) bestsc = sc;
	if ((sc = oth->t[TYY] + othdp->ymx[i-1]) > bestsc) bestsc = sc;
      }
      bestsc += oth->yem[cur_y];
    }
    othdp->ymx[i] = bestsc;
    
    /* state E(i)
     */
    bestsc = -BIGFLOAT;
    if ((sc = oth->t[TME] + othdp->mmx[i]) > bestsc) bestsc = sc;
    if ((sc = oth->t[TXE] + othdp->xmx[i]) > bestsc) bestsc = sc;
    if ((sc = oth->t[TYE] + othdp->ymx[i]) > bestsc) bestsc = sc;
    othdp->emx[i] = bestsc;


    
    /* v[i] = \max_{d=0}^{d=i} E(d)*FRN(d+1 to i)
     *
     * test: v[i] = ViterbiOTHDiag_L(start, L=i+1)
     */
    bestsc = -BIGFLOAT;
    for (d = 1; d <= l; d++) {      
      fr = ScoreWithNullDiag(seqX, seqY, start+d, l-d, oth->FRN);

      if ((sc = othdp->emx[d-1]  + oth->t[TEFR]  + fr) > bestsc) bestsc = sc;
      if ((sc = othdp->flmx[d-1] + oth->t[TFLFR] + fr) > bestsc) bestsc = sc; 
    }
    v[i] = bestsc;

  } /* while i */
}



/* Function: forwB()
 * Date:     ER, Wed Dec  1 17:14:58 CST 1999 [St. Louis]
 *
 * Purpose:  fills B(i,j) matrix full forward dp.
 *
 * Args:     seqX
 *           seqY
 *           dp   -- dp matrix structure for OTH model
 *           null -- oth_model structure
 *
 * Returns:  B matrix value
 */
double
forwB(int i, int j, int Lx, int Ly, struct othdpf_s *dp, struct othmodel_s *oth)
{
  int     idx = 0;
  int     dx, dy;
  int     dimX, dimY;
  double *sc;
  double  score;

  dimX = Lx + 1;
  dimY = Ly + 1;
  
  sc = (double *) MallocOrDie (sizeof(double) * dimX * dimY);

  if (i == Lx && j == Ly) {
    /* B(-,-)
     */
    free(sc);
    return score = oth->t[TFLB] + dp->flxmx[Lx] + dp->flymx[Ly];
  }
  else if (j == Ly) {
    /* B(i,-)
     */
    sc[idx++] = oth->t[TFLB] + dp->flxmx[i] + dp->flymx[Ly];
    for (dx = 0; dx < i; dx++)
      sc[idx++]  = dp->emx[dx*dimY+Ly] + oth->t[TEFJ] + oth->t[TFJB] + 
	dp->fjxmx[i][i-dx] + dp->fjymx[Ly][0];
    free(sc);
    return score = DLog2Sum(sc, idx);
  }
  else if (i == Lx) {
    /* B(-,j)
     */
    sc[idx++] = oth->t[TFLB] + dp->flymx[j] + dp->flxmx[Lx];
    
    for (dy = 0; dy < j; dy++) 
      sc[idx++]  = dp->emx[Lx*dimY+dy] + oth->t[TEFJ] + oth->t[TFJB] + 
	dp->fjymx[j][j-dy] + dp->fjxmx[Lx][0];

    free(sc);
    return score = DLog2Sum(sc, idx);
  }
  else {
    /* B(i,j)
     */
    sc[idx++] = oth->t[TFLB] + dp->flxmx[i] + dp->flymx[j];
    for (dx = 0; dx < i; dx++)
      for (dy = 0; dy < j; dy++) 
	sc[idx++]  = dp->emx[dx*dimY+dy] + oth->t[TEFJ] + oth->t[TFJB] +
	  dp->fjxmx[i][i-dx] + dp->fjymx[j][j-dy];
    
    free(sc);
    return score = DLog2Sum(sc, idx);
  }
}

/* Function: forwBdiag()
 * Date:     ER, Tue Nov  2 14:30:07 CST 1999 [St. Louis]
 *
 * Purpose:  fills B matrix forward dp.
 *
 * Args:     seqX
 *           seqY
 *           dp   -- dp matrix structure for OTH model
 *           null -- oth_model structure
 *
 * Returns:  B matrix value
 */
double
forwBdiag(int i, struct othdpd_s *dp, struct othmodel_s *oth)
{
  int     idx = 0;
  int     d;
  double *sc;
  double  score;

  sc = (double *) MallocOrDie (sizeof(double) * (i+2));

  sc[idx++] = oth->t[TFLB] + dp->flmx[i];

  for (d = 0; d < i; d++) 
    sc[idx++] = dp->emx[d] + oth->t[TEFJ] + oth->t[TFJB] + dp->fjmx[i][i-d];
  
  score = DLog2Sum(sc, idx);

  free(sc);
  return score;
}

/* Function: forwMdiag()
 * Date:     ER, Mon Jul 26 12:18:47 CDT 1999 [St. Louis]
 *
 * Purpose:  fills M matrix forward dp.
 *
 * Args:     cur_x   -- position in seqX
 *           cur_y   -- position in seqY
 *           dp       -- dp matrix structure for OTH model
 *           othmodel -- oth_model structure
 *
 * Returns:  M matrix value
 */
double
forwMdiag(int i, int L, int cur_x, int cur_y, struct othdpd_s *dp, struct othmodel_s *oth)
{
  int    idx = 0;
  double sc[4];
  double score;

  if (cur_x == 4 || cur_y == 4) 
    score = -BIGFLOAT;
  else {
    if (i == 0)
      score = oth->t[TBM] + oth->t[TFLB] + dp->flmx[L];
    else {
      sc[idx++] = oth->t[TBM] + dp->bmx[i-1];
      sc[idx++] = oth->t[TMM] + dp->mmx[i-1];
      sc[idx++] = oth->t[TXM] + dp->xmx[i-1];
      sc[idx++] = oth->t[TYM] + dp->ymx[i-1];

      score = DLog2Sum(sc, idx);
    }
    score += oth->mem[idx(cur_x,cur_y)];
  }
  
  return score;
}


/* Function: forwXdiag()
 * Date:     ER, Mon Jul 26 12:21:36 CDT 1999 [St. Louis]
 *
 * Purpose:  fills X matrix forward dp.
 *
 * Args:     cur_x   -- position in seqX
 *           cur_y   -- position in seqY
 *           dp       -- dp matrix structure for OTH model
 *           othmodel -- oth_model structure
 *
 * Returns:  X matrix value
 */
double
forwXdiag(int i, int L, int cur_x, int cur_y, struct othdpd_s *dp, struct othmodel_s *oth)
{
  int    idx = 0;
  double sc[4];
  double score;

  if (cur_x < 4 && cur_y == 4) {
    if (i == 0)
      score = oth->t[TBX] + oth->t[TFLB] + dp->flmx[L];
    else {
      sc[idx++] = oth->t[TBX] + dp->bmx[i-1]; 
      sc[idx++] = oth->t[TMX] + dp->mmx[i-1];
      sc[idx++] = oth->t[TXX] + dp->xmx[i-1];
      sc[idx++] = oth->t[TYX] + dp->ymx[i-1];
      
      score = DLog2Sum(sc, idx);
    }
    score += oth->xem[cur_x];
  }
  else 
    score = -BIGFLOAT;
  
  return score;
}

/* Function: forYdiag()
 * Date:     ER, Mon Jul 26 12:22:04 CDT 1999 [St. Louis]
 *
 * Purpose:  fills Y matrix forward dp.
 *
 * Args:     cur_x   -- position in seqX
 *           cur_y   -- position in seqY
 *           dp       -- dp matrix structure for OTH model
 *           othmodel -- oth_model structure
 *
 * Returns:  Y matrix value
 */
double
forwYdiag(int i, int L, int cur_x, int cur_y, struct othdpd_s *dp, struct othmodel_s *oth)
{
  int    idx = 0;
  double sc[4];
  double score;

  if (cur_x == 4 && cur_y < 4) {
    if (i == 0)
      score = oth->t[TBY] + oth->t[TFLB] + dp->flmx[L];
    else {
      sc[idx++] = oth->t[TBY] + dp->bmx[i-1]; 
      sc[idx++] = oth->t[TMY] + dp->mmx[i-1];
      sc[idx++] = oth->t[TXY] + dp->xmx[i-1];
      sc[idx++] = oth->t[TYY] + dp->ymx[i-1];
      
      score = DLog2Sum(sc, idx);
    }
    score += oth->yem[cur_y];
  }
  else 
    score = -BIGFLOAT;
  
  return score;
}

/* Function: forwEdiag()
 * Date:     ER, Mon Jul 26 12:22:30 CDT 1999 [St. Louis]
 *
 * Purpose:  fills E matrix forward dp.
 *
 * Args:     cur_x   -- position in seqX
 *           cur_y   -- position in seqY
 *           dp       -- dp matrix structure for OTH model
 *           othmodel -- oth_model structure
 *
 * Returns:  E matrix value
 */
double
forwEdiag(int i, int cur_x, int cur_y, struct othdpd_s *dp, struct othmodel_s *oth)
{
  int    idx = 0;
  double sc[4];
  double score;

  sc[idx++] = oth->t[TME] + dp->mmx[i];
  sc[idx++] = oth->t[TXE] + dp->xmx[i];
  sc[idx++] = oth->t[TYE] + dp->ymx[i];
  
  score = DLog2Sum(sc, idx);
  return score;
}

/* Function: tracebackOTH_L2()
 * Date:     ER, Fri Oct  8 14:14:02 CDT 1999 [St. Louis]
 *
 * Purpose:  Traceback of best align with viterbi algorith for OTH model.
 *
 * Args:     seqX, seqY   -- equal length sequences, ACGT and gaps (-) 
 *           L            -- lengths of sX,sY 
 *           othmodel     -- oth_model structure
 *
 * Returns:  void. prints the traceback for the vitebi algorithm.
 */
void
tracebackOTH_L2(FILE *ofp, SQINFO sqinfoX, int *seqX, SQINFO sqinfoY, int *seqY, 
		int startX, int startY, int Lx, int Ly, int *ret_len_ali, double score, 
		struct othdpf_s *dp, struct othmodel_s *oth, struct ali_s *ali, 
		int alignment, int traceback, int doends, struct  end_s *othends)
{
  struct tracer_s      *tr;      /* the traceback tree under construction         */
  struct tracer_s      *cur_tr;  /* ptr to node of tr we're working on            */
  struct tracerstack_s *dolist;  /* pushdown stack of active tr nodes             */
  int    pos, prv_pos;           /* current and previous positions (pos=i*dimY+j) */
  int    posi, posj, posij;      /* posi=(i-1,j), posj=(i,j-1), posij=(i-1,j-1)   */
  int    i, iabs, k, endx;       /* positions in seqX                             */
  int    j, jabs, l, endy;       /* positions in seqY                             */
  int    cur_x, cur_y;           /* nucleotides at those positions                */
  int    cur_st, prv_st;
  int    dimX, dimY;
  int    len_end;                /* length of flanking end scored with FRN        */
  int    Ltot;                   /* sum of both lengths                           */
  int    z=1;
  int    n, nx, ny;              /* to count positions in the alignment           */
  int    dx, dy;
  int    flag = FALSE;
  float  sc, cur_sc, prv_sc;     /* do the comparisons as floats (for precision reasons)   */

  if (traceback) fprintf(ofp, "\nOTH traceback (full viterbi)\n");

  dimX = Lx + 1;
  dimY = Ly + 1;

  Ltot = (Lx > Ly)? 2*Lx : 2*Ly;

  endx = startX + Lx - 1;
  endy = startY + Ly - 1;

  if (traceback) 
    fprintf(ofp,"tracing T (%d,%d) [%d %d] %f \n", endx, endy, seqX[endx], seqY[endy], score);
  
  if (Lx == 0 && Ly == 0) pos = Lx*dimY + Ly;
  else if (Lx == 0)
    {
      flag = FALSE;
      for (l = 0; l < Ly; l++) {
	pos = Lx*dimY + (Ly-1-l);
	if ((float)score == (float)(dp->emx[pos] + oth->t[TEFR] + dp->frymx[Ly-l] +
				    dp->frxmx[Lx])) 
	  { 
	    endy -= l; 
	    
	    len_end = startY+Ly-1-endy;
	    for (k = len_end; k > 0; k--) {
	      z++;
	      ali->charX[Ltot-z] = '-';
	      ali->charY[Ltot-z] = tolower(DNAAlphabet[seqY[endy+k]]);
	    }
	    
	    prv_st = stE;
	    prv_sc = dp->emx[pos];
	    if (traceback) fprintf(ofp," E->T, %f\n", score - prv_sc);
	    flag = TRUE;
	    break;
	  }
      }
      if (!flag) {
	endy -= Ly;
	if (traceback) fprintf(ofp," FL->FR, %f\n", score);
	prv_st = -1;
      }
    }

  else if (Ly == 0)
    {
      flag = FALSE;
      for (k = 0; k < Lx; k++) {
	pos = (Lx-1-k)*dimY + Ly;
	if ((float)score == (float)(dp->emx[pos] + oth->t[TEFR] + dp->frxmx[Lx-l] + dp->frymx[Ly])) 
	  { 
	    endx -= k; 
	    
	    len_end = startX+Lx-1-endx;
	    for (k = len_end; k > 0; k--) {
	      z++;
	      ali->charX[Ltot-z] = tolower(DNAAlphabet[seqX[endx+k]]);
	      ali->charY[Ltot-z] = '-';
	    }
	    
	    prv_st = stE;
	    prv_sc = dp->emx[pos];
	    if (traceback) fprintf(ofp," E->T, %f\n", score - prv_sc);
	    flag =  TRUE;
	    break;
	  }
      }
      if (!flag) {
	endx -= Lx;
 	if (traceback) fprintf(ofp," FL->FR, %f\n", score);
	prv_st = -1;
      }
    }

  flag = FALSE;
  for (k = 0; k < Lx; k++) {
    for (l = 0; l < Ly; l++) {
      pos = (Lx-1-k)*dimY + (Ly-1-l);
      
      if ((float)score == (float)(dp->emx[pos] + oth->t[TEFR] + dp->frxmx[Lx-k] + dp->frymx[Ly-l])) 
	{ 
	  endx -= k; 
	  endy -= l; 
	  
	  prv_st = stE;
	  prv_sc = dp->emx[pos];
	  if (traceback) fprintf(ofp," E->T, %f %d %d\n", score - prv_sc, endx, endy);
	  flag = TRUE;
	  break;
	}
    }
    if (flag) break;
  }

  if (!flag) {
    endx -= Lx;
    endy -= Ly;
    if (traceback) fprintf(ofp," FL->FR, %f\n", score);
    prv_st = -1;
  }
  
  len_end = (k >= l)? startX+Lx-endx : startY+Ly-endy;
  z = len_end - 1; 


  for (n = 1, nx = n, ny = n; n < len_end; n++) {
    if (seqX[endx+n] < 4 && endx+n < startX+Lx)
      { ali->charX[Ltot-z] = tolower(DNAAlphabet[seqX[endx+n]]); nx++; }
    if (seqY[endy+n] < 4 && endy+n < startY+Ly) 
      { ali->charY[Ltot-z] = tolower(DNAAlphabet[seqY[endy+n]]); ny++; }
    z--;
  }
  
  if (nx < len_end) {
   z = len_end-nx; 
   for (n = nx; n < len_end; n++) { ali->charX[Ltot-z] = '-'; z--; }
 }
 else if (ny < len_end) {
   z = len_end-ny;
   for (n = ny; n < len_end; n++) { ali->charY[Ltot-z] = '-'; z--; }
 }
  if (z != 0) Die ("wrong count down (z = %d)", z);
  
  /* Initialize
   * Start at pos "(endx,endy)" with state "stE"
   */
  tr     = InitTracer();       /* start a trace tree */
  dolist = InitTracerstack();  /* start a stack for traversing the trace tree */
  
  z = len_end-1;
  
  if (prv_st != -1) {
    cur_tr = AttachTracer(tr, pos, stE); 
    PushTracerstack(dolist, cur_tr);
  }
  
  while ((cur_tr = PopTracerstack(dolist)) != NULL)
   {
     pos = cur_tr->emit;

     i = pos/dimY;
     j = pos%dimY;
     
     iabs = i + startX;
     jabs = j + startY;
     

     if (i > -1 && i < Lx) cur_x = seqX[iabs];
     else                  cur_x = -1;
     if (j > -1 && j < Ly) cur_y = seqY[jabs];
     else                  cur_y = -1;

     cur_sc = prv_sc;
     cur_st = cur_tr->type;

     if (i == Lx && j == Ly) 
       {
	 posi  = -1;
	 posj  = -1;
	 posij = -1;
       }
     else if (i == Lx)
       {
	 posi  = -1;
	 posj = (j>0)? Lx*dimY + j-1 : Lx*dimY + Ly;
	 posij = -1;
       }
     else if (j == Ly)
       {
	 posi = (i>0)? (i-1)*dimY + Ly : Lx*dimY + Ly; 
	 posj  = -1;
	 posij = -1;
       }
     else if (i == 0 && j == 0) 
       {
	 posi  = Lx*dimY + j;
	 posj  = i *dimY + Ly;
	 posij = Lx*dimY + Ly;
       }
     else if (i == 0)
       {
	 posi  = Lx*dimY + j;
	 posj  = i *dimY + j-1;
	 posij = Lx*dimY + j-1;
       }
     else if (j == 0)
       {
	 posi  = (i-1)*dimY + j;
	 posj  = i    *dimY + Ly;
	 posij = (i-1)*dimY + Ly;
       }
     else
       {
	 posi  = (i-1)*dimY + j;
	 posj  = i    *dimY + j-1;
	 posij = (i-1)*dimY + j-1;
       }

     if      (cur_st == stE) prv_pos = pos;
     else if (cur_st == stM) prv_pos = posij;
     else if (cur_st == stX) prv_pos = posi;
     else if (cur_st == stY) prv_pos = posj;

 
    if (traceback) fprintf(ofp,"tracing %s (%d,%d) [%d %d] %f \n", 
			   ostNAME[cur_st], i, j, cur_x, cur_y, cur_sc);

     switch (cur_st){
     case stB: 
       flag = FALSE;
       if (posi == -1 && posj == -1 &&
	   cur_sc == (sc = oth->t[TFLB] + dp->flxmx[Lx] + dp->flymx[Ly])) {
	 prv_st = -1;
	 break;	   
       }
       else if (posi == -1 &&
	   cur_sc == (sc = oth->t[TFLB] + dp->flymx[j] + dp->flxmx[Lx])) {
	 prv_st = -1;

	 for (n = jabs; n >= startY; n--) {
	   z++;
	   if (seqY[n] < 4) { 
	     ali->charX[Ltot-z] = '-'; 
	     ali->charY[Ltot-z] = tolower(DNAAlphabet[seqY[n]]); 
	   }
	 }  
	 break;	   
       }
       else if (posi == -1) {
	 for (dy = 0; dy <= j; dy++) 
	   if (cur_sc == (sc = dp->emx[Lx*dimY+dy] + oth->t[TEFJ] + oth->t[TFJB] + 
			  dp->fjymx[j][j-dy] + dp->fjxmx[0][0])) {
	     prv_st  = stE;
	     prv_pos = Lx*dimY+dy;
	     prv_sc = dp->emx[prv_pos];
	     
	     for (n = jabs; n >= dy; n--) {
	       z++;
	       if (seqY[n] < 4) { 
		 ali->charX[Ltot-z] = '-'; 
		 ali->charY[Ltot-z] = tolower(DNAAlphabet[seqY[n]]);
	       }
	     }
	     break;
	   }
       }
       else if (posj == -1 &&
		cur_sc == (sc = oth->t[TFLB] + dp->flxmx[i] + dp->flymx[Ly])) {
	 prv_st = -1;
	 
	 for (n = iabs; n >= startX; n--) {
	   z++;
	   if (seqX[n] < 4) { 
	     ali->charX[Ltot-z] = tolower(DNAAlphabet[seqX[n]]); 
	     ali->charY[Ltot-z] = '-'; 
	   }
	 }	 
	 break;
       }
       else if (posj == -1) {
	 for (dx = 0; dx <= i; dx++) 
	   if (cur_sc == (sc = dp->emx[dx*dimY+Ly] + oth->t[TEFJ] + oth->t[TFJB] + 
			  dp->fjxmx[i][i-dx] + dp->fjymx[0][0])) {
	     prv_st  = stE;
	     prv_pos = dx*dimY+Ly;
	     prv_sc = dp->emx[prv_pos];
     
	     for (n = jabs; n >= dx; n--) {
	       z++;
	       if (seqX[n] < 4) { 
		 ali->charX[Ltot-z] = tolower(DNAAlphabet[seqX[n]]); 
		 ali->charY[Ltot-z] = '-';
	       }
	     }
	     break;
	   }
       }
       else if (posi > -1 && posj > -1 &&
		cur_sc == (sc = oth->t[TFLB] + dp->flxmx[i] + dp->flymx[j])) {
	 prv_st = -1;
	 
	 for (n = (i<j)? i:j, nx = iabs, ny = jabs; n >= 0; n--) {
	   z++;
	   if (seqX[nx] < 4) { ali->charX[Ltot-z] = tolower(DNAAlphabet[seqX[nx]]); nx--; }
	   if (seqY[ny] < 4) { ali->charY[Ltot-z] = tolower(DNAAlphabet[seqY[ny]]); ny--; }
	 }
	 
	 for (n = nx; n >= startX; n--) { 
	   z++; 
	   ali->charX[Ltot-z] = tolower(DNAAlphabet[seqX[n]]); 
	   ali->charY[Ltot-z] = '-'; 
	 }
	 for (n = ny; n >= startY; n--) { 
	   z++; 
	   ali->charX[Ltot-z] = '-'; 
	   ali->charY[Ltot-z] = tolower(DNAAlphabet[seqY[n]]); 
	 }
	 break;
       }
       else if (posi > -1 && posj > -1) {
	 for (dx = 0; dx <= i; dx++) {
	   for (dy = 0; dy <= j; dy++) {
	     if ((dx < i || dy < j)  &&
		 cur_sc == (sc = dp->emx[dx*dimY+dy] + oth->t[TEFJ] + oth->t[TFJB] + 
			    dp->fjxmx[i][i-dx] +  dp->fjymx[j][j-dy])) {
	       prv_st  = stE;
	       prv_pos = dx*dimY + dy;
	       prv_sc = dp->emx[prv_pos];
	       
	       for (n = (i-dx<j-dy)? i-dx:j-dy, nx = iabs, ny = jabs; n >= 0; n--) {
		 z++;
		 if (seqX[nx] < 4) { ali->charX[Ltot-z] = tolower(DNAAlphabet[seqX[nx]]); nx--; }
		 if (seqY[ny] < 4) { ali->charY[Ltot-z] = tolower(DNAAlphabet[seqY[ny]]); ny--; }
	       }
	       
	       for (n = nx; n > dx; n--) { 
		 z++; 
		 ali->charX[Ltot-z] = tolower(DNAAlphabet[seqX[n]]); 
		 ali->charY[Ltot-z] = '-'; 
	       }
	       for (n = ny; n > dy; n--) { 
		 z++; 
		 ali->charX[Ltot-z] = '-'; 
		 ali->charY[Ltot-z] = tolower(DNAAlphabet[seqY[n]]); 
	       }
	       
	       flag = TRUE;
	       break;
	     }
	   }
	   if (flag) break;
	 }
	 if (flag) break;
       }
       else Die("ViterbiOTH_L2() wrong traceback in %s (%d,%d)", ostNAME[cur_st], iabs, jabs);      
       
     case stM:
       z++;
       ali->charX[Ltot-z] = DNAAlphabet[cur_x];
       ali->charY[Ltot-z] = DNAAlphabet[cur_y];
       
       if (i == 0 && j == 0) {
	 if (cur_sc == (sc = oth->t[TBM] + oth->t[TFLB] + oth->mem[idx(cur_x,cur_y)] + dp->flxmx[Lx] + dp->flymx[Ly])) {
	   prv_st = stB;
	   prv_sc = oth->t[TFLB] + dp->flxmx[Lx] + dp->flymx[Ly];
	   break;
	 }
	 else Die("ViterbiTraceOTH_L2() wrong traceback in %s(i==0,j==0)", ostNAME[cur_st]);
       }
       else if (cur_sc == (sc = oth->t[TBM] + oth->mem[idx(cur_x,cur_y)] + dp->bmx[prv_pos])) {
	 prv_st = stB; 
	 prv_sc = dp->bmx[prv_pos];
	 break;
       }
       else if (cur_sc == (sc = oth->t[TMM] + oth->mem[idx(cur_x,cur_y)] + dp->mmx[prv_pos])) {
	 prv_st = stM; 
	 prv_sc = dp->mmx[prv_pos];
	 break;
       }
       else if (cur_sc == (sc = oth->t[TXM] + oth->mem[idx(cur_x,cur_y)] + dp->xmx[prv_pos])) {
	 prv_st = stX;
	 prv_sc = dp->xmx[prv_pos];
	 break;
       }
       else if (cur_sc == (sc = oth->t[TYM] + oth->mem[idx(cur_x,cur_y)] + dp->ymx[prv_pos])) {
	 prv_st = stY;
	 prv_sc = dp->ymx[prv_pos];
	 break;
       }
       else Die("ViterbiOTH_L2() wrong traceback in %s (%d,%d) \n", ostNAME[cur_st], iabs, jabs);
       
     case stX: 
       z++;
       ali->charX[Ltot-z] = DNAAlphabet[cur_x];
       ali->charY[Ltot-z] = '.';
       
       if (i == 0 && j == -1) {
	 if (cur_sc == (sc = oth->t[TBX] + oth->t[TFLB] + oth->xem[cur_x] + dp->flxmx[Lx] + dp->flymx[Ly])) {
	   prv_st = -1;
	   break;
	 }
	 else Die("ViterbiTraceOTH_L2() wrong traceback in %s(i==0,j==0)", ostNAME[cur_st]);
       }
       else if (cur_sc == (sc = oth->t[TBX] + oth->xem[cur_x] + dp->bmx[prv_pos])) {
	 prv_st = stB; 
	 prv_sc = dp->bmx[prv_pos];
	 break;
       }
       else if (cur_sc == (sc = oth->t[TMX] + oth->xem[cur_x] + dp->mmx[prv_pos])) {
	 prv_st = stM; 
	 prv_sc = dp->mmx[prv_pos];
	 break;
       }
       else if (cur_sc == (sc = oth->t[TXX] + oth->xem[cur_x] + dp->xmx[prv_pos])) {
	 prv_st = stX; 
	 prv_sc = dp->xmx[prv_pos];
	 break;
       }
       else if (cur_sc == (sc = oth->t[TYX] + oth->xem[cur_x] + dp->ymx[prv_pos])) {
	 prv_st = stY; 
	 prv_sc = dp->ymx[prv_pos];
	 break;
       }
       else Die("ViterbiOTH_L2() wrong traceback in %s (%d,%d)", ostNAME[cur_st], iabs, jabs);
       
     case stY: 
       z++;
       ali->charX[Ltot-z] = '.';
       ali->charY[Ltot-z] = DNAAlphabet[cur_y];
       
       if (i == -1 && j == 0) {
	 if (cur_sc == (sc = oth->t[TBY] + oth->t[TFLB] + oth->yem[cur_y] + dp->flxmx[Lx] + dp->flymx[Ly])) {
	   prv_st = -1;
	   break;
	 }
	 else Die("ViterbiTraceOTH_L2() wrong traceback in %s(i==0,j==0)", ostNAME[cur_st]);
       }
       else if (cur_sc == (sc = oth->t[TBY] + oth->yem[cur_y] + dp->bmx[prv_pos])) {
	 prv_st = stB; 
	 prv_sc = dp->bmx[prv_pos];
	 break;
       }
       else if (cur_sc == (sc = oth->t[TMY] + oth->yem[cur_y] + dp->mmx[prv_pos])) {
	 prv_st = stM; 
	 prv_sc = dp->mmx[prv_pos];
	 break;
       }
       else if (cur_sc == (sc = oth->t[TXY] + oth->yem[cur_y] + dp->xmx[prv_pos])) {
	 prv_st = stX;
	 prv_sc = dp->xmx[prv_pos];
	 break;
       }
       else if (cur_sc == (sc = oth->t[TYY] + oth->yem[cur_y] + dp->ymx[prv_pos])) {
	 prv_st = stY;
	 prv_sc = dp->ymx[prv_pos];
	 break;
       }
       else Die("ViterbiOTH_L2() wrong traceback in %s (%d,%d)", ostNAME[cur_st], iabs, jabs);
       
     case stE:
       if      (cur_sc == (sc = oth->t[TME] + dp->mmx[prv_pos])) {
	 prv_st = stM; 
	 prv_sc = dp->mmx[prv_pos];
	 break;
       }
       else if (cur_sc == (sc = oth->t[TXE] + dp->xmx[prv_pos])) {
	 prv_st = stX;
	 prv_sc = dp->xmx[prv_pos];
	 break;
       }
       else if (cur_sc == (sc = oth->t[TYE] + dp->ymx[prv_pos])) {
	 prv_st = stY;
	 prv_sc = dp->ymx[prv_pos];
	 break;
       }
       else Die("ViterbiOTH_L2() wrong traceback in %s (%d,%d)", ostNAME[cur_st], iabs, jabs);
       
     default:
       Die("invalid state in ViterbiOTH_L2()");
     }

     if (prv_st != -1) {
       if (traceback) fprintf(ofp," %s->%s, %f\n", ostNAME[prv_st], ostNAME[cur_st], cur_sc - prv_sc);
       PushTracerstack(dolist, AttachTracer(cur_tr, prv_pos, prv_st));
     }
     else if (traceback) fprintf(ofp," %s->%s, %f\nEND OTH traceback\n\n", ostNAME[0], ostNAME[cur_st], cur_sc);
       
   }
  
  if (alignment) PrintAlign(ofp, sqinfoX, sqinfoY, Ltot-z, Ltot, ali);
  *ret_len_ali = z;

  FreeTracer(tr);
  FreeTracer(cur_tr);
  FreeTracerstack(dolist);
}

/* Function: tracebackOTHdiag_L()
 * Date:     ER, Tue Nov  2 14:30:33 CST 1999 [St. Louis]
 *
 * Purpose:  Traceback of best align with viterbi algorith for OTH model.
 *
 * Args:     seqX, seqY   -- equal length sequences, ACGT and gaps (-) 
 *           L            -- lengths of sX,sY 
 *           othmodel     -- oth_model structure
 *
 * Returns:  void. prints the traceback for the vitebi algorithm.
 */
void
tracebackOTHdiag_L(FILE *ofp, SQINFO sqinfoX, int *seqX, SQINFO sqinfoY, int *seqY, int start, int L, 
		   double score, struct othdpd_s *dp, struct othmodel_s *oth, struct ali_s *ali, 
		   int alignment, int traceback, int voutput, struct end_s *othends)
{
  struct tracer_s      *tr;      /* the traceback tree under construction  */
  struct tracer_s      *cur_tr;  /* ptr to node of tr we're working on     */
  struct tracerstack_s *dolist;  /* pushdown stack of active tr nodes      */
  int    i, iabs, prv_i;         /* position in seqX, seqY                 */
  int    k, end;                 /* position in seqX, seqY                 */
  int    cur_x, cur_y;           /* nucleotides at those positions         */
  int    cur_st, prv_st;
  float  sc, cur_sc, prv_sc;     /* do the comparisons as floats (for precision reasons) */
  int    n;                      /* to count positions in the alignment                  */
  int    d;
  int    flag = FALSE; 
  int    lc = 0;
  int    x;

  /* initialize ends
   */
  PatternEnd(othends);

  end = start + L - 1;
  if (traceback) 
    fprintf(ofp, "\nOTH traceback (diagonal viterbi) [start= %d, end = %d]\n", start, end);

  if (traceback) fprintf(ofp,"tracing T (%d) [%d %d] %f \n", end, seqX[end], seqY[end], score);
  
  flag = FALSE;

  if ((float)score == (float)(dp->flmx[L]   + oth->t[TFLFR] + dp->frmx[0])) { 
    flag = TRUE; prv_st = -1;  
    if (voutput) { othends->lend[lc] = end;  othends->rend[lc] = end; } 
  }
      
  for (k = 0; k < L; k++) {
    if ((float)score == (float)(dp->emx[L-1-k] + oth->t[TEFR] + dp->frmx[L-k])) 
      { 
	end -= k; 
	prv_sc = (float)dp->emx[end-start];
	if (traceback) fprintf(ofp," E->FR, %f\n", score - prv_sc);
	if (voutput) { othends->rend[lc] = L-k; }
	flag = TRUE;
	break;
      }
    else if ((float)score == (float)(dp->flmx[L-1-k] + oth->t[TFLFR] + dp->frmx[L-k])) 
      { 
	end = -1; 
	if (traceback) fprintf(ofp," FL->FR, %f\n", score);
	prv_st = start;
	if (voutput) { othends->lend[lc] = L-k; othends->rend[lc] = L-k; }
	flag = TRUE;
	break;
      }
  }
  /* check that it grabed at least one position
   */
  if (!flag) Die("ViterbiTraceOTHDiag_L() wrong traceback in FR");
  flag = FALSE;

  for (n = end+1; n < L+start; n++) {
    if (seqX[n] < 4) ali->charX[n] = tolower(DNAAlphabet[seqX[n]]); 
    else             ali->charX[n] = '-';
    if (seqY[n] < 4) ali->charY[n] = tolower(DNAAlphabet[seqY[n]]); 
    else             ali->charY[n] = '-';
  }

  if (L == 0) {
    cur_st = stFR;
    cur_sc = score;
    if (cur_sc == (sc = oth->t[TFLFR] +
		   dp->flmx[L] + dp->frmx[L])) 
      {
	prv_st = stFL;
	if (traceback) fprintf(ofp,"tracing %s (%d) %f \n", ostNAME[cur_st], end, cur_sc);
	if (traceback) fprintf(ofp," %s->%s, %f\n", ostNAME[prv_st], ostNAME[cur_st], cur_sc);  
	return;
      }
    else Die("ViterbiTraceOTHDiag_L() wrong traceback in stE(L=0)");
  }

  /* Initialize
   * Start at pos "end" with state "stE"
   */
  tr     = InitTracer();       /* start a trace tree */
  dolist = InitTracerstack();  /* start a stack for traversing the trace tree */
  
  if (end > start) {
    cur_tr = AttachTracer(tr, end, stE); 
    PushTracerstack(dolist, cur_tr);
  }

  while ((cur_tr = PopTracerstack(dolist)) != NULL)
    {
      iabs = cur_tr->emit;
      i    = iabs - start;

      if (i > -1) {
	cur_x = seqX[iabs];
	cur_y = seqY[iabs];
      }
      else {
	cur_x = -1;
	cur_y = -1;
      }
      
      cur_sc = prv_sc;
      cur_st = cur_tr->type;
      
      if (cur_st == stE) prv_i = i;
      else               prv_i = i-1;
      
      if (traceback) fprintf(ofp,"tracing %s (%d) [%d %d] %f \n", ostNAME[cur_st], iabs, 
			     cur_x, cur_y, cur_sc);
      
      switch (cur_st) {
      case stB:
	flag = FALSE;

	sc = oth->t[TFLB] + dp->flmx[L];
	if (i == -1 && 
	    cur_sc < sc+MARGIN && cur_sc > sc-MARGIN) 
	  {
	    prv_st = -1;  
	    if (voutput) othends->lend[lc] = start;
	    
	    break;	   
	  }
	else {
	  sc = oth->t[TFLB] + dp->flmx[i];
	  if (cur_sc < sc+MARGIN && cur_sc > sc-MARGIN) 
	    {
	      prv_st = -1;
	      
	      for (n = iabs; n >= start; n--) {
		if (seqX[n] < 4) ali->charX[n] = tolower(DNAAlphabet[seqX[n]]); 
		else             ali->charX[n] = '-';
		if (seqY[n] < 4) ali->charY[n] = tolower(DNAAlphabet[seqY[n]]); 
		else             ali->charY[n] = '-';
	      }
	      
	      flag = TRUE;	  
	      if (voutput) { othends->lend[lc] = iabs+1; lc ++; }
 
	      break;
	    }
	}
	
	for (d = 0; d < i; d++) {
	  sc = dp->emx[d] + oth->t[TEFJ] + oth->t[TFJB] + dp->fjmx[i][i-d];
	  if (!flag &&
	      cur_sc < sc+MARGIN && cur_sc > sc-MARGIN) 
	    {
	      prv_st = stE;
	      prv_sc = dp->emx[d];
	      prv_i  = d;
	      
	      for (n = iabs; n > start+d; n--) {
		if (seqX[n] < 4) ali->charX[n] = tolower(DNAAlphabet[seqX[n]]); 
		else             ali->charX[n] = '-';
		if (seqY[n] < 4) ali->charY[n] = tolower(DNAAlphabet[seqY[n]]); 
		else             ali->charY[n] = '-';
	      }
	      flag = TRUE;	
	      if (voutput) { othends->lend[lc] = iabs+1;  lc ++; }
	      
	      break;
	    }
	}
	
	if (!flag) Die("ViterbiOTHDiag_L() wrong traceback in %s (%d)", ostNAME[cur_st], iabs);
	break;
	
      case stM: 
	flag = FALSE;
	ali->charX[iabs] = DNAAlphabet[cur_x];
	ali->charY[iabs] = DNAAlphabet[cur_y];

	if (i == 0) {
	  sc = oth->t[TBM] + oth->t[TFLB] + oth->mem[idx(cur_x,cur_y)] + dp->flmx[L];
	  if (cur_sc < sc+MARGIN && cur_sc > sc-MARGIN) 
	    {
	      prv_st = stB;
	      prv_sc = oth->t[TFLB] + dp->flmx[L];
	      flag   = TRUE;
	      break;
	    }
	  else Die("ViterbiTraceOTHDiag_L() wrong traceback in %s(i==0)", ostNAME[cur_st]);
	}
	else {
	  sc = oth->t[TMM] + oth->mem[idx(cur_x,cur_y)] + dp->mmx[prv_i];
	  if (cur_sc < sc+MARGIN && cur_sc > sc-MARGIN) 
	    {
	      prv_st = stM;
	      prv_sc = dp->mmx[prv_i];
	      flag   = TRUE;
	      break;
	    }
	  sc = oth->t[TXM] + oth->mem[idx(cur_x,cur_y)] + dp->xmx[prv_i];
	  if (!flag &&
	      cur_sc < sc+MARGIN && cur_sc > sc-MARGIN) 
	    {
	      prv_st = stX;
	      prv_sc = dp->xmx[prv_i];
	      flag   = TRUE;
	      break;
	    }
	  sc = oth->t[TYM] + oth->mem[idx(cur_x,cur_y)] + dp->ymx[prv_i];
	  if (!flag &&
	      cur_sc < sc+MARGIN && cur_sc > sc-MARGIN) 
	    {
	      prv_st = stY;
	      prv_sc = dp->ymx[prv_i];
	      flag   = TRUE;
	      break;
	    }
	  sc = oth->t[TBM] + oth->mem[idx(cur_x,cur_y)] + dp->bmx[prv_i];
	  if (!flag &&
	      cur_sc < sc+MARGIN && cur_sc > sc-MARGIN) 
	    {
	      prv_st = stB;
	      prv_sc = dp->bmx[prv_i];
	      flag   = TRUE;
	      break;
	    }
	  if (!flag) Die("ViterbiOTHDiag_L() wrong traceback in %s (%d)", ostNAME[cur_st], iabs);
	}
	
      case stX: 
	flag = FALSE;
	ali->charX[iabs] = DNAAlphabet[cur_x];
	ali->charY[iabs] = '.';
	
	if (i == 0) {
	  sc = oth->t[TBX] + oth->t[TFLB] + oth->xem[cur_x] + dp->flmx[L];
	  if (cur_sc < sc+MARGIN && cur_sc > sc-MARGIN) 
	    {
	      prv_st = stB;
	      prv_sc = oth->t[TFLB] + dp->flmx[L];
	      flag   = TRUE;
	      break;
	    }
	  else Die("ViterbiTraceOTHDiag_L() wrong traceback in %s(i==0)", ostNAME[cur_st]);
	}
	else {
	  sc = oth->t[TMX] + oth->xem[cur_x] + dp->mmx[prv_i];
	  if (cur_sc < sc+MARGIN && cur_sc > sc-MARGIN) 
	    {
	      prv_st = stM;
	      prv_sc = dp->mmx[prv_i];
	      flag   = TRUE;
	      break;
	    }
	  sc = oth->t[TXX] + oth->xem[cur_x] + dp->xmx[prv_i];
	  if (!flag &&
	      cur_sc < sc+MARGIN && cur_sc > sc-MARGIN) 
	    {
	      prv_st = stX;
	      prv_sc = dp->xmx[prv_i];
	      flag   = TRUE;
	      break;
	    }
	  sc = oth->t[TYX] + oth->xem[cur_x] + dp->ymx[prv_i];
	  if (!flag &&
	      cur_sc < sc+MARGIN && cur_sc > sc-MARGIN) 
	    {
	      prv_st = stY;
	      prv_sc = dp->ymx[prv_i];
	      flag   = TRUE;
	      break;
	    }
	  sc = oth->t[TBX] + oth->xem[cur_x] + dp->bmx[prv_i];
	  if (!flag &&
	      cur_sc < sc+MARGIN && cur_sc > sc-MARGIN) 
	    {
	      prv_st = stB;
	      prv_sc = dp->bmx[prv_i];
	      flag   = TRUE;
	    break;
	    }
	  if (!flag) Die("ViterbiOTHDiag_L() wrong traceback in %s (%d)", ostNAME[cur_st], iabs);
	}
	
      case stY: 
	flag = FALSE;
	ali->charX[iabs] = '.';
	ali->charY[iabs] = DNAAlphabet[cur_y];
	
	if (i == 0) {
	  sc = oth->t[TBY] + oth->t[TFLB] + oth->yem[cur_y] + dp->flmx[L];
	  if (cur_sc < sc+MARGIN && cur_sc > sc-MARGIN) 
	    {
	      prv_st = stB;
	      prv_sc = oth->t[TFLB] + dp->flmx[L];
	      flag   = TRUE;
	      break;
	    }
	  else Die("ViterbiTraceOTHDiag_L() wrong traceback in %s(i==0)", ostNAME[cur_st]);
	}
	else {
	  sc = oth->t[TMY] + oth->yem[cur_y] + dp->mmx[prv_i];
	  if (cur_sc < sc+MARGIN && cur_sc > sc-MARGIN) 
	    {
	      prv_st = stM;
	      prv_sc = dp->mmx[prv_i];
	      flag   = TRUE;
	      break;
	    }
	  sc = oth->t[TXY] + oth->yem[cur_y] + dp->xmx[prv_i];
	  if (!flag &&
	      cur_sc < sc+MARGIN && cur_sc > sc-MARGIN) 
	    {
	      prv_st = stX;
	      prv_sc = dp->xmx[prv_i];
	      flag   = TRUE;
	      break;
	    }
	  sc = oth->t[TYY] + oth->yem[cur_y] + dp->ymx[prv_i];
	  if (!flag &&
	      cur_sc < sc+MARGIN && cur_sc > sc-MARGIN) 
	    {
	      prv_st = stY;
	      prv_sc = dp->ymx[prv_i];
	      flag   = TRUE;
	      break;
	    }
	  sc = oth->t[TBY] + oth->yem[cur_y] + dp->bmx[prv_i];
	  if (!flag &&
	      cur_sc < sc+MARGIN && cur_sc > sc-MARGIN) 
	    {
	      prv_st = stB;
	      prv_sc = dp->bmx[prv_i];
	      flag   = TRUE;
	      break;
	    }
	  if (!flag) Die("ViterbiOTHDiag_L() wrong traceback in %s (%d)", ostNAME[cur_st], iabs);
	}
	
      case stE:  
	flag = FALSE;
	sc = oth->t[TME] + dp->mmx[prv_i];
	if (cur_sc < sc+MARGIN && cur_sc > sc-MARGIN) 
	  {
	    prv_st = stM; 
	    prv_sc = dp->mmx[prv_i];
	    flag   = TRUE;	   
	    if(voutput) othends->rend[lc] = iabs;

	    break;
	  }
	sc = oth->t[TXE] + dp->xmx[prv_i];
	if (!flag &&
	    cur_sc < sc+MARGIN && cur_sc > sc-MARGIN) 
	  {
	    prv_st = stX;
	    prv_sc = dp->xmx[prv_i];
	    flag   = TRUE;
	    if(voutput) othends->rend[lc] = iabs;
	    break;
	  }
	sc = oth->t[TYE] + dp->ymx[prv_i];
	if (!flag &&
	    cur_sc < sc+MARGIN && cur_sc > sc-MARGIN) 
	  {
	    prv_st = stY;
	    prv_sc = dp->ymx[prv_i];
	    flag   = TRUE;
	    if(voutput) othends->rend[lc] = iabs;
	    break;
	  }
	if (!flag) Die("ViterbiOTHDiag_L() wrong traceback in %s (%d)", ostNAME[cur_st], iabs);
	
      default:
	Die("invalid state in ViterbiOTHDiag_L()");
      }
      
      if (prv_st != -1) {
	if (traceback) 
	  fprintf(ofp," %s->%s, %f\n", ostNAME[prv_st], ostNAME[cur_st], cur_sc - prv_sc);
	PushTracerstack(dolist, AttachTracer(cur_tr, prv_i+start, prv_st));
      }
      else 
	if (traceback) 
 	  fprintf(ofp," %s->%s %f\nEND OTH traceback\n\n", ostNAME[0],  ostNAME[cur_st], cur_sc);
     
       if (lc >= MAX_NUM_ENDS) Die(" Too many ends in OTH traceback. Increase parameter MAX_NUM_ENDS");
   }
  if (alignment) PrintAlign(ofp, sqinfoX, sqinfoY, start, L, ali);
 
  if (traceback) {
    printf("OTH ends [%d %d]\n", start, start+L-1);
    for (x = 0; x < MAX_NUM_ENDS; x++)
      if (othends->lend[x] > -1)
	printf("(%d..%d) ", othends->lend[x], othends->rend[x]);
    printf("\n");
  }

  FreeTracer(tr);
  FreeTracer(cur_tr);
  FreeTracerstack(dolist);
}









