#! /usr/bin/perl 
#
# Use HTK for a variety of tasks on any data
#
# Version 1.3
# Modified by Sarah Borys
# Jan. 7, 2009
#
# Origional Code
# Written by Mark Hasegawa-Johnson
# Jan. 3, 2006
     
########################################################
# Section 0: GLOBAL VARIALBES AND PARAMETERS
#
# All modifications to train.pl should only be made in this section.  If you have problems 
# running this code, contact Sarah by email at sborys@uiuc.edu.

# ------------------ Run HCopy to generate acoustic features ---------------------#
# Store the configuration files  
$CONFIGDIR = "cfg";
# 
# The script file directory
$SCPDIR = "scp";
#
# Where should HCopy store the acoustic data
$DATADIR = "data";
#
# The configuration file for HCopy (generated by train.pl)
$HCOPYCFG = "HCopy.cfg";
#
# HCopy script file (generated by train.pl) 
$HCOPYSCP = "HCopy.scp";
# 
# List of training files (generated by train.pl, see $TESTFILES)
$TRAINSCP = "train.scp";
#
# List of test files (generated by train.pl, see $TESTFILES)
$TESTSCP = "test.scp";
#
# Tell train.pl which audio files to put in the test set.  May contain a regular expression.
# For example $TESTFILES = "sw48\d\d" or $TESTFILES = "M0[1-3]".
$TESTFILES = "SA";
#
# set the file extention of HCopy acoustic files based on TARGETKIND 
%HCEXT = ( "MFCC" =>          "mfc",
           "PLP" =>           "plp",
           "WAVEFORM" =>      "wfm",
           "LPC" =>           "lpc",
           "LPREFC" =>        "lpr",
           "LPDELCEP" =>      "ldc",
           "IREFC" =>         "irc",
           "FBANK" =>         "fbk",
           "MELSPEC" =>       "msp",
           "USER" =>          "usr",
           "DISCRETE" =>      "dsc"
    );
#
# Set the options for HCopy
# Set the trace option (-T) for HCopy.  Set $HCOPYTRACE = 'inf' to turn off the trace
$HCOPYTRACE = "1";
#
# Set any other options for HCopy not explicitly supported
$HCOPYOPTS = "";
#
# Root of the directory tree containing data files  
$DATATREE="/workspace/fluffy1/ntimit-train/wav_chopped8k";
#
# Set the extension of the audio data files incase it is something other than "wav." 
$AUDIOEXT = "wav";
#
# Set the HTK configureation parameters
# IMPORTANT NOTE: if TARGETKIND == USER, set NUMCEPS to be equal to the length of
# your feature vector! 
%HTKCFG = ( "SOURCEFORMAT" => "WAV",
                "TARGETKIND" => "MFCC_E_D_A_T",
                "TARGETRATE" => "100000.0",
                "SAVECOMPRESSED" => "T",
                "WINDOWSIZE" => "320000.0",
                "NUMCHANS" => "26",
                "CEPLIFTER" => "22",
                "NUMCEPS" => "12",
                'HADAPT:BLOCKS' => '3',
                'HADAPT:OCCTHRESH' => '300.0',
                'USEPOWER' => 'T'
    );

# ----------------- Transcriptions and Dictionary Perl Processing ------------ #
# Directory where train.pl will write the MLF files
$MLFDIR = "mlf";
#
# Directory where the mmf files are stored.  train.pl also stores phone lists 
# in this directory
$MMFDIR = "mmf";
#
# Tell train.pl where the dictionary is stored and where to write additional 
# dictionary files 
$DICTDIR = "dict";
#
# Tell train.pl where to write all language model-related files
$GRAMDIR = "gram";
#
# Set the name of the pre-generated dictionary.  This dictionary should not include 
# entries with sp at the end
$DICTIONARY = "NTIMIT.dict";
#
# Set the name of the monophone dictionary with short pauses (generated by train.pl)
$MONOSPDICT = "monosp.dict";
#
# Set the name of the monophone dictionary without short pauses (generated by train.pl, should not 
# be equal to $DICTIONARY)
$MONONOSPDICT = "mononosp.dict";
#
# Set the name of the triphone dictionary with short pauses (generated by train.pl)
$TRISPDICT = "trisp.dict";
#
# Set the name of the triphone dictionary without short pauses (generated by train.pl)
$TRINOSPDICT = "trinosp.dict";
#
# The name of the monophone list (generated by train.pl)
$MONOPHONELIST = "monophones.phf";
#
# The name of the monophone list with no non-speech sounds included (generated by train.pl)
$NOSPPHONELIST = "aligned_phones.phf";
#
# train.pl will generate three word-level mlf files.  
# $WORDUTTMLF is the label file with pronounciation variants and utterance boundary markers, for HERest 
$WORDUTTMLF = "word_utt.mlf";
#
# $WORDNUTTMLF is the mlf with pronounce variants but no utterance boundary markers, for HLStats 
$WORDNUTTMLF = "word_nutt.mlf";
#
# $WORDSNORMLF is the label file with no pronunciation variants or silences, for HResults 
$WORDSNORMLF = "word_snor.mlf";
#
# Set the path to transcriptions files of the corpus if they are not already in HTK mlf format. 
$TRANSPATH = "/workspace/fluffy1/ntimit-train/nist_wrd";
#
# Set the extension of the non-HTK transcription files located at $TRANSPATH 
$TRANSEXT = "wrd";
#
# Select the corpus of your transcriptions, i.e., set your transcription format to 
# Only listed formats are supported. 
# ESPS format has lines like ->    start_time color label 
# TIMIT format has lines like ->   start_time end_time label 
# NIST format has lines like ->     filename start_time end_time label
# Note that start_time and end_time do not need to be in 100ns units (hnsu).  train.pl can convert 
# times to hnsu.  See %TSCALE.
%TRANSFORMAT = ( "ESPS" => '0',
                 "TIMIT" => '1',
                 "NIST" => '0',
                 "MLF" => '0',
    );
#
# Time unit conversion factor for corpus transcription files   
# Set $TSCALE = 10000000 to convert from seconds to hnsu 
# Set $TSCALE = 625 to convert from TIMIT 16,000 samples per second to hnsu 
$TSCALE = 625;
#
# List of phones that train.pl should not add an 'sp' (short pause) after
@NOTPHONES = ( 'laff', 
               'noiz', 
               'sil', 
               'sp', 
               'UTT_END', 
               'pv', 
               'pn', 
               'unk'
    );

# ------------------- Initialize monophone models with HCompV ---------------- #
# Other variables from above that must be defined to run HCompV:
# $MMFDIR
# $HTKCFG
# %HTKCFG
# $TRAINSCP
# $SCPDIR
# $CFGDIR
#
# The directory where HCompV will store the output
$HCOMPVOUTDIR = "protophones";
# 
# The configuration file needed for training and testing with the HTK tools 
# (generated by train.pl)
$HTKCFG = "HTK.cfg";
#
# The name of the prototype MMF file
$PROTOMMF = "proto";
#
# Set the trace option of HCompV.  Set $HCVTRACE = 'inf' to turn off this option 
$HCOMPVTRACE = "1";
#
# Set the variance floor for HCompV.  Set $HCOMPVFLOOR = 'none' to turn off this option 
$HCOMPVFLOOR = 0.001;
# 
# Other options for HCompV 
$HCOMPVOPTS = "-m";
#
# The following variables are for customizing the structure of the HMM
# Set the number of states in the prototype HMM.  This number includes start and end states.
$NPROTOSTATES = 5;
#
# List the number of states for each phone model if certain models have different
# numbers of states than $NPROTOSTATES
%NSTATES = ( 'UTT_END' => 3,
             'sp' => 3,
             'dx' => 4
    );
# Set the number of initial mixtures in each state
$NMIXES = 1;
#
# Set the hmm to be either ergodic ($ERGODIC = 1) or left to right ($ERGODIC = 0)
$ERGODIC = 0;

# --------------------------- HHEd and HLEd processing ----------------------- #
# Other variables from above needed by HHEd and HLEd
# $MMFDIR
# $MLFDIR
# $HCOMPVOUTDIR
# $HTKCFG
# %HTKCFG
# $MONOPHONELIST
# %NSTATES
# $NPROTOSTATES
#
# The edit directory 
$EDITDIR = "ed";
#
# This file will contain the instructions for HHEd to tie silence states together.
$SILHHED = "sil.hed";
# 
# List the sounds that are not speech sounds.  These sounds will have some parameters tied together
# when HHEd exacutes the commands in $SILHHED.  The 'sp' and 'UTT_END' are handled separately and 
# should not be included in this array. 
@NOTSPEECH = ('sil',
              'noiz',
              'laff',
              'unk'
    );
#
# Contains instructions for HLEd for expanding the word MLF with the dictionary 
$EXPANDHLED = "expand.hled";
#
# Set trace option.  Set $HHEDTRACE = 'inf' to turn off the trace 
$HHEDTRACE = 1;
# 
# Set the subdirectory where HHEd will output the tied silence models
# This directory will be created in $MMFDIR
$HHEDTIEDSILDIR = "withsilence";
#
# Set the trace option.  Set $HLEDTRACE = 'inf' to turn off the trace 
$HLEDTRACE = 'inf';
# 
# Where should HLEd say that the label files are? (the -l option)
$HLEDLABLOC = "\'*\'";
#     
# Name of output MMF file generated by tools like HERest and HHEd 
$MMFOUTFILE = "mmf";
#
# The monophone level mlf 
$MONOPHONEMLF = "monophones.mlf";

# ------------------- Train unaligned monophone models using HERest ---------- #
# The following variables from above are also required for this section of code
# $CFGDIR
# $SCPDIR
# $MMFDIR
# $MLFDIR
# $HTKCFG
# $WORDUTT
# $TRAINSCP
# $MONOPHONELIST
#
# Set the subdirectory where HERest will write its output for monophone model estimation.
# Note that the actually directories created by train.pl will have the form : 
# $MMFDIR/${FLATSTARTDIR}${NMIXES}G${i} where $NMIXES is the number of mixtures in your hmm and 
# $i tells you which iteration of HERest created the files in the directory 
$FLATSTARTDIR = 'flat';
#
# Set the trace option.  Set $HERESTTRACE = 'inf' to turn off this option
$HERESTTRACE = 'inf';
# 
# Set the number of times train.pl will run HERest 
$HERESTITER = 5;
# 
# Set the pruning threshold of HERest.  Set $HREPRUNE = "inf" to turn this option off  
$HERESTPRUNE = "50.0 100.0 10000.0";
# 
# Set the name of the stats file created by HERest (-s option, automatically used on final iteration of
# HERest) 
$HERESTSTATS = 'stats';
# 
# Set any other options for HERest.  For example, to set the minimum number of examples per 
# phone to 4 and to update only the mixture weights and transition probabilities, set  
# $HERESTOPTS = "-n 4 -u uw"; 
$HERESTOPTS = "";

# -------------------------------- Forced alignment -------------------------- #
# Other variables defined above needed for this section 
# $CFGDIR
# $SCPDIR
# $MMFDIR
# $MLFDIR
# $DICTDIR
# #MONONOSPDICT
# $TRAINSCP
# $WORDUTT
# $NOSPPHONELIST
# $HTKCONFIG
#
# The directory $ALIGDIR is used for forced alignment.
# $ALIGNMMFDIR contains MMF files created from model estimation using HRest and HInit 
$ALIGNDIR = "align";
# 
# Base name of the directories that will store various files from each iteration of
# forced alignment.  Four different kinds of directories will be created from this base name. 
# 1.  ${ALIGNITERDIR}V${i} - stores the MLFs generated by HVite at each iteration i 
# 2.  ${ALIGNITERDIR}H${i} - stores the mmf models generated by HRest at each iteration i 
# 3.  ${ALIGNITERDIR}I${i} - stores the mmf models generated by HInit at each iteration i 
# 4.  ${ALIGNITERDIR}D${i} - stores the aligned phone dictionary genereated by train.pl at
# each iteration i 
# Directory type 1 will automatically be stored in $MMFDIR/$ALIGNMMFDIR 
# Dorectories of type 2 and 3 will automatically be stored in $MMFDIR/$ALIGNMMFDIR
# Directories of type 4 will automatically be stored in $DICTDIR/$ALIGNDIR
$ALIGNITERDIR = "iter";
# What scripts (files) do we want for forecd alignment?  This is included here incase 
# the files to be forced aligned are not part of the training, test, or copy set
@ALIGNSCP = ("$SCPDIR/train.scp",
             "$SCPDIR/test.scp"
    );
# 
# Subdirectory where HVite store the individual label files (needed by HVite for forced alignment,
# do not set this to $LABDIR = '\'*\'')  This directory will be located in $MLFDIR
$LABDIR = "labs";
#
# What is the actual name of the script that is going to be used for forced alignment 
$ALIGNSCP = "align.scp";
#
# Set the trace for HInit.  Set HINITTRACE = 'inf' to turn off the trace 
$HINITTRACE = 'inf';
# 
# Set the minimum number of phones HInit needs for model estimation  
# Set $HINITMINPHONES = "" to use the default value (3 examples) 
$HINITMINPHONES = "1";
# 
# How many iterations of the training algorithm will HInit run? 
# Set $HINITITER = "" to use the default optioin (100 iterations) 
$HINITITER = 100;
# 
# Set any other options HInit might need 
$HINITOPTS = "";
#
# Set the trace option.  To turn off the trace, set $HRESTTRACE = 'inf'
$HRESTTRACE = 'inf';
# 
# How many iterations of the BW algorigm does HRest need to do?  Set 
# $HRESTITER = "" to use the default value (100 iterations) 
$HRESTITER = 100;
# 
# Wet the minimum number of phones that HRest needs for model estimation
$HRESTMINPHONES = 1;
#
# Set any other options HRest might need 
$HRESTOPTS = "";
# 
# Set the trace option.  Set $HVITETRACE = 'inf' to turn off the trace.  
$HVITETRACE = 'inf';
# 
# Set the -p option.  Set $P = "" to use the HTK default 
$P = -20.0;
# 
# Set the -s option.  Set $S = "" to use the HTK default  
$S = 10.0;
# 
# Set the mlf file extension (-l option)
$HVITELABLOC = "\'*\'";
# 
# Set the label file extension (-y option).  This is needed for forced alignment 
# and for lattice generation.  Lattice generation is not implemented yet 
$HVITELAB = "lab";
# 
# Set the pruning option for HVite 
$HVITEPRUNE = "250.0 100.0 1000.0";
#
# Other options for HVite not explicitly implemented  
$HVITEOPTS = "";
#
# How many iterations will train.pl do of the forced alignment algorithm
$ALIGNITER = 7;

# ------------------- Train monophone models with HInit and HRest ------------ #
# Other required variables
# (list pending)
#
# The following two directories are where HInit and HRest write their output for monophone 
# model re-estimation. 
# HInit output (written to directory $MMFDIR/$HINITOUTDIR)
$HINITOUTDIR = "init";
# 
# HRest output (written to directory $MMFDIR/$HRESTOUTDIR 
$HRESTOUTDIR = "rest";
# These options can be used to merge aligned phones 
# in transcriptions aligned using HVite. 
# Set $MERGE = 1 to merge phones under duration $DUR.  Set $MERGE = 0 to not use this option 
$MERGE = 1;
# 
# Set $DUR = min phone duration.  If $MERGE == 1 then all phones <= $DUR will be merged 
# after forced alignment.  Must be in 100 ns units 
$DUR = 500000;

# ------ Build a grammar for monophone recognition with HParse and then test the --------- #
# --------------- time-aligned phone models using HVite and HResults --------------------- #
# Other variables needed 
# (this needs to be written)
#
# Directory where the results will be stored 
$RESULTSDIR = "results";
# 
# Subdirectories for the different recognition tasks
# Forced (time aligned) monophone recognition
$ALIGNRECDIR = "align";
#
# Create the monophone -> monophone dictionary 
$MONOPHONEMAP = "phonemap.dict";
#
# The name of the grammar file for phone recognition 
$PHONEGRAMMAR = "grammar";
# 
# Phone network generated by HParse 
$PHONENET = "phnet";
#
# The mlf created by HVite during forced (time) aligned monophone recognition 
$ALIGNRECMLF = "alignrec.mlf";

# ---------------------------- Create triphone mlf and mmf files ----------------------------- #
# other variables
# $EDITDIR
# $MLFDIR
# $MMFDIR
#
# Directory where HHEd will store the triphone models
$HHEDTRIPHONEDIR = "cloned_triphones";
#
# The list of the triphones 
$TRIPHONELIST = "triphones.phf";
# 
# The name of the triphone mlf created by HLEd
$TRIPHONEMLF = "triphones.mlf";
# 
# Name of the edit file to create word internal or cross-word triphone mlfs 
$TRIPHONESHLED = "triphones.hled";
#
# Name of the edit file to create triphone mmf files
$TRIPHONESHHED = "triphones.hhed";
#
# Combined list of monophones and triphones
$ALLPHONESLIST = "allphones.phf";
#
# Add any label in your database to this array that HLEd should not expand into triphones 
@DONTEXPAND = ( 'laff', 
                'noiz', 
                'sil', 
                'sp', 
                'UTT_END', 
                'pv', 
                'pn', 
                'unk'
);

# -------------------- Training triphone models with HERest --------------- #
# Other variables
#
# Base name of the directory where HERest will write the triphone mmf models.  The actual
# directory will appear as $MMFDIR/${HERESTTRIDIR}${NMIXES}G${i}, where i is the iteration 
# number 
$HERESTTRIDIR = "tri";

# ------------------ Creating clustered triphone models with HHEd ----------------- #
# Other variables needed 
#

# The directory where HHEd will write the clustered triphone output
$CLUSTERDIR = "clus";

# the file  HHEd uses to cluster the triphones
$CLUSTERHHED = "cluster.hhed";

# Tree cluster file generated by HHEd
$CLUSTERLIST = "clusters.phf";

# Tree file generated by HHEd
$TRIPHONETREE = "triphone_tree.trf";

# Distinctive Feature Definitions for triphone clustering
# This is fragile; it must be changed whenever the dictionary's
#     phoneset is changed!!
%QS = ();
# 1. Vowel, Semivowels, and Flap
#   1a. manner
$QS{'syllabic_liquid'} = ['er', 'el'];
$QS{'nonsyllabic_liquid'} = ['l', 'r'];
$QS{'liquid'} = [ @{$QS{'syllabic_liquid'}}, @{$QS{'nonsyllabic_liquid'}} ];
$QS{'glide'} = ['w', 'y'];
$QS{'semivowel'} = [ @{$QS{'liquid'}}, @{$QS{'glide'}} ];
$QS{'semivowel_or_flap'} = [ @{$QS{'semivowel'}}, 'dx' ];
$QS{'diphthong'} = [ 'ay', 'oy', 'aw' ];
$QS{'monophthong'} = ['aa','ao','ah','ow','uh','uw','ae','eh','ey','ih','iy','ax'];
$QS{'vowel'} = [ @{$QS{'monophthong'}}, @{$QS{'diphthong'}} ];
$QS{'vowel_or_flap'} = [ @{$QS{'vowel'}}, 'dx' ];
$QS{'syllabic_consonant'} = [ @{$QS{'syllabic_liquid'}}, 'en' ];
$QS{'syllabic'} = [ @{$QS{'syllabic_consonant'}}, @{$QS{'vowel'}} ];
$QS{'vocalic'} = [ @{$QS{'semivowel_or_flap'}}, @{$QS{'vowel'}} ];
#   1b. low, high, back, front, tense, lax
$QS{'lowback_tense'} = [ 'aa', 'ao' ];
$QS{'lowback'} = [ 'aa', 'ao', 'ah' ];
$QS{'highback'} = [ 'uw', 'uh' ];
$QS{'lowfront'} = [ 'ae', 'eh' ];
$QS{'highfront'} = [ 'ih', 'iy' ];
$QS{'highfront_final'} = [ @{$QS{'highfront'}}, 'ay', 'oy' ];
$QS{'lowfront_initial'} = [ @{$QS{'lowfront'}}, 'aw' ];
$QS{'highback_final'} = [ @{$QS{'highback'}}, 'aw' ];
$QS{'lowback_initial'} = [ @{$QS{'lowback'}}, 'ay' ];
$QS{'midback_initial'} = [ @{$QS{'midback'}}, 'oy' ];
$QS{'back'} = [ @{$QS{'highback'}}, 'ow', @{$QS{'lowback'}} ];
$QS{'back_initial'} = [ @{$QS{'back'}}, @{$QS{'diphthong'}} ];
$QS{'back_final'} = [ @{$QS{'back'}}, 'aw' ];
$QS{'front'} = [ @{$QS{'highfront'}}, 'ey', @{$QS{'lowfront'}} ];
$QS{'front_final'} = [ @{$QS{'front'}}, 'ay', 'oy' ];
$QS{'low'} = [ @{$QS{'lowback'}}, @{$QS{'lowfront'}} ];
$QS{'low_initial'} = [ @{$QS{'low'}}, @{$QS{'diphthong'}} ];
$QS{'high'} = [ @{$QS{'highback'}}, @{$QS{'highfront'}} ];
$QS{'high_final'} = [ @{$QS{'high'}}, @{$QS{'diphthong'}} ];
$QS{'lowlax'} = [ 'eh', 'ah' ];
$QS{'lowtense'} = [ 'aa', 'ao', 'ae' ];
$QS{'lax'} = [ 'eh', 'ah', 'ih', 'uh' ];
$QS{'nontense'} = [ 'eh', 'ah', 'ih', 'uh', 'ax' ];
$QS{'midback'} = [ 'ow'] ; 
#   1c. rounded vs. unrounded
$QS{'rounded_monophthong'} = [ 'ao', 'uw', 'uh', 'ow' ];
$QS{'rounded_vowel'} = [ @{$QS{'rounded_monophthong'}}, 'aw' ];
$QS{'rounded'} = [ @{$QS{'rounded_vowel'}}, 'w' ];
$QS{'semirounded'} = [ @{$QS{'rounded'}}, 'r', 'er' ];
$QS{'w_final'} = [ 'uw', 'ow', 'aw', 'w' ];
#  1d. place
$QS{'retroflex'} = [ 'r', 'er' ];
$QS{'palatal_tense'} = ['iy', 'ey' ];
$QS{'palatal_lax'} = ['ih', 'eh' ];
$QS{'palatal_vowel'} = [ @{$QS{'palatal_tense'}}, @{$QS{'palatal_lax'}} ];
$QS{'palatal_vocalic'} = [ @{$QS{'palatal_vowel'}}, 'y' ];
$QS{'y_final'} = [ 'y', 'iy', 'ey', 'ay', 'oy' ];
$QS{'uvular_vowel'} = [ 'ow', 'uh' ];
$QS{'uvular_syllabic'} = [ 'ow', 'uh', 'el' ];
$QS{'uvular_vocalic'} = [ 'ow', 'uh', 'el', 'l' ];
$QS{'uvular_high'} = [ 'uh', 'el', 'l' ];
$QS{'pharyngeal_tense'} = [ 'aa', 'ao', 'ae' ];
$QS{'pharyngeal'} = [ 'aa', 'ah', 'ao', 'ae' ];
$QS{'schwalike'} = [ 'ax', 'ah' ];
$QS{'central'} = [ 'ax', 'ah', 'uh' ];
# 2. Consonants, including semivowels: most important division is probably place
$QS{'labiovelar'} = [ 'f', 'v' ];
$QS{'bilabial'} = ['m', 'p', 'b'];
$QS{'labial'}  = [ @{$QS{'labiovelar'}}, @{$QS{'bilabial'}} ];
$QS{'dental'} = ['th', 'dh'];
$QS{'alveolar'} = [ 't','d','n','s','z' ];
$QS{'palatal'} = ['sh','zh','ch', 'jh', 'y'];
$QS{'palatal_nonaffricate'} = ['sh','zh', 'y'];
$QS{'retroflex'} = [ 'r', 'er' ];
$QS{'anterior'} = [ @{$QS{'dental'}}, @{$QS{'alveolar'}} ];
$QS{'posterior'} = [ @{$QS{'palatal'}}, @{$QS{'retroflex'}} ];
$QS{'distributed'} = [ @{$QS{'dental'}}, @{$QS{'palatal'}} ];
$QS{'apical'} = [ @{$QS{'alveolar'}}, @{$QS{'retroflex'}} ];
$QS{'coronal'} = [ @{$QS{'apical'}}, @{$QS{'distributed'}} ];
$QS{'velar'} = [ 'ng', 'k', 'g' ];
$QS{'palatovelar'} = [ @{$QS{'palatal'}}, @{$QS{'velar'}} ];
$QS{'glottal'} = [ 'q', 'hh' ];
# 3. Consonants: manner and voicing
$QS{'vcd_stop'} = [ 'b', 'd', 'g' ];
$QS{'unv_stop'} = [ 'p', 't', 'k' ];
$QS{'aspirated'} = [ @{$QS{'unv_stop'}}, 'hh' ];
$QS{'stop'} = [ @{$QS{'unv_stop'}},  @{$QS{'vcd_stop'}} ];
$QS{'affricate'} = ['jh', 'ch'];
$QS{'stop_or_affricate'} = [ @{$QS{'stop'}},  @{$QS{'affricate'}} ];
$QS{'unv_nonstrident'} = [ 'f', 'th' ];
$QS{'vcd_nonstrident'} = [ 'v', 'dh' ];
$QS{'unv_stridentf'} = [ 's', 'sh' ];
$QS{'vcd_stridentf'} = [ 'z', 'zh' ];
$QS{'unv_strident'} = [ 's', 'sh', 'ch' ];
$QS{'vcd_strident'} = [ 'z', 'zh', 'jh' ];
$QS{'vcd_fricative'}=[@{$QS{'vcd_nonstrident'}},@{$QS{'vcd_stridentf'}} ];
$QS{'unv_fricative'}=[@{$QS{'unv_nonstrident'}},@{$QS{'unv_stridentf'}}];
$QS{'stridentf'} = [ @{$QS{'vcd_stridentf'}},  @{$QS{'unv_stridentf'}} ];
$QS{'strident'} = [ @{$QS{'vcd_strident'}},  @{$QS{'unv_strident'}} ];
$QS{'nonstrident'} = [ @{$QS{'unv_nonstrident'}},  @{$QS{'vcd_nonstrident'}} ];
$QS{'fricative'} = [ @{$QS{'stridentf'}},  @{$QS{'nonstrident'}} ];
$QS{'nonsyl_nasal'} = [ 'm', 'n', 'ng' ];
$QS{'nasal'} = [ 'm', 'n', 'ng', 'en' ];
$QS{'continuant'} = [ @{$QS{'fricative'}},  @{$QS{'vocalic'}} ];
$QS{'sonorant'} = [ @{$QS{'nasal'}},  @{$QS{'vocalic'}} ];
$QS{'vcd_obstruent'} = [ @{$QS{'vcd_stop'}},  @{$QS{'vcd_fricative'}}, 'jh' ];
$QS{'unv_obstruent'} = [ @{$QS{'unv_stop'}},  @{$QS{'unv_fricative'}}, 'ch' ];
$QS{'voiced'} = [ @{$QS{'vcd_obstruent'}},  @{$QS{'sonorant'}} ];
# 4. Stress: 
#  4a. All other questions:
#   push the stressed counterparts of all phones in the list
foreach $c (keys %QS) { push(@{$QS{$c}}, map {$_.'1';} @{$QS{$c}}); }
#  4b. The stress question: 
#   a phone is stressed if in the context of any stressed phone
my(@stressedphones) = map { $_ . '1' } @PHONELIST;
$QS{'stress'} = \@stressedphones;

# ---------------- Build the language model with HLStats and HBuild ---------------- #
# Other variables needed 

# The list of all words in the corpus 
$WORDLIST = "wordlist.txt";
#
# Set the name of the bigram created by HLStats 
$BIGRAM = "bigram.ll";
#
# Set the name of the lattice created by HBuild  
$LATTICE = "bigram.lat";
#
# The trace option for HLStats.  Set $HLSTATSTRACE = "inf" to turn off this option. 
$HLSTATSTRACE = "1";
#
# Set any other options for HLStats 
$HLSTATSOPTS = "-t 1 -h 1";
#
# The trace option for HBuild.  Set HBUILDTRACE = "inf" to turn off this option. 
$HBUILDTRACE = "1";
#
# Set any options for HBuild 
$HBUILDOPTS = "";

# ---------------- Create scripts for talker dependent recognition ---------- #  

# directory where the script files will be written 
$TALKDEPDIR = "tlkdep";

# ------------------ Upmix and do talker dependent recognition ------------ # 
# Other variables

# Directory where the talker independent results will be stored
$ALLRECDIR = "all";
#
# HVite configuration file 
$HVITECFG = "HVite.cfg";
#
# Base name of the HHEd upmix file
$UPMIXHHED = "upmix";
#
# Set the max number of mixtures to upmix to 
$MAXMIX = 37;
# 
# How many mixtures do you increase the model by during each iteration? 
$UPMSTEP = 2;
#
# Base file name of the regression class trees
$REGCLSTREE = "rc";

#####################################################################
# Section 1: PERL SUBROUTINES
#
# This section contains all auxilary perl subroutines written for processing 
# MLFs, MMF, token lists, and other HTK-related other files that may need 
# pre-processing or modification before they are able to be used with HTK.  
#
# word2snor: convert word to its SNOR
#   (standard normalized orthographic representation).
# The purpose of SNOR: to make sure that the only differences between
#   two transcriptions are the ones that you want HResults to count.
# Standard definition for SNOR is available at http://www.nist.gov/speech/.
# The SNOR implemented in this function is NOT QUITE NIST'S DEFINITION.
# Usage: @snor = word2snor($word).
# The length of @snor is:
#   0 - if $word should not be counted (e.g. silence)
#   1 - most of the time
#   2 or more - if $word should be divided into multiple words, e.g.,
#       a compound word, acronym, or number
# This function is used many places, so don't comment it out...
sub word2snor {
    my(@snor) = $_[0]; 
    # Return a zero-length list for silence, noise, ...
    if ($snor[0] =~ /^\[(noise|silence|vocalized-noise)/) { return (); } 
    # ... laughter that is not masking a word ...
    if ($snor[0] =~ /^\[laughter(_|\])/) { return (); }
    # ... utterance boundaries ...
    if ($snor[0] =~ /^!(ENTER|EXIT)$/) { return (); }
    # ... filled pauses ...
    if ($snor[0] =~ /^(uh|um|umm)$/i) { return (); }

    # Remove the laughter cover
    $snor[0] =~ s/^\[laughter\-(.*)\]/\1/;

    # Remove irrelevant detail from the word
    $snor[0] =~ s/_.*//; # Remove pronunciation variants
    $snor[0] =~ s:^.*/(.*)\]$:\1:; # Malaprop => intended word
    $snor[0] =~ s/[\.\']//g; # Ignore period or apostrophe
    
    # The NIST standard says that fragments should count in 
    # the numerator of the WER computation, but not the denominator.
    # HResults doesn't implement that.
    # Here's a heuristic approach:
    #  Keep only the acoustically implemented part.
    #  Disadvantage: "-[De]troit" becomes "troit", hard to read
    #  Advantage: "-[de]pending" becomes "pending", identical to the
    #   word "pending" that HVite would most correctly recognize.
    $snor[0] =~ s/\[[^\]]*\]\-//;  # Eliminate word-final unpronounced phones
    $snor[0] =~ s/\-\[[^\]*]\]//;  # Eliminate word-initial unpron phones

    # Change & to the subword "And", e.g., A&E becomes AAndE
    $snor[0] =~ s/&/And/g;
    
    # Treat any remaining slash (20/20) or dash (AK-47, eighty-six),
    #   as a divider for a compound word.
    my($word) = $snor[0];
    @snor = split(/[\-\/]/, $word);

    # Divide acronyms, e.g. ATT becomes A T T
    for($n=$#snor; $n >= 0; $n = $n-1) {
	# Heuristic: Separate around capitalized sub-words & digit strings:
	#  GrandPrixAAndE10fifty => Grand Prix A And E 10 fifty
	my(@bar)=split(/([A-Z][a-z]*|\d+)/,$snor[$n]);
	splice(@snor,$n,1,@bar);
    }

    # Goal: convert digit sequence into its component words
    # Problem: same digit sequence can be pronounced different ways
    # These rules match 43 out of 50 cases in the Switchboard dictionary
    # Warning to hackers: Early rules feed later rules
    for($n=0; $n <= $#snor; $n = $n+1) {
	# Double-oh ...
	if($snor[$n]=~/^00(.*)$/) { splice(@snor,$n,1,("double","oh",$1)); }
	# One thousand, twelve thousand, ...
	if($snor[$n]=~/^(\d?\d)000$/) { splice(@snor,$n,1,($1,"thousand")); }
	# One hundred, twelve hundred, ...
	if($snor[$n]=~/^(\d?\d)00$/) { splice(@snor,$n,1,($1,"hundred")); }
	# Any other three or four digit string: split into 2-digit substrings
	if($snor[$n] =~ /^(\d?\d)(\d\d)$/) { splice(@snor,$n,1,($1,$2)); }
	# Any other string longer than four digits: pronounce each digit
	if($snor[$n]=~/\d\d\d\d\d/) { splice(@snor,$n,1,split(//,$snor[$n])); }
	
	# 10..19
	my(@w) = ('ten','eleven','twelve','thirteen','fourteen',
		  'fifteen','sixteen','seventeen','eighteen','nineteen');
	if($snor[$n]=~/^1\d$/) { $snor[$n] = $w[$snor[$n]-10]; }
	# Any other two-digit string
	@w = ('oh','ten','twenty','thirty','forty','fifty',
	      'sixty','seventy','eighty','ninety');
	if($snor[$n]=~/^(\d)(\d)$/) { splice(@snor,$n,1,($w[$1],$2)); }
	# Single digits
	@w = ('zero','one','two','three','four','five','six',
	      'seven','eight','nine');
	if($snor[$n]=~/^(\d)$/) { $snor[$n] = $w[$1]; }
    }

    return(@snor);
}   # END of function definition word2snor

# peruse_tree: a recursive subroutine to search the data tree
# Takes two arguements.  The first arguement is the path to the sound 
# files.  The second arguement is the extension of the file type you are
# searching for.
# Used in section(s): 2
sub peruse_tree {
    my(@rval)=();
    # If $_[0] is a directory, open it and recurse
    if ( -d $_[0] ) {
        opendir(TOPDIR, $_[0]) || die "Can't opendir $_[0]: $!";
        foreach $filename (readdir(TOPDIR)) {
            # If $filename doesn't end in period, append recursion to @rval
            unless( $filename =~ /\.$/ ) { push(@rval, peruse_tree("$_[0]/$filename", $_[1])); }
        }
        closedir(TOPDIR);
    }
    # Otherwise: if $_[0] matches the pattern, return it
    elsif ( $_[0] =~ $_[1] )  { @rval = ( $_[0] ); }
    return(@rval);
}   # END of function definition peruse_tree

# transMatGen's first arguement is the number of states in an HMM and prints
# a transition matrix to \*FILE, the second arguement of the function.  
# Uses the global variable $ERGODIC.
sub transMatGen {
    my($nstates) = $_[0];
    my($FILE) = $_[1];
    print $FILE "<TransP> $nstates\n";
    if ($ERGODIC) {
	my($EmitStates) = $nstates - 1;
	my($transp) = 1/$EmitStates;
        $transp =~ m/(0\.\d\d)/;
        $transp = $1;
        my($sumProb) = 0;
	foreach $m (1..$EmitStates) { $sumProb = $sumProb + $transp }
	if ((my($PError) = 1 - $sumProb) > 0)  { $transp2 = $transp + $PError }
	else { $transp2 = $transp }
	foreach $m (1..$nstates) {
	    if ($m == 1)  {
		foreach $k (1..$nstates)  {
		    if ($k == 2)  { print $FILE "1.0 " }
		    else { print $FILE "0.0 " }
		}
	    }
	    elsif ($m == $nstates) { foreach $k (1..$nstates) { print $FILE "0.0 " } }
	    else {
		$sumProb = 0;
		foreach $k (1..$nstates) { 
                    if ($k == 1) { print $FILE "0.0 " }
      		    else {
		        if ($k == $m) { print $FILE "$transp2 " }
		        else { print $FILE "$transp " }
		    }
		} # END foreach $k (1..$nstates)                                                        
	    } # END else                                                                                     
	    print MMF "\n";
	}  # END foreach $m (1..$nstates)                                                               
    } # END if($ERGODIC)                                                                                     
    else {
         #we make a left-to-right HMM instead                                                                 
	foreach $m (1..$nstates)  {
	    foreach $k (1..$nstates) {
		if ($m == 1) {
		    if ($k == 2) { print $FILE "1.0 " }
		    else { print $FILE "0.0 " }
		}
		elsif (($k == $m) || ($k == $m+1)) {
                    if ($m == $nstates) { print $FILE "0.0 " }
                    else { print $FILE "0.5 " }
		}
		else { print $FILE "0.0 " }
	    }
	    print $FILE "\n";
	}
    }
    print $FILE "<EndHMM>\n";

}   # END of function transMatGen

sub ProtoGen {
    
    my($phone) = $_[0]; 
    my($ref) = $_[1];

    print $ref '~o <VecSize> ',$VECTORDIM,' <', $HTKCFG{'TARGETKIND'}, ">\n";
    print $ref "~h \"$phone\"\n<BeginHMM>\n <NumStates> $NPROTOSTATES\n";
    foreach $m (2..$NPROTOSTATES-1) {
        print $ref " <State> $m ";
        if ($NMIXES > 1) {
	   print $ref "<NumMixes> $NMIXES ";
	   $mixProb = 1/$NMIXES;
           $mixProb =~ m/(0\.\d\d)/;
           $mixProb = $1;
           $sumProb = 0;
        }
        foreach $k (1..$NMIXES) {
            if ($NMIXES > 1)  {
                if ($k == $NMIXES)  {
                    if (($PError = 1 - $sumProb) > 0) {
                        $mixProb = $mixProt + $PError;
                    }
                }
                $sumProb = $sumProb + $mixProb;
                print $ref "\n  <Mixture> $k $mixProb";
	    }
            print $ref "\n   <Mean> $VECTORDIM\n   ";
            foreach (1..$VECTORDIM) { print ${ref} " 0.0" or die "$!";}
            print $ref "\n   <Variance> $VECTORDIM\n   ";
            foreach (1..$VECTORDIM) { print ${ref} " 1.0" }
        }
        print $ref "\n";
    }

    #write the transition matrix to file MMF                                                                  
    transMatGen($NPROTOSTATES, $ref);
} # end of function ProtoGen

sub mergePhones {
    my($inmlf) = $_[0];

    # read in the input mlf file
    open(INMLF, "$inmlf") or die "can't open $inmlf : $!";
    my(@mlf) = <INMLF>;
    close INMLF;
     
    #eliminate phones that have a duration under $DUR
    my($i)=0;
    while ($i <= $#mlf) {
    # check to make sure our line starts with a number
	if ($mlf[$i] =~ /^\d/)  {
	    @line = split(/ /,$mlf[$i]);        
            #check the phone duration
	    $dif = $line[1] - $line[0]; #print STDERR " $dif\n";
	    if ($dif <= $DUR) {
                # if it's the first entry in the file transcription
                if ($mlf[$i-1] =~ /^"/) {
                    $mlf[$i+1] =~ s/^(\d+)/0/;
                    push @new, $mlf[$i+1];
                    $i = $i+2;                
                }
                # if it's the last entry in the file transcription
                elsif ($mlf[$i+1] =~ /\./) {
                    # modify the previous line
                    $new[$#new] =~ s/\s(\d+)/ $line[1]/;
                    $i = $i+1;
                }
                # any other line
                else {
                    # modify the previous line
                    $dif = $dif/2;
                    $dif =~ s/\.\d+//;
                    $new[$#new] =~ /\d+\s(\d+)/;
                    $end = $1;
                    $new = $end + $dif;
                    $new[$#new] =~ s/$end/$new/;
                    # modify the next line 
                    $mlf[$i+1] =~ s/^(\d+)/$new/;
                    # if the next line is the start of a word                                                                    # print STDERR "$#line\n"; 
                    if ($#line == 3) {
                        $mlf[$i+1] =~ s/\n/ $line[3]/;
	    	        push @new, $mlf[$i+1];
		    }
                    # otherwise  
                    else {
			push @new, $mlf[$i+1];
		    }
                    $i = $i+2;
                }
	    }
	    else { push @new, $mlf[$i]; $i=$i+1; }
	}
        # if it doesn't start with a number, just put it in the new mlf file
	else { push @new, $mlf[$i]; $i=$i+1; }
    }
    
    # write the output mlf file
    open (OUTMLF, ">mlf/temp.mlf") or die "can't open mlf/temp.mlf : $!";
    for $i (@new) {
        print OUTMLF "$i";
    }
    close(OUTMLF);
    system("mv mlf/temp.mlf $inmlf");

} # end of function mergePhones

sub extractDict {
    my($mlf) = $_[0];
    my($aligndict) = $_[1];
    my(@dict) = ();
    open (MLF, "$mlf") or die "can't open $mlf : $!";
    for (<MLF>)  {
	chomp; chomp;
	@line = split;
	if ($#line == 3) {
	    if ($word =~ /\w/) {
		push @dict, $word;
	    }
	    $word = "$line[3] $line[2]";
	}
	elsif ($#line == 2) {
	    $word = $word . " $line[2]";
	}
	elsif ($line[0] eq ".") {
	    push @dict, $word;
	    $word = "";
	}
	else {}
    }
    close MLF;

    my(%hash)  = map { $_, 1 } @dict;
    @dict = keys %hash;
    @dict = sort @dict;

    open (NEW, ">$aligndict") or die "can't open $aligndict : $!";
    for $i (@dict) {
	print NEW "$i\n";
    }
    close NEW
} # end of function extractDict

###################################################################
# Section 2: Perl pre-processing

# Process and save the command line input
$USAGE = "USAGE: perl train.pl string\
where string can contain any one of the following letters.  Note that \
some options may require output from other options.  Options are listed\
in the order that train.pl will run them.  Before running train.pl, remember\
to set the user defined variables in section 0 of the code.\

\tC\tCreate acoustic observations with HCopy\
\tT\tUse perl to generate MLF transcriptions\
\tI\tInitialize a prototype HMM using HCompV\
\tl\tCreate monophone HMMs and unaligned monophone transcriptions using HHEd and HLEd\
\tr\tRe-estimate monophone HMMs using HERest\
\ta\tForce align monophone transcriptions using HERest, HRest, and HVite\
\ti\tInitialize and re-estimate monophone models using HInit and HRest\ 
\tv\tRun time-aligned monophone recognition using HParse and HVite\
\tL\tCreate triphone HMMs and unaligned triphone transcriptions using HHEd and HHLed\
\tR\tRe-estimate un-clustered triphone models using HERest\
\tc\tCluster triphone models using HHEd\
\tE\tRe-estimate clustered triphone models using HERest\
\tm\tBuild the language model using HLStats and HBuild\
\td\tUse perl to create talker-dependent test scripts\
\tU\tUpmix and test speaker dependent models with HHEd, HERest, and HVite\
\tu\tUpmix and test speaker independent models with HHEd, HERest, and HVite\
\n\n";

if ($#ARGV == -1) { die "$USAGE\n" }
else { 
    print "train.pl to do list:\n";
    $item = 0;
    #check the command line arguements and confim what train.pl will do
    if ($ARGV[0] =~ /C/) { 
        $item = $item + 1;
        $cmdline{"C"} = "";
        print "$item. Create acoustic observation files (HCopy)\n";
    }
    if ($ARGV[0] =~ /T/)  {
        $item = $item + 1;
        $cmdline{"T"} = "";
        print "$item. Process dictionary and transcription files (Perl processing)\n";
    }
    if ($ARGV[0] =~ /I/) {
        $item = $item + 1;
        $cmdline{"I"} = "";
        print "$item. Initialize hidden Markov models (HCompV)\n";
    }
    if ($ARGV[0] =~ /l/) { 
        $item = $item + 1;
        $cmdline{"l"} = "";
        print "$item. Create monophone prototype HMMs and transcriptions (HLEd, HHEd)\n";
    }
    if ($ARGV[0] =~ /r/) {
        $item = $item + 1;
        $cmdline{"r"} = "";
        print "$item. Re-estimate monophone models (HERest)\n";
    }
    if ($ARGV[0] =~ /a/) {
        $item = $item+1;
        $cmdline{"a"} = "";
        print "$item. Force align transcriptions (HInit, HRest, HVite)\n";
    }
    if ($ARGV[0] =~ /i/) {
        $item = $item + 1;
        $cmdline{"i"} = "";
        print "$item. Initialize and re-train time aligned monophone models (HInit, HRest)\n";
    }
    if ($ARGV[0] =~ /v/) {
        $item = $item + 1;
        $cmdline{"v"} = "";
        print "$item. Build grammar and test time-aligned phone models (HParse, HVite)\n";
    }
    if ($ARGV[0] =~ /L/) {
        $item = $item + 1;
        $cmdline{"L"} = "";
        print "$item. Create triphone HMMs and transcriptions (HLEd, HHEd)\n";
    }
    if ($ARGV[0] =~ /R/) {
        $item = $item + 1;
        $cmdline{"R"} = "";
        print "$item. Re-estimate triphone models (HERest)\n";
    }
    if ($ARGV[0] =~ /c/) {
        $item = $item + 1;
        $cmdline{"c"} = "";
        print "$item. Cluster triphones (HHEd)\n";
    }
    if ($ARGV[0] =~ /E/) {
        $item = $item + 1;
        $cmdline{"E"} = "";
        print "$item. Re-estimate clustered triphones (HERest)\n";
    }
    if ($ARGV[0] =~ /m/) {
        $item = $item + 1;
        $cmdline{"m"} = "";
        print "$item. Build the language model (HLStats, HBuild)\n";
    }
    if ($ARGV[0] =~ /d/) {
        $item = $item + 1;
        $cmdline{"d"} = "";
        print "$item. Create talker-dependent test scripts (Perl Processing)\n";
    }
    if ($ARGV[0] =~ /U/) {
        $item = $item + 1;
        $cmdline{"U"} = "";
        print "$item. Upmix and test speaker dependent models (HHEd, HERest, HVite)\n";
    }
    if ($ARGV[0] =~ /u/) {
        $item = $item + 1;
        $cmdline{"u"} = "";
        print "$item. Upmix and test speaker independent models (HHEd, HERest, HVite)\n";
    }
    print "\n";
}

# find all the necessary file names and options
# ------------------------------------- Sec 3 -------------------------------- #
# The HCopy configuration file
$hcopycfg = "$CONFIGDIR/$HCOPYCFG";

# training and testing configuration file
$htkcfg = "$CONFIGDIR/$HTKCFG";

# HCopy scp file
$hcopyscp = "$SCPDIR/$HCOPYSCP";

# training script file
$trainscp = "$SCPDIR/$TRAINSCP";

# test script file
$testscp = "$SCPDIR/$TESTSCP";

# Find the appropriate file extension for the acoustic data files generated by HCopy 
$target = $HTKCFG{TARGETKIND};
$target =~ s/_\w//g;
$DataExt = $HCEXT{"$target"};

# Create the search pattern for the audio file extension 
$searchAudExt = qr/\.$AUDIOEXT$/i;

# Set the HCopy trace option 
if (!($HCOPYTRACE eq "inf")) { $HCopyTrace = "-T " . $HCOPYTRACE }
else { $HCopyTrace = "" }

# ----------------------------------------- Sec 4 ------------------------------- #
# The master label files
$wordutt = "$MLFDIR/$WORDUTTMLF";
$wordnutt = "$MLFDIR/$WORDNUTTMLF";
$wordsnor = "$MLFDIR/$WORDSNORMLF";

# Dictionary files
$basedict = "$DICTDIR/$DICTIONARY";

# Monophone list
$monophonelist = "$MMFDIR/$MONOPHONELIST";

# No sp monophonelist
$nospphonelist = "$MMFDIR/$NOSPPHONELIST";

# Monophone dictionary with sp's 
$monospdict = "$DICTDIR/$MONOSPDICT";

# Monophone dictionary without sp's
$mononospdict = "$DICTDIR/$MONONOSPDICT";

# triphone dictionary with sp's
$trispdict = "$DICTDIR/$TRISPDICT";

# triphone dictionary without sp's
$trinospdict = "$DICTDIR/$TRINOSPDICT";

# Create a search pattern for the transcription file extension 
$searchTransExt = qr/\.$TRANSEXT$/i;

# ----------------------------------- Sec 5 ----------------------------- #
# Set the trace option for HCompV
if (!($HCOMPVTRACE eq "inf")) { $HCompVTrace = "-T " . $HCOMPVTRACE }
else { $HCompVTrace = "" }
#
# Set the HCompV variance floor option
if (!($HCOMPVFLOOR eq 'none')) { $HCompVFloor = "-f " . $HCOMPVFLOOR }
else { $HCompVFloor = "" }
#
# Set the name of the output mmf file
$protommf = "$MMFDIR/$PROTOMMF";

# ---------------------------------- Sec 6 ----------------------------- #
# The file that was generated by HCompV
$hcompvproto = "$MMFDIR/$HCOMPVOUTDIR/$PROTOMMF";
#
# The mmf file created from $hcompvproto
$hcompvmmf = "$MMFDIR/$HCOMPVOUTDIR/$MMFOUTFILE";
#
# The variance floors generated by HCompV
$vfloors="$MMFDIR/$HCOMPVOUTDIR/vFloors";
#
# The mmf file generated by HHEd
$hhedmmf = "$MMFDIR/$HHEDTIEDSILDIR/$MMFOUTFILE";
#
# The HHEd edit file
$silhhed = "$EDITDIR/$SILHHED";
#
# The HLEd expand file
$expandhled = "$EDITDIR/$EXPANDHLED";
# 
# Set the HHEd trace option 
if (!($HHEDTRACE eq 'inf')) { $HHEdTrace = "-T " . $HHEDTRACE }
else { $HHEdTrace = "" }
#
# Set the HLEd trace option 
if (!($HLEDTRACE eq 'inf')) { $HLEdTrace = "-T " . $HLEDTRACE }
else { $HLEdTrace = "" }
#
# The monophone mlf
$monophonemlf = "$MLFDIR/$MONOPHONEMLF";

# ---------------------------- Sec 7 and Sec 8 ------------------------------ #
# Set the HERest pruning option 
if (!($HERESTPRUNE eq "inf")) { $HERestPrune = "-t " . $HERESTPRUNE }
else { $HERestPrune = "" }
#
# Set the HERest trace option 
if (!($HERESTTRACE eq 'inf')) { $HERestTrace = "-T " . $HERESTTRACE }
else { $HERestTrace = "" }

# Set the mmf file incase we need it
$herestmmf = "$MMFDIR/${FLATSTARTDIR}${NMIXES}G${HERESTITER}/$MMFOUTFILE";

# Script used during forced alignment
$alignscp = "$SCPDIR/$ALIGNSCP";

# Set the HInit trace option 
if (!($HINITTRACE eq 'inf')) { $HInitTrace = "-T " . $HINITTRACE }
else { $HInitTrace = "" };

# Set the HRest trace option 
if (!($HRESTTRACE eq 'inf')) { $HRestTrace = "-T " . $HRESTTRACE } 
else { $HRestTrace = "" }

# Set the HVite pruning option 
if (!($HVITEPRUNE eq "inf")) { $HVitePrune = "-t " . $HVITEPRUNE }
else { $HVitePrune = "" }

# Set the HVite trace option
if (!($HVITETRACE eq 'inf')) { $HViteTrace = "-T " . $HVITETRACE }
else { $HViteTrace = "" }

# Set the name of the script for forced alignment
$alignscp = "$SCPDIR/$ALIGNSCP";

# ----------------------------------- Sec 9 ----------------------------- #
# the automatically generated phone grammar
$grammar = "$GRAMDIR/$PHONEGRAMMAR";

# The network generated by HParse
$phonenet = "$GRAMDIR/$PHONENET";

# The monophone map dictionary
$phonemap = "$DICTDIR/$MONOPHONEMAP";

# ---------------------------------- Sec 10 ---------------------------- #
# file that makes the triphone 
$triphoneshled = "$EDITDIR/$TRIPHONESHLED";

# the triphone mlf file 
$triphonemlf = "$MLFDIR/$TRIPHONEMLF";

# the list of triphones 
$triphonelist = "$MMFDIR/$TRIPHONELIST";

# the list of all the phones in the dataset
$allphoneslist = "$MMFDIR/$ALLPHONESLIST";

# name of the triphones hhed file
$triphoneshhed = "$EDITDIR/$TRIPHONESHHED";

# ---------------------------------- Sec 12 ------------------------------- #
# stuff needed

# HHEd cluster file
$clusterhhed = "$EDITDIR/$CLUSTERHHED";

# cluster list file
$clusterlist = "$MMFDIR/$CLUSTERLIST";

# triphone tree file
$triphonetree = "$MMFDIR/$TRIPHONETREE";

# ---------------------------------- Sec 14 ------------------------------- #
# Other stuff

# List of all the words in the corpus
$wordlist = "$GRAMDIR/$WORDLIST";

# the bigram model
$bigram = "$GRAMDIR/$BIGRAM";

# set the name of the lattice file
$lattice = "$GRAMDIR/$LATTICE";

# Set the trace option for HLStats
if (!($HLSTATSTRACE eq 'inf')) { $HLStatsTrace = "-T " . $HLSTATSTRACE }
else { $HLStatsTrace = "" }

# Set the trace option for HBuild
if (!($HBUILDTRACE eq "inf")) { $HBuildTrace = "-T " . $HBUILDTRACE }
else { $HBuildTrace = "" }
 
# ------------------------------ Sec 16 --------------------------- #
# HVite config file
$hvitecfg = "$CONFIGDIR/$HVITECFG";

####################################################################
# Section 3: Create the observation files
#

# First, check the user input and see if this section even needs to be run
if (exists $cmdline{"C"})  {

    # Create a configuration file
    mkdir "$CONFIGDIR" unless -d "$CONFIGDIR";
    open(CFG,">$hcopycfg") || die "Unable to write to $hcopycfg: $!";
    foreach $key (keys %HTKCFG) 
    { print CFG "$key = $HTKCFG{$key}\n"; }
    close(CFG);

    # Now, search the datatree for wav files...
    print "Looking for source files in $DATATREE\n";
    @SOURCEFILE_LIST = peruse_tree($DATATREE, $searchAudExt);

    # Print input and output files to $SCPF and scp/.scp
    mkdir "$SCPDIR" unless -d "$SCPDIR";

    # Write the necessary script files for HCopy and other tools
    open(SCP1,">$hcopyscp") || die "Unable to write to scp/$hcopyscp: $!";
    open(SCPTRAIN,">$trainscp") || die "Unable to write to $trainscp: $!";
    open(SCPTEST,">$testscp") || die "Unable to write to $testscp: $!";
    foreach $n (0..$#SOURCEFILE_LIST) {
        my($filename) = $SOURCEFILE_LIST[$n];
        my($plp)=$filename;
        $plp =~ s/\..*/\.$DataExt/;
        $plp =~ s:.*/::g;
        print SCP1 "$filename data/$plp\n";
        # Assume that separate eval test data exist.
        # Put talkers 49xx in dev test, all others in training.
        # This may be too few dev test speakers, but HVite is too slow to 
        #  to handle very many devtest speakers, so this will do for now
        if($plp !~ /$TESTFILES/) { print SCPTRAIN "$DATADIR/$plp\n"; }
        else { print SCPTEST "$DATADIR/$plp\n"; }
    }
    close SCP1; close SCPTRAIN; close SCPTEST;

    # Now: run HCopy
    mkdir "$DATADIR" unless -d "$DATADIR";
    system("HCopy -A $HCopyTrace $HCOPYOPTS -C $hcopycfg -S $hcopyscp");
}

#######################################################################
# Section 4: process the dictionary and transcription files

# Read the dictionary
# Ignore comment lines (# ...).
# Assume that lines in dictionary have form "word phone phone phone ..."
#  and that every phone in the dictionary should have its own HMM.

if (exists $cmdline{"T"}) {
    print "Reading dictionary \n";
    %WORDLIST = ();
    open(DICT,"$basedict") || die "Unable to read from $basedict: $!";
    foreach $line (<DICT>) {
        ($line =~ /^\#/) && next;
        my(@recs) = split(/\s+/,$line);
        $WORDLIST{$recs[0]} = "@recs[1..$#recs]";
    }
    close DICT;

    if (!($TRANSFORMAT{MLF})) {
        # Create three MLFs: 
        #   MLFU with pronounce variants and utterance boundary markers, for HERest
        #   MLFN with pronounce variants but no utterance boundary markers, for HLStats
        #   MLFT with no pronunciation variants or silences, for HResults
        print "Writing MLFs\n";
        mkdir "$MLFDIR" unless -d "$MLFDIR";
        open(MLFU,">$wordutt") || die "Unable to write $wordutt: $!";
        open(MLFN,">$wordnutt") || die "Unable to write $wordnutt: $!";
        open(MLFT,">$wordsnor") || die "Unable to write $wordsnor: $!";
        print MLFU '#!MLF!#',"\n";
        print MLFN '#!MLF!#',"\n";
        print MLFT '#!MLF!#',"\n";

        # Search for the source file list
        @SOURCEFILE_LIST = peruse_tree("$TRANSPATH",$searchTransExt);

        # Read in each of the label files
        foreach $lfile (@SOURCEFILE_LIST) {
            $searchLab = qr:(.*)/(.*)\.$TRANSEXT:;
            $lfile =~ m:$searchLab:;

            #needed for NIST files
            $foundPound = 1;

	    if(open(LFILE,$lfile)) {
	        # If open was successful: create an MLF entry
	        print MLFU '"*/',"$2.lab\"\n";
	        print MLFN '"*/',"$2.lab\"\n";
	        print MLFT '"*/',"$2.lab\"\n";
	        # Read in the transcription
	        my(@lines) = <LFILE>;
	        close(LFILE);
	        # Process and write it out
	        foreach $n (0..$#lines) {
                    $lines[$n] =~ s/^\s*//;    #eliminate initial spaces (NIST format)
		    @recs = split(/\s+/,$lines[$n]);
                    if ($TRANSFORMAT{NIST})  {
		        # Scale t1 and t2 to integer multiples of 100ns
		        $t1 = $recs[1] * $TSCALE; $t1 =~ s/\..*//;
		        $t2 = $recs[2] * $TSCALE; $t2 =~ s/\..*//;
                    }
                    elsif ($TRANSFORMAT{TIMIT}) {
                        # Scale t0 and t1 to integer multiples of 100ns 
                        $t1 = $recs[0] * $TSCALE; $t1 =~ s/\..*//;
                        $t2 = $recs[1] * $TSCALE; $t2 =~ s/\..*//;
                    }
                    elsif ($TRANSFORMAT{ESPS}) {
                        if ($foundPound) { 
                            if ($lines[$n] =~ /\#/) { $foundPound = 0; $NISTstart = 0; }
			    next;
                        }
                        else {
                            $t1 = $NISTstart;
                            $t2 = $recs[1] * $TSCALE; $t2 =~ s/\..*//;
                            $NISTstart = $t2;
                        }
	   	    }
                    else { die "Unknown transcription format!" }
		    # Remove chars from the word that are not used in dict
                    if (($TRANSFORMAT{NIST}) || ($TRANSFORMAT{ESPS}))  {
		        $word = $recs[3]; $word =~ s/[\{\}]//g;
                    }
                    else { $word = $recs[2]; $word =~ s/[\{\}]//g; }
		    # Divide non-speech words into short, medium, and long
		    if($word =~ /\[(noise|laughter|silence|vocalized-noise)\]/) {
		        if($t2-$t1 < 900000) { $word =~ s/\]/_short\]/; }
		        elsif($t2-$t1 > 4000000) { $word =~ s/\]/_long\]/; }
		        else { $word =~ s/\]/_medium\]/; }
		    }

		    if ($word !~ /^\s*$/) {
		        # First, make sure that this word exists in the dictionary
		        unless(exists($WORDLIST{$word})) {
		            # Check to see if an uncapitalized version exists...
		            my($lcword) = $word;
		            $lcword =~ tr/A-Z/a-z/;
		            # If so, change $word to $lcword
		            if(exists($WORDLIST{$lcword})) { $word = $lcword; }
		            # Otherwise, add word to the dictionary
		            # Pronunciation is "noiz" for any type of noise, else "unk"
		            elsif($word =~/^\[.*noise.*\]$/) {$WORDLIST{$word}="noiz";}
		            else { $WORDLIST{$word} = "unk"; }
	                }

		        # Print non-null word to MLF without utterance boundary markers
		        print MLFN "$t1\t$t2\t$word\n";

   		        # If this is the first entry in MLFU, put an !ENTER before it
		        if ($TRANSFORMAT{TIMIT}) {
			    if($n==0) {print MLFU "0\t",$t1,"\t!ENTER\n"; $t1=$t1+1;}
		        }
		         else { if($n==0) {print MLFU "$t1\t",$t1+1,"\t!ENTER\n"; $t1=$t1+1;} }
		         # If this is the last entry in MLFU, leave time for the !EXIT
		         if($n==$#lines) { $t2=$t2-1; }
		         # Finally, print this word and its time to the MLFU
		         print MLFU "$t1\t$t2\t$word\n";		    
		    
		         # Finally, compute and print to MLFT the 
		         # standard normalized orthographic (SNOR) representation
		         my(@SNOR) = word2snor($word);
	  	         foreach $w (@SNOR) { if($w !~ /^\s*$/) { print MLFT "$w\n"; } }
 
	            } # if ($word !~ /^\s*$/)
	        } # foreach $n (0..$#lines)
    
	        # Add one more to MLFU: an !EXIT utterance boundary marker
	        # with negligible but nonzero length (100ns)
	        print MLFU "$t2\t", $t2+1, "\t!EXIT\n";
	        print MLFU '.',"\n"; # Last line of each label file
	        print MLFN '.',"\n"; # Last line of each label file
	        print MLFT '.',"\n"; # Last line of each label file
            }
            else { print STDERR "WARNING: Unable to open $lfile\n"; }
        }

        # Close the MLFs
        close(MLFU);
        close(MLFN);
        close(MLFT);
    }
    # Print out the new dictionaries; while we're at it, get a PHONELIST
    print "Writing $monospdict, $mononospdict, $trispdict, $trinospdict\n";
    # Initialize the PHONELIST to contain the special phones "sp" and "UTT_END"
    %PHONELIST = ( "sp" => 1, "UTT_END" => 1 );
    open(DMS,">$monospdict")||die "Unable write $monospdict: $!";
    open(DMN,">$mononospdict")||die "Unable write $mononospdict: $!";
    open(DTS,">$trispdict")||die "Unable write $trispdict: $!";
    open(DTN,">$trinospdict")||die "Unable write $trinospdict: $!";
    # End every entry in "sp" short pause, to block cross-word triphone formation.
    # Since "sp" is a T-model with its PDF tied to silence, it doesn't (help or) 
    # hurt to add it after every word, even pseudo-words [noise] and [laughter]
    $startpatt = "^(";
    foreach $i (@NOTPHONES) {
        $startpatt = $startpatt . "${i}|";
    }
    $startpatt =~ s/\|$/\)/;
    foreach $word (sort keys %WORDLIST) {
	my(@p) = split(/\s+/,$WORDLIST{$word});
        # Dictionary with sp and with stress
	print DMS "$word $WORDLIST{$word} sp\n";
        # Dictionary without sp and with stress
        print DMN "$word $WORDLIST{$word}\n";
        # Triphone dictionary with/without sp and with stress
	my(@t) = @p;
	unless($p[0] =~ /$startpatt/) {
	    foreach $n (1..$#p) { $t[$n] = $p[$n-1] .'-' . $t[$n]; }
	    foreach $n (0..($#p-1)) { $t[$n] = $t[$n] . '+' . $p[$n+1]; }
        }
	print DTS "$word @t sp\n";
        print DTN "$word @t\n"; 
         
        # PHONELIST (unstressed monophones)
	foreach $n (0..$#p) { $PHONELIST{$p[$n]}=1; }
    }
    # Print, at last, the !ENTER and !EXIT utterance boundary markers.
    #    HBuild requires that !ENTER and !EXIT be in the dictionary, but 
    # doesn't require that they have any associated acoustic model (i.e.,
    # it would be sufficient to print "!ENTER\n!EXIT\n").
    #    HERest requires that each utterance end with a non-tee acoustic model;
    # one way to satisfy this is to give !EXIT a one-frame acoustic model.
    #    HVite has been unable to compute the -K transform file, in my 
    # experiments so far, unless both !ENTER and !EXIT have acoustic models.
    #    The following implementation "pronounces" both !ENTER and !EXIT
    # using the UTT_END pseudo-phone.  We will define UTT_END to have just
    # one emitting state, and we will use sil.hhed to remove the self-loop
    # from that state, so that the UTT_END acoustic model always consumes
    # one and only one frame of data at the beginning and end of every 
    # utterance.  That one frame is often silence, but not always:
    # all Switchboard utterance units (defined by the -utt.text transcriptions)
    # begin with silence, but many end at the end of a word.
    #
    print DMN "!ENTER UTT_END\n!EXIT UTT_END\n"; close(DMNS);
    print DTN "!ENTER UTT_END\n!EXIT UTT_END\n"; close(DTNS);
    print DMS "!ENTER UTT_END\n!EXIT UTT_END\n"; close(DMSS);
    print DTS "!ENTER UTT_END\n!EXIT UTT_END\n"; close(DTSS);

    # Create the list of monophones...
    mkdir "$MMFDIR" unless -d "$MMFDIR";
    print "Writing monophone list\n";
    open(PF,">$monophonelist") || die "Unable to write $monophonelist: $!";
    open(NSPF,">$nospphonelist") || die "Unable to write to $nospphonelist: $!";
    foreach $phone (sort keys %PHONELIST) { 
        print PF "$phone\n"; 
        if (!($phone =~ /(UTT_END|sp)/)) { print NSPF "$phone\n" }
    }
    close PF;
    close NSPF;

}

##############################################################
# Section 5: Initialize models using HCompV

if (exists $cmdline{"I"})  {
    # First, create the HMM training config file, with subset of %HTKCFG
    open(CFG,">$htkcfg") || die "Unable to write to $htkcfg: $!";
    foreach $entry ('TARGETKIND','TARGETRATE','SAVECOMPRESSED','NUMCEPS') 
        { print CFG "$entry = $HTKCFG{$entry}\n"; }
    close(CFG);

    # Do a simple parse of the TARGETKIND to get the vector dimension
    $VECTORDIM = $HTKCFG{'NUMCEPS'};
    if($HTKCFG{'TARGETKIND'} =~ /_E/) { $VECTORDIM = $VECTORDIM+1; }
    if($HTKCFG{'TARGETKIND'} =~ /_O/)  { $VECTORDIM = $VECTORDIM+1; }
    if($HTKCFG{'TARGETKIND'} =~ /_T/) { $VECTORDIM = $VECTORDIM*4; }
    elsif($HTKCFG{'TARGETKIND'} =~ /_A/) { $VECTORDIM = $VECTORDIM*3; }
    elsif($HTKCFG{'TARGETKIND'} =~ /_D/) { $VECTORDIM = $VECTORDIM*2; }

    # Create the "proto" HMM 
    mkdir "$MMFDIR" unless -d "$MMFDIR";
    open(MMF,">$protommf") || die "Unable to write to $protommf: $!";
    ProtoGen("$PROTOMMF",\*MMF);
    close MMF;

    # Now run HCompV with the prototype
    mkdir "$MMFDIR/$HCOMPVOUTDIR" unless -d "$MMF/$HCOMPVOUTDIR";
    system("HCompV -A $HCompVTrace -I $wordutt -C $htkcfg $HCOMPVOPTS $HCompVFloor -S $trainscp -M $MMFDIR/$HCOMPVOUTDIR $protommf");

}

#########################################################################
# Section 6: Create phone models and  unaligned transcriptions
# HHEd and HLEd

if (exists $cmdline{"l"}) {
    # Read in the mean and variance vector from mmf/protophones/proto
    open(PROTO,"$hcompvproto") || die "Unable to read $hcompvproto: $!";
        while($l=<PROTO>) {
            if($l =~ /Mean/i) { $PROTOMEAN = <PROTO>; }
            elsif($l =~ /Variance/i) { $PROTOVAR = <PROTO>; last; }
        }
    close(PROTO);

    # Read in the variance floors
    open(VFLOORS,"$vfloors") || die "Unable to open $vfloors: $!";
    @VFLOORS=<VFLOORS>;
    close(VFLOORS);

    # Read back in the phonelist
    open(PHONEF,"$monophonelist") || die "Unable to open $monophonelist: $!";
    @PHONELIST=();
    while($l=<PHONEF>) { chomp $l; chomp $l; push(@PHONELIST,$l); }
    close PHONEF;

    # Do a simple parse of the TARGETKIND to get the vector dimension
    $VECTORDIM = $HTKCFG{'NUMCEPS'};
    if($HTKCFG{'TARGETKIND'} =~ /_E/) { $VECTORDIM = $VECTORDIM+1; }
    if($KTHCFG{'TARGETKIND'} =~ /_O/) { $VECTORDIM = $VECTORDIM+1; }
    if($HTKCFG{'TARGETKIND'} =~ /_T/) { $VECTORDIM = $VECTORDIM*4; }
    elsif($HTKCFG{'TARGETKIND'} =~ /_A/) { $VECTORDIM = $VECTORDIM*3; }
    elsif($HTKCFG{'TARGETKIND'} =~ /_D/) { $VECTORDIM = $VECTORDIM*2; }

    # Create the HMM prototypes -- all the same
    open(MMF,">$hcompvmmf") || die "Unable to write to $hcompvmmf: $!";
    # Write the ~observation macro
    print MMF '~o <VecSize> ',$VECTORDIM,' <', $HTKCFG{'TARGETKIND'}, ">\n";
    # Write the ~vfloors macro
    print MMF @VFLOORS;
    # Write all of the ~hmm macros
    foreach $phone (@PHONELIST) {
        # Number of states: $NPROTOSTATES, except for special cases
        if (exists $NSTATES{$phone}) {
            print MMF "~h \"$phone\"\n", "<BeginHMM>\n", " <NumStates> $NSTATES{$phone}\n";
            foreach $state (2..($NSTATES{$phone}-1)) {
                print MMF " <State> $state\n";
                print MMF "  <Mean> $VECTORDIM\n   $PROTOMEAN";
                print MMF "  <Variance> $VECTORDIM\n   $PROTOVAR";
            }
            # Print the transition probability matrix for the HMM
	    transMatGen($NSTATES{$phone},\*MMF);
        }
        else { 
            open(PROTO,"$hcompvproto") or die "can't open file $hcompvproto : $!";
            $protopatt = qr/$PROTOMMF/;
            foreach $line (<PROTO>) {
                $line =~ s/proto/$phone/;
                print MMF "$line";
            }
            close PROTO;
        }
    }
    close MMF;

    # Tie together semantically distinct but acoustically identical states:
    #  All states of silence are tied to the one state of sp,
    #  All other non-speech phones are tied down to a single state each.
    mkdir "$EDITDIR" unless -d "$EDITDIR";
    open(HEDF,">$silhhed") || die "Unable to write to $silhhed: $!";
    foreach $phone (@NOTSPEECH) {
        print HEDF "TI ${phone}st \{${phone}.state\[2\]";
        foreach $n (3..($NSTATES{$phone}-1)) { print HEDF ",${phone}.state\[$n\]"; }
        # If the one state in SP is used, require it to be a silence
        ($phone =~ /sil/) && print HEDF ',sp.state[2]';
        print HEDF "\}\n";
    }
    print HEDF "AT 1 3 0.3 \{sp.transP\}\n"; # Add the option of skipping through sp
    print HEDF "AT 2 2 0.0 \{UTT_END.transP\}\n"; # Make UTT_END exactly one frame
    close HEDF;

    # Use HHEd 
    mkdir "$MMFDIR/$HHEDTIEDSILDIR" unless -d "$MMFDIR/$HHEDTIEDSILDIR";
    system("HHEd -A $HHEdTrace -H $hcompvmmf -M $MMFDIR/$HHEDTIEDSILDIR $silhhed  $monophonelist");

    # First expand the label file into phonemes, using the dictionary  
    open(ED,">$expandhled") || die "Unable to write to $expandhled: $!";
    print ED "EX\n";
    close ED;

    # Create a version without stress marking, for training of monophone models
    system("HLEd -A $HLEdTrace -d $monospdict -l $HLEDLABLOC -i $monophonemlf $expandhled $wordutt");

}

#################################################################################
# Section 7: Now run HERest for the Gaussian monophone models

if (exists $cmdline{"r"}) {
    $herestmmf = "$MMFDIR/$HCOMPVOUTDIR/$MMFOUTFILE";
    foreach $i (1..$HERESTITER) {
        mkdir "$MMFDIR/${FLATSTARTDIR}${NMIXES}G${i}" unless -d "$MMFDIR/${FLATSTARTDIR}${NMIXES}G${i}";
        # Compute occupancy statistics only for $i==$HERESTITER
        $occ = ($i==$HERESTITER) ? "-s $MMFDIR/${FLATSTARTDIR}${NMIXES}G${i}/$HERESTSTATS" : "";         
        system("HERest -A $HERestTrace -I $monophonemlf $occ -C $htkcfg $HERestPrune -S $trainscp -H $herestmmf -M $HERESTOPTS $MMFDIR/$FLATSTARTDIR${NMIXES}G${i} $monophonelist");
        $herestmmf = "$MMFDIR/${FLATSTARTDIR}${NMIXES}G${i}/$MMFOUTFILE";
    }
}

##################################################################################
# Section 8: Force align models (HERest, HRest, HVite)

if (exists $cmdline{"a"}) {
    
    # create some directories
    mkdir "$DICTDIR/$ALIGNDIR" unless -d "$DICTDIR/$ALIGNDIR";
    mkdir "$DICTDIR/$ALIGNDIR/${ALIGNITERDIR}D0" unless -d "$DICTDIR/$ALIGNDIR/${ALIGNITERDIR}D0";
    mkdir "$MLFDIR/$ALIGNDIR" unless -d "$MLFDIR/$ALIGNDIR";
    mkdir "$MLFDIR/$ALIGNDIR/${ALIGNITERDIR}V0" unless -d "$MLFDIR/$ALIGNDIR/${ALIGNITERDIR}V0";

    #read in the monophone list 
    open(PHONEF,"$nospphonelist") || die "Unable to open $nospphonelist : $!";
    @PHONELIST=();
    foreach $line (<PHONEF>) {
        chomp $line; chomp $line;
        push(@PHONELIST,$line);
    }
    close PHONEF;

    # Create a script for alignment
    $scripts = "";
    foreach $s (@ALIGNSCP) {
        $scripts = $scripts . " $s";
    }
    system("cat $scripts > $alignscp");

    mkdir "$MLFDIR/$ALIGNDIR/$LABDIR" unless -d "$MLFDIR/$ALIGNDIR/$LABDIR";
    # force align the transcriptions with HVite
    # don't specify a beam width because we need transcriptions for all files
    # Use input from the re-estimation from HERest
    system("HVite -A -a -m -o S -l $HVITELABLOC -y $HVITELAB -I $wordutt -i $MLFDIR/$ALIGNDIR/${ALIGNITERDIR}V0/$MONOPHONEMLF -H $herestmmf -C $htkcfg $mononospdict $nospphonelist -S $alignscp");

    # Create the directory where the mmf file will be stored
    mkdir "$MMFDIR" unless -d "$MMFDIR";
    mkdir "$MMFDIR/$ALIGNDIR" unless -d "$MMFDIR/$ALIGNDIR";

    # Compile the HInit and HRest options
    $HInitOpts = "";
    $HRestOpts = "";
    if (!($HINITITER eq "")) { $HInitOpts = $HInitOpts . "-i " . "$HINITITER " }
    if (!($HINITMINPHONES eq "")) { $HInitOpts = $HInitOpts . "-m " . "$HINITMINPHONES " }
    if (!($HRESTITER eq "")) { $HRestOpts = $HRestOpts . "-i " . "$HRESTITER " }
    if (!($HRESTMINPHONES eq "")) { $HRestOpts = $HRestOpts . "-m " . "$HRESTMINPHONES " }
    if (!($HRESTVFLOOR eq "")) { $HRestOpts = $HRestOpts . "-v " . "$HRESTVFLOOR " }

    # set the dictionary
    $aligndict = "$mononospdict";

    # Do forced alignment
    foreach $i (1..$ALIGNITER) {
        $j = $i - 1;
        $MLF =~ s/\d+/$i/;
	mkdir "$MMFDIR/$ALIGNDIR/${ALIGNITERDIR}I${i}" unless -d "$MMFDIR/$ALIGNDIR/${ALIGNITERDIR}I${i}";
        mkdir "$MMFDIR/$ALIGNDIR/${ALIGNITERDIR}H${i}" unless -d "$MMFDIR/$ALIGNDIR/${ALIGNITERDIR}H${i}";
        mkdir "$DICTDIR/$AKIGNDIR/${ALIGNITERDIR}D${i}" unless -d "$DICTDIR/$ALIGNDIR/${ALIGNITERDIR}D${i}";
        foreach $n (0..$#PHONELIST) {
            # run HInit
            system("HInit -A $HInitTrace -I $MLFDIR/$ALIGNDIR/${ALIGNITERDIR}V${j}/$MONOPHONEMLF -S $alignscp -C $htkcfg $HInitOpts $HINITOPTS -M $MMFDIR/$ALIGNDIR/${ALIGNITERDIR}I${i} -l $PHONELIST[$n] $hcompvproto");
            # edit HInit output models
            system("sed \"s/$PROTOMMF/$PHONELIST[$n]/\" $MMFDIR/$ALIGNDIR/${ALIGNITERDIR}I${i}/$PROTOMMF > $MMFDIR/$ALIGNDIR/${ALIGNITERDIR}I${i}/$PHONELIST[$n]");  
            # run HRest         
            system("HRest -A $HRestTrace -I $MLFDIR/$ALIGNDIR/${ALIGNITERDIR}V${j}/$MONOPHONEMLF $HRestOpts $HRESTOPTS -S $alignscp $HRESTOPTS -C $htkcfg -M $MMFDIR/$ALIGNDIR/${ALIGNITERDIR}H${i} -l $PHONELIST[$n] $MMFDIR/$ALIGNDIR/${ALIGNITERDIR}I${i}/$PHONELIST[$n]");
        }
        system("rm -f $MMFDIR/$ALIGNDIR/${ALIGNITERDIR}H${i}/$MMFOUTFILE");
        system("cat $MMFDIR/$ALIGNDIR/${ALIGNITERDIR}H${i}/* > $MMFDIR/$ALIGNDIR/${ALIGNITERDIR}H${i}/$MMFOUTFILE");
        mkdir "$MLFDIR/$ALIGNDIR/${ALIGNITERDIR}V${i}" unless -d "$MLFDIR/$ALIGNDIR/${ALIGNITERDIR}V${i}";
        system("HVite -A $HViteTrace -a -m -l $HVITELABLOC -y $HVITELAB -o SW -i $MLFDIR/$ALIGNDIR/${ALIGNITERDIR}V${i}/$MONOPHONEMLF -I $wordutt -H $MMFDIR/$ALIGNDIR/${ALIGNITERDIR}H${i}/$MMFOUTFILE -C $htkcfg $aligndict $nospphonelist -S $alignscp");
      #  if ($MERGE == 1) {
         #   unless ($i == $ALIGNITER) {
         #       mergePhones("$MLFDIR/$ALIGNDIR/${ALIGNITERDIR}V${i}/$MONOPHONEMLF");
                # if the dictionary has changed, we want to get a new dictionary from the MLF
	#        extractDict("$MLFDIR/$ALIGNDIR/${ALIGNITERDIR}V${i}/$MONOPHONEMLF", "$DICTDIR/$ALIGNDIR/${ALIGNITERDIR}D${j}/$MONONOSPDICT");
 #               $aligndict = "$DICTDIR/$ALIGNDIR/${ALIGNITERDIR}D${i}/$MONONOSPDICT";
#	    }
#	}
    }
    # copy the forced aligned mlf
    system("cp $MLFDIR/$ALIGNDIR/${ALIGNITERDIR}V${ALIGNITER}/$MONOPHONEMLF $monophonemlf");
  #  system("cp $aligndict $mononospdict");

}

##################################################################################
# Section 9: Re-estimate time aligned monophone models using HInit and HRest

if (exists $cmdline{"i"}) {

    #read in the monophone list
    open(PHONEF,"$nospphonelist") || die "Unable to open $nospphonelist : $!";
    @PHONELIST=();
    foreach $line (<PHONEF>) { 
        chomp $line; chomp $line; 
        push(@PHONELIST,$line);
    }
    close PHONEF;

    #make the directories to store the HInit re-estimated models    
    mkdir "$MMFDIR" unless -d "$MMFDIR";                                          
    mkdir "$MMFDIR/$HINITOUTDIR" unless -d "$MMFDIR/$HINITOUTDIR";
    mkdir "$MMFDIR/$HRESTOUTDIR" unless -d "$MMFDIR/$HRESTOUTDIR";

    $HInitOpts = "";
    if (!($HINITITER eq "")) { $HInitOpts = $HInitOpts . "-i " . $HINITITER }
    if (!($HINITMINPHONES eq "")) { $HInitOpts = $HInitOpts . " -m " . $HINITMINPHONES }
    $HRestOpts = "";
    if (!($HRESTITER eq "")) { $HRestOpts = $HRestOpts . "-i " . $HRESTITER }
    if (!($HRESTMINPHONES eq "")) { $HRestOpts = $HRestOpts . " -m " . $HRESTMINPHONES }
    if (!($HRESTVFLOOR eq "")) { $HRestOpts = $HRestOpts . " -v " . $HRESTVFLOOR }

    #train models using HInit
    foreach $n (0..$#PHONELIST) {
         system("HInit -A $HInitTrace -I $monophonemlf -S $trainscp -C $htkcfg $HInitOpts $HINITOPTS -M $MMFDIR/$HINITOUTDIR/ -l $PHONELIST[$n] $hcompvproto");
         # edit HInit output models 
         system("sed \"s/$PROTOMMF/$PHONELIST[$n]/\" $MMFDIR/$HINITOUTDIR/$PROTOMMF > $MMFDIR/$HINITOUTDIR/$PHONELIST[$n]");     
         #run HRest
         system("HRest -A $HRestTrace -I $monophonemlf $HRestOpts -S $trainscp -C $htkcfg -M $MMFDIR/$HRESTOUTDIR -l $PHONELIST[$n] $MMFDIR/$HINITOUTDIR/$PHONELIST[$n]");
    }
    system("rm -f $MMFDIR/$HRESTOUTDIR/$MMFOUTFILE");
    system("cat $MMFDIR/$HRESTOUTDIR/* > $MMFDIR/$HRESTOUTDIR/$MMFOUTFILE");
}

##################################################################################
# Section 10: Build A monophone dictionary, LM, and run HVite on the monophone models

if (exists $cmdline{"v"}) {
    # Create grammar and monophone dictionary
    mkdir "$GRAMDIR" unless -d "$GRAMDIR";
    open(PHF,"$nospphonelist") or die "Can't open $nospphonelist : $!";
    open(GRAMMAR,">$grammar") or die "Can't open $grammar : $!";
    open(MDICT,">$phonemap") or die "Can't open $phonemap : $!";
    $string = "";
    foreach $phn (<PHF>) {
        chomp $phn; chomp $phn;
        $string = $string . " $phn |";
        print MDICT "$phn $phn\n";
    }
    $string =~ s/\|$//;
    print GRAMMAR "\$phone =${string};\n\( < \$phone > )\n";
    close PHF;
    close GRAMMAR;
    close MDICT;

    # Run HParse 
    system("HParse -c -l $grammar $phonenet");

    # Set the HVite options
    $HViteOpts = "";
    if (!($HVITELABLOC eq "")) { $HViteOpts = $HViteOpts . "-l " . "$HVITELABLOC  " }
    if (!($P eq "")) { $HViteOpts = $HViteOpts . "-p " .  "$P " } 

    # Run HVite
    mkdir "$RESULTSDIR" unless -d "$RESULTSDIR";
    mkdir "$RESULTSDIR/$ALIGNRECDIR" unless -d "$RESULTSDIR/$ALIGNRECDIR";

    system("HVite -A $HViteTrace -C $htkcfg -H $MMFDIR/$HRESTOUTDIR/$MMFOUTFILE -I $monophonemlf -S $testscp $HViteOpts $HVITEOPTS $HVitePrune -i $RESULTSDIR/$ALIGNRECDIR/$ALIGNRECMLF -w $phonenet $phonemap $monophonelist"); 
    system("HResults -A -I $wordsnor $monophonelist $RESULTSDIR/$ALIGNRECDIR/$ALIGNRECMLF > $RESULTSDIR/$ALIGNRECDIR/results");
}

###################################################################################
# Section 11: Create unclustered, stress-dependent triphone models
# HHEd and HLEd

if (exists $cmdline{"L"}) {

    # First, expand the label file into stress-dependent triphones
    open(LEDF,">$triphoneshled") || die "Unable to write $triphoneshled: $!";
    #  Tell the system to ignore any phone that never occurs in the same 
    #   word as a real speech phone: these are 
    #   laff, noiz, sil, sp, UTT_END, pv, pn, and unk.
    foreach $p (@NOTPHONES)
    { print LEDF "WB $p\n"; }
    print LEDF "TC\n";
    close LEDF;    
    # Expand to get stress-dependent triphones by using mlf/monostress.mlf
    system("HLEd -A $HLedTrace -l $HLEDLABLOC -i $triphonemlf -n $triphonelist $triphoneshled $monophonemlf");

    # Make a list including all monophones and all triphones
    system("cat $monophonelist $triphonelist | sort | uniq > $allphoneslist");

    # Clone stress-independent HMMs to get all phones listed in mmf/triphones.phf
    # This is possible because stress has been removed from center phone
    open(HEDF,">$triphoneshhed") || die "Unable to write to $triphoneshhed: $!";
    print HEDF "CL $triphonelist\n";
    close HEDF;
    mkdir "$MMFDIR" unless -d "$MMFDIR";
    mkdir "$MMFDIR/$HHEDTRIPHONEDIR" unless -d "$MMFDIR/$HHEDTRIPHONEDIR";
    system("HHEd -A $HHEdTrace -M $MMFDIR/$HHEDTRIPHONEDIR -H $MMFDIR/${FLATSTARTDIR}${NMIXES}G$HERESTITER/$MMFOUTFILE $triphoneshhed $monophonelist");
}

####################################################################################################
# Section 12:  Run HERest on Triphone models
# Run HERest on the separated triphones, allowing them to diverge,
#   so that the clustering algorithm can decide how to cluster them.
# The -m option sets minimum required number of toks to 1.
# The -u m option specifies that only means should be updated, not variances.
# Collect residency stats for each.

if (exists $cmdline{"R"})  {
    $herestmmf="$MMFDIR/$HHEDTRIPHONEDIR/$MMFOUTFILE";
    foreach $i (1..$HERESTITER) {
        mkdir "$MMFDIR/${HERESTTRIDIR}${NMIXES}G${i}" unless -d "$MMFDIR/${HERESTTRIDIR}${NMIXES}G${i}";
        # Compute occupancy statistics only for $i==$HERESTITER
        $occ = ($i==$HERESTITER) ? "-s $MMFDIR/${HERESTTRIDIR}${NMIXES}G${i}/$HERESTSTATS" : "";
        system("HERest -A $HERestTrace $HERESTOPTS $occ -I $triphonemlf -C $htkcfg $HERestPrune -S $trainscp -H $herestmmf -M $MMFDIR/${HERESTTRIDIR}${NMIXES}G${i} $triphonelist");
    $herestmmf="$MMFDIR/${HERESTTRIDIR}${NMIXES}G${i}/$MMFOUTFILE";
    }
}
##############################################################################
# Section 13: Cluster triphones

if (exists $cmdline{"c"}) {
    # Read back in the phonelist
    open(PHONEF,"$monophonelist") || die "Unable to open $monophonelist: $!";
    @PHONELIST=();
    while($l=<PHONEF>) { chomp $l; chomp $l; push(@PHONELIST,$l); }
    close PHONEF;

    $dontexpand = "(^";
    foreach $phn (@DONTEXPAND) { $dontexpand = $dontexpand . "${phn}|" }
    $dontexpand =~ s/\|$/\)/;
    $dontexpand = qr/$dontexpand/;

    # Create the HHEd file
    open(HEDF,">$clusterhhed") || die "Unable to write to $clusterhhed: $!";
    print HEDF "RO 100.0 \"$MMFDIR/${HERESTTRIDIR}${NMIXES}G${HERESTITER}/stats\"\n";
    print HEDF "TR 0\n"; # Turn off trace while reading in questions
    # Formats for left-context and right-context questions
    $QS_FORMAT{'L'} = '%s-*';
    $QS_FORMAT{'R'} = '*+%s';
    # First, write out all questions defined by the %QS hash
    foreach $class (sort keys %QS) {
        # Print left-context and right-context questions for each class
	$" = ',';       # Comma-separated list items in these print statements
	foreach $direction ('L','R') {
	    # Print the stress-independent questions
	    my(@q)=map { sprintf($QS_FORMAT{$direction},$_); } @{$QS{$class}};
	    print HEDF "QS ${direction}_$class \{@q\}\n";
	}
	$" = ' ';       # Restore the default list-item separator
    }
    # Now print out singletons: is (L,R) = particular monophone X?
    foreach $phone (@PHONELIST) { 
    # Exclude phones that can never be in the same word with a speech phone:
	unless ($phone =~ /$dontexpand/) {
	    # For this question, make sure to ignore stress
	    print HEDF "QS L_$phone \{$phone-\*,${phone}1-\*\}\n";
	    print HEDF "QS R_$phone \{\*+$phone,\*+${phone}1\}\n";
	}
    }
    print HEDF "TR 2\n"; # Turn on trace while performing tree-based clustering

    # Create a search pattern to search for all entries in @NOTPHONES
    $notphones = "^(";
    foreach $m (@NOTPHONES) {
        $notphones = $notphones . "${m}|";
    }
    $notphones =~ s/\|$/\)/;
    $notphones =~ qr/$notphones/; 

    foreach $phone (@PHONELIST) { 
    # Exclude all phones that can never be in the same word with speech phone:
	unless ($phone =~ /$notphones/) {
	    $NSTATES{$phone}=5 unless exists($NSTATES{$phone});
	    foreach $state (2..($NSTATES{$phone}-1)) {
		print HEDF "TB 350.0 \"${phone}S${state}\" \{($phone,*-${phone},${phone}+*,*-${phone}+*).state\[${state}\]\}\n";
	    }
	}
    }
    # Print out the trees and the phone set, then close
    print HEDF "CO $clusterlist\n";
    print HEDF "ST $triphonetree\n";
    close HEDF;

    # Perform clustering
    mkdir "$MMFDIR/$CLUSTERDIR${NMIXES}G0" unless -d "$MMFDIR/$CLUSTERDIR${NMIXES}G0";
    system("HHEd -A $HHEdTrace -H $MMFDIR/$HERESTTRIDIR${NMIXES}G${HERESTITER}/$MMFOUTFILE -M $MMFOUTFILE/$CLUSTERDIR${NMIXES}G0 $clusterhhed $triphonelist\n");

}
#####################################################################################################
# Section 14: Run HERest on clustered triphone models

if (exists $cmdline{"E"}) {
    # Now run HERest for the Gaussian clustered triphone models
    $herestmmf="$MMFDIR/${CLUSTERDIR}${NMIXES}G0/$MMFOUTFILE"; 
    foreach $i (1..$HERESTITER) {
        mkdir "$MMFDIR/$CLUSTERDIR${NMIXES}G${i}" unless -d "$MMFDIR/${CLUSTERDIR}${NMIXES}G${i}";
        # Compute occupancy statistics only for $i==$HERESTITER
        $occ = ($i==$HERESTITER) ? "-s $MMFDIR/${CLUSTERDIR}${NMIXES}G${i}/stats" : "";
        system("HERest -A $HERestTrace $HERESTOPTS $occ -I $triphonemlf -C $htkcfg $HERestPrune -S $trainscp -H $herestmmf -M $MMFDIR/${CLUSTERDIR}${NMIXES}G${i} $clusterlist");
        $herestmmf="$MMFDIR/$CLUSTERDIR${NMIXES}G${i}/$MMFOUTFILE";
    }
}
###################################################################################################
# Section 15: Train the language model (HLStats and HBuild)

if (exists $cmdline{"m"}) {
    # Find out exactly which words and boundary markers are used in word_utt.mlf
    mkdir "$GRAMDIR" unless -d "$GRAMDIR";
    open(MLF,"$wordutt") || die "Unable to read $wordutt: $!";
    %WL=();
    foreach (<MLF>) { 
        chomp; chomp; split; 
        if (@_ > 2) { $WL{$_[2]}=1; } 
        elsif (!(/[\.\#]/)) { $WL{$_[0]}=1; } }
    close MLF;
    # Write exactly those words to gram/wordlist.txt, plus UTT_START
    open(WL,">$wordlist") || die "Unable to write to $wordlist: $!";
    foreach (sort keys %WL) { print WL "$_\n"; }
    close WL;

    # Get LM stats
    # This should be changed so that it reads only from the training words,
    #   and so that the LM is then backed-off to include other words in the DICT
    # This must use the MLF without utterance boundary markers; 
    #   HLStats will automatically re-insert !ENTER and !EXIT in the stats.
    system("HLStats -A $HLStatsTrace $HLSTATSOPTS -b $bigram -o $wordlist $wordnutt");

    # Convert the statistics file into a search lattice
    # !ENTER and !EXIT are the default utterance boundary markers
    system("HBuild -A $HBuildTrace -n $bigram $wordlist $lattice");

}
#############################################################################################
# Section 16: # Compute the talker-dependent test scripts

if (exists $cmdline{"d"}) {
    %TESTTLKR = ();
    $extpatt = "\.(";
    foreach $key (keys %HCEXT) {
        $extpatt = $extpatt . "${keys}|";
    }
    $extpatt =~ s/\|$/)/;
    $extpatt = qr/$extpatt/;
    open(TSCP,"$testscp") || die "Unable to read $testscp: $!";
    while ($line=<TSCP>) {
        chomp $line; chomp $line;
        my($filename) = $line;
        # Get the talker name: eliminate pre-slash and post-dash modifiers
        $line =~ s:.*/::;
        $line =~ s/\-.*//;   
        # Save filename in the list @{$TESTTLKR{$tlkr}}
        if(exists $TESTTLKR{$line}) { push(@{$TESTTLKR{$line}}, $filename); }
        else { $TESTTLKR{$line} = [ $filename ]; }
    }
    # Print talker-dependent test scripts
    mkdir "$SCPDIR/$TALKDEPDIR" unless -d "$SCPDIR/$TALKDEPDIR";
    foreach $tlkr (keys %TESTTLKR) {
        open(SCP,">$SCPDIR/$TALKDEPDIR/$tlkr.scp") || 
	    die "Unable to write $SCPDIR/$TALKDEPDIR/$tlkr.scp: $!";
        print SCP map { sprintf("%s\n",$_); } @{$TESTTLKR{$tlkr}};
        close(SCP);
    }
}

###################################################################################
# Section 17: UPMIX and TEST
# This is where we try to really improve recognition accuracy.
# The for($nmix) loop should be run until
#   the user gets tired of waiting and kills the batch job.
# Each loop upmixes the models, re-estimates, then tests.
# This loop depends on %TESTTLKR from previous section, and
#   on no other global variables.

if (exists $cmdline{"U"}) {

    open(TSCP,"$testscp") || die "Unable to read $testscp: $!";
    while ($line=<TSCP>) {
        chomp $line; chomp $line;
        my($filename) = $line;
        # Get the talker name: eliminate pre-slash and post-dash modifiers 
        $line =~ s:.*/::;
        $line =~ s/\-.*//;
        # Save filename in the list @{$TESTTLKR{$tlkr}} 
        if(exists $TESTTLKR{$line}) { push(@{$TESTTLKR{$line}}, $filename); }
        else { $TESTTLKR{$line} = [ $filename ]; }
    }  

    mkdir "$RESULTSDIR" unless -d "$RESULTSDIR";
    mkdir "$RESULTSDIR/$ALLRECDIR" unless -d "$RESULTSDIR/$ALLRECDIR";

    # First, create an HMM config file with options for speaker adaptation
    open(CFG,">$hvitecfg") || die "Unable to write $hvitecfg: $!";
    foreach $entry ('TARGETKIND','TARGETRATE','HADAPT:BLOCKS','HADAPT:OCCTHRESH')
        { print CFG "$entry = $HTKCFG{$entry}\n"; }
    close(CFG);

    #DEBUG for($nmix=1; $nmix <= 37; $nmix += 2) {
    for($nmix=$NMIXES+$UPMSTEP; $nmix <= $MAXMIX; $nmix += $UPMSTEP) {

        # UPMIX and TEST for each type of model separately (monophones, clusters)
        foreach $type ("$CLUSTERDIR", "$FLATSTARTDIR") {
	    # Specify the list of HMMs (PHONEF) and label file (MLF) and DICT
	    #  DICT is only necessary because of the odd way stress is handled --
	    #  otherwise, HTK would perform triphone expansion for us
	    $PHONEF=($type eq "$FLATSTARTDIR")?"$monophonelist":"$clusterlist";
	    $MLF=($type eq "$FLATSTARTDIR")?"$monophonemlf":"$triphonemlf";
 	    $DICT=($type eq "$FLATSTARTDIR")?"$monospdict":"$trispdict";
	
            # If $nmix > 1, upmix from a lower number of mixtures
	    if($nmix > 1) {
	        $pmix = $nmix-$UPMSTEP;
	        # Create the HHEd script for upmixing
	        open(HEDF,">$EDITDIR/${UPMIXHHED}${nmix}.hhed") || die "Unable to write to $EDITDIR/${UPMIXHHED}${nmix}.hhed: $!";
	        print HEDF "MU $nmix \{ *.state\[2-4\].mix \}\n";
	        close HEDF;
	        
	        # Upmix the MMF
	        mkdir "$MMFDIR/${type}${nmix}G0" unless -d "$MMFDIR/${type}${nmix}G0";
	        system("HHEd -A $HHEdTrace -H $MMFDIR/${type}${pmix}G$HERESTITER/$MMFOUTFILE -M $MMFDIR/${type}${nmix}G0 $EDITDIR/${UPMIXHHED}${nmix}.hhed $PHONEF");
	        $MMF="$MMFDIR/${type}${nmix}G0/$MMFOUTFILE";
	        
	        # Re-estimate the upmixed MMF
	        foreach $i (1..$HERESTITER) {
		    my($next)="$MMFDIR/${type}${nmix}G${i}";
		    $nextdir = $next;
		    mkdir $next unless -d $next;
		    # Compute occupancy statistics only for $i==$HERESTITER
		    $occ = ($i==$HERESTITER) ? "-s $MMFDIR/${type}${nmix}G${i}/stats" : "";
		    system("HERest -A $HERestTrace $occ -I $MLF -C $htkcfg $HERestPrune -S $trainscp -H $MMF -M $next $PHONEF");
		    $MMF="$next/$MMFOUTFILE";
		    if (0) {
		    # Ensure that all mixture wts in each ~s macro add to one.
		    # HERest should do this, but often fails
		    open(MMFI,$MMF) || die "Unable to read from $MMF: $!";
		    open(MMFO,">$next/m2") || die "Unagle to write $next/m2: $!";
		    $line=<MMFI>;
		    # Each loop reads and processes one macro definition
		    until(eof(MMFI)) {
		        $macro=$line;
		        # Append lines to macro until another macro reached, or EOF
		        until(eof(MMFI)||(($line=<MMFI>)=~/^~/)) {$macro.=$line;}
		        # If state macro, normalize the weights:
		        if($macro =~ /^~s/) {
			    # Split $macro: $c[0]=header, @c[1..$#c] are Gaussians
			    my(@c) = split(/<MIXTURE>/i,$macro);
			    # Pull out the mixture weights
			    my(@w) = map { /^\s+\d+\s+([\d\.e\-]+)/i; $1; } @c;
			    # Find their sum
			    for($sum=0,$n=1; $n<=$#w; $n+=1) { $sum+=$w[$n]; }
			    # Normalize weights, and put them back into @c
			    for($n=1; $n<=$#w; $n+=1) { 
			        $w[$n] /= $sum;   # Normalize
			        $w[$n] = sprintf("%8.8f",$w[$n]); # Format
			        $c[$n] =~ s/^(\s+\d+\s+)([\d\.e\-]+)/\1$w[$n]/;
			    }
			    $macro = join('<MIXTURE>',@c); # The result
		        }
		        print MMFO $macro;
		    } # until(eof(MMFI))
		    close(MMFI);
		    close(MMFO);
		
	 	    # Now move normalized macro m2 to mmf
		    system("mv $next/mmf $next/mmf_unnorm");
		    system("mv $next/m2 $next/mmf");
	        } # foreach $i (1..$HERESTITER)
	    } # if ($nmix > 1)
	} # end if (0)
	
            # Construct the regression class tree for this set of HMMs
	    # Here is the HHEd target: the MMF that will be used for testing
	    $MMF="$MMFDIR/${type}${nmix}${REGCLSTREE}/$MMFOUTFILE";
	    open(HED,">$EDITDIR/${REGCLSTREE}${type}${nmix}.hhed") || die "Unable to write to $EDITDIR/${REGCLSTREE}${type}${nmix}.hhed: $!";
	    print HED "RN \"${type}${nmix}\"\n";
	    print HED "LS \"$MMFDIR/${type}${nmix}G${HERESTITER}/stats\"\n";
	    print HED 'RC 32 "rtree" {sil.state[2-4].mix,noiz.state[2-4].mix,laff.state[2-4].mix,UTT_END.state[2].mix}',"\n";
	    close(HED);
	    mkdir "$MMFDIR/${type}${nmix}${REGCLSTREE}" unless -d "$MMFDIR/${type}${nmix}${REGCLSTREE}";
	    system("HHEd -A $HHEdTrace -H $MMFDIR/${type}${nmix}G${HERESTITER}/$MMFOUTFILE -w $MMF $EDITDIR/${REGCLSTREE}${type}${nmix}.hhed $PHONEF");

	    # $i==0 is talker-independent recognition, and creates an MLLR tfm.
	    # $i==1 loads the tfm, and performs talker-adapted recognition.
	    # Initially, I thought that we would get better performance
	    #  by iterating the adaptation (better transcription should
	    #  mean that HVite learns a better MLLR tfm, right?), but 
	    #  in practice, performance drops for $i>1. Perhaps I'm doing
	    #  this wrong?
	    # TODO list: exactly this same method should be used to perform
	    #  vocal tract length normalization.  
	    # TODO list: get lattice outputs from HVite, merge them using
	    #  fsmunion, and then combine their likelihoods to get a better
	    #  recognition result.
	    foreach $i (0,1) {
	        $previ = $i - 1;

		mkdir "$RESULTSDIR" unless -d "$RESULTSDIR";
	        # First, run this iteration for each talker
	        foreach $tlkr (keys %TESTTLKR) {
		    mkdir "$RESULTSDIR/$tlkr" unless -d "$RESULTSDIR/$tlkr";

		    # $i==0 is speaker-independent recognition (no -J option).
		    $JOPT=($i>0)?"-J $RESULTDIR/$tlkr/${type}${nmix}T$previ.tmf":"";
		    # find out how many files there are for this talker
		    $NFILES = @{$TESTTLKR{$tlkr}};
		    # Run HVite
		    system("HVite -A $HViteTrace -C $hvitecfg -p $P -s $S -S $SCPDIR/$TALKDEPDIR/$tlkr.scp -H $MMF -l \'*\' -j $NFILES -K $RESULTSDIR/$tlkr/${type}${nmix}T$i.tmf -k uid $DATADIR/$tlkr -k uname $DATADIR/$tlkr -k chan switchboard_12hour -k desc unsupervised_adaptation_iter_$i $JOPT -i $RESULTSDIR/$tlkr/${type}${nmix}T$i.mlf -w $lattice $HVitePrune $DICT $PHONEF");

		    # Normalize the output transcription
		    open(MI,"$RESULTSDIR/$tlkr/${type}${nmix}T$i.mlf") || die "Unable to read $RESULTSDIR/$tlkr/${type}${nmix}T$i.mlf: $!";
		    open(MO,">$RESULTSDIR/$tlkr/${type}${nmix}T${i}s.mlf") || die "Unable to write $RESULTSDIR/$ALLRECDIR/${type}${nmix}T${i}s.mlf: $!";
		    foreach (<MI>) {
		        # Header or footer line: Print without change
		        if(/^[\#\"\.]/) { print MO $_; }
                        # All other lines: get the word, word2snor it, print
                        else {
                            chomp; chomp; my(@r) = split;
                            my(@snor)=word2snor($r[2]);
                            foreach $w (@snor) {if($w!~/^\s+$/) {print MO "$w\n";}}
                        }
                    } 
                    close(MI); close(MO);

                    # Compute the talker-dependent HResults
                    system("HResults -A -c -I $wordsnor $PHONEF $RESULTSDIR/$tlkr/${type}${nmix}T${i}s.mlf > $RESULTSDIR/$tlkr/${type}${nmix}T${i}.results");
                } # foreach $tlkr

                # Now compute HResults for all talkers together
                my(@TLKRMLFS)=map {"$RESULTSDIR/$_/${type}${nmix}T${i}s.mlf";} keys %TESTTLKR;
                system("HResults -A -c -I $wordsnor $PHONEF @TLKRMLFS");
                system("HResults -A -c -I $wordsnor $PHONEF @TLKRMLFS > $RESULTSDIR/$ALLRECDIR/${type}${nmix}T${i}.results");
            } # foreach $i
        } # foreach $type
    } # foreach $nmix
}

###############################################################################################
# Section 18: Upmix and do speaker independent recognition using HVite

if (exists $cmdline{"u"}) {

    # mike the directories if they don't already exist   
    mkdir "$RESULTSDIR" unless -d "$RESULTSDIR";
    mkdir "$RESULTSDIR/$ALLRECDIR" unless -d "$RESULTSDIR/$ALLRECDIR";

    for($nmix=$NMIXES+$UPMSTEP; $nmix <= $MAXMIX; $nmix += $UPMSTEP) {

        # UPMIX and TEST for each type of model separately (monophones, clusters) 
        foreach $type ("$CLUSTERDIR", "$FLATSTARTDIR") {

            # Specify the list of HMMs (PHONEF) and label file (MLF) and DICT
            #  DICT is only necessary because of the odd way stress is handled -- 
            #  otherwise, HTK would perform triphone expansion for us 
            $PHONEF=($type eq "$FLATSTARTDIR")?"$monophonelist":"$clusterlist";
            $MLF=($type eq "$FLATSTARTDIR")?"$monophonemlf":"$triphonemlf";
            $DICT=($type eq "$FLATSTARTDIR")?"$monospdict":"$trispdict";

            # If $nmix > 1, upmix from a lower number of mixtures 
            if($nmix > 1) {
                $pmix = $nmix-$UPMSTEP;
                # Create the HHEd script for upmixing 
                open(HEDF,">$EDITDIR/${UPMIXHHED}${nmix}.hhed") || die "Unable to write to $EDITDIR/${UPMIXHHED}${nmix}.hhed: $!";
                print HEDF "MU $nmix \{ *.state\[2-4\].mix \}\n";
		close HEDF;
                
                # Upmix the MMF
                mkdir "$MMFDIR/${type}${nmix}G0" unless -d "$MMFDIR/${type}${nmix}G0";
                system("HHEd -A $HHEdTrace -H $MMFDIR/${type}${pmix}G$HERESTITER/$MMFOUTFILE -w $MMFDIR/${type}${nmix}G0/$MMFOUTFILE $EDITDIR/${UPMIXHHED}${nmix}.hhed $PHONEF");
                $MMF="$MMFDIR/${type}${nmix}G0/$MMFOUTFILE";
                
                # Re-estimate the upmixed MMF
                foreach $i (1..$HERESTITER) {
                    my($next)="$MMFDIR/${type}${nmix}G${i}";
                    $nextdir = $next;
                    mkdir $next unless -d $next;
                    # Compute occupancy statistics only for $i==$HERESTITER
                    $occ = ($i==$HERESTITER) ? "-s $MMFDIR/${type}${nmix}G${i}/stats" : "";
                    system("HERest -A $HERestTrace $occ -I $MLF -C $htkcfg $HERestPrune -S $trainscp -H $MMF -M $next $PHONEF");
                    $MMF="$next/$MMFOUTFILE";
               } # end foreach $i
            }  # end if $mix > 1

            mkdir "$RESULTSDIR" unless -d "$RESULTSDIR";
            # First, run this iteration for each talker
            mkdir "$RESULTSDIR/$ALLRECDIR" unless -d "$RESULTSDIR/$ALLRECDIR";

            # Run HVite
            system("HVite -A $HViteTrace -C $htkcfg -p $P -s $S -S $testscp -H $MMF -l \'*\' -i $RESULTSDIR/$ALLRECDIR/${type}${nmix}.mlf $HVitePrune $HVITEOPTS -w $lattice $DICT $PHONEF");

	    if (0) {
            # Normalize the output transcription
            open(MI,"$RESULTSDIR/$ALLRECDIR/${type}${nmix}.mlf") || die "Unable to read $RESULTSDIR/$ALLRECDIR/${type}${nmix}.mlf: $!";
            open(MO,">$RESULTSDIR/$ALLRECDIR/${type}${nmix}s.mlf") || die "Unable to write $RESULTSDIR/$ALLREC/${type}${nmix}s.mlf: $!";
            foreach (<MI>) {
                # Header or footer line: Print without change
                if(/^[\#\"\.]/) { print MO $_; }
                # All other lines: get the word, word2snor it, print
                else {
		    chomp; chomp; my(@r) = split;
                    my(@snor)=word2snor($r[2]);
                    foreach $w (@snor) {if($w!~/^\s+$/) {print MO "$w\n";}}
                }
                close(MI); close(MO);
	    }
            # Compute the talker-independent HResults
            system("HResults -A -c -I $wordsnor $PHONEF $RESULTSDIR/$ALLRECDIR/${type}${nmix}s.mlf > $RESULTSDIR/$ALLRECDIR/${type}${nmix}s.results");
	    } # if (0)
            system("HResults -A -c -I $wordsnor $PHONEF $RESULTSDIR/$ALLRECDIR/${type}${nmix}.mlf > $RESULTSDIR/$ALLRECDIR/${type}${nmix}.results");
        } # foreach $type
    } # foreach $nmix

}
