%--------------------------------------------------------------------------
%
% check installation
%
%--------------------------------------------------------------------------
cfg = config();
if(isempty(cfg))
    error('config failed...');
end

%--------------------------------------------------------------------------
%
% dataset settings
%
%--------------------------------------------------------------------------
jobs   = {'2016_09_22_DE_50VP_F5_40x40_x5', ...
          '2016_09_22_IT_50VP_F5_42x42_x5', ...
          '2016_09_22_GR_50VP_F5_42x42_x5', ...
          '2018_03_01_ES_50VP_F5_40x40_x5', ...
          '2015_11_13_AR_50VP_F5_40x40_x5', ...
          '2014_10_09_EN_50VP_F5_35x35_x5' };
    
dics   = {'DE_50VP_F5', ...
          'IT_50VP_F5', ...  
          'GR_50VP_F5_v2', ...
          'ES_50VP_F5', ...
          'AR_VP_F5', ...
          'EN_VP50_F5'};

% datasets to be trained: set to 0 the elements you do not want to train
% (maybe becaus you've already trained them!)
% (the order is the same of "jobs" above)
jobRun = [ 1, ... % DE
           1, ... % IT
           1, ... % GR
           1, ... % ES
           1, ... % AR
           1];    % EN

jobNodes   = [40, 42, 42, 40, 40, 35];
jobSymbols = [33, 33, 29, 36, 34, 33];

jobLabels = {'DE', ...
             'IT', ...
             'GR', ...
             'ES', ...
             'AR', ...
             'EN'};

jobLangs = {'German', ...
            'Italian', ...
            'Greek', ...
            'Spanish', ...
            'Arabic', ...
            'English'};

jobsCount = length(jobs);

%--------------------------------------------------------------------------
%
% load tsoms
%
%--------------------------------------------------------------------------
allEpochIndexes = [1:50 100];
epochIndexes = 100;
cumulateHomographs = true;

%
t = cell(jobsCount, 1);
for ji=1:jobsCount
    t{ji} = TSOM(job(jobs{ji}), epochIndexes, []);
end

% cumulate homographs
for ji=1:jobsCount
    t{ji}.LoadWords(1, dics{ji}, '', cumulateHomographs);
end

% fix column names for GR
for ji = 3
    repsCount = length(t{ji});
    for ri=1:repsCount
        tbl = t{ji}(ri).dic{1}.wordsTable;
        wordsCount = t{ji}(ri).dic{1}.wordsCount;
        
        if(isempty(TABLE_FindHeader(tbl, 'I/R')))
            tbl = TABLE_AddCol(tbl, 'I/R', 'I');
        end
        
        if(isempty(TABLE_FindHeader(tbl, 'I/R type')))
            tbl = TABLE_AddCol(tbl, 'I/R type', 'I');
        end
        
        afx = TABLE_GetColString(tbl, 'affixation');
        msa = TABLE_GetColString(tbl, 'morphological stem alternation');
        psa = TABLE_GetColString(tbl, 'phonological stem alternation');
        sa = TABLE_GetColDouble(tbl, 'stem alternatives');
        for wi=1:wordsCount
            if(strcmp(afx{wi}, 'A') && strcmp(msa{wi}, 'U') && strcmp(psa{wi}, 'Null'))
                ir = 'I';
                ir_type = 'I';
            elseif(strcmp(afx{wi}, 'A') && strcmp(msa{wi}, 'U') && strcmp(psa{wi}, 'S'))
                ir = 'I';
                ir_type = 'I';
            elseif(strcmp(afx{wi}, 'S') && strcmp(msa{wi}, 'S') && strcmp(psa{wi}, 'S'))
                ir = 'R';
                ir_type = 'RMA';
            elseif(strcmp(afx{wi}, 'S') && strcmp(msa{wi}, 'S') && strcmp(psa{wi}, 'Null'))
                ir = 'R';
                ir_type = 'RMA';
            elseif(strcmp(afx{wi}, 'S') && strcmp(msa{wi}, 'Null') && strcmp(psa{wi}, 'S'))
                ir = 'R';
                ir_type = 'RPA';
            else
                ir = 'none';
                ir_type = 'none';
            end
            tbl = TABLE_SetString(tbl, 'I/R', wi, ir);
            tbl = TABLE_SetString(tbl, 'I/R type', wi, ir_type);
        end
        t{ji}(ri).dic{1}.wordsTable = tbl;
    end
end

%--------------------------------------------------------------------------
%
% load tsom data
%
%--------------------------------------------------------------------------
fileSuffix = '2019_07_25';

%
ans_gwsi_AE = cell(jobsCount, 1);
ans_gwspb_AE = cell(jobsCount, 1);
for ji = 1:jobsCount
    fn = [job(jobs{ji}) '\ans_gwsi_AE_' fileSuffix '.mat'];
    if(~FILE_Exist(fn))
        tt = TSOM(job(jobs{ji}), allEpochIndexes, []);
        tt.LoadWords(1, dics{ji}, '', cumulateHomographs);
        
        ans_gwsi = GetWordsISR(-1, tt);
        SaveAns(tt, ans_gwsi, 'ans_gwsi_AE', fileSuffix);
        
        ans_gwspb = GetWordsPredBMU(-1, tt, 1);
        SaveAns(tt, ans_gwspb, 'ans_gwspb_AE', fileSuffix);
    
    end
    
    ans_gwsi_AE{ji} = LoadAns(t{ji}, 'ans_gwsi_AE', fileSuffix);
    ans_gwspb_AE{ji} = LoadAns(t{ji}, 'ans_gwspb_AE', fileSuffix);
end

%
ans_grnsd = cell(jobsCount, 1);
for ji = 1:jobsCount
    ans_grnsd{ji} = GetRootNeighboursDic(-1, t{ji}, 1, 0);
end

%
ans_gwspb = cell(jobsCount, 1);
for ji = 1:jobsCount
    ans_gwspb{ji} = GetWordsPredBMU(-1, t{ji}, 1, epochIndexes);
end

%
addPrefixToStem = false;
ans_gups = cell(jobsCount, 1);
for ji = 1:jobsCount
    ans_gups{ji} = GetUniquenessPoints(-1, t{ji}, 1, addPrefixToStem);
end

%
ans_gfph = cell(jobsCount, 1);
for ji = 1:jobsCount
    ans_gfph{ji} = DIC_GetFPH(t{ji}, 1, [], 'AddPrefixToStem', addPrefixToStem);
end

% BUILD per-symbol learning epoch
isr_epochsFirst = cell(jobsCount, 1);
isr_epochsSteady = cell(jobsCount, 1);

for ji = 1:jobsCount
    repsCount = length(t{ji});
    wordsCount = ans_gwsi_AE{ji}.pws.wordsCount;
    epochsCount = length(allEpochIndexes);
    
    isr_epochsFirst{ji} = cell(wordsCount, repsCount);
    isr_epochsSteady{ji} = cell(wordsCount, repsCount);
    
    msg = sprintf('%d rep(s), %d word(s), %d epoch(s)...', repsCount, wordsCount, epochsCount);
    msg_begin(msg);
    nn = wordsCount*repsCount*epochsCount;
    ii = 1;
    
    for wi = 1:wordsCount
        wordSymbols = ans_gwsi_AE{ji}.pws.wordSymbols{wi};
        wordLength = length(wordSymbols);
        for ri = 1:repsCount
            epochsFirst = nan(wordLength, 1);
            epochsSteady = nan(wordLength, 1);
            for ei = 1:epochsCount
                isr_wordSymbols = ans_gwsi_AE{ji}.gwi{wi,ei,ri}.isr_wordSymbols;    

                for pos = 1:wordLength
                    if(wordSymbols(1:pos) == isr_wordSymbols(1:pos))
                        if(isnan(epochsFirst(pos)))
                            epochsFirst(pos) = ei;
                        end
                        if(isnan(epochsSteady(pos)))
                            epochsSteady(pos) = ei;
                        end
                    else
                        epochsSteady(pos) = nan;
                    end
                end
                
                msg_update(100*ii/nn);
                ii = ii + 1;
            end
            isr_epochsFirst{ji}{wi, ri} = epochsFirst;
            isr_epochsSteady{ji}{wi, ri} = epochsSteady;
        end
    end
    msg_end();
end

%--------------------------------------------------------------------------
%
% #1: export TBL per-letter (at epoch 100)
%
%--------------------------------------------------------------------------
headers = { 'jobId', 'jobLang', 'repId', ...
            'paradigm', 'IR', 'Rp', ...
            'word', 'len', 'freq', ...
            'actOk', 'isrOk', ...
            'typNNB', 'tokNNB', 'SZs', 'NBH', ...
            'stem', 'ending', ...
            'Lw', 'Lp', 'Ls', 'Le', ...
            'UP1', 'CUP', 'MB', ...
            'symbol', 'pos', ...
            'ant', ...
            'actSurprise', 'isrSurprise', ...
            'actSurprisal', 'isrSurprisal', ...
            'isrLE'};

rowsCount = 0;
for ji = 1:jobsCount
    repsCount = length(t{ji});
    rowsCount = rowsCount + sum(ans_gwsi_AE{ji}.pws.wordLengths) * repsCount;
end

tw = TBL_Writer('headers', 'rowsCount', 'row', 'tbl');

eval(tw.Open);
for ji = 1:jobsCount
    repsCount = length(t{ji});
    wordsCount = ans_gwsi_AE{ji}.pws.wordsCount;
    words = DIC_GetColString(t{ji}, 1, 'word form');
    if(addPrefixToStem)
        prefixLengths = zeros(1, wordsCount);
    else
        prefixLengths = DIC_GetColDouble(t{ji}, 1, 'prefix length');
    end
    ir = DIC_GetColString(t{ji}, 1, 'I/R');
    for wi = 1:wordsCount

        pi = ans_gfph{ji}.wi2pi(wi);
        si = ans_gfph{ji}.wi2si(wi);
        ei = ans_gfph{ji}.wi2ei(wi);

        Rp = ans_gfph{ji}.Rp(pi);
        
        Lw = ans_gfph{ji}.Lw(wi);
        Lp = prefixLengths(wi);
        Ls = ans_gfph{ji}.Ls(si);
        Le = ans_gfph{ji}.Le(ei);

        len = ans_gwsi_AE{ji}.pws.wordLengths(wi);
        freq = ans_gwsi_AE{ji}.pws.wordFreqs(wi);
        
        typNNB = ans_grnsd{ji}.dic_typesNeighbourhoodDensityNSW(wi);
        tokNNB = ans_grnsd{ji}.dic_tokensNeighbourhoodDensityNSW(wi);

        NBH = ans_grnsd{ji}.dic_neighbourhoodEntropyNSW(wi);
        
        SZs = length(ans_gfph{ji}.si2wis{si});
        wordSymbols = ans_gwsi_AE{ji}.pws.wordSymbols{wi};
        
        for ri = 1:repsCount 
            for pos = 1:ans_gwsi_AE{ji}.pws.wordLengths(wi)

                row = { ji, jobLangs{ji}, ri, ...
                        ans_gfph{ji}.paradigms{pi}, ir{wi}, Rp, ...
                        words{wi}, len, freq, ...
                        ans_gwsi_AE{ji}.act_corrects(wi, end, ri), ans_gwsi_AE{ji}.isr_corrects(wi, end, ri), ...
                        typNNB, tokNNB, SZs, NBH, ...
                        ans_gfph{ji}.stems{si}, ans_gfph{ji}.endings{ei}, ...
                        Lw, Lp, Ls, Le, ...
                        ans_gups{ji}.up1(wi), ans_gups{ji}.cup(wi), ans_gups{ji}.mb2(wi), ...
                        t{ji}(ri).symbols{wordSymbols(pos)}, pos, ...
                        ans_gwspb_AE{ji}.anticipation{wi}(pos, end, ri), ...
                        ans_gwsi_AE{ji}.gwi{wi, end, ri}.act_ySurprise(pos), ans_gwsi_AE{ji}.gwi{wi, end, ri}.isr_ySurprise(pos), ...
                        ans_gwsi_AE{ji}.gwi{wi, end, ri}.act_ySurprisal(pos), ans_gwsi_AE{ji}.gwi{wi, end, ri}.isr_ySurprisal(pos), ...
                        isr_epochsSteady{ji}{wi, ri}(pos)};

                eval(tw.Write);    
            end
        end
    end
end
eval(tw.Close);
tbl = TBL_SetAutoCategorical(tbl);

%
tbl.distToMB = tbl.pos - tbl.MB;
tbl.distToUP1 = tbl.pos - tbl.UP1;
tbl.distToCUP = tbl.pos - tbl.CUP;

%
tbl.segment = cell(height(tbl), 1);
tbl.distToS = zeros(height(tbl), 1);

i = (tbl.pos <= tbl.UP1);
tbl.segment(i) = {'1xBtoUP1'};
tbl.distToS(i) = tbl.pos(i) - 1;

i = (tbl.pos > tbl.UP1) & (tbl.pos < tbl.MB);
tbl.segment(i) = {'2xUP1toMB'};
tbl.distToS(i) = tbl.pos(i) - tbl.UP1(i) - 1;

i = (tbl.pos >= tbl.MB) & (tbl.pos <= tbl.CUP);
tbl.segment(i) = {'3xMBtoCUP'};
tbl.distToS(i) = tbl.pos(i) - tbl.MB(i);

i = (tbl.pos >= tbl.MB) & (tbl.pos > tbl.CUP);
tbl.segment(i) = {'4xCUPtoE'};
tbl.distToS(i) = tbl.pos(i) - tbl.CUP(i) - 1;

tbl.segment = categorical(tbl.segment);

tbl.pred = 1*(tbl.ant > 0);

tbl.isEnding = categorical(tbl.distToMB >= 0);

% cumulative suprisal
n = height(tbl);
v = nan(n, 1);
msg_begin('...');
for i = 1:n
    pos = tbl.pos(i);
    if(pos == 2)
        v(i) = tbl.isrSurprisal(i);
    elseif(tbl.pos(i) > 2)
        v(i) = v(i-1) + tbl.isrSurprisal(i);
    end 
    msg_update(100*i/n);
end
msg_end();

tbl.isrSurprisalSum = v;

tbl.isrSurprisalLPF = v;
i = (tbl.pos > 1);
tbl.isrSurprisalLPF(i) = tbl.isrSurprisalLPF(i) ./ (tbl.pos(i) - 1);

% OK
TBL_Save(tbl, '.\tbl_perLetter.txt');

%--------------------------------------------------------------------------
%
% #2: export TBL per-letter, per-epoch (every 5 epochs)
%
%--------------------------------------------------------------------------

epochs = 5:5:50;
headers = { 'jobId', 'jobLang', 'repId', 'epoch', ...
            'paradigm', 'IR', 'Rp', ...
            'word', 'len', 'freq', ...
            'actOk', 'isrOk', ...
            'typNNB', 'tokNNB', 'SZs', ...
            'stem', 'ending', ...
            'Lw', 'Lp', 'Ls', 'Le', ...
            'UP1', 'CUP', 'MB', ...
            'symbol', 'pos', ...
            'ant', ...
            'actSurprise', 'isrSurprise', ...
            'actSurprisal', 'isrSurprisal'};

rowsCount = 0;
for ji = 1:jobsCount
    repsCount = length(t{ji});
    rowsCount = rowsCount + sum(ans_gwsi_AE{ji}.pws.wordLengths) * repsCount * length(epochs);
end

tw = TBL_Writer('headers', 'rowsCount', 'row', 'tbl');

eval(tw.Open);
for ji = 1:jobsCount
    repsCount = length(t{ji});
    wordsCount = ans_gwsi_AE{ji}.pws.wordsCount;
    words = DIC_GetColString(t{ji}, 1, 'word form');
    if(addPrefixToStem)
        prefixLengths = zeros(1, wordsCount);
    else
        prefixLengths = DIC_GetColDouble(t{ji}, 1, 'prefix length');
    end
    ir = DIC_GetColString(t{ji}, 1, 'I/R');
    for wi = 1:wordsCount

        pi = ans_gfph{ji}.wi2pi(wi);
        si = ans_gfph{ji}.wi2si(wi);
        ei = ans_gfph{ji}.wi2ei(wi);

        Rp = ans_gfph{ji}.Rp(pi);
        
        Lw = ans_gfph{ji}.Lw(wi);
        Lp = prefixLengths(wi);
        Ls = ans_gfph{ji}.Ls(si);
        Le = ans_gfph{ji}.Le(ei);

        len = ans_gwsi_AE{ji}.pws.wordLengths(wi);
        freq = ans_gwsi_AE{ji}.pws.wordFreqs(wi);
        
        typNNB = ans_grnsd{ji}.dic_typesNeighbourhoodDensityNSW(wi);
        tokNNB = ans_grnsd{ji}.dic_tokensNeighbourhoodDensityNSW(wi);

        SZs = length(ans_gfph{ji}.si2wis{si});
        wordSymbols = ans_gwsi_AE{ji}.pws.wordSymbols{wi};
        
        for ri = 1:repsCount 
            for epoch = epochs
                for pos = 1:ans_gwsi_AE{ji}.pws.wordLengths(wi)
                
                    row = { ji, jobLangs{ji}, ri, epoch, ...
                            ans_gfph{ji}.paradigms{pi}, ir{wi}, Rp, ...
                            words{wi}, len, freq, ...
                            ans_gwsi_AE{ji}.act_corrects(wi, epoch, ri), ans_gwsi_AE{ji}.isr_corrects(wi, epoch, ri), ...
                            typNNB, tokNNB, SZs, ...
                            ans_gfph{ji}.stems{si}, ans_gfph{ji}.endings{ei}, ...
                            Lw, Lp, Ls, Le, ...
                            ans_gups{ji}.up1(wi), ans_gups{ji}.cup(wi), ans_gups{ji}.mb2(wi), ...
                            t{ji}(ri).symbols{wordSymbols(pos)}, pos, ...
                            ans_gwspb_AE{ji}.anticipation{wi}(pos, epoch, ri), ...
                            ans_gwsi_AE{ji}.gwi{wi, epoch, ri}.act_ySurprise(pos), ans_gwsi_AE{ji}.gwi{wi, epoch, ri}.isr_ySurprise(pos), ...
                            ans_gwsi_AE{ji}.gwi{wi, epoch, ri}.act_ySurprisal(pos), ans_gwsi_AE{ji}.gwi{wi, epoch, ri}.isr_ySurprisal(pos)};

                    eval(tw.Write);    
                end
            end
        end
    end
end
eval(tw.Close);
tbl = TBL_SetAutoCategorical(tbl);

%
tbl.distToMB = tbl.pos - tbl.MB;
tbl.distToUP1 = tbl.pos - tbl.UP1;
tbl.distToCUP = tbl.pos - tbl.CUP;

%
tbl.segment = cell(height(tbl), 1);
tbl.distToS = zeros(height(tbl), 1);

i = (tbl.pos <= tbl.UP1);
tbl.segment(i) = {'1xBtoUP1'};
tbl.distToS(i) = tbl.pos(i) - 1;

i = (tbl.pos > tbl.UP1) & (tbl.pos < tbl.MB);
tbl.segment(i) = {'2xUP1toMB'};
tbl.distToS(i) = tbl.pos(i) - tbl.UP1(i) - 1;

i = (tbl.pos >= tbl.MB) & (tbl.pos <= tbl.CUP);
tbl.segment(i) = {'3xMBtoCUP'};
tbl.distToS(i) = tbl.pos(i) - tbl.MB(i);

i = (tbl.pos >= tbl.MB) & (tbl.pos > tbl.CUP);
tbl.segment(i) = {'4xCUPtoE'};
tbl.distToS(i) = tbl.pos(i) - tbl.CUP(i) - 1;

tbl.segment = categorical(tbl.segment);

tbl.pred = 1*(tbl.ant > 0);

% OK
TBL_Save(tbl, '.\tbl_perLetter_perEpoch.txt');

%--------------------------------------------------------------------------
%
% #3: export TBL per-word (at epoch 100)
%
%--------------------------------------------------------------------------
headers = { 'jobId', 'jobLang', 'repId', ...
            'word', 'paradigm', 'IR', 'Rp', ...
            'stem', 'ending', ...
            'len', 'freq', ...
            'UP1', 'CUP', 'MB', ...
            'typNNB', 'tokNNB', ...
            'Lw', 'Lp', 'Ls', 'Le', ...
            'SZs', 'SZe', ...
            'Fs', 'Fe', ...
            'HeGsPs', 'HsGePe', 'DKLs', 'DKLe', ...
            'NBH', ...
            'act', 'isr', ...
            'actFE', 'isrFE', ...
            'actLE', 'isrLE', ...
            'actLS', 'isrLS', ...
            'ant', 'preds', 'props', ...
            'actSurprisal', 'isrSurprisal'};   
        
rowsCount = 0;
for ji = 1:jobsCount
    repsCount = length(t{ji});
    rowsCount = rowsCount + ans_gwsi_AE{ji}.pws.wordsCount * repsCount;
end

tw = TBL_Writer('headers', 'rowsCount', 'row', 'tbl');

eval(tw.Open);
for ji = 1:jobsCount
    repsCount = length(t{ji});
    wordsCount = ans_gwsi_AE{ji}.pws.wordsCount;
    words = DIC_GetColString(t{ji}, 1, 'word form');
    if(addPrefixToStem)
        prefixLengths = zeros(1, wordsCount);
    else
        prefixLengths = DIC_GetColDouble(t{ji}, 1, 'prefix length');
    end
    paradigms = DIC_GetColString(t{ji}, 1, 'lexical exponent');
    ir = DIC_GetColString(t{ji}, 1, 'I/R');
    for ri = 1:repsCount
        for wi = 1:wordsCount
            actFE = ans_gwsi_AE{ji}.act_epochsFirst(wi, ri);
            isrFE = ans_gwsi_AE{ji}.isr_epochsFirst(wi, ri);
            
            actLE = ans_gwsi_AE{ji}.act_epochsSteady(wi, ri);
            isrLE = ans_gwsi_AE{ji}.isr_epochsSteady(wi, ri);
            
            if(actLE > 50)
                actLE = nan;
            end
            if(isrLE > 50)
                isrLE = nan;
            end

            actLS = actLE - ans_gwsi_AE{ji}.act_epochsFirst(wi, ri);
            isrLS = isrLE - ans_gwsi_AE{ji}.isr_epochsFirst(wi, ri);
            
            freq = ans_gwsi_AE{ji}.pws.wordFreqs(wi);
            len = ans_gwsi_AE{ji}.pws.wordLengths(wi);
            
%             if(strcmp(jobDistrs{ji}, 'uniform'))
%                 NNBst0 = ans_grnsd{ji}.dic_tokensNeighbourhoodDensity(wi)/5 - 1;
%             else
%                 NNBst0 = ans_grnsd{ji}.dic_tokensNeighbourhoodDensity(wi) - freq;
%             end
            
            typNNB = ans_grnsd{ji}.dic_typesNeighbourhoodDensityNSW(wi);
            tokNNB = ans_grnsd{ji}.dic_tokensNeighbourhoodDensityNSW(wi);
            
            si = ans_gfph{ji}.wi2si(wi);
            ei = ans_gfph{ji}.wi2ei(wi);
            pi = ans_gfph{ji}.wi2pi(wi);
            
            ant = mean(ans_gwspb{ji}.anticipation{wi}(:, end, ri));
            preds = ans_gwspb{ji}.predictionsCount(wi, end, ri);
            props = ans_gwspb{ji}.propagationsCount(wi, end, ri);
            
            nbh = ans_grnsd{ji}.dic_neighbourhoodEntropyNSW(wi);
            
            row = { ji, jobLangs{ji}, ri, ...
                    words{wi}, paradigms{wi}, ir{wi}, ans_gfph{ji}.Rp(pi), ...
                    ans_gfph{ji}.stems{si}, ans_gfph{ji}.endings{ei}, ...
                    len, freq, ...
                    ans_gups{ji}.up1(wi), ans_gups{ji}.cup(wi), ans_gups{ji}.mb2(wi), ...
                    typNNB, tokNNB, ...
                    ans_gfph{ji}.Lw(wi), prefixLengths(wi), ans_gfph{ji}.Ls(si), ans_gfph{ji}.Le(ei), ...
                    length(ans_gfph{ji}.si2wis{si}), length(ans_gfph{ji}.ei2wis{ei}), ...
                    ans_gfph{ji}.Fs(si), ans_gfph{ji}.Fe(ei), ...
                    ans_gfph{ji}.HeGsPs(si), ans_gfph{ji}.HsGePe(ei), ans_gfph{ji}.DKLs(si), ans_gfph{ji}.DKLe(ei), ...
                    nbh, ...
                    ans_gwsi_AE{ji}.act_corrects(wi, end, ri), ans_gwsi_AE{ji}.isr_corrects(wi, end, ri), ...
                    actFE, isrFE, ... 
                    actLE, isrLE, ... 
                    actLS, isrLS, ...
                    ant, preds, props, ...
                    nanmean(ans_gwsi_AE{ji}.gwi{wi,end,ri}.act_ySurprisal), nanmean(ans_gwsi_AE{ji}.gwi{wi,end,ri}.isr_ySurprisal)};

            eval(tw.Write);    
        end
    end
end
eval(tw.Close);

tbl.logFreq = log10(tbl.freq);
tbl = TBL_SetAutoCategorical(tbl);

% OK
TBL_Save(tbl, '.\tbl_perWord.txt');

%--------------------------------------------------------------------------
%
% #4: export TBL per-word, per-epoch
%
%--------------------------------------------------------------------------
headers = { 'jobId', 'jobLang', 'repId', 'epoch', ...
            'paradigm', 'IR', 'Rp', ...
            'word', 'len', 'freq', ...
            'UP1', 'CUP', 'MB', ...
            'typNNB', 'tokNNB', 'SZs', ...
            'act', 'isr', ...
            'actSurprise', 'isrSurprise', ...
            'actSurprisal', 'isrSurprisal', ...
            'ant', 'props'};
        
rowsCount = 0;
for ji = 1:jobsCount
    repsCount = length(t{ji});
    rowsCount = rowsCount + ans_gwsi_AE{ji}.pws.wordsCount * repsCount * 50;
end

tw = TBL_Writer('headers', 'rowsCount', 'row', 'tbl');
eval(tw.Open);

for ji = 1:jobsCount
    jobLang = jobLangs{ji};
    
    repsCount = length(t{ji});
    wordsCount = ans_gwsi_AE{ji}.pws.wordsCount;
    
    words = DIC_GetColString(t{ji}, 1, 'word form');
    if(addPrefixToStem)
        prefixLengths = zeros(1, wordsCount);
    else
        prefixLengths = DIC_GetColDouble(t{ji}, 1, 'prefix length');
    end
    irs = DIC_GetColString(t{ji}, 1, 'I/R');
    
    for wi = 1:wordsCount

        len = ans_gwsi_AE{ji}.pws.wordLengths(wi);
        freq = ans_gwsi_AE{ji}.pws.wordFreqs(wi);
        
        pi = ans_gfph{ji}.wi2pi(wi);
        si = ans_gfph{ji}.wi2si(wi);
        ei = ans_gfph{ji}.wi2ei(wi);

        paradigm = ans_gfph{ji}.paradigms{pi};
        ir = irs{wi};
        Rp = ans_gfph{ji}.Rp(pi);
        
        word = words{wi};
        typNNB = ans_grnsd{ji}.dic_typesNeighbourhoodDensityNSW(wi);
        tokNNB = ans_grnsd{ji}.dic_tokensNeighbourhoodDensityNSW(wi);
        SZs = length(ans_gfph{ji}.si2wis{si});

        UP1 = ans_gups{ji}.up1(wi);
        CUP = ans_gups{ji}.cup(wi);
        MB = ans_gups{ji}.mb2(wi);
        
        
        for epoch = 1:50
            for ri = 1:repsCount
                
                row = { ji, jobLang, ri, epoch, ...
                        paradigm, ir, Rp, ...
                        word, len, freq, ...
                        UP1, CUP, MB, ...
                        typNNB, tokNNB, SZs, ...
                        ans_gwsi_AE{ji}.act_corrects(wi,epoch,ri), ans_gwsi_AE{ji}.isr_corrects(wi,epoch,ri), ...
                        nansum(ans_gwsi_AE{ji}.gwi{wi,epoch,ri}.act_ySurprise), nanmean(ans_gwsi_AE{ji}.gwi{wi,epoch,ri}.isr_ySurprise), ...
                        nanmean(ans_gwsi_AE{ji}.gwi{wi,epoch,ri}.act_ySurprisal), nanmean(ans_gwsi_AE{ji}.gwi{wi,epoch,ri}.isr_ySurprisal), ...
                        mean(ans_gwspb_AE{ji}.anticipation{wi}(2:end, epoch, ri)), ...
                        ans_gwspb_AE{ji}.propagationsCount(wi, epoch, ri) };
                    
                eval(tw.Write);
            end
        end
    end
end

eval(tw.Close);
tbl = TBL_SetAutoCategorical(tbl);

% OK
TBL_Save(tbl, '.\tbl_perWord_perEpoch.txt');

%--------------------------------------------------------------------------
%
% #5: export TBL per-paradigm (at epoch 100)
%
%--------------------------------------------------------------------------
headers = { 'jobId', 'jobLang', 'repId', ...
            'paradigm', 'IR', 'Rp', 'Hp', ...
            'freq', 'len', 'lenMax', 'H', 'size', ...
            'actLE', 'isrLE', ...
            'actLS', 'isrLS'};   
        
rowsCount = 0;
for ji = 1:jobsCount
    repsCount = length(t{ji});
    pars = unique(DIC_GetColString(t{ji}, 1, 'lexical exponent'));
    rowsCount = rowsCount + length(pars) * repsCount;
end

tw = TBL_Writer('headers', 'rowsCount', 'row', 'tbl');

eval(tw.Open);
for ji = 1:jobsCount
    repsCount = length(t{ji});

    for pi = 1:ans_gfph{ji}.paradigmsCount
        
        wis = ans_gfph{ji}.pi2wis{pi};
        ir = DIC_GetColString(t{ji}, 1, 'I/R', wis(1));

        freq = sum(ans_gwsi_AE{ji}.pws.wordFreqs(wis));
        len = mean(ans_gwsi_AE{ji}.pws.wordLengths(wis));
        lenMax = max(ans_gwsi_AE{ji}.pws.wordLengths(wis));
        H = vec2ent(ans_gwsi_AE{ji}.pws.wordFreqs(wis));
        
        Rp = ans_gfph{ji}.Rp(pi);
        
        
        %Rp = regPar; 
        %
        for ri = 1:repsCount
            actLE = ans_gwsi_AE{ji}.act_epochsSteady(wis, ri);
            actLE(actLE > 50) = nan;
            actLS = nanmax(actLE) - nanmin(actLE);
            actLE = nanmean(actLE); 
            
            isrLE = ans_gwsi_AE{ji}.isr_epochsSteady(wis, ri);
            isrLE(isrLE > 50) = nan;
            isrLS = nanmax(isrLE) - nanmin(isrLE);
            isrLE = nanmean(isrLE); 

            row = { ji, jobLangs{ji}, ri, ...
                    ans_gfph{ji}.paradigms{pi}, ir{1}, Rp, ans_gfph{ji}.Hp(pi), ...
                    freq, len, lenMax, H, length(wis), ...
                    actLE, isrLE, ...
                    actLS, isrLS};
            eval(tw.Write);    
        end
    end
end
eval(tw.Close);

tbl = TBL_SetAutoCategorical(tbl);

% OK
TBL_Save(tbl, '.\tbl_perParadigm.txt');