#!/usr/bin/perl -w

#############################################################################
#
#  compare_engines.cgi  -  CGI script for comparing selected engines
#
#  KCEC scripts
#
#  Copyright (C) 2005-2008 by Kirill Kryukov
#
#  This software is provided 'as-is', without any express or implied
#  warranty. In no event will the author be held liable for any damages
#  arising from the use of this software.
#
#  Any usage, modification or redistribution of this script requires author's
#  permission.
#
#############################################################################


use strict;
use sigtrap;
use warnings;

use CGI qw/:standard/;
use URI::Escape;

use KCEC_utils;
use KCEC_engines;
use KCEC_rating_list;
use KCEC_tables_2;

$| = 1;


print header(-charset=>'utf-8');

#print Dump;


# Clean up the dump files.
unlink(<core.*>);


$verbose = 0;
my $debug_on = 0;
read_options();

if ($debug_on) { print "Reading engine info .."; }
read_engine_info();
if ($debug_on) { print " OK<br>\n"; }

if ($debug_on) { print "Reading statistics .."; }
read_simple_statistics();
if ($debug_on) { print " OK<br>\n"; }


my $rating_list_file = "../$options{study_file_tag}.rating_list";
if (!-e $rating_list_file) { print "Rating list file can't be found!\n"; exit; }
if ($debug_on) { print "Reading rating list .."; }
read_rating_list($rating_list_file,'');
if ($debug_on) { print " OK<br>\n"; }



foreach my $i (@engines)
{
    $i->{details_link} =~ s#href="cgi/#href="#;
    if (defined($i->{archive_name}) and $i->{archive_name} ne '')
        { $i->{archive_name} = '../' . $i->{archive_name}; }
    $i->{selected} = 0;
}


my $big_header = read_file_1('generated-header.html');
my $big_footer = read_file_1('generated-footer.html');

while ($big_header =~ m#href="(.*?)"#g)
{
    my $found = $1;
    if ($found !~ m#/# and $found !~ m#http#) { $big_header =~ s#$found#../$found#; }
}




###################################
#
# Reading print options.
#

my %print = ();
my $max_table_size = 100;
my $diagonal_width = 0;
my $cross_tables_for_best_versions_only = 0;
my $only_public_releases = 0;
my $only_stable_versions = 0;
my $only_default_settings = 0;
my $only_new_versions = 0;
my $only_best_in_class = 0;
my $num_best_in_class = 1;
my ($from_elo, $to_elo) = (0, 10000);
my ($ct_from_elo, $ct_to_elo) = (0, 10000);
my $match_length = int($options{min_games_for_a_pair});
my $sort_tables = '';
my $sort_tables_tag = 'by_rating';

my $reference_list = '';
my $reference_list_file = '';
my $recalibrate = 'no';

my $profile_step = 50;
my $profile_numbers = 0;

# Output selection and print options
{
    for (param('print')) { $print{lc($_)} = 1; }
    #if (param('print')) { %print = map { $_ => 1 } lc(param('print')); }

    my @valid_print_items_array =
    (
        'rating list','rating list (text)','profile',
        'results table','performance table','score table','los table',
        'ponder hit table','eval difference table',
        'draw percentage table','white score table',
        'number of games table','overlap table','comopp gamenum table',
        'testers table','performance and testers table',

        'score with common opponents',
        'score with all opponents',
        'performance with common opponents',
        'performance with all opponents',
        'los with common opponents',
        'los with all opponents',
        'ponder hit with common opponents',
        'ponder hit with all opponents',
        'eval difference with common opponents',
        'eval difference with all opponents',
        'testers with common opponents',
        'testers with all opponents',
        'performance and testers with common opponents',
        'performance and testers with all opponents',

        'ponder hit - most similar pairs',
        'ponder hit - most similar pairs (different families only)',
        'ponder hit - most different pairs',
        'ponder hit - most different pairs (same families only)',
        'eval difference - most similar pairs',
        'eval difference - most similar pairs (different families only)',
        'eval difference - most different pairs',
        'eval difference - most different pairs (same families only)'
    );
    my %valid_print_items = ();
    for (@valid_print_items_array) { $valid_print_items{lc($_)} = 1; }
    my ($num_valid, $num_invalid) = (0, 0);

    #print "Original print options:<br>\n";
    #foreach (keys %print) { print "Printing: \"$_\"<br>\n"; }
    #print "<br>\n";

    foreach my $item (keys %print)
    {
        if ($valid_print_items{$item}) { $num_valid++; }
        else 
        {
            print "Invalid print option: \"$item\"<br>\n";
            $num_invalid++;
            delete $print{$item};
        }
    }

    if ($num_valid <= 0) { %print = ( 'rating list' => 1, 'results table' => 1, 'los table' => 1 ); }

    #foreach (keys %print) { print "Printing: \"$_\"<br>\n"; }


    if (defined(param('table_size')))
    {
        my ($table_size_str) = param('table_size');
        if ($table_size_str =~ /(^\d{1,3})/) { $max_table_size = $1; }
        else { $max_table_size = 100; }
        if ($max_table_size < 2) { $max_table_size = 2; }
        if ($max_table_size > 100) { $max_table_size = 100; }
    }

    if (defined(param('diag')))
    {
        my ($diag_str) = param('diag');
        if ($diag_str =~ /^(\d{1,2})$/)
        {
            $diagonal_width = $1;
        }
    }

    if (param('cross_tables_for_best_versions_only')) { $cross_tables_for_best_versions_only = 1; }
    if (param('only_public_releases')) { $only_public_releases = 1; }
    if (param('only_stable_versions')) { $only_stable_versions = 1; }
    if (param('only_default_settings')) { $only_default_settings = 1; }
    if (param('only_new')) { $only_new_versions = 1; }
    if (param('only_best_in_class')) { $only_best_in_class = 1; }

    if (defined(param('num_best_in_class')))
    {
        ($num_best_in_class) = int(param('num_best_in_class'));
        if ($num_best_in_class <= 0) { $num_best_in_class = 1; }
    }


    if (defined(param('from_elo')))
    {
        my $a = param('from_elo');
        if ($a =~ /-?\d{1,5}/) { $from_elo = $a; }
    }
    if (defined(param('to_elo')))
    {
        my $a = param('to_elo');
        if ($a =~ /-?\d{1,5}/) { $to_elo = $a; }
    }
    if ($from_elo > $to_elo) { $from_elo = 0; $to_elo = 10000; }


    if (defined(param('ct_from_elo')))
    {
        my $a = param('ct_from_elo');
        if ($a =~ /-?\d{1,5}/) { $ct_from_elo = $a; }
    }
    if (defined(param('ct_to_elo')))
    {
        my $a = param('ct_to_elo');
        if ($a =~ /-?\d{1,5}/) { $ct_to_elo = $a; }
    }
    if ($ct_from_elo > $ct_to_elo) { $ct_from_elo = 0; $ct_to_elo = 10000; }



    if (defined(param('match_length')))
    {
        my ($match_length_str) = param('match_length');
        if ($match_length_str =~ /^(\d{1,5})$/)
        {
            $match_length = $1;
            if ($match_length == 0) { $match_length = 1; }
            if ($match_length > 1000) { $match_length = 999; }
        }
    }

    if (param('sort_tables')) { ($sort_tables) = param('sort_tables'); }
    if ($sort_tables ne 'by rating' and $sort_tables ne 'by name') { $sort_tables = 'by rating'; }
    foreach my $sort_tag (keys %engine_order)
        { if ($engine_order{$sort_tag}->{name} eq $sort_tables) { $sort_tables_tag = $sort_tag; last; } }

    if (defined(param('profile_step')))
    {
        my ($step) = param('profile_step');
        if ($step >= 5 and $step <= 1000) { $profile_step = $step; }
    }

    if (param('profile_numbers')) { $profile_numbers = 1; }



    if (param('reference_list'))
    {
        ($reference_list) = param('reference_list');
        if (lc($reference_list) eq 'none') { $reference_list = ''; }
        else
        {
            if (-e "../rating-lists/$reference_list.rating_list") { $reference_list_file = "$reference_list.rating_list"; }
            elsif (-e "../rating-lists/$reference_list") { $reference_list_file = $reference_list; }
        }
    }

    if ($reference_list_file)
    {
        read_rating_list("../rating-lists/$reference_list_file",'ref_');
        if (param('recalibrate')) { ($recalibrate) = param('recalibrate'); }
    }
}


my $print_with_common_opponents =
    ( $print{'score with common opponents'} or
      $print{'performance with common opponents'} or
      $print{'los with common opponents'} or
      $print{'ponder hit with common opponents'} or
      $print{'eval difference with common opponents'} or
      $print{'testers with common opponents'} or
      $print{'performance and testers with common opponents'} ) ? 1 : 0;

my $print_with_all_opponents =
    ( $print{'score with all opponents'} or
      $print{'performance with all opponents'} or
      $print{'los with all opponents'} or
      $print{'ponder hit with all opponents'} or
      $print{'eval difference with all opponents'} or
      $print{'testers with all opponents'} or
      $print{'performance and testers with all opponents'} ) ? 1 : 0;

my $print_side_by_side = ( $print_with_common_opponents or $print_with_all_opponents or
                           $print{'overlap table'} or $print{'comopp gamenum table'} ) ? 1 : 0;

my $need_los =
    ( $print{'rating list'} or
      $print{'los table'} or
      $print{'los with common opponents'} or
      $print{'los with all opponents'} ) ? 1 : 0;

my $need_testers_data =
    ( $print{'testers table'} or
      $print{'performance and testers table'} or
      $print{'testers with common opponents'} or
      $print{'testers with all opponents'} or
      $print{'performance and testers with common opponents'} or
      $print{'performance and testers with all opponents'} ) ? 1 : 0;

my $need_correlation_data =
    ( $print{'ponder hit table'} or
      $print{'eval difference table'} or
      $print{'ponder hit with common opponents'} or
      $print{'ponder hit with all opponents'} or
      $print{'eval difference with common opponents'} or
      $print{'eval difference with all opponents'} or
      $print{'ponder hit - most similar pairs'} or
      $print{'ponder hit - most similar pairs (different families only)'} or
      $print{'ponder hit - most different pairs'} or
      $print{'ponder hit - most different pairs (same families only)'} or
      $print{'eval difference - most similar pairs'} or
      $print{'eval difference - most similar pairs (different families only)'} or
      $print{'eval difference - most different pairs'} or
      $print{'eval difference - most different pairs (same families only)'} ) ? 1 : 0;

my $select_pairs =
    ( $print{'ponder hit - most similar pairs'} or
      $print{'ponder hit - most similar pairs (different families only)'} or
      $print{'ponder hit - most different pairs'} or
      $print{'ponder hit - most different pairs (same families only)'} or
      $print{'eval difference - most similar pairs'} or
      $print{'eval difference - most similar pairs (different families only)'} or
      $print{'eval difference - most different pairs'} or
      $print{'eval difference - most different pairs (same families only)'} ) ? 1 : 0;



init_tables('');
set_match_length($match_length);
set_max_table_size($max_table_size);
make_color_legends();


foreach my $i (@engines)
{
    $i->{details_link} =~ s/engine_details\.cgi\?/engine_details.cgi?match_length=$match_length&amp;each_game=0&amp;/g;
}



######################################################
#
# Reading selection of engines.
#


my $selection_uri = '';
my $selection_uri_from_class_only = '';
my $comparison_title = '';
my $comparison_is_for_class_only = 1;


if (param('class'))
{
    my ($class) = param('class');
    if (lc($class) eq 'selected engines' or lc($class) eq 'best selected engines') {}
    elsif (lc($class) eq 'none' or lc($class) eq 'nothing') {}
    else
    {
        my $class_tag = '';
        foreach my $tag (keys %engine_selection)
        {
            if (lc($engine_selection{$tag}->{name}) eq lc($class)) { $class_tag = $tag; last; }
        }

        if ($class_tag)
        {
            my $filter = $engine_selection{$class_tag}->{filter};
            foreach my $eng (@engines)
            {
                if ( &$filter($eng) and $eng->{elo} >= $from_elo and $eng->{elo} <= $to_elo)
                {
                    $eng->{selected} = 1;
                }
            }

            if ($only_public_releases)
            {
                foreach my $eng (@engines)
                {
                    if (!engine_is_public($eng)) { $eng->{selected} = 0; }
                }
            }

            if ($only_stable_versions)
            {
                foreach my $eng (@engines)
                {
                    if (!engine_is_stable($eng)) { $eng->{selected} = 0; }
                }
            }

            if ($only_default_settings)
            {
                foreach my $eng (@engines)
                {
                    if (!engine_is_default_setting($eng)) { $eng->{selected} = 0; }
                }
            }

            if ($only_new_versions)
            {
                foreach my $eng (@engines)
                {
                    if (!engine_is_new($eng)) { $eng->{selected} = 0; }
                }
            }

            if ($only_best_in_class)
            {
                #mark_best_versions('selected','elo','best_selected');
                mark_n_best_versions('selected','elo','best_selected',$num_best_in_class);
                foreach my $eng (@engines)
                {
                    if (!$eng->{best_selected}) { $eng->{selected} = 0; }
                    $eng->{best_selected} = 0;
                }

            }

            #if ($debug_on)
            #{
            #    print "Selected engines:<br>\n";
            #    foreach my $eng (sort { $b->{elo} <=> $a->{elo} } @engines)
            #    {
            #        if ($eng->{selected})
            #        {
            #            print "--- $eng->{signature} --- $eng->{games_played} games<br>\n";
            #        }
            #    }
            #}

            $selection_uri .= '&amp;class=' . uri_escape($class);
            $comparison_title = $engine_selection{$class_tag}->{name};
        }
    }
}


$selection_uri_from_class_only = $selection_uri;


if (param('fam') or param('family'))
{
    my @family_list = ();
    if (param('fam')) { push @family_list, param('fam'); }
    if (param('family')) { push @family_list, param('family'); }
    foreach my $fam (@family_list)
    {
        foreach my $eng (@engines)
        {
            if ($fam eq $eng->{family} and $eng->{elo} >= $from_elo and $eng->{elo} <= $to_elo) { $eng->{selected} = 1; }
        }
        $selection_uri .= '&amp;family=' . uri_escape($fam);
    }
}


if (param('e') or param('eng') or param('engine') or param('engines'))
{
    my @engine_list = ();
    if (param('e')) { push @engine_list, param('e'); }
    if (param('eng')) { push @engine_list, param('eng'); }
    if (param('engine')) { push @engine_list, param('engine'); }
    if (param('engines')) { push @engine_list, param('engines'); }
    foreach my $name (@engine_list)
    {
        if (defined($engine_index_by_signature{$name}))
        {
            my $i = $engine_index_by_signature{$name};
            my $eng = $engines[$i];
            if ($eng->{elo} >= $from_elo and $eng->{elo} <= $to_elo) { $eng->{selected} = 1; }
        }
        $selection_uri .= '&amp;e=' . uri_escape($name);
    }
}


#
# Fixing title of this engine selection.
#
if ($comparison_title ne '' and $selection_uri eq $selection_uri_from_class_only)
{
    $comparison_is_for_class_only = 1;

    if ($only_public_releases)
    {
        if (lc($comparison_title) eq 'all engines')
            { $comparison_title = 'Public engines'; }
        else
            { $comparison_title .= ', public releases'; }
    }

    if ($only_stable_versions)
    {
        if (lc($comparison_title) eq 'all engines')
            { $comparison_title = 'Stable engines'; }
        elsif (lc($comparison_title) eq 'public engines')
            { $comparison_title = 'Public stable engines'; }
        else
            { $comparison_title .= ', stable versions'; }
    }

    if ($only_default_settings)
    {
        if (lc($comparison_title) eq 'all engines')
            { $comparison_title = 'Default settings'; }
        else
            { $comparison_title .= ', default settings'; }
    }

    if ($only_new_versions)
    {
        if (lc($comparison_title) eq 'all engines')
            { $comparison_title = 'New versions'; }
        else
            { $comparison_title .= ', new versions'; }
    }

    if ($only_best_in_class)
    {
        $comparison_title .= ', ';
        if ($num_best_in_class > 1) { $comparison_title .= $num_best_in_class . ' '; }
        $comparison_title .= 'best versions only';
    }
}
else
{
    $comparison_is_for_class_only = 0;
    $comparison_title = 'Custom engine selection';
}
substr($comparison_title,0,1) = uc(substr($comparison_title,0,1));


if ($from_elo != 0 and $to_elo != 10000)
{
    $comparison_title .= "<br>(only those rated from $from_elo to $to_elo Elo points)";
}
else
{
    if ($from_elo != 0)
    {
        $comparison_title .= "<br>(only those rated at least $from_elo Elo points)";
    }
    if ($to_elo != 10000)
    {
        $comparison_title .= "<br>(only those rated at most $to_elo Elo points)";
    }
}


my $crosstable_engine_selection_title = $comparison_title;

if ( $cross_tables_for_best_versions_only and
     (not $comparison_is_for_class_only or not $only_best_in_class) )
{
    $crosstable_engine_selection_title .= ' (best versions only)';
}


if ($ct_from_elo != 0 and $ct_to_elo != 10000)
{
    $crosstable_engine_selection_title .= "<br>(only those rated from $ct_from_elo to $ct_to_elo Elo points)";
}
else
{
    if ($ct_from_elo != 0)
    {
        $crosstable_engine_selection_title .= "<br>(only those rated at least $ct_from_elo Elo points)";
    }
    if ($ct_to_elo != 10000)
    {
        $crosstable_engine_selection_title .= "<br>(only those rated at most $ct_to_elo Elo points)";
    }
}




#
# Finding out number of selected engines, and number of best versions among selected engines.
#
my $num_engines_selected = 0;
my $num_best_engines_selected = 0;
my %all_selected_engines = ();
my $list_of_engines_uri = '';
my $list_of_best_versions_uri = '';



#
# Recalibration
#
my $num_engines_used_for_recalibration = 0;
if ($recalibrate ne 'no')
{
    if ($debug_on) { print "Recalibrating .."; }

    my $diff_sum = 0;
    my $use_selected_engines = ($recalibrate =~ /_using_selection/) ? 1 : 0;
    foreach my $eng (@engines)
    {
        if ( !defined($eng->{ref_elo}) or $eng->{ref_elo} == -32000) { next; }
        if ( $use_selected_engines and !$eng->{selected}) { next; }
        $diff_sum += $eng->{elo} - $eng->{ref_elo};
        $num_engines_used_for_recalibration++;
    }

    if ( $num_engines_used_for_recalibration > 0 )
    {
        my $diff = int( $diff_sum / $num_engines_used_for_recalibration );

        if ($recalibrate =~ 'ref_to_current')
        {
            foreach my $eng (@engines)
            {
                if (defined($eng->{ref_elo}) and $eng->{ref_elo} != -32000)
                {
                    $eng->{ref_elo} += $diff;
                    $eng->{ref_elo_html} = $eng->{ref_elo};
                    $eng->{ref_elo_html} =~ s/\-/&#8722;/g;
                }
            }
        }
        elsif ($recalibrate =~ 'current_to_ref')
        {
            foreach my $eng (@engines)
            {
                $eng->{elo} -= $diff;
                $eng->{elo_html} = $eng->{elo};
                $eng->{elo_html} =~ s/\-/&#8722;/g;
            }
        }
    }

    if ($debug_on) { print " OK<br>\n"; }
}





{
    mark_n_best_versions('selected','elo','best_selected',1);

    foreach my $eng (@engines)
    {
        if ($eng->{selected})
        {
            $num_engines_selected++;
            $all_selected_engines{$eng->{signature}}->{selected} = 1;
            $list_of_engines_uri .= '&amp;eng=' . uri_escape($eng->{signature});
        }

        if ($eng->{best_selected})
        {
            $num_best_engines_selected++;
            $list_of_best_versions_uri .= '&amp;e=' . uri_escape($eng->{signature});
        }
    }
}



#
# Constructing special index array for engines that we have in crosstables.
#
my @engine_indexes_in_crosstables = ();

{
    my $sel_tag = $cross_tables_for_best_versions_only ? 'best_selected' : 'selected';
    foreach my $a (@engines)
    {
        if ($a->{$sel_tag} and $a->{games_played} > 0 and $a->{elo} >= $ct_from_elo and $a->{elo} <= $ct_to_elo)
        {
            push @engine_indexes_in_crosstables, $a->{index};
        }
    }
}


#
# Sorting engines.
#
if ($sort_tables eq 'by rating')
{
    @engine_indexes_in_crosstables =
        sort { sort_engines_by_rating($engines[$a],$engines[$b]) } @engine_indexes_in_crosstables;
}
else
{
    @engine_indexes_in_crosstables =
        sort { sort_engines_by_name($engines[$a],$engines[$b]) } @engine_indexes_in_crosstables;
}


#
# Checking if we have too many engines, removing extra ones.
#
#if ( scalar(@engine_indexes_in_crosstables) > $max_table_size )
#{
#    $#engine_indexes_in_crosstables = $max_table_size+1;
#    $crosstable_engine_selection_title .= ', top $max_table_size';
#}



#
# Finishing with the crosstable index contsruction.
#
my $num_engines_in_crosstables = scalar(@engine_indexes_in_crosstables);

for (my $i=0; $i<=$#engine_indexes_in_crosstables; $i++)
{
    $engines[$engine_indexes_in_crosstables[$i]]->{index_in_crosstable} = $i;
}


#
# Initializing crosstable data.
#
my @crosstable_data = ();

for (my $i=0; $i<=$#engine_indexes_in_crosstables; $i++)
{
    for (my $j=0; $j<=$#engine_indexes_in_crosstables; $j++)
    {
        if ($i==$j) { next; }
        $crosstable_data[$i][$j]->{num} = 0;
    }
}


my @side_by_side_data = ();

my $num_games_in_crosstable = 0;


#
# Reading basic pair data.
#
{
    if ($debug_on) { print "Reading basic pair data .."; }

    my @index = ();
    open (my $INDEX,"basic_pair_data_index") or die;
    while (<$INDEX>)
    {
        if (/^(\d+)::(\S+)\s*$/) { $index[$1] = $2; }
    }
    close $INDEX;

    for (my $ii=0; $ii<=$#engine_indexes_in_crosstables; $ii++)
    {
        my $i = $engine_indexes_in_crosstables[$ii];

        my %data = ();
        my $all_data = '';

        if (!defined $index[$i]) { die; }
        if ($index[$i] !~ /^(\d+)::(\d+)$/) { die; }
        my ($pos,$size) = ($1,$2);

        open (my $BASIC,"basic_pair_data") or die;
        binmode $BASIC;
        seek $BASIC, $pos, 0;
        read $BASIC, $all_data, $size;
        close $BASIC;

        while ($all_data =~ /^\s*(\d+)-(\d+)\s+(.+?)\s*$/gm)
        {
            my ($n1,$n2,$line) = ($1,$2,$3);
            if ($n1 ne $i) { die; }
            $data{$n2} = $line;

            if ($print_side_by_side)
            {
                if ($line =~ /^\+(\d+)-(\d+)=(\d+)\s+(\S+)/)
                {
                    my ($wins,$losses,$draws,$perf) = ($1,$2,$3,$4);
                    $side_by_side_data[$ii]->{$n2}->{num} = $wins+$draws+$losses;
                    $side_by_side_data[$ii]->{$n2}->{wins} = $wins;
                    $side_by_side_data[$ii]->{$n2}->{losses} = $losses;
                    $side_by_side_data[$ii]->{$n2}->{draws} = $draws;
                    $side_by_side_data[$ii]->{$n2}->{score} = 2*$wins + $draws;

                    $side_by_side_data[$ii]->{$n2}->{perf} = $perf;
                    $perf =~ s/\-/&#8722;/g;
                    $side_by_side_data[$ii]->{$n2}->{perf_html} = $perf;
                }
            }
        }

        for (my $jj=0; $jj<=$#engine_indexes_in_crosstables; $jj++)
        {
            my $j = $engine_indexes_in_crosstables[$jj];
            if ($j == $i) { next; }
            if (!exists $data{$j}) { next; }

            if ($data{$j} =~ /^\+(\d+)-(\d+)=(\d+)\s+(\S+)/)
            {
                my ($wins,$losses,$draws,$perf) = ($1,$2,$3,$4);
                my $num = $wins+$draws+$losses;
                $crosstable_data[$ii][$jj]->{num} = $num;
                $crosstable_data[$ii][$jj]->{wins} = $wins;
                $crosstable_data[$ii][$jj]->{losses} = $losses;
                $crosstable_data[$ii][$jj]->{draws} = $draws;
                $crosstable_data[$ii][$jj]->{score} = 2*$wins + $draws;

                $crosstable_data[$ii][$jj]->{perf} = $perf;
                $perf =~ s/\-/&#8722;/g;
                $crosstable_data[$ii][$jj]->{perf_html} = $perf;

                if ($jj < $ii) { $num_games_in_crosstable += $num; }
            }
        }
    }

    if ($debug_on) { print " OK<br>\n"; }
}



#
# Constructing overlap data
#
if ($print{'overlap table'} or $print{'comopp gamenum table'})
{
    for (my $ii=0; $ii<=$#engine_indexes_in_crosstables-1; $ii++)
    {
        my $i = $engine_indexes_in_crosstables[$ii];

        for (my $jj=$ii+1; $jj<=$#engine_indexes_in_crosstables; $jj++)
        {
            my $j = $engine_indexes_in_crosstables[$jj];

            my $num = 2 * $crosstable_data[$ii][$jj]->{num};
            #my $realnum = $num;
            foreach my $k (keys %{$side_by_side_data[$ii]})
            {
                if (exists $side_by_side_data[$jj]->{$k})
                {
                    my $a = $side_by_side_data[$ii]->{$k}->{num};
                    my $b = $side_by_side_data[$jj]->{$k}->{num};
                    #$realnum += ($a + $b);

                    my ($min,$max);
                    if ($a <= $b) { ($min,$max) = ($a,$b); } else { ($min,$max) = ($b,$a); }
                    if ($max > $min * 2) { $max = $min * 2; }
                    $num += ($min + $max);
                }
            }

            $crosstable_data[$ii][$jj]->{'comgamenum'} = $num;
            $crosstable_data[$jj][$ii]->{'comgamenum'} = $num;

            my $total = $engines[$i]->{games_played} + $engines[$j]->{games_played};
            $num = $num / $total;

            $crosstable_data[$ii][$jj]->{'overlap'} = $num;
            $crosstable_data[$jj][$ii]->{'overlap'} = $num;
        }
    }
}



#
# Reading tester stats if we need it.
#
if ($need_testers_data)
{
    if ($debug_on) { print "Reading testers data .."; }

    read_tester_list();

    my @pair_tester_index = ();
    open (my $PAIR_TESTER_INDEX,"pair_tester_index") or die;
    while (<$PAIR_TESTER_INDEX>)
    {
        if (/^(\d+)::(\S+)\s*$/) { $pair_tester_index[$1] = $2; }
    }
    close $PAIR_TESTER_INDEX;

    for (my $ii=0; $ii<=$#engine_indexes_in_crosstables; $ii++)
    {
        my $i = $engine_indexes_in_crosstables[$ii];

        my %data = ();
        my $all_data = '';

        if (!defined $pair_tester_index[$i]) { die; }
        if ($pair_tester_index[$i] !~ /^(\d+)::(\d+)$/) { die; }
        my ($pos,$size) = ($1,$2);

        open (my $PAIR_TESTER,"pair_tester") or die;
        binmode $PAIR_TESTER;
        seek $PAIR_TESTER, $pos, 0;
        read $PAIR_TESTER, $all_data, $size;
        close $PAIR_TESTER;

        while ($all_data =~ /^\s*(\d+)-(\d+)\s+(.+?)\s*$/gm)
        {
            my ($n1,$n2,$line) = ($1,$2,$3);
            if ($n1 ne $i) { die; }
            $data{$n2} = $line;

            if ($print_side_by_side)
            {
                while ($line =~ /(\d+):(\d+)/g)
                {
                    $side_by_side_data[$ii]->{$n2}->{testers}->{$1}->{n} = $2;
                }
            }
        }

        for (my $jj=0; $jj<=$#engine_indexes_in_crosstables; $jj++)
        {
            my $j = $engine_indexes_in_crosstables[$jj];
            if ($j == $i) { next; }
            if (!exists $data{$j}) { next; }

            while ($data{$j} =~ /(\d+):(\d+)/g)
            {
                $crosstable_data[$ii][$jj]->{testers}->{$1}->{n} = $2;
            }
        }
    }

    if ($debug_on) { print " OK<br>\n"; }
}



#
# Reading correlation data if we need it.
#
if ($need_correlation_data and $options{compute_correlation})
{
    if ($debug_on) { print "Reading correlation data .."; }

    my @index = ();
    open (my $INDEX,"correlation_data_index") or die;
    while (<$INDEX>)
    {
        if (/^(\d+)::(\S+)\s*$/) { $index[$1] = $2; }
    }
    close $INDEX;

    for (my $ii=0; $ii<=$#engine_indexes_in_crosstables; $ii++)
    {
        my $i = $engine_indexes_in_crosstables[$ii];

        my %data = ();
        my $all_data = '';

        if (!defined $index[$i]) { die; }
        if ($index[$i] !~ /^(\d+)::(\d+)$/) { die; }
        my ($pos,$size) = ($1,$2);

        open (my $CORR,"correlation_data") or die;
        binmode $CORR;
        seek $CORR, $pos, 0;
        read $CORR, $all_data, $size;
        close $CORR;

        while ($all_data =~ /^\s*(\d+)-(\d+)\s+(.+?)\s*$/gm)
        {
            my ($n1,$n2,$line) = ($1,$2,$3);
            if ($n1 ne $i) { die; }
            $data{$n2} = $line;

            if ($print_side_by_side)
            {
                if ($line =~ /^(\d+)\+(\d+):(\d+)/)
                {
                    my ($exp_pl,$unexp_pl,$eval_diff_sum) = ($1,$2,$3);
                    my $corr_pl = $exp_pl + $unexp_pl;
                    $side_by_side_data[$ii]->{$n2}->{exp_pl} = $exp_pl;
                    $side_by_side_data[$ii]->{$n2}->{unexp_pl} = $unexp_pl;
                    $side_by_side_data[$ii]->{$n2}->{corr_pl} = $corr_pl;
                    $side_by_side_data[$ii]->{$n2}->{eval_diff_sum} = $eval_diff_sum;
                    $side_by_side_data[$ii]->{$n2}->{ponder_hit} = ($corr_pl == 0) ? 0 : ($exp_pl * 100 / $corr_pl);
                    $side_by_side_data[$ii]->{$n2}->{eval_diff} = ($exp_pl == 0) ? 0 : ($eval_diff_sum / ($exp_pl * 100));
                }
            }
        }

        for (my $jj=0; $jj<=$#engine_indexes_in_crosstables; $jj++)
        {
            my $j = $engine_indexes_in_crosstables[$jj];
            if ($j == $i) { next; }
            if (!exists $data{$j}) { next; }

            if ($data{$j} =~ /^(\d+)\+(\d+):(\d+)/)
            {
                my ($exp_pl,$unexp_pl,$eval_diff_sum) = ($1,$2,$3);
                my $corr_pl = $exp_pl + $unexp_pl;
                $crosstable_data[$ii][$jj]->{exp_pl} = $exp_pl;
                $crosstable_data[$ii][$jj]->{unexp_pl} = $unexp_pl;
                $crosstable_data[$ii][$jj]->{corr_pl} = $corr_pl;
                $crosstable_data[$ii][$jj]->{eval_diff_sum} = $eval_diff_sum;
                $crosstable_data[$ii][$jj]->{ponder_hit} = ($corr_pl == 0) ? 0 : ($exp_pl * 100 / $corr_pl);
                $crosstable_data[$ii][$jj]->{eval_diff} = ($exp_pl == 0) ? 0 : ($eval_diff_sum / ($exp_pl * 100));
            }
        }
    }

    if ($debug_on) { print " OK<br>\n"; }
}


#if ($print_side_by_side)
#{
#    for (my $i=0; $i<=$#engine_indexes_in_crosstables; $i++)
#    {
#        $engines[$engine_indexes_in_crosstables[$i]]->{
#    }
#}


#
# Reading LOS data if we need it.
#
my @los_data = ();

if ($need_los)
{
    if ($debug_on) { print "Reading LOS data .."; }

    open (my $LOS,'los_data') or die;
    binmode $LOS;
    for (my $i=0; $i<=$#engines; $i++)
    {
        my $s = '';
        read ( $LOS, $s, 4*($#engines+1) );
        @{$los_data[$i]} = unpack("I*",$s);
    }
    close $LOS;

    for (my $i=0; $i<=$#engine_indexes_in_crosstables; $i++)
    {
        for (my $j=0; $j<=$#engine_indexes_in_crosstables; $j++)
        {
            if ($j==$i) { next; }
            $crosstable_data[$i][$j]->{los} = $los_data[$engine_indexes_in_crosstables[$i]][$engine_indexes_in_crosstables[$j]];
        }
    }

    if ($print{'los with common opponents'} or $print{'los with all opponents'})
    {
        my %all_opp = ();

        for (my $i=0; $i<=$#engine_indexes_in_crosstables; $i++)
        {
            foreach my $j (keys %{$side_by_side_data[$i]})
            {
                $all_opp{$j} = 1;
            }
        }

        for (my $i=0; $i<=$#engine_indexes_in_crosstables; $i++)
        {
            foreach my $j (keys %all_opp)
            {
                if (!exists($side_by_side_data[$i]->{$j}))
                {
                    $side_by_side_data[$i]->{$j}->{num} = 0;
                }
                $side_by_side_data[$i]->{$j}->{los} = $los_data[$engine_indexes_in_crosstables[$i]][$j];
            }
        }
    }

    if ($debug_on) { print " OK<br>\n"; }
}



#
# Creating index of opponents
#
my @indexes_of_opponents = ();

if ($print_side_by_side)
{
    foreach my $eng (@engines)
    {
        $eng->{opponent_of_num} = 0;
    }

    for (my $ii=0; $ii <= $#engine_indexes_in_crosstables and $ii < $max_table_size; $ii++)
    {
        my $i = $engine_indexes_in_crosstables[$ii];
        foreach my $o (keys %{$side_by_side_data[$ii]})
        {
            if ($side_by_side_data[$ii]->{$o}->{num})
            {
                $engines[$o]->{opponent_of_num}++;
            }
        }
    }

    foreach my $eng (@engines)
    {
        if ($eng->{opponent_of_num})
        {
            push @indexes_of_opponents, $eng->{index};
        }
    }

    @indexes_of_opponents = sort { sort_engines_by_rating($engines[$a],$engines[$b]) } @indexes_of_opponents;

    for (my $i=0; $i<=$#indexes_of_opponents; $i++)
    {
        $engines[$indexes_of_opponents[$i]]->{index_in_opponents} = $i;
    }
}



#
# Creating list of pairs if we need it.
#
my @pair_list_data = ();

if ($select_pairs and $options{compute_correlation})
{
    if ($debug_on) { print "Creating list of pairs .."; }

    if ($print_side_by_side)
    {
        for (my $i=0; $i<=$#engine_indexes_in_crosstables; $i++)
        {
            for (my $j=0; $j<=$#indexes_of_opponents; $j++)
            {
                my $jn = $indexes_of_opponents[$j];
                if ( defined $side_by_side_data[$i]->{$jn} and
                     defined $side_by_side_data[$i]->{$jn}->{num} and $side_by_side_data[$i]->{$jn}->{num} > 0 )
                {
                    push @pair_list_data, $side_by_side_data[$i]->{$jn};
                    $pair_list_data[$#pair_list_data]->{a_index} = $engine_indexes_in_crosstables[$i];
                    $pair_list_data[$#pair_list_data]->{b_index} = $jn;
                }
            }
        }
    }
    else
    {
        for (my $i=0; $i<=$#engine_indexes_in_crosstables-1; $i++)
        {
            for (my $j=$i+1; $j<=$#engine_indexes_in_crosstables; $j++)
            {
                if ( defined $crosstable_data[$i][$j] and
                     defined $crosstable_data[$i][$j]->{num} and $crosstable_data[$i][$j]->{num} > 0 )
                {
                    push @pair_list_data, $crosstable_data[$i][$j];
                    $pair_list_data[$#pair_list_data]->{a_index} = $engine_indexes_in_crosstables[$i];
                    $pair_list_data[$#pair_list_data]->{b_index} = $engine_indexes_in_crosstables[$j];
                }
            }
        }
    }

    for (my $i=0; $i<$#pair_list_data; $i++)
    {
        my $p = $pair_list_data[$i];
        $pair_list_data[$i]->{same_family} = 
            ($engines[$p->{a_index}]->{family} eq $engines[$p->{b_index}]->{family}) ? 1 : 0;
    }

    if ($debug_on) { print " OK<br>\n"; }
}



sub sort_pairs_by_ponder_hit
{
    my ($a,$b) = @_;
    return $a->{ponder_hit} <=> $b->{ponder_hit};
}

sub sort_pairs_by_eval_diff
{
    my ($a,$b) = @_;
    return $a->{eval_diff} <=> $b->{eval_diff};
}

sub get_pair_ponder_hit
{
    my $a = shift;
    return sprintf("%.1f",$a->{ponder_hit});
}

sub get_pair_eval_diff
{
    my $a = shift;
    return sprintf("%.2f",$a->{eval_diff});
}


sub move_counter_for_ponder_hit
{
    my $a = shift;
    return $a->{corr_pl};
}


sub move_counter_for_eval_diff
{
    my $a = shift;
    return $a->{exp_pl};
}


sub make_list_of_pairs($$$$$$$$$$)
{
    my ( $sorter, $getter, $move_counter, $use_same_families, $use_diff_families,
         $min_plies, $gradient_id, $measure_name, $asc, $relation ) = @_;

    if (!$use_same_families and !$use_diff_families) { return ''; }

    my $s = '';
    $s .= "<table border=0 cellspacing=0 cellpadding=0 style=\"margin-top: 0pt;\"><tr><td style=\"padding: 5pt;\">" .
          "<h3 style=\"margin-bottom: 0pt;\">$measure_name: Most $relation pairs";
    if (!$use_same_families) { $s .= ' (different families only)'; }
    if (!$use_diff_families) { $s .= ' (same families only)'; }
    $s .= "</h3></td></tr><tr><td>\n";
    $s .= "<table border=1 cellspacing=0 cellpadding=0 class=rating_table>\n";
    $s .= "<tr><td class=header>#</td><td class=header>Pair</td><td class=header>$measure_name</td>";
    $s .= "<td class=header>Moves<br>counted</td></tr>";
    my $num = 1;
    foreach my $p (sort { $asc ? &$sorter($a,$b) : - &$sorter($a,$b) } @pair_list_data)
    {
        if (!$use_same_families and $p->{same_family}) { next; }
        if (!$use_diff_families and !$p->{same_family}) { next; }

        my $plies = &$move_counter($p);
        if ($plies < $min_plies) { next; }
        my $bgcol = interpolate_color(&$getter($p),$gradient{$gradient_id});
        $s .= '<tr>';
        $s .= "<td class=number style=\"background-color: $bgcol;\">$num</td>";
        $s .= "<td class=name style=\"background-color: $bgcol;\">" .
              $engines[$p->{a_index}]->{signature} . ' &ndash; ' .
              $engines[$p->{b_index}]->{signature} . '</td>';
        $s .= "<td class=number style=\"background-color: $bgcol;\">" . &$getter($p) . '</td>';
        $s .= "<td class=number style=\"background-color: $bgcol;\">" . $plies . '</td>';
        $s .= "</tr>\n";
        $num++;
        if ($num > 30) { last; }
    }
    $s .= "</table></td></tr></table>\n";
    return $s;
}




$big_header =~ s/\{COMPARISON_TITLE\}/$comparison_title/g;
print $big_header;
print "<center>\n";

if ($num_engines_selected > 0)
{
    print_all();
}
else
{
    print "No engines selected!\n";
}

print "</center>\n";
print $big_footer;


sub print_profile
{
    my ($diff_step,$rating_step,$print_numbers) = @_;
    if (!defined($diff_step)) { $diff_step = $profile_step; }
    if (!defined($rating_step)) { $rating_step = $diff_step; }
    if (!defined($print_numbers)) { $print_numbers = $profile_numbers; }

    my @matrix_plus;
    my @matrix_minus;
    my $max_diff = 0;
    my $max_rating = 0;
    my $min_rating = 10000;

    for (my $a=0; $a<=$#engine_indexes_in_crosstables; $a++)
    {
        my $ai = $engine_indexes_in_crosstables[$a];
        my $a_eng = $engines[$ai];
        my $a_elo = $a_eng->{elo};
        for (my $b=0; $b<=$#engine_indexes_in_crosstables; $b++)
        {
            my $bi = $engine_indexes_in_crosstables[$b];
            if ($bi <= $ai) { next; }
            my $b_eng = $engines[$bi];
            my $b_elo = $b_eng->{elo};

            my $num = $crosstable_data[$a][$b]->{num};
            if ($num <= 0) { next; }

            my $diff = int ( abs($a_elo - $b_elo) / $diff_step );
            if ($diff > $max_diff) { $max_diff = $diff; }

            my ($r_low, $r_high);
            if ($a_elo <= $b_elo)
            {
                $r_low = int ( $a_elo / $rating_step );
                $r_high = int ( $b_elo / $rating_step );
            }
            else
            {
                $r_low = int ( $b_elo / $rating_step );
                $r_high = int ( $a_elo / $rating_step );
            }

            for ($r_low, $r_high)
            {
                if ($_ > $max_rating) { $max_rating = $_; }
                if ($_ < $min_rating) { $min_rating = $_; }
            }

            $matrix_plus[$diff][$r_low] += $num;
            $matrix_minus[$diff][$r_high] += $num;
        }
    }

    my @total_for_rating_difference = ();
    my $total_game_num = 0;
    my $largest = 0;

    for (my $diff = 0; $diff <= $max_diff; $diff++)
    {
        $total_for_rating_difference[$diff] = 0;
        for (my $r = $min_rating; $r <= $max_rating; $r++)
        {
            if (defined($matrix_plus[$diff][$r]))
            {
                $total_for_rating_difference[$diff] += $matrix_plus[$diff][$r];
                if ($matrix_plus[$diff][$r] > $largest) { $largest = $matrix_plus[$diff][$r]; }
            }
        }
        $total_game_num += $total_for_rating_difference[$diff];
    }

    my @cumulative_total_for_rating_difference = ();
    $cumulative_total_for_rating_difference[0] = $total_for_rating_difference[0];
    for (my $diff = 1; $diff <= $max_diff; $diff++)
    {
        $cumulative_total_for_rating_difference[$diff] = $cumulative_total_for_rating_difference[$diff-1] + $total_for_rating_difference[$diff];
    }


    if ($min_rating % 4) { $min_rating -= ($min_rating % 4); }
    if (($max_rating+1) % 4) { $max_rating += 4 - (($max_rating+1) % 4); }
    my $col_num = $max_rating - $min_rating + 1;

    my $grad = "0:FFFFFF " . int($largest/2) . ":2CDC4D $largest:2FA6DE";


    print "<center><table border=0 cellspacing=0 cellpadding=0 style=\"margin-top: 0pt;\">\n";
    print "<tr><td><h3 style=\"margin-bottom: 0pt;\">Testing Profile</h3></td></tr>\n";
    #print "<tr><td>Diff: 0 - $max_diff, Rating: $min_rating - $max_rating, Largest: $largest</td></tr>\n";
    print "<tr><td>See how games played by selected engines are distributed<br>according to the rating (horizontal axis) and rating difference (vertical axis)</td></tr>\n";
    print "<tr><td>\n";
    print "<table cellspacing=0 cellpadding=0 class=\"profile\">\n";
    print "<tr class=\"header\">";
    print "<td rowspan=2>Rating<br>difference</td>";
    print "<td colspan=$col_num>Rating</td>";
    print "<td rowspan=2>Total</td>";
    print "<td rowspan=2>Cumulative<br>Total</td>";
    print "<td rowspan=2>Rating<br>difference</td>";
    print "</tr>\n";

    print "<tr>";
    for (my $r = $min_rating; $r <= $max_rating; $r += 4)
    {
        print "<td colspan=4 class=\"header_number\">", $r*$rating_step, '&#8722;', ($r+4)*$rating_step-1, "</td>";
    }
    print "</tr>\n";

    for (my $diff = $max_diff; $diff >= 0; $diff--)
    {
        print "<tr>";
        print "<td class=\"header_number\">&#8722;", ($diff+1)*$diff_step-1, '...', ($diff?'&#8722;':''), $diff*$diff_step, "</td>";
        for (my $r = $min_rating; $r <= $max_rating; $r++)
        {
            if (!defined($matrix_minus[$diff][$r])) { $matrix_minus[$diff][$r] = 0; }
            if ($matrix_minus[$diff][$r] > 0)
                { print "<td style=\"background-color: ", interpolate_color($matrix_minus[$diff][$r],$grad), "\">"; }
            else { print "<td>"; }
            print( ($profile_numbers and ($matrix_minus[$diff][$r] > 0)) ? $matrix_minus[$diff][$r] : "&nbsp;");
            print "</td>";
        }
        print "<td>", $total_for_rating_difference[$diff], "&nbsp;(", sprintf("%.1f",$total_for_rating_difference[$diff]*100/$total_game_num), "%)</td>";
        print "<td>", $cumulative_total_for_rating_difference[$diff], "&nbsp;(", sprintf("%.1f",$cumulative_total_for_rating_difference[$diff]*100/$total_game_num), "%)</td>";
        print "<td class=\"header_number\">&#8722;", ($diff+1)*$diff_step-1, '...', ($diff?'&#8722;':''), $diff*$diff_step, "</td>";
        print "</tr>\n";
    }

    #print "<tr class=\"header\">";
    #print "<td>&nbsp;</td>";
    #for (my $r = $min_rating; $r <= $max_rating; $r += 4)
    #{
    #    print "<td colspan=4 class=\"header_number\">", $r*$rating_step, '&#8722;', ($r+4)*$rating_step-1, "</td>";
    #}
    #print "<td colspan=3>&nbsp;</td>";
    #print "</tr>\n";

    for (my $diff = 0; $diff <= $max_diff; $diff++)
    {
        print "<tr>";
        print "<td class=\"header_number\">", $diff*$diff_step, '...', ($diff+1)*$diff_step-1, "</td>";
        for (my $r = $min_rating; $r <= $max_rating; $r++)
        {
            if (!defined($matrix_plus[$diff][$r])) { $matrix_plus[$diff][$r] = 0; }
            if ($matrix_plus[$diff][$r] > 0)
                { print "<td style=\"background-color: ", interpolate_color($matrix_plus[$diff][$r],$grad), "\">"; }
            else { print "<td>"; }
            print( ($profile_numbers and ($matrix_plus[$diff][$r] > 0)) ? $matrix_plus[$diff][$r] : "&nbsp;");
            print "</td>";
        }
        print "<td>", $total_for_rating_difference[$diff], "&nbsp;(", sprintf("%.1f",$total_for_rating_difference[$diff]*100/$total_game_num), "%)</td>";
        print "<td>", $cumulative_total_for_rating_difference[$diff], "&nbsp;(", sprintf("%.1f",$cumulative_total_for_rating_difference[$diff]*100/$total_game_num), "%)</td>";
        print "<td class=\"header_number\">", $diff*$diff_step, '...', ($diff+1)*$diff_step-1, "</td>";
        print "</tr>\n";
    }

    print "<tr class=\"header\">";
    print "<td rowspan=2>Rating<br>difference</td>";
    for (my $r = $min_rating; $r <= $max_rating; $r += 4)
    {
        print "<td colspan=4 class=\"header_number\">", $r*$rating_step, '&#8722;', ($r+4)*$rating_step-1, "</td>";
    }
    print "<td rowspan=2>Total</td>";
    print "<td rowspan=2>Cumulative<br>Total</td>";
    print "<td rowspan=2>Rating<br>difference</td>";
    print "</tr>\n";
    print "<tr class=\"header\">";
    print "<td colspan=$col_num>Rating</td>";
    print "</tr>\n";


    print "</table>\n";
    print "</td></tr>\n";
    print "</table></center>\n";
}


sub print_all
{
    print "Comparing ", $num_engines_selected, " engines!<br>\n";

    if ($cross_tables_for_best_versions_only)
        { print "$num_best_engines_selected best versions of selected engines"; }
    else
        { print "$num_engines_selected selected engines"; }
    print " played $num_games_in_crosstable games with each other<br>\n";


    my $selection = $cross_tables_for_best_versions_only ? 'best_selected' : 'selected';
    #my $table_best_versions = ($only_best_in_class or $cross_tables_for_best_versions_only);

                                                   
    if ($print{'rating list (text)'})
    {
        print make_rating_table_text('selected',$comparison_title,'',$stat{number_of_games});
        return;
    }

    if ($print{'rating list'})
    {
        print make_rating_table_2('selected',$comparison_title, 
                                ($comparison_is_for_class_only) ? $only_public_releases : 0,
                                ($comparison_is_for_class_only) ? $only_stable_versions : 0,
                                ($comparison_is_for_class_only) ? $only_default_settings : 0,
                                ($comparison_is_for_class_only) ? $only_best_in_class : 0,
                                $num_best_in_class, '',
                                ($comparison_is_for_class_only) ? $selection_uri : $list_of_engines_uri,
                                $stat{number_of_games}, \@los_data,
                                $reference_list, $recalibrate, $num_engines_used_for_recalibration );
    }

    if ($print{'profile'})
    {
        print_profile();
    }





    if ($print{'results table'})
    {
        print make_pairwise_table_2('results',$crosstable_engine_selection_title,'',5,0,
                                    \@engine_indexes_in_crosstables,\@crosstable_data,$diagonal_width);
    }

    if ($print{'performance table'})
    {
        print make_pairwise_table_2('performance',$crosstable_engine_selection_title,'',3,0,
                                    \@engine_indexes_in_crosstables,\@crosstable_data,$diagonal_width);
    }

    if ($print{'score table'})
    {
        print make_pairwise_table_2('score',$crosstable_engine_selection_title,'',3,0,
                                    \@engine_indexes_in_crosstables,\@crosstable_data,$diagonal_width);
    }

    if ($print{'los table'})
    {
        print make_pairwise_table_2('los',$crosstable_engine_selection_title,'',3,0,
                                    \@engine_indexes_in_crosstables,\@crosstable_data,$diagonal_width);
    }

    if ($options{compute_correlation})
    {
        if ($print{'ponder hit table'})
        {
            print make_pairwise_table_2('ponder_hit',$crosstable_engine_selection_title,'',3,0,
                                        \@engine_indexes_in_crosstables,\@crosstable_data,$diagonal_width);
        }

        if ($print{'eval difference table'})
        {
            print make_pairwise_table_2('eval_diff',$crosstable_engine_selection_title,'',3,0,
                                        \@engine_indexes_in_crosstables,\@crosstable_data,$diagonal_width);
        }
    }

    if ($print{'draw percentage table'})
    {
        print make_pairwise_table_2('drawnum',$crosstable_engine_selection_title,'',3,0,
                                    \@engine_indexes_in_crosstables,\@crosstable_data,$diagonal_width);
    }

    if ($print{'white score table'})
    {
        #print make_pairwise_table('adv',$selection,$comparison_title,$sort_tables_tag,'',
        #                          $max_table_size,$cross_tables_for_best_versions_only,3,0);
    }

    if ($print{'number of games table'})
    {
        print make_pairwise_table_2('gamenum',$crosstable_engine_selection_title,'',3,0,
                                    \@engine_indexes_in_crosstables,\@crosstable_data,$diagonal_width);
    }

    if ($print{'comopp gamenum table'})
    {
        print make_pairwise_table_2('comgamenum',$crosstable_engine_selection_title,'',3,0,
                                    \@engine_indexes_in_crosstables,\@crosstable_data,$diagonal_width);
    }

    if ($print{'overlap table'})
    {
        print make_pairwise_table_2('overlap',$crosstable_engine_selection_title,'',3,0,
                                    \@engine_indexes_in_crosstables,\@crosstable_data,$diagonal_width);
    }



    if ($print{'testers table'})
    {
        print make_pairwise_table_2('testers',$crosstable_engine_selection_title,'',5,0,
                                    \@engine_indexes_in_crosstables,\@crosstable_data,$diagonal_width);
    }

    if ($print{'performance and testers table'})
    {
        print make_pairwise_table_2('perf_and_testers',$crosstable_engine_selection_title,'',5,0,
                                    \@engine_indexes_in_crosstables,\@crosstable_data,$diagonal_width);
    }






    if ($print{'score with common opponents'})
    {
        print make_side_by_side_table_2('score',$crosstable_engine_selection_title,'',2,0,
                                        \@engine_indexes_in_crosstables,\@indexes_of_opponents,\@side_by_side_data);
    }

    if ($print{'score with all opponents'})
    {
        print make_side_by_side_table_2('score',$crosstable_engine_selection_title,'',1,0,
                                        \@engine_indexes_in_crosstables,\@indexes_of_opponents,\@side_by_side_data);
    }

    if ($print{'performance with common opponents'})
    {
        print make_side_by_side_table_2('performance',$crosstable_engine_selection_title,'',2,0,
                                        \@engine_indexes_in_crosstables,\@indexes_of_opponents,\@side_by_side_data);
    }

    if ($print{'performance with all opponents'})
    {
        print make_side_by_side_table_2('performance',$crosstable_engine_selection_title,'',1,0,
                                        \@engine_indexes_in_crosstables,\@indexes_of_opponents,\@side_by_side_data);
    }

    if ($print{'los with common opponents'})
    {
        print make_side_by_side_table_2('los',$crosstable_engine_selection_title,'',2,0,
                                        \@engine_indexes_in_crosstables,\@indexes_of_opponents,\@side_by_side_data);
    }

    if ($print{'los with all opponents'})
    {
        print make_side_by_side_table_2('los',$crosstable_engine_selection_title,'',1,0,
                                        \@engine_indexes_in_crosstables,\@indexes_of_opponents,\@side_by_side_data);
    }

    if ($options{compute_correlation})
    {
        if ($print{'ponder hit with common opponents'})
        {
            print make_side_by_side_table_2('ponder_hit',$crosstable_engine_selection_title,'',2,0,
                                            \@engine_indexes_in_crosstables,\@indexes_of_opponents,\@side_by_side_data);
        }

        if ($print{'ponder hit with all opponents'})
        {
            print make_side_by_side_table_2('ponder_hit',$crosstable_engine_selection_title,'',1,0,
                                            \@engine_indexes_in_crosstables,\@indexes_of_opponents,\@side_by_side_data);
        }

        if ($print{'eval difference with common opponents'})
        {
            print make_side_by_side_table_2('eval_diff',$crosstable_engine_selection_title,'',2,0,
                                            \@engine_indexes_in_crosstables,\@indexes_of_opponents,\@side_by_side_data);
        }

        if ($print{'eval difference with all opponents'})
        {
            print make_side_by_side_table_2('eval_diff',$crosstable_engine_selection_title,'',1,0,
                                            \@engine_indexes_in_crosstables,\@indexes_of_opponents,\@side_by_side_data);
        }
    }

    if ($print{'testers with common opponents'})
    {
        print make_side_by_side_table_2('testers',$crosstable_engine_selection_title,'',2,0,
                                        \@engine_indexes_in_crosstables,\@indexes_of_opponents,\@side_by_side_data);
    }

    if ($print{'testers with all opponents'})
    {
        print make_side_by_side_table_2('testers',$crosstable_engine_selection_title,'',1,0,
                                        \@engine_indexes_in_crosstables,\@indexes_of_opponents,\@side_by_side_data);
    }

    if ($print{'performance and testers with common opponents'})
    {
        print make_side_by_side_table_2('perf_and_testers',$crosstable_engine_selection_title,'',2,0,
                                        \@engine_indexes_in_crosstables,\@indexes_of_opponents,\@side_by_side_data);
    }

    if ($print{'performance and testers with all opponents'})
    {
        print make_side_by_side_table_2('perf_and_testers',$crosstable_engine_selection_title,'',1,0,
                                        \@engine_indexes_in_crosstables,\@indexes_of_opponents,\@side_by_side_data);
    }






    if ($need_correlation_data)
    {
        if ($options{compute_correlation})
        {
            if ($print{'ponder hit - most similar pairs'})
            {
                print make_list_of_pairs (\&sort_pairs_by_ponder_hit, \&get_pair_ponder_hit, \&move_counter_for_ponder_hit, 1, 1,
                                          $options{min_correlation_moves}, 'ponder_hit', 'Ponder hit', 0, 'similar' );
            }

            if ($print{'ponder hit - most similar pairs (different families only)'})
            {
                print make_list_of_pairs (\&sort_pairs_by_ponder_hit, \&get_pair_ponder_hit, \&move_counter_for_ponder_hit, 0, 1,
                                          $options{min_correlation_moves}, 'ponder_hit', 'Ponder hit', 0, 'similar' );
            }

            if ($print{'ponder hit - most different pairs'})
            {
                print make_list_of_pairs (\&sort_pairs_by_ponder_hit, \&get_pair_ponder_hit, \&move_counter_for_ponder_hit, 1, 1,
                                          $options{min_correlation_moves}, 'ponder_hit', 'Ponder hit', 1, 'different' );
            }

            if ($print{'ponder hit - most different pairs (same families only)'})
            {
                print make_list_of_pairs (\&sort_pairs_by_ponder_hit, \&get_pair_ponder_hit, \&move_counter_for_ponder_hit, 1, 0,
                                          $options{min_correlation_moves}, 'ponder_hit', 'Ponder hit', 1, 'different' );
            }

            if ($print{'eval difference - most similar pairs'})
            {
                print make_list_of_pairs (\&sort_pairs_by_eval_diff, \&get_pair_eval_diff, \&move_counter_for_eval_diff, 1, 1,
                                          $options{min_correlation_moves}, 'eval_diff', 'Evaluation difference', 1, 'similar' );
            }

            if ($print{'eval difference - most similar pairs (different families only)'})
            {
                print make_list_of_pairs (\&sort_pairs_by_eval_diff, \&get_pair_eval_diff, \&move_counter_for_eval_diff, 0, 1,
                                          $options{min_correlation_moves}, 'eval_diff', 'Evaluation difference', 1, 'similar' );
            }

            if ($print{'eval difference - most different pairs'})
            {
                print make_list_of_pairs (\&sort_pairs_by_eval_diff, \&get_pair_eval_diff, \&move_counter_for_eval_diff, 1, 1,
                                          $options{min_correlation_moves}, 'eval_diff', 'Evaluation difference', 0, 'different' );
            }

            if ($print{'eval difference - most different pairs (same families only)'})
            {
                print make_list_of_pairs (\&sort_pairs_by_eval_diff, \&get_pair_eval_diff, \&move_counter_for_eval_diff, 1, 0,
                                          $options{min_correlation_moves}, 'eval_diff', 'Evaluation difference', 0, 'different' );
            }
        }
        #else
        #{
        #    print "<div class=\"text\"><br><h3>Correlation statistics</h3>" .
        #          "<p>This is an interim \"Live\" update, it does not contain correlation statistics. " .
        #          "For latest correlation statistics please check our" .
        #          " <a href=\"" . $options{homepage_stable} . "\">latest stable update</a>.</p></div>\n";
        #}
    }


    my $options = "table_size=$max_table_size&amp;match_length=$match_length";
    if ($cross_tables_for_best_versions_only) { $options .= "&amp;cross_tables_for_best_versions_only=1"; }
    if ($only_best_in_class) { $options .= "&amp;only_best_in_class=1"; }




    print "<br><div class=\"text\">\n";
    print "<form action=\"compare_engines.cgi\" method=\"get\">\n";



    print "<h3 class=rating_table>Alter engine selection</h3>\n";

    print "<center>\n";
    print "<select name=e size=" . $options{engine_selection_size} . " multiple>\n";
    foreach my $i ( sort { $a->{signature} cmp $b->{signature} } @engines)
    {
        if ($i->{games_played} <= 0) { next; }
        print $i->{selected} ? '<option selected="selected">' : '<option>', $i->{signature}, "</option>\n";
    }
    print "</select>\n";
    print "<br><br><input type=\"submit\" value=\"Compare them!\">\n";
    print "</center>\n";



    print "<h3 class=rating_table>Alter output selection</h3>\n";
    print "<center><table cellpadding=0 cellspacing=0 border=0>\n";
    print "<tr><td style=\"padding-right: 25pt; text-align: left;\">\n";

    print "<input type=\"checkbox\" name=\"print\" value=\"Rating list\"" .
              ($print{'rating list'} ? ' checked' : '') . "> Rating list<br>\n";
    print "<input type=\"checkbox\" name=\"print\" value=\"Profile\"" .
              ($print{'profile'} ? ' checked' : '') . "> Testing profile<br>\n";
    print "&nbsp;&nbsp;(Step: <input type=\"text\" name=\"profile_step\" size=\"3\" maxlength=\"3\" value=\"" .
          int($profile_step) . "\">, Numbers: <input type=\"checkbox\" name=\"profile_numbers\" value=\"1\"" .
          ($profile_numbers ? ' checked' : '') . ">)<br><br>\n";
    print "<input type=\"checkbox\" name=\"print\" value=\"Results table\"" .
              ($print{'results table'} ? ' checked' : '') . "> Results table<br>\n";
    print "<input type=\"checkbox\" name=\"print\" value=\"Performance table\"" .
              ($print{'performance table'} ? ' checked' : '') . "> Performance table<br>\n";
    print "<input type=\"checkbox\" name=\"print\" value=\"Score table\"" .
              ($print{'score table'} ? ' checked' : '') . "> Score table<br>\n";
    print "<input type=\"checkbox\" name=\"print\" value=\"LOS table\"" .
              ($print{'los table'} ? ' checked' : '') . "> LOS table<br>\n";

    if ($options{compute_correlation})
    {
        print "<input type=\"checkbox\" name=\"print\" value=\"Ponder hit table\"" .
                  ($print{'ponder hit table'} ? ' checked' : '') . "> Ponder hit table<br>\n";
        print "<input type=\"checkbox\" name=\"print\" value=\"Eval difference table\"" .
                  ($print{'eval difference table'} ? ' checked' : '') . "> Eval difference table<br>\n";
    }

    print "<input type=\"checkbox\" name=\"print\" value=\"Draw percentage table\"" .
              ($print{'draw percentage table'} ? ' checked' : '') . "> Proportion of draws<br>\n";
    #print "<input type=\"checkbox\" name=\"print\" value=\"White score table\"" .
    #          ($print{'white score table'} ? ' checked' : '') . "> White score<br>\n";
    print "<input type=\"checkbox\" name=\"print\" value=\"Number of games table\"" .
              ($print{'number of games table'} ? ' checked' : '') . "> Number of games<br>\n";
    print "<input type=\"checkbox\" name=\"print\" value=\"Comopp gamenum table\"" .
              ($print{'comopp gamenum table'} ? ' checked' : '') . "> Number of connecting games<br>\n";
    print "<input type=\"checkbox\" name=\"print\" value=\"Overlap table\"" .
              ($print{'overlap table'} ? ' checked' : '') . "> Percentage of connecting games<br>\n";


    print "</td><td style=\"padding-left: 25pt; text-align: left;\">\n";

    print "<input type=\"checkbox\" name=\"print\" value=\"Score with common opponents\"" .
              ($print{'score with common opponents'} ? ' checked' : '') . "> Score with common opponents<br>\n";
    print "<input type=\"checkbox\" name=\"print\" value=\"Score with all opponents\"" .
              ($print{'score with all opponents'} ? ' checked' : '') . "> Score with all opponents<br>\n";
    print "<input type=\"checkbox\" name=\"print\" value=\"Performance with common opponents\"" .
              ($print{'performance with common opponents'} ? ' checked' : '') . "> Performance with common opponents<br>\n";
    print "<input type=\"checkbox\" name=\"print\" value=\"Performance with all opponents\"" .
              ($print{'performance with all opponents'} ? ' checked' : '') . "> Performance with all opponents<br>\n";
    print "<input type=\"checkbox\" name=\"print\" value=\"LOS with common opponents\"" .
              ($print{'los with common opponents'} ? ' checked' : '') . "> LOS with common opponents<br>\n";
    print "<input type=\"checkbox\" name=\"print\" value=\"LOS with all opponents\"" .
           ($print{'los with all opponents'} ? ' checked' : '') . "> LOS with all opponents<br>\n";


    if ($options{compute_correlation})
    {
        print "<input type=\"checkbox\" name=\"print\" value=\"Ponder hit with common opponents\"" .
                  ($print{'ponder hit with common opponents'} ? ' checked' : '') . "> Ponder hit with common opponents<br>\n";
        print "<input type=\"checkbox\" name=\"print\" value=\"Ponder hit with all opponents\"" .
                  ($print{'ponder hit with all opponents'} ? ' checked' : '') . "> Ponder hit with all opponents<br>\n";
        print "<input type=\"checkbox\" name=\"print\" value=\"Eval difference with common opponents\"" .
                  ($print{'eval difference with common opponents'} ? ' checked' : '') . "> Eval difference with common opponents<br>\n";
        print "<input type=\"checkbox\" name=\"print\" value=\"Eval difference with all opponents\"" .
                  ($print{'eval difference with all opponents'} ? ' checked' : '') . "> Eval difference with all opponents<br>\n";

        print "<br>";

        print "<input type=\"checkbox\" name=\"print\" value=\"Ponder hit - most similar pairs\"" .
                  ($print{'ponder hit - most similar pairs'} ? ' checked' : '') . "> Ponder hit: most similar pairs<br>\n";
        print "<input type=\"checkbox\" name=\"print\" value=\"Ponder hit - most similar pairs (different families only)\"" .
                  ($print{'ponder hit - most similar pairs (different families only)'} ? ' checked' : '') . "> Ponder hit: most similar pairs (different families only)<br>\n";
        print "<input type=\"checkbox\" name=\"print\" value=\"Ponder hit - most different pairs\"" .
                  ($print{'ponder hit - most different pairs'} ? ' checked' : '') . "> Ponder hit: most different pairs<br>\n";
        print "<input type=\"checkbox\" name=\"print\" value=\"Ponder hit - most different pairs (same families only)\"" .
                  ($print{'ponder hit - most different pairs (same families only)'} ? ' checked' : '') . "> Ponder hit: most different pairs (same families only)<br>\n";
        print "<input type=\"checkbox\" name=\"print\" value=\"Eval difference - most similar pairs\"" .
                  ($print{'eval difference - most similar pairs'} ? ' checked' : '') . "> Eval diff: most similar pairs<br>\n";
        print "<input type=\"checkbox\" name=\"print\" value=\"Eval difference - most similar pairs (different families only)\"" .
                  ($print{'eval difference - most similar pairs (different families only)'} ? ' checked' : '') . "> Eval diff: most similar pairs (different families only)<br>\n";
        print "<input type=\"checkbox\" name=\"print\" value=\"Eval difference - most different pairs\"" .
                  ($print{'eval difference - most different pairs'} ? ' checked' : '') . "> Eval diff: most different pairs<br>\n";
        print "<input type=\"checkbox\" name=\"print\" value=\"Eval difference - most different pairs (same families only)\"" .
                  ($print{'eval difference - most different pairs (same families only)'} ? ' checked' : '') . "> Eval diff: most different pairs (same families only)<br>\n";
    }


    print "</td></tr>\n";
    print "</table></center>\n";

    print "<br><p>\n";
    print "Maximum size of cross-tables (from 2 to 100):\n";
    print "<input type=\"text\" name=\"table_size\" size=\"3\" maxlength=\"3\" value=\"$max_table_size\"><br>\n";

    print "Limit crosstables to engines in Elo range: <input type=\"text\" name=\"ct_from_elo\" size=\"6\" maxlength=\"6\" value=\"$ct_from_elo\">\n";
    print "to <input type=\"text\" name=\"ct_to_elo\" size=\"6\" maxlength=\"6\" value=\"$ct_to_elo\">\n";
    if ($options{comparison_option_match_length})
    {
        print "<br>\n";
        print "Match length (Minimum number of games to highlight with color):\n";
        print "<input type=\"text\" name=\"match_length\" size=\"3\" maxlength=\"3\" value=\"$match_length\">\n";
    }
    print "</p>\n";
    print "<p>\n";
    print "<input type=\"checkbox\" name=\"cross_tables_for_best_versions_only\" value=\"1\"" .
              ($cross_tables_for_best_versions_only ? ' checked' : '') . ">\n";
    print "Cross-tables show only best version of each engine<br>\n";
    if ($options{comparison_option_crosstables_sort_order})
    {
        print "Sort cross-tables by: &nbsp;\n";
        print "<input type=\"radio\" name=\"sort_tables\" value=\"by rating\"" .
                  ($sort_tables eq 'by rating' ? ' checked' : '') ."> Rating &nbsp;\n";
        print "<input type=\"radio\" name=\"sort_tables\" value=\"by name\"" .
                  ($sort_tables eq 'by name' ? ' checked' : '') . "> Name<br>\n";
    }
    print "Highlight diagonal of <input type=\"text\" name=\"diag\" size=\"2\" maxlength=\"2\" value=\"$diagonal_width\"> cells wide.";
    print " (0 to highlight everything)<br>\n";
    print "</p>\n";


    print "<p>Reference rating list: ";
    print "<select name=reference_list size=1>\n";
    print $reference_list ? "<option>None</option>\n" : "<option selected=\"selected\">None</option>\n";
    my @list_of_lists = (<../rating-lists/*>);
    foreach my $name (sort @list_of_lists)
    {
        $name =~ s#^\.\./rating-lists/##;
        $name =~ s#\.rating_list$##;
        if ($name eq $options{study_file_tag} . '.' . $stat{date_of_last_addition_short}) { next; }
        if ($reference_list eq $name) { print "<option selected=\"selected\">$name</option>\n"; }
        else { print "<option>$name</option>\n"; }
    }
    print "</select><br>\n";


    print "Recalibrate:<br>\n";
    print " &nbsp; <input type=\"radio\" name=\"recalibrate\" value=\"no\"" .
          ( $recalibrate eq 'no' ? ' checked' : '') .
          "> No recalibration (reference and current list are compared as they are)<br>\n";
    print " &nbsp; <input type=\"radio\" name=\"recalibrate\" value=\"ref_to_current_using_selection\"" .
          ( $recalibrate eq 'ref_to_current_using_selection' ? ' checked' : '') .
          "> Recalibrate reference list to current one using selected engines only<br>\n";
    print " &nbsp; <input type=\"radio\" name=\"recalibrate\" value=\"ref_to_current_using_all\"" .
          ( $recalibrate eq 'ref_to_current_using_all' ? ' checked' : '') .
          "> Recalibrate reference list to current one using all common engines<br>\n";
    print " &nbsp; <input type=\"radio\" name=\"recalibrate\" value=\"current_to_ref_using_selection\"" .
          ( $recalibrate eq 'current_to_ref_using_selection' ? ' checked' : '') .
          "> Recalibrate current list to reference using selected engines only<br>\n";
    print " &nbsp; <input type=\"radio\" name=\"recalibrate\" value=\"current_to_ref_using_all\"" .
          ( $recalibrate eq 'current_to_ref_using_all' ? ' checked' : '') .
          "> Recalibrate current list to reference using all common engines<br>\n";
    print "</p>\n";


    print "<center><input type=\"submit\" value=\"Show selected data!\"></center>\n";



    print "</form>";
    print "</div>\n";
}