#!/usr/bin/perl -w
################################################################
# chi.pl : Kullback's Chi Test for two texts                   #
#--------------------------------------------------------------#
# Test if two texts over the alphabet [A..Z] belong to the     #
#   same language -- i. e. have the same distribution of       #
#   letters.                                                   #
#--------------------------------------------------------------#
# Input from Text1.txt : first text                            #
# Input from Text2.txt : second text                           #
# Output to STDOUT : chi statistic (cross-product sum)         #
#--------------------------------------------------------------#
# Transliteration of special characters according to English   #
#   rules                                                      #
# For special characters of other languages use preptext.pl    #
#   first.                                                     #
#--------------------------------------------------------------#
# Klaus Pommerening 2013-December-23                           #
#                                                              #
# This program is freeware. Use it as you like.                #
# Usual disclaimers apply.                                     #
################################################################

require "auxcrypt.pl";     # Use standard alphabet
                           #   as global variables $alphabet,
                           #   @alphabet, and %sa.
                           # Use procedure &to26eng

open(FILE1,"Text1.txt");          # Read first text.
chomp(@inlist1 = <FILE1>);        # Read input line by line,
                                  #   chop trailing CR.
foreach $line (@inlist1) {        # Change each line
  $line = &to26eng($line); }      #   to Alphabet [A..Z].
$text1 = join("",@inlist1);       # Join lines as 1 string.
close(FILE1);
$len1 = length($text1);
@textlist1 = split(//,$text1);
for ($j = 0; $j < @alphabet; $j++) {
  $count1{$alphabet[$j]} = 0;        # Initialize counters.
  }
foreach $i (@textlist1) {            # Count letters.
  $count1{$i}++;
  }
# for ($j = 0; $j < @alphabet; $j++) {
#   print "$alphabet[$j]: $count1{$alphabet[$j]}\n";
#   }
# print "Length = ", $len1, "\n";

open(FILE2,"Text2.txt");          # Read second text.
chomp(@inlist2 = <FILE2>);        # Read input line by line,
                                  #   chop trailing CR.
foreach $line (@inlist2) {        # Change each line
  $line = &to26eng($line); }      #   to Alphabet [A..Z].
$text2 = join("",@inlist2);       # Join lines as 1 string.
close(FILE2);
$len2 = length($text2);
@textlist2 = split(//,$text2);
for ($j = 0; $j < @alphabet; $j++) {
  $count2{$alphabet[$j]} = 0;        # Initialize counters.
  }
foreach $i (@textlist2) {            # Count letters.
  $count2{$i}++;
  }
# for ($j = 0; $j < @alphabet; $j++) {
#   print "$alphabet[$j]: $count2{$alphabet[$j]}\n";
#   }
# print "Length = ", $len2, "\n";

$chi = 0;                            # sum of squares
for ($j = 0; $j < @alphabet; $j++) {
  $chi += $count1{$alphabet[$j]} * $count2{$alphabet[$j]}
  }
$chi = $chi/($len1*$len2);           # normalization

print "Chi = ", $chi, "\n";
