xref: /third_party/icu/icu4j/perf-tests/perftests.pl

#!/usr/local/bin/perl
# * © 2016 and later: Unicode, Inc. and others.
# * License & terms of use: http://www.unicode.org/copyright.html
# *******************************************************************************
# * Copyright (C) 2002-2012 International Business Machines Corporation and     *
# * others. All Rights Reserved.                                                *
# *******************************************************************************

use XML::LibXML;

# Assume we are running within the icu4j root directory
use lib '.';
use Dataset;
my $OS=$^O;

my $CLASSPATH;
if ($OS eq "linux" || $OS eq "darwin") {
	$CLASSPATH="../icu4j.jar:../tools/misc/out/lib/icu4j-tools.jar:out/bin";
} else {
	$CLASSPATH="../icu4j.jar;../tools/misc/out/lib/icu4j-tools.jar;out/bin";
}
#---------------------------------------------------------------------

# Methods to be tested.  Each pair represents a test method and
# a baseline method which is used for comparison.
my @METHODS  = (
                 ['TestJDKConstruction',     'TestICUConstruction'],
                 ['TestJDKParse',            'TestICUParse'],
                 ['TestJDKFormat',           'TestICUFormat']
               );
# Patterns which define the set of characters used for testing.
my @OPTIONS = (
#                 locale    pattern              date string
                [ "en_US",  "dddd MMM yyyy",     "15 Jan 2007"],
                [ "sw_KE",  "dddd MMM yyyy",     "15 Jan 2007"],
                [ "en_US",  "HH:mm",             "13:13"],
                [ "en_US",  "HH:mm zzzz",        "13:13 Pacific Standard Time"],
                [ "en_US",  "HH:mm z",           "13:13 PST"],
                [ "en_US",  "HH:mm Z",           "13:13 -0800"],
              );

my $THREADS;        # number of threads (input from command-line args)
my $CALIBRATE = 2;  # duration in seconds for initial calibration
my $DURATION  = 10; # duration in seconds for each pass
my $NUMPASSES = 4;  # number of passes.  If > 1 then the first pass
                    # is discarded as a JIT warm-up pass.

my $TABLEATTR = 'BORDER="1" CELLPADDING="4" CELLSPACING="0"';

my $PLUS_MINUS = "±";

if ($NUMPASSES < 3) {
    die "Need at least 3 passes.  One is discarded (JIT warmup) and need two to have 1 degree of freedom (t distribution).";
}


# run all tests with the specified number of threads from command-line input
# (if there is no arguments, use $THREADS = 1)
foreach my $arg ($#ARGV >= 0 ? @ARGV : "1") {
  $THREADS = $arg;
  main();
}


#---------------------------------------------------------------------
sub main {

#-----------DATE FORMAT PERFORMANCE TESTS-----------------------------
    my $testclass = 'com.ibm.icu.dev.test.perf.DateFormatPerformanceTest';
    #my $threads = ($THREADS > 1) ? "($THREADS threads)" : "";

    my $doc = XML::LibXML::Document->new('1.0', 'utf-8');
    my $root = $doc->createElement("perfTestResults");

 #   my $raw = "";
    my @shortNames = ( "open" , "parse", "fmt");
    my $index=0;

    for my $methodPair (@METHODS) {

        my $testMethod = $methodPair->[0];
        my $baselineMethod = $methodPair->[1];
	my $testname = $shortNames[$index];
	$index++;

        $OUT = '';
	my $patternCounter=1;

        for my $pat (@OPTIONS) {

            # measure the test method
            my $t = measure2($testclass, $testMethod, $pat, -$DURATION);
	    my $testResult = $t->getMean();
	    my $jdkElement = $doc->createElement("perfTestResult");
	    my $testName = "DateFmt-$testname-pat$patternCounter-JDK";
	    $jdkElement->setAttribute("test" => $testName);
	    $jdkElement->setAttribute("iterations" => 1);
	    $jdkElement->setAttribute("time" => $testResult);
	    $root->appendChild($jdkElement);

            # measure baseline method
            my $b = measure2($testclass, $baselineMethod, $pat, -$DURATION);
            my $baseResult = $b->getMean();
	    my $icuElement = $doc->createElement("perfTestResult");
	    my $testName = "DateFmt-$testname-pat$patternCounter";
	    $patternCounter++;
	    $icuElement->setAttribute("test"=> $testName);
 	    $icuElement->setAttribute("iterations" => 1);
	    $icuElement->setAttribute("time" => $baseResult);
	    $root->appendChild($icuElement);

       }
    }

#------------------DECIMAL FORMAT TESTS---------------------------------

    my $testclass = 'com.ibm.icu.dev.test.perf.DecimalFormatPerformanceTest';
    my @OPTIONS = (
#		locale	    pattern	date string
		[ "en_US", "#,###.##", "1,234.56"],
		[ "de_DE", "#,###.##", "1.234,56"],
		);
    my $index=0;
    for my $methodPair (@METHODS) {

        my $testMethod = $methodPair->[0];
        my $baselineMethod = $methodPair->[1];
	my $testname = $shortNames[$index];
	$index++;


        for my $pat (@OPTIONS) {
	       my $patternName = $pat->[0];

            # measure the test method
            my $t = measure2($testclass, $testMethod, $pat, -$DURATION);
	    my $testResult = $t->getMean();
	    my $jdkElement = $doc->createElement("perfTestResult");
	    my $testName = "NumFmt-$testname-$patternName-JDK";
	    $jdkElement->setAttribute("test" => $testName);
	    $jdkElement->setAttribute("iterations"=>1);
	    $jdkElement->setAttribute("time" => $testResult);
	    $root->appendChild($jdkElement);

            # measure baseline method
            my $b = measure2($testclass, $baselineMethod, $pat, -$DURATION);
            my $baseResult = $b->getMean();
	    my $icuElement = $doc->createElement("perfTestResult");
	    my $testName = "NumFmt-$testname-$patternName";
	    $icuElement->setAttribute("test"=> $testName);
	    $icuElement->setAttribute("iterations"=>1);
	    $icuElement->setAttribute("time" => $baseResult);
	    $root->appendChild($icuElement);
	}
    }

#----------------COLLATION PERFORMANCE TESTS--------------------------_

    %dataFiles = (
   	   "en_US",         "TestNames_Latin.txt",
	   "da_DK",         "TestNames_Latin.txt",
	   "de_DE",         "TestNames_Latin.txt",
	   "de__PHONEBOOK", "TestNames_Latin.txt",
	   "fr_FR",         "TestNames_Latin.txt",
	   "ja_JP",         "TestNames_Latin.txt TestNames_Japanese_h.txt TestNames_Japanese_k.txt TestNames_Asian.txt",
	   "zh_CN",         "TestNames_Latin.txt TestNames_Chinese.txt",
	   "zh_TW",         "TestNames_Latin.txt TestNames_Chinese.txt",
	   "zh__PINYIN",    "TestNames_Latin.txt TestNames_Chinese.txt",
	   "ru_RU", 	    "TestNames_Latin.txt TestNames_Russian.txt",
	   "th",            "TestNames_Latin.txt TestNames_Thai.txt",
	   "ko_KR",         "TestNames_Latin.txt TestNames_Korean.txt",
	   );

    #  Outer loop runs through the locales to test
    #     (Edit this list directly to make changes)
    #
    foreach  $locale (
	   "en_US",
	   "da_DK",
	   "de_DE",
	   "de__PHONEBOOK",
	   "fr_FR",
	   "ja_JP",
           "zh_CN",
	   "zh_TW",
	   "zh__PINYIN",
           "ko_KR",
	   "ru_RU",
	   "th",
                   )
       {


       #
       # Inner loop runs over the set of data files specified for each locale.
       #    (Edit the %datafiles initialization, above, to make changes.
       #
        $ff = $dataFiles{$locale};
        @ff = split(/[\s]+/, $ff);
        $counter = 1;
        foreach  $data (@ff) {
          #
          # Run ICU Test for this (locale, data file) pair.
          #
           $iStrCol = `java -classpath $CLASSPATH com.ibm.icu.dev.test.perf.CollationPerformanceTest -terse -file data/collation/$data -locale $locale -loop 1000 -binsearch`;
print "java -classpath $CLASSPATH com.ibm.icu.dev.test.perf.CollationPerformanceTest -terse -file data/collation/$data -locale $locale -loop 1000 -binsearch\n";
  $iStrCol =~s/[,\s]*//g;  # whack off the leading "  ," in the returned result.
          doKeyTimes("java -classpath $CLASSPATH com.ibm.icu.dev.test.perf.CollationPerformanceTest -terse -file data/collation/$data -locale $locale -loop 1000 -keygen",
                    my $iKeyGen, my $iKeyLen);

          #
          # Run Windows test for this (locale, data file) pair.  Only do if
          #    we are not on Windows 98/ME and we hava a windows langID
          #    for the locale.
          #
           $wStrCol =  $wKeyGen =  $wKeyLen = 0;
          my $wStrCol = `java -classpath $CLASSPATH com.ibm.icu.dev.test.perf.CollationPerformanceTest -terse -file data/collation/$data -locale $locale -loop 1000 -binsearch -java`;
          $wStrCol =~s/[,\s]*//g;  # whack off the leading "  ," in the returned result.
          doKeyTimes("java -classpath $CLASSPATH com.ibm.icu.dev.test.perf.CollationPerformanceTest -terse -file data/collation/$data -locale $locale -loop 1000 -keygen -java",
                     $wKeyGen, $wKeyLen);

           $collDiff =  $keyGenDiff =  $keyLenDiff = 0;
          if ($wKeyLen > 0) {
               $collDiff   = (($wStrCol - $iStrCol) / $iStrCol) * 100;
               $keyGenDiff = (($wKeyGen - $iKeyGen) / $iKeyGen) * 100;
               $keyLenDiff = (($wKeyLen - $iKeyLen) / $iKeyLen) * 100;
          }

	my $ICU = $doc->createElement("perfTestResult");
	my $testname = "Coll-$locale-data$counter-StrCol";
	#write the results corresponding to this local,data pair
	$ICU->setAttribute("test"=> $testname);
	$ICU->setAttribute("iterations"=>1000);
	$ICU->setAttribute("time"=> $iStrCol);
	$root->appendChild($ICU);

	my $Key = $doc->createElement("perfTestResult");
	my $testname = "Coll-$locale-data$counter-keyGen";
	$Key->setAttribute("test"=> $testname);
	$Key->setAttribute("iterations"=>1000);
	$Key->setAttribute("time"=>$iKeyGen);
	$root->appendChild($Key);

	my $JDK = $doc->createElement("perfTestResult");
	my $testname = "Coll-$locale-data$counter-StrCol-JDK";
	$JDK->setAttribute("test"=>$testname);
	$JDK->setAttribute("iterations"=>1000);
	$JDK->setAttribute("time"=>$wStrCol);
	$root->appendChild($JDK);

	my $Key = $doc->createElement("perfTestResult");
	my $testname = "Coll-$locale-data$counter-keyGen-JDK";
	$Key->setAttribute("test"=>$testname);
	$Key->setAttribute("iterations"=>1000);
	$Key->setAttribute("time"=>$wKeyGen);
	$root->appendChild($Key);
	$counter++;
     }
   }



#----------WRITE RESULTS TO perf.xml-----------------------
    $doc->setDocumentElement($root);
    open my $out_fh, '>', "perf.xml";
    print {$out_fh} $doc->toString;
}


#---------------------------------------------------------------------
# Append text to the global variable $OUT
sub out {
   $OUT .= join('', @_);
}


#---------------------------------------------------------------------
# Measure a given test method with a give test pattern using the
# global run parameters.
#
# @param the method to run
# @param the pattern defining characters to test
# @param if >0 then the number of iterations per pass.  If <0 then
#        (negative of) the number of seconds per pass.
#
# @return a Dataset object, scaled by iterations per pass and
#         events per iteration, to give time per event
#
sub measure2 {
    my @data = measure1(@_);
    my $iterPerPass = shift(@data);
    my $eventPerIter = shift(@data);

    shift(@data) if (@data > 1); # discard first run

    my $ds = Dataset->new(@data);
    $ds->setScale(1.0e-3 / ($iterPerPass * $eventPerIter));
    $ds;
}

#---------------------------------------------------------------------
# Measure a given test method with a give test pattern using the
# global run parameters.
#
# @param the method to run
# @param the pattern defining characters to test
# @param if >0 then the number of iterations per pass.  If <0 then
#        (negative of) the number of seconds per pass.
#
# @return array of:
#         [0] iterations per pass
#         [1] events per iteration
#         [2..] ms reported for each pass, in order
#
sub measure1 {
    my $testclass = shift;
    my $method = shift;
    my $pat = shift;
    my $iterCount = shift; # actually might be -seconds/pass

    # is $iterCount actually -seconds/pass?
    if ($iterCount < 0) {

        # calibrate: estimate ms/iteration
        print "Calibrating...";
        my @t = callJava($testclass, $method, $pat, -$CALIBRATE, 1);
        print "done.\n";

        my @data = split(/\s+/, $t[0]->[2]);
        $data[0] *= 1.0e+3;

        my $timePerIter = 1.0e-3 * $data[0] / $data[1];

        # determine iterations/pass
        $iterCount = int(-$iterCount / $timePerIter + 0.5);
   }

    # run passes
    print "Measuring $iterCount iterations x $NUMPASSES passes...";
    my @t = callJava($testclass, $method, $pat, $iterCount, $NUMPASSES);
    print "done.\n";
    my @ms = ();
    my @b; # scratch
    for my $a (@t) {
        # $a->[0]: method name, corresponds to $method
        # $a->[1]: 'begin' data, == $iterCount
        # $a->[2]: 'end' data, of the form <ms> <loops> <eventsPerIter>
        # $a->[3...]: gc messages from JVM during pass
        @b = split(/\s+/, $a->[2]);
        push(@ms, $b[0] * 1.0e+3);
    }
    my $eventsPerIter = $b[2];

    my @ms_str = @ms;
    $ms_str[0] .= " (discarded)" if (@ms_str > 1);

    ($iterCount, $eventsPerIter, @ms);
}

#---------------------------------------------------------------------
# Invoke java to run $TESTCLASS, passing it the given parameters.
#
# @param the method to run
# @param the number of iterations, or if negative, the duration
#        in seconds.  If more than on pass is desired, pass in
#        a string, e.g., "100 100 100".
# @param the pattern defining characters to test
#
# @return an array of results.  Each result is an array REF
#         describing one pass.  The array REF contains:
#         ->[0]: The method name as reported
#         ->[1]: The params on the '= <meth> begin ...' line
#         ->[2]: The params on the '= <meth> end ...' line
#         ->[3..]: GC messages from the JVM, if any
#
sub callJava {
    my $testclass = shift;
    my $method = shift;
    my $pat = shift;
    my $n = shift;
    my $passes = shift;

    my $n = ($n < 0) ? "-t ".(-$n) : "-i ".$n;

    my $cmd = "java -classpath $CLASSPATH $testclass $method $n -p $passes -L @$pat[0] \"@$pat[1]\" \"@$pat[2]\" -r $THREADS";
    print "[$cmd]\n"; # for debugging
    open(PIPE, "$cmd|") or die "Can't run \"$cmd\"";
    my @out;
    while (<PIPE>) {
        push(@out, $_);
    }
    close(PIPE) or die "Java failed: \"$cmd\"";

    @out = grep(!/^\#/, @out);  # filter out comments

    #print "[", join("\n", @out), "]\n";

    my @results;
    my $method = '';
    my $data = [];
    foreach (@out) {
        next unless (/\S/);

        if (/^=\s*(\w+)\s*(\w+)\s*(.*)/) {
            my ($m, $state, $d) = ($1, $2, $3);
            #print "$_ => [[$m $state $data]]\n";
            if ($state eq 'begin') {
                die "$method was begun but not finished" if ($method);
                $method = $m;
                push(@$data, $d);
                push(@$data, ''); # placeholder for end data
            } elsif ($state eq 'end') {
                if ($m ne $method) {
                    die "$method end does not match: $_";
                }
                $data->[1] = $d; # insert end data at [1]
                #print "#$method:", join(";",@$data), "\n";
                unshift(@$data, $method); # add method to start

                push(@results, $data);
                $method = '';
                $data = [];
            } else {
                die "Can't parse: $_";
           }
        }
       elsif (/^\[/) {
            if ($method) {
                push(@$data, $_);
            } else {
                # ignore extraneous GC notices
            }
        }
        else {
            die "Can't parse: $_";
        }
    }

    die "$method was begun but not finished" if ($method);

    @results;
}

#-----------------------------------------------------------------------------------
#  doKeyGenTimes($Command_to_run, $time, $key_length)
#       Do a key-generation test and return the time and key length/char values.
#
sub doKeyTimes($$$) {
   # print "$_[0]\n";
   local($x) = `$_[0]`;                  # execute the collperf command.
   ($_[1], $_[2]) = split(/\,/, $x);     # collperf returns "time, keylength" string.
}


#eof