###########################################################################
#
# sorttools.pm --
# A component of the Greenstone digital library software
# from the New Zealand Digital Library Project at the 
# University of Waikato, New Zealand.
#
# Copyright (C) 1999 New Zealand Digital Library Project
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
#
###########################################################################

# various subroutines to format strings for sorting, and for customising the sort order (structured_sort)

# To customise metadata formatting functions for a particular collection, create a customsorttools.pm inside
# collection's perllib folder, and implement any of the functions you want to change.

# To customise the structured sort order, either edit structured_sort_definition.pm, or put a modified copy of it into
# your collection's perllib folder.
package sorttools;

use strict;
use Unicode::Normalize;
use Sort::ArbBiLexGS;

# this is defined in structured_sort_definition.pm
our $structured_grouped_order;

require "structured_sort_definition.pm";

#https://interglacial.com/tpj/14/
#https://metacpan.org/pod/Sort::ArbBiLex
#https://dev.to/drhyde/a-brief-guide-to-perl-character-encoding-if7
#https://stackoverflow.com/questions/14303677/languages-supported-by-latin-vs-latin-extended-glyphs-in-fonts-on-google-web


# remove new lines to make the one level ordering - this was used when the structure was
# a single string
#my $structured_order = $structured_grouped_order;
#$structured_order =~ s/[\cm\cj]/ /g;

# join the arrays to make the one level ordering
my $structured_order = join " ", map {@$_ } @$structured_grouped_order;

# printing this here breaks GLI.
#binmode(STDERR, ":utf8");
#print STDERR "sorttools::structured grouped ordering = $structured_grouped_order\n";

# single level sort function - order specified with no grouping
sub structured_sort;
*structured_sort = Sort::ArbBiLexGS::maker(0, $structured_order);
# cmp using this new sort
sub structured_cmp {Sort::ArbBiLexGS::xcmp(\&structured_sort, @_) };

# double level sort function - each category (line in the declaration) sorts together, unless a tie break is needed, in which case
# internal ordering is used.
sub structured_grouped_sort;
*structured_grouped_sort = Sort::ArbBiLexGS::maker(0, $structured_grouped_order);

sub structured_grouped_cmp {Sort::ArbBiLexGS::xcmp(\&structured_grouped_sort, @_) };

# force a single level sort based on the two level declaration - when deciding partition buckets, want a == A == \N{LATIN SMALL LETTER A WITH ACUTE} etc
sub structured_grouped_sort_one_level;
*structured_grouped_sort_one_level = Sort::ArbBiLexGS::maker(1, $structured_grouped_order);
sub structured_grouped_1_cmp {Sort::ArbBiLexGS::xcmp(\&structured_grouped_sort_one_level, @_) };

# make a hash to use for normalising
my @decl = Sort::ArbBiLexGS::generate_decl_array($structured_grouped_order);
my %structured_grouped_hash = ();
foreach my $group (@decl) {
    my $first = @$group[0];
    foreach my $item (@$group) {
        $structured_grouped_hash{$item} = $first;
    }
}

my $structured_grouped_glyph_re = join "|", map(quotemeta,
                             sort {length($b) <=> length($a)} keys %structured_grouped_hash);

sub structured_grouped_normalise {
    
    my $input = shift;
    #print STDERR "normalising $input to ".join('', map $structured_grouped_hash{ $_}, $input =~ m<($structured_grouped_glyph_re)>go)."\n";
    return join('', map $structured_grouped_hash{ $_}, $input =~ m<($structured_grouped_glyph_re)>go);
}

my $has_custom_sort = 0;

sub setup_custom_sort {

    my $collectdir = $ENV{'GSDLCOLLECTDIR'};
    my $customperllibfolder = &FileUtils::filenameConcatenate($collectdir,  'perllib');
    my $customsortfile = &FileUtils::filenameConcatenate($customperllibfolder, 'customsorttools.pm');
    if (&FileUtils::fileExists($customsortfile)) {
	# add perllib folder to INC, if its not already there
	my $found_perllibfolder = 0;
	foreach my $path (@INC)
	{
	    if ($path eq $customperllibfolder)
	    {
		$found_perllibfolder = 1;
		last;
	    }
	}
	if (!$found_perllibfolder)
	{
	    unshift (@INC, $customperllibfolder);
	}
	
	require customsorttools;
	$has_custom_sort = 1;
    }

}

# moved here from BasClas so import can share it
sub format_metadata_for_sorting_old {
    my ($metaname, $metavalue, $doc_obj, $casefold, $accentfold) = @_;

    if (!defined $metaname || $metaname !~ /\S/ || ! defined $metavalue || $metavalue !~ /\S/) {
	return "";
    }
    
    if ($has_custom_sort && defined (&customsorttools::format_metadata_for_sorting)) {
	return &customsorttools::format_metadata_for_sorting($metaname, $metavalue, $doc_obj, $casefold, $accentfold);
    }

    if ($metaname eq "Language") {
	$metavalue = $iso639::fromiso639{$metavalue};
	return $metavalue;
    } 
    
    my $lang;
    if (defined $doc_obj) {
	$lang = $doc_obj->get_metadata_element ($doc_obj->get_top_section(), 'Language');
    }
    $lang = 'en' unless defined $lang;

    if (defined $casefold && $casefold eq "true") {
	$metavalue = lc($metavalue);
    }
    if (defined $accentfold && $accentfold eq "true") {
	$metavalue = NFKD($metavalue);
	$metavalue =~ s/\p{NonspacingMark}//g;
    }

    # is this metadata likely to be a name?
    my $function_name="format_string_name_$lang";
    if ($metaname =~ /^(?:\w+\.)?(?:Creators?|Authors?|Editors?)(?:[:,].*)?$/ 
	&& exists &$function_name) {
	no strict 'refs';
	&$function_name(\$metavalue);
    } else {
	$function_name="format_string_$lang";	
	if (exists &$function_name) {
	    no strict 'refs';
	    &$function_name(\$metavalue);
	}
    }

    return $metavalue;
}

#$options is a hash ref (generally $self of a classifier)
# thie options this is looking for are:
#  no_metadata_formatting (flag)
#  casefold_metadata_for_sorting true/false
#  accentfold_metadata_for_sorting true/false
# format_metadata_as_name_for_sorting true/false/undefined (or empty string)
sub format_metadata_for_sorting {
    my ($metaname, $metavalue, $doc_obj, $options) = @_;

    $options //= {}; # options may be undefined, so make an empty hashref if it is
    
    return $metavalue if $options->{'no_metadata_formatting'}; 

    if (!defined $metaname || $metaname !~ /\S/ || ! defined $metavalue || $metavalue !~ /\S/) {
        return "";
    }

    if ($has_custom_sort && defined (&customsorttools::format_metadata_for_sorting)) {
	return &customsorttools::format_metadata_for_sorting($metaname, $metavalue, $doc_obj, $options);
    }

    my $casefold = 0;
    if (defined $options->{'casefold_metadata_for_sorting'} && $options->{'casefold_metadata_for_sorting'} eq "true") {
        $casefold = 1;
    }
    my $accentfold = 0;
    if (defined $options->{'accentfold_metadata_for_sorting'} && $options->{'accentfold_metadata_for_sorting'} eq "true") {
        $accentfold = 1;
    }
    my $nameformat eq undef;
    if (defined $options->{'format_metadata_as_name_for_sorting'}) {
        if ($options->{'format_metadata_as_name_for_sorting'} eq "true") {
            $nameformat = 1;
        } elsif ($options->{'format_metadata_as_name_for_sorting'} eq "false") {
            $nameformat = 0;
        }
        # otherwise we leave nameformat as undef
    }
 
 
    if ($metaname eq "Language") {
	$metavalue = $iso639::fromiso639{$metavalue};
	return $metavalue;
    } 
    
    my $lang;
    if (defined $doc_obj) {
	$lang = $doc_obj->get_metadata_element ($doc_obj->get_top_section(), 'Language');
    }
    $lang = 'en' unless defined $lang;

    if ($casefold) {
	$metavalue = lc($metavalue);
    }
    if ($accentfold) {
	$metavalue = NFKD($metavalue);
	$metavalue =~ s/\p{NonspacingMark}//g;
    }

    # is this metadata a name?
    my $isaname = 0;
    if (defined $nameformat) {
        if ($nameformat) {
            $isaname = 1;
        }
    } else {
        # we haven't specified name or not, so guess
        if ($metaname =~ /^(?:\w+\.)?(?:Creators?|Authors?|Editors?)(?:[:,].*)?$/) {
            $isaname = 1;
        }
    }


    my $function_name="format_string_name_$lang";
    if ($isaname) {
        if (!exists &$function_name) {
            $function_name="format_string_name_generic";
        }
	no strict 'refs';
	&$function_name(\$metavalue);
    } else {
	$function_name="format_string_$lang";	
	if (!exists &$function_name) {
            $function_name = "format_string_generic";
        }
        no strict 'refs';
        &$function_name(\$metavalue);
    }


    return $metavalue;
}



sub format_string_generic {
    my $stringref = shift;
    
    if ($has_custom_sort && defined (&customsorttools::format_string_generic)) {
	return &customsorttools::format_string_generic($stringref);
    }

    $$stringref =~ s/&[^;]+;//g; # html entities
    $$stringref =~ s/^[^[:alnum:]]//g; # remove any non-alphanumeric chars from start of the string
    $$stringref  =~ s/[^[:alnum:]]+/ /g; # replace any non alphanum chars with space
    $$stringref =~ s/\s+/ /g;
    $$stringref =~ s/^\s+//;
    $$stringref =~ s/\s+$//;

}

## format_string_$lang() converts to lowercase (where appropriate), and
# removes punctuation, articles from the start of string, etc
## format_string_name_$lang() converts to lowercase, puts the surname first,
# removes punctuation, etc


sub format_string_name_generic {
    my ($stringref) = @_;

    if ($has_custom_sort && defined (&customsorttools::format_string_name_generic)) {
	return &customsorttools::format_string_name_generic($stringref);
    }

    $$stringref =~ s/&[^;]+;//g; # html entities
    
    my $comma_format = ($$stringref =~ m/^.+,.+$/);
    $$stringref =~ s/[[:punct:]]//g; # or should we do the alnum things used above?
    $$stringref =~ s/\s+/ /g;
    $$stringref =~ s/^\s+//;
    $$stringref =~ s/\s+$//;

    
    if (!$comma_format) {
	# No commas in name => name in 'firstname surname' format
	# need to sort by surname
	my @names = split / /, $$stringref;
	my $surname = pop @names;
	while (scalar @names && $surname =~ /^(jnr|snr)$/i) {
	    $surname = pop @names;
	}
	$$stringref = $surname . " " . $$stringref;
    }
}

### language-specific sorting functions (called by format_metadata_for_sorting)
sub format_string_en {
    my $stringref = shift;

    if ($has_custom_sort && defined (&customsorttools::format_string_en)) {
	return &customsorttools::format_string_en($stringref);
    }

    &format_string_generic($stringref);
    $$stringref =~ s/^\s*(the|a|an)\b\s*//; # articles
}

sub format_string_fr {
    my $stringref = shift;

    if ($has_custom_sort && defined (&customsorttools::format_string_fr)) {
	return &customsorttools::format_string_fr($stringref);
    }

    &format_string_generic($stringref);
    $$stringref =~ s/^\s*(les?|la|une?)\b\s*//; # articles
}

sub format_string_es {
    my $stringref = shift;

    if ($has_custom_sort && defined (&customsorttools::format_string_es)) {
	return &customsorttools::format_string_es($stringref);
    }

    &format_string_generic($stringref);
    $$stringref =~ s/^\s*(la|el)\b\s*//; # articles
}

### end of language-specific functions

# takes arguments of day, month, year and converts to
# date of form yyyymmdd. month may be full (e.g. "January"),
# abbreviated (e.g. "Jan"), or a number (1-12). Years like "86" 
# will be assumed to be "1986".
sub format_date {
    my ($day, $month, $year) = @_;

    if ($has_custom_sort && defined (&customsorttools::format_date)) {
	return &customsorttools::format_date($day, $month, $year);
    }
    
    my %months = ('january' => '01', 'jan' => '01', 'february' => '02', 'feb' => '02',
		  'march' => '03', 'mar' => '03', 'april' => '04', 'apr' => '04',
		  'may' => '05', 'june' => '06', 'jun' => '06', 'july' => '07', 
		  'jul' => '07', 'august' => '08', 'aug' => '08', 'september' => '09', 
		  'sep' => '09', 'october' => '10', 'oct' => '10', 'november' => '11', 
		  'nov' => '11', 'december' => '12', 'dec' => '12');

    $month =~ tr/A-Z/a-z/;
    
    if ($day < 1) { 
	print STDERR "sorttools::format_date WARNING day $day out of range\n";
	$day = "01";
    } elsif ($day > 31) {
	print STDERR "sorttools::format_date WARNING day $day out of range\n";
	$day = "31";
    }

    $day = "0$day" if (length($day) == 1);

    if ($month =~ /^\d\d?$/) {
	if ($month < 1) {
	    print STDERR "sorttools::format_date WARNING month $month out of range\n";
	    $month = "01";
	} elsif ($month > 12) {
	    print STDERR "sorttools::format_date WARNING month $month out of range\n";
	    $month = "12";
	}
	if ($month =~ /^\d$/) {
	    $month = "0" . $month;
	}
    } elsif (!defined $months{$month}) {
	print STDERR "sorttools::format_date WARNING month $month out of range\n";
	$month = "01";
    } else {
	$month = $months{$month};
    }
    
    if ($year !~ /^\d\d\d\d$/) {
	if ($year !~ /^\d\d$/) {
	    my $newyear = 1900 + $year;
	    print STDERR "sorttools::format_date WARNING year $year assumed to be $newyear\n";
	    $year=$newyear;
	} else {
	    print STDERR "sorttools::format_date WARNING year $year out of range - reset to 1900\n";
	    $year = "1900";
	}
    }

    return "$year$month$day";
}


1;