###########################################################################
#
# solrbuilder.pm -- perl wrapper for building index with Solr
# A component of the Greenstone digital library software
# from the New Zealand Digital Library Project at the
# University of Waikato, New Zealand.
#
# Copyright (C) 2026 New Zealand Digital Library Project
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
#
###########################################################################


package solrbuilder;

use strict; 
no strict 'refs';

use lucenebuilder;
use solrserver;

sub BEGIN {
    @solrbuilder::ISA = ('lucenebuilder');
}


sub new {
    my $class = shift(@_);
    my $self = new lucenebuilder (@_);
    $self = bless $self, $class;

    $self->{'buildtype'} = "solr";

    my $solr_passes_script = "solr_passes.pl";

    $self->{'solr_passes'} = "$solr_passes_script";
    # Tack perl on the beginning to ensure execution
    $self->{'solr_passes_exe'} = "\"".&util::get_perl_exec()."\" -S \"$solr_passes_script\"";
    return $self;
}


sub default_buildproc {
    my $self  = shift (@_);

    return "solrbuildproc";
}

# in other indexes this combines the index defs with subcolls and langs.
# here, we want to add filters to the index
sub combine_indexes_and_subindexes {
    my $self  = shift (@_);

    ##TODO!
}

# solr now gets all text back from the index - we add it during the
# indexing stage, so this phase need do nothing
sub compress_text
{
    my $self = shift (@_);
    return;
}


#----



sub filter_in_out_file
{
    my ($in_filename,$out_filename,$replace_rules) = @_;

    if (open(SIN,"<$in_filename")) {

	if (open(SOUT,">$out_filename")) {

	    my $line;
	    while (defined ($line=<SIN>)) {
		chomp $line;

		my $done_insert = 0;
		foreach my $rule (@$replace_rules) {
		    my $line_re = $rule->{'regexp'};
		    my $insert  = $rule->{'insert'};

		    if ($line =~ m/$line_re/) {
			print SOUT $insert;
			$done_insert = 1;
			last;
		    }
		}
		if (!$done_insert) {
		    print SOUT "$line\n";;
		}
	    }

	    close(SOUT);
	}
	else {
	    print STDERR "Error: Failed to open $out_filename\n";
	    print STDERR "       $!\n";
	}

	close(SIN);
    }
    else {
	print STDERR "Error: Failed to open $in_filename\n";
	print STDERR "       $!\n";
    }

}

# Generate solr schema.xml file based on indexmapfield and other associated
# config files 
#
# Unlike make_auxiliary_files(), this needs to be done up-front (rather
# than at the end) so the data-types in schema.xml are correctly set up
# prior to document content being pumped through solr_passes.pl

# we first put a copy of gs3-default-conf into the collection's etc folder
# - if its not already there
sub premake_solr_auxiliary_files 
{
    my $self = shift (@_);

    my $has_allfields = $self->{'has_allfields_index'};
    my $doc_index = $self->{'doc_level_indexing'};
    my $sec_index = $self->{'sec_level_indexing'};
    my $text_index = $self->{'has_text_index'};
    my $field_list = $self->{'all_field_list'};
    
    # Replace the following marker: 
    #
    #   <!-- ##GREENSTONE-FIELDS## -->
    #
    # with lines of the form:
    #
    #   <field name="<field>" type="string" ... /> 
    #
    # for each <field> in 'indexfieldmap'

    # the document and section schemas may differ
    my $didx_schema_insert_xml = "";
    my $sidx_schema_insert_xml = "";

    foreach my $fullfieldname(@$field_list) {
        my $field_insert = "    "; # indent
        my $field =  $self->{'buildproc'}->{'indexfieldmap'}->{$fullfieldname};
        $field_insert .= "<field name=\"$field\" ";

        my $customfieldtype = undef;
        if(defined $self->{'collect_cfg'}->{'indexfieldoptions'}->{$fullfieldname}->{'solrfieldtype'}) {	
            $customfieldtype = $self->{'collect_cfg'}->{'indexfieldoptions'}->{$fullfieldname}->{'solrfieldtype'};
            print STDERR "@@@@#### found custom TYPE: $customfieldtype for field $field\n";
        }
        my $thisfieldtype;
        my $multivalued = "true";
        
        # we assume all meta fields are small, so we store them to get snippets, instead of storing offsets which increases index size
        if($field eq "CD" || $field eq "CS" || $field eq "ML") {
            # Coordinate and CoordShort meta should not be split but treated as a whole string for searching. So type=string, not type=text_en_splitting			
            # Can't set to type="location", which uses solr.LatLonType, since type=location fields "must not be multivalued" as per conf/schema.xml.in.
            # And we can have multiple Coordinate (and multiple CoordShort) meta for one doc, so multivalued=true.
            $thisfieldtype = $customfieldtype // "string";
            
            # mapLabel: same attributes as for coord meta CD and CS above
            # mapLabel is also like facets with type="string" to not get tokenized, and multiValued="true" to allow each shape's label to be stored distinctly
        }
        elsif($field eq "ID") {
            # Identifier - needs to be treated as a single string, and can only have one value
            # we'll use our new text_id type, as we want lowercasing
            $thisfieldtype = $customfieldtype // "text_id";
            $multivalued = "false";
        }
        elsif($field eq "LT" || $field eq "LO") {
            # Latitude and Longitude is being phased out in favour of using Coord meta.
            # However, if ever returning to using Lat and Lng instead of Coord meta, then the way the Lat Lng meta is currently written out for type="location"
            # is in the wrong format. Lat and Lng shouldn't get written out separately but as: Lat,Lng
            # It gets written out in solrbuildproc.pm, I think, so that would be where it needs to be corrected.
            # For more info on type=location for our solr 4.7.2 or thereabouts, see https://web.archive.org/web/20160312154250/https://wiki.apache.org/solr/SpatialSearchDev
            # which states:
            #    When indexing, the format is something like:
            #       <field name="store_lat_lon">12.34,-123.45</field>
            $thisfieldtype = $customfieldtype // "location";
            $multivalued = "false";
        } else {
            $thisfieldtype = $customfieldtype // "text_en_splitting";
        }
        $field_insert .= "type=\"$thisfieldtype\" indexed=\"true\" stored=\"true\" multiValued=\"$multivalued\" />\n";
        if ($has_allfields) {
            $field_insert .="    <copyField source=\"$field\" dest=\"ZZ\"/>\n";
        }

        if ($doc_index) {
            $didx_schema_insert_xml .= $field_insert;
        }
        if ($sec_index) {
            $sidx_schema_insert_xml .= $field_insert;
        }
    } # foreach field

    # catch all metadata for allfields
    if ($has_allfields) {
        my $field_insert = "<copyField source=\"*_m\" dest=\"ZZ\"/>\n";
        if ($doc_index) {
            $didx_schema_insert_xml .= $field_insert;
        }
        if ($sec_index) {
            $sidx_schema_insert_xml .= $field_insert;
        }
    }

    # text field
    # note, this won't work for index specs like text,TI
    my $customfieldtype = undef;
    if(defined $self->{'collect_cfg'}->{'indexfieldoptions'}->{'text'}->{'solrfieldtype'}) {	
        $customfieldtype = $self->{'collect_cfg'}->{'indexfieldoptions'}->{'text'}->{'solrfieldtype'};
        print STDERR "@@@@#### found custom TYPE: $customfieldtype for field TX\n";
    }

    my $thisfieldtype = $customfieldtype // "text_en_splitting";
    if ($doc_index && $text_index) {
        # we add a TX field, indexed but not stored
        my $field_insert =  "     <field name=\"TX\" type=\"$thisfieldtype\" indexed=\"true\" stored=\"false\" multiValued=\"false\" termVectors=\"true\" termPositions=\"true\" termOffsets=\"true\" storeOffsetsWithPositions=\"true\"/>\n";
        if ($has_allfields) {
            $field_insert .="    <copyField source=\"TX\" dest=\"ZZ\"/>\n";
        }
        $didx_schema_insert_xml .= $field_insert;
    }
    # always put text field for sec index. but only indexed if user specified a text index
    my $indexed = "false";
    if ($text_index) {
        $indexed = "true";
    }
    my $field_insert =  "     <field name=\"TX\" type=\"$thisfieldtype\" indexed=\"$indexed\" stored=\"true\" multiValued=\"false\"";
    if ($text_index) {
        $field_insert .= " termVectors=\"true\" termPositions=\"true\" termOffsets=\"true\" storeOffsetsWithPositions=\"true\"";
    }
    $field_insert .="/>\n";

    if ($has_allfields && $sec_index && $text_index) {
        $field_insert .= "<copyField source=\"TX\" dest=\"ZZ\"/>\n";
    }
    $sidx_schema_insert_xml .= $field_insert;

    #print STDERR "sidx schema = **********************8\n";
    #print STDERR "$sidx_schema_insert_xml\n";

    #print STDERR "didx schema = **********************8\n";
    #print STDERR "$didx_schema_insert_xml\n";

    my $site = $self->{'site'};
    my $collection = $self->{'collection'};
    my $core_names = $self->{'solrfullcorenames'};
    
    my $solr_src_home = $ENV{'GEXT_SOLR9'};
    my $default_conf_dirname = &FileUtils::filenameConcatenate($solr_src_home,"gs3-default-conf");

   
    my $collect_home = $ENV{'GSDLCOLLECTDIR'};
    my $in_dirname = &FileUtils::filenameConcatenate($collect_home,"etc","solr-conf");

    # have we done the original copy of solr conf to collection's etc?
    # in not, copy it now - this will create the dir
    if (!FileUtils::directoryExists($in_dirname)) {
        FileUtils::copyFilesRecursiveNoSVN($default_conf_dirname, $in_dirname);
    }
        
    my $schema_in_filename = &FileUtils::filenameConcatenate($in_dirname,"schema.xml.in");

    my @in_file_list = ( "solrconfig.xml", "synonyms.txt", "protwords.txt", "stopwords.txt");
    my @in_dir_list = ( "lang" );
    
    #my $collect_home = $ENV{'GSDLCOLLECTDIR'};
    #my $out_dirname = &FileUtils::filenameConcatenate($collect_home,"etc","conf");

    my $build_dir = $self->{'build_dir'};
    my $solr_data_home = $ENV{'SOLR_HOME'};
    my $collection_out_dirname = &FileUtils::filenameConcatenate($solr_data_home, "cores", $site, $collection);
    foreach my $core (@$core_names) {
        my $core_short="";
        my $out_dirname = &FileUtils::filenameConcatenate($collection_out_dirname, $core, "conf");
        &FileUtils::makeAllDirectories($out_dirname);
        
        my $schema_out_filename = &FileUtils::filenameConcatenate($out_dirname,"schema.xml");
    
        my $insert_rules;
        if ($core =~ /didx$/) {
            $core_short = "didx";
            $insert_rules = [ { 'regexp' => "^\\s*<!--\\s*##GREENSTONE-FIELDS##\\s*-->\\s*\$",
                                'insert' => $didx_schema_insert_xml } ];
        } else {
            $core_short = "sidx";
            $insert_rules = [ { 'regexp' => "^\\s*<!--\\s*##GREENSTONE-FIELDS##\\s*-->\\s*\$",
                                'insert' => $sidx_schema_insert_xml } ];
        }
        &filter_in_out_file($schema_in_filename,$schema_out_filename,$insert_rules);

        # now do the same for solrconfig.xml, stopwords, ...
        # these are simpler, as they currently do not need any filtering
        
        
        foreach my $file ( @in_file_list ) {
            my $in_filename = &FileUtils::filenameConcatenate($in_dirname,$file);
            my $out_filename = &FileUtils::filenameConcatenate($out_dirname,$file);
            
            if(&FileUtils::fileExists($in_filename)) {
                &filter_in_out_file($in_filename,$out_filename,[]);
            }
        }
        
        my @in_dir_list = ( "lang" );
        foreach my $dir ( @in_dir_list ) {
            
            my $full_subdir_name = &FileUtils::filenameConcatenate($in_dirname,$dir);
            
            if(&FileUtils::directoryExists($full_subdir_name)) {
                &FileUtils::copyFilesRecursiveNoSVN($full_subdir_name, $out_dirname);
            }
        }
        # now make a copy of new conf into build_dir
        my $backup_conf_dir = &FileUtils::filenameConcatenate($build_dir, $core_short, "conf");
        if (&FileUtils::directoryExists($backup_conf_dir)) {
            &FileUtils::removeFilesRecursive($backup_conf_dir);
        }
        &FileUtils::copyFilesRecursive($out_dirname, $backup_conf_dir);
            
    } # foreach $core

    
    
    
}


# we write our own version here as we don't want subcols and langs into index defs
# TODO - maybe here we add the fields to the index??
sub create_index_mapping {
    my $self = shift (@_);
    my ($index) = @_;

    my %mapping = ();

    $mapping{'indexmaporder'} = [];

    # only one index and its name is idx
    $mapping{$index} = 'idx';
    $mapping{'indexmap'}{$index} = 'idx';
    push (@{$mapping{'indexmaporder'}}, $index);

    return \%mapping;
}

sub pre_build_indexes
{
    my $self = shift (@_);
    my ($indexname) = @_; # optional, if we havemanually specified an index to build
    my $outhandle = $self->{'outhandle'};

    # If the Solr server is not already running, the following starts
    # it up, and only returns when the server is "reading and listening"

    # build_dir is only passed in for the 'text' phase
    my $solr_server = new solrserver();
    $solr_server->start();
    $self->{'solr_server'} = $solr_server;

    #currently we only have one index specified. mgppbuilder has mashed all field names into one csv index
    
    if (defined $indexname && $indexname =~ /\w/) {
        $self->{'original_index'} = $indexname;
    } else {
        $self->{'original_index'} = $self->{'collect_cfg'}->{'indexes'}->[0];
    }

    return unless $self->want_built($self->{'original_index'}); # in case we have a dontbuild line in config file

    my $index_dir = "idx";
    # create the mapping between the index descriptions 
    # and their directory names (includes subcolls and langs)
    # this no longer needs to do much
    $self->{'index_mapping'} = $self->create_index_mapping ($self->{'original_index'});

    $self->{'all_metadata_specified'} = 0; # has the user added a 'metadata' index?
    $self->{'has_allfields_index'} = 0; # has the user added allfields index
    $self->{'has_text_index'} = 0; # is there a 'text' index

    my @all_fields = ();
    my $done_fields = {};
    my $buildproc = $self->{'buildproc'};

    # go through all fields and make sure we have a shortname made
    foreach my $field (split (/;/, $self->{'original_index'})) {

        if ($field eq "metadata") {
            $self->{'all_metadata_specified'} = 1;
        }
        elsif ($field eq "allfields") {
            $self->{'has_allfields_index'} = 1;
            $buildproc->{'indexfieldmap'}->{'allfields'} = "ZZ";
            $buildproc->{'indexfieldmap'}->{'ZZ'} = 1;
        }
        elsif($field eq "text") {
            $self->{'has_text_index'} = 1;
            $buildproc->{'indexfieldmap'}->{'text'} = "TX";
            $buildproc->{'indexfieldmap'}->{'TX'} = 1;
        }
            
        else {
            next if (defined $done_fields->{$field});
            
            push(@all_fields,$field);
            $done_fields->{$field} = 1;
            if (!defined $buildproc->{'indexfieldmap'}->{$field})
            {
                my $shortname = $buildproc->get_or_create_shortname($field);
                $buildproc->{'indexfieldmap'}->{$field} = $shortname;
                $buildproc->{'indexfieldmap'}->{$shortname} = 1;
            }
        }
    }
    $self->{'all_field_list'} = \@all_fields;


    # now go through index fields again and create the ordered mapping
    # do we need to do this here???
    my @indexfieldmap = ();
    my @indexfields = ();


    # Now update the solr-core information in solr.xml
    # => at most two cores <colname>-Doc and <colname>-Sec

    my $site        = $self->{'site'};
    my $collection     = $self->{'collection'};
    my $build_dir = $self->{'build_dir'};

    $self->{'solrcores'} = [];
    # force_removeold == opposite of being run in 'incremental' mode
    my $force_removeold = ($self->{'incremental'}) ? 0 : 1;

    my $core_prefix = (defined $site) ? "$site-$collection" : $collection;
    if ($force_removeold) {
        $core_prefix = "building-$core_prefix-";
    }
    $self->{'doc_level_indexing'} = 0;
    $self->{'sec_level_indexing'} = 0;
    
    if ($self->{'levels'}->{'document'}) {
        $self->{'doc_level_indexing'} = 1;
    }
    if ($self->{'levels'}->{'section'}) {
        $self->{'sec_level_indexing'} = 1;
    }
    # we always have a section index, for text retrieval
    push (@{$self->{'solrcores'}}, "sidx");
    $self->{'solrcoreforretrieval'} = "sidx"; # always this for now
    push (@{$self->{'solrfullcorenames'}}, $core_prefix."sidx");  #? 
    if ($self->{'doc_level_indexing'}) {
        # we only make a doc level index if its specified in collconfig
        push (@{$self->{'solrcores'}}, "didx");
        push (@{$self->{'solrfullcorenames'}}, $core_prefix."didx"); #?   
    }

    # set up the schemas etc
    
    $self->premake_solr_auxiliary_files();
    
    foreach my $core (@{$self->{'solrfullcorenames'}}) {
        if ($force_removeold) {
            # now go on and create new index
            print $outhandle "Creating Solr core: $core\n";
            $solr_server->admin_create_core($core, $site, $collection );
            
        }
        else {
            # if collect==core is already in solr.xml (check with STATUS)
            # => use RELOAD* call to refresh fields now expressed in schema.xml
            # 
            # else 
            # => use CREATE API to add to solr.xml
            #
            # No longer calling RELOAD, because Georgy documented a memory leak with it (svn r32178)
            # Using unload + create to get the same effect as RELOAD without its side-effects.
            # 
            #TODO the above comment was for solr4. Is it still relevant?
            # check if this is still true? can we use reload? what exactly are we doing here??
            my $check_core_exists = $solr_server->admin_ping_core($core);
            
            if ($check_core_exists) {	    
                print $outhandle "Unloading Solr core: $core\n";
                $solr_server->admin_unload_core($core);
            }
            
            print $outhandle "Creating Solr core: $core\n";
            $solr_server->admin_create_core($core, $site, $collection);
        }
    } #foreach core
            
}

sub build_indexes {
    my $self = shift (@_);
    my ($indexname, $indexlevel) = @_;
    my $outhandle = $self->{'outhandle'};

    $self->pre_build_indexes($indexname); #indexname is optional

    if (defined $indexname && $indexname =~ /\w/ && defined $indexlevel && $indexlevel =~ /\w/ ) {
        print STDERR "calling build index on $indexname and $indexlevel\n";
        $self->build_index($indexname, $indexlevel);
        return;
    }

    my $fields_list = join(";",@{$self->{'all_field_list'}}); # all the specifid fields
    if ($self->{'all_metadata_specified'}) {
        $fields_list .= ";metadata";
    }
    
    if ($self->{'doc_level_indexing'}) {
        # build up the index definition for Doc index
        my $index = $fields_list;
        $self->build_index($index, 'document', $self->{'has_text_index'});
    }
    # we always build a section level index of text
    my $index = "";
    if ($self->{'sec_level_indexing'}) {
        # add in the fields if we are searching at this level
        $index = "$fields_list"; 
    }
    # the 1 says 'has_text_index'
    $self->build_index($index, 'section', 1);

    $self->post_build_indexes();
}


sub build_index {
    my $self = shift (@_);
    my ($index,$llevel, $has_text_index) = @_;
    my $outhandle = $self->{'outhandle'};
    my $build_dir = $self->{'build_dir'};

    # get the full index directory path and make sure it exists
#    my $indexdir = $self->{'index_mapping'}->{$index};
 #   &FileUtils::makeAllDirectories(&FileUtils::filenameConcatenate($build_dir, $indexdir));

    # Find the perl script to call to run solr
    my $solr_passes = $self->{'solr_passes'};
    my $solr_passes_exe = $self->{'solr_passes_exe'};

    # define the section names for solrpasses
    # define the section names and possibly the doc name for solrpasses
    my $solr_passes_sections = $llevel;

    my $osextra = "";
    if ($ENV{'GSDLOS'} =~ /^windows$/i) {
	$build_dir =~ s@/@\\@g;
    } else {
	if ($outhandle ne "STDERR") {
	    # so solr_passes doesn't print to stderr if we redirect output
	    $osextra .= " 2>/dev/null";
	}
    }

    # get the index expression if this index belongs
    # to a subcollection
    my $indexexparr = [];
    my $langarr = [];

    # there may be subcollection info, and language info.
    my ($fields, $subcollection, $language) = split (":", $index);
    my @subcollections = ();
    @subcollections = split /,/, $subcollection if (defined $subcollection);

    foreach $subcollection (@subcollections) {
	if (defined ($self->{'collect_cfg'}->{'subcollection'}->{$subcollection})) {
	    push (@$indexexparr, $self->{'collect_cfg'}->{'subcollection'}->{$subcollection});
	}
    }

    # add expressions for languages if this index belongs to
    # a language subcollection - only put languages expressions for the
    # ones we want in the index
    my @languages = ();
    my $languagemetadata = "Language";
    if (defined ($self->{'collect_cfg'}->{'languagemetadata'})) {
	$languagemetadata = $self->{'collect_cfg'}->{'languagemetadata'};
    }
    @languages = split /,/, $language if (defined $language);
    foreach my $language (@languages) {
	my $not=0;
	if ($language =~ s/^\!//) {
	    $not = 1;
	}
	if($not) {
	    push (@$langarr, "!$language");
	} else {
	    push (@$langarr, "$language");
	}
    }

    # Build index dictionary. Uses verbatim stem method
    print $outhandle "\n    creating index dictionary (solr_passes -I1)\n"  if ($self->{'verbosity'} >= 1);
    print STDERR "<Phase name='CreatingIndexDic'/>\n" if $self->{'gli'};
    my ($handle);

    if ($self->{'debug'}) {
	$handle = *STDOUT;
    } else {
	my $site        = $self->{'site'};
	my $collection     = $self->{'collection'};
	my $core_prefix = (defined $site) ? "$site-$collection" : $collection;
	my $ds_idx      = ($llevel eq 'document')? "didx" : "sidx";#$self->{'index_mapping'}->{$index};
	my $core        = "$core_prefix-$ds_idx";

	$core = "building-".$core unless $self->{'incremental'}; # core points to building only for force_removeold

        # we should be able to remove build dir from here as solr_passes should now only pipe stuff to solr
	print STDERR "Cmd: $solr_passes_exe $core index \"$build_dir\"   $osextra\n";
	if (!open($handle, "| $solr_passes_exe $core index \"$build_dir\"   $osextra")) {
	    print STDERR "<FatalError name='NoRunSolrPasses'/>\n</Stage>\n" if $self->{'gli'};
	    die "solrbuilder::build_index - couldn't run $solr_passes_exe\n!$\n";
	}
    }

    my $store_levels = $self->{'levels'};
    my $db_level = "section"; #always - do we still need this?
    my $local_levels = { $llevel => 1 }; # work on one level at a time

    # set up the document processr
    $self->{'buildproc'}->set_output_handle ($handle);
    $self->{'buildproc'}->set_mode ('text');
    $self->{'buildproc'}->set_index ($index, $indexexparr);
    $self->{'buildproc'}->set_text_index($has_text_index); # 1 or 0
    $self->{'buildproc'}->set_index_languages ($languagemetadata, $langarr) if (defined $language);
    $self->{'buildproc'}->set_indexing_text (1);
    #$self->{'buildproc'}->set_indexfieldmap ($self->{'indexfieldmap'});
    $self->{'buildproc'}->set_levels ($local_levels);
    if (defined $self->{'collect_cfg'}->{'sortfields'}) {
	$self->{'buildproc'}->set_sortfields ($self->{'collect_cfg'}->{'sortfields'});
    }
    if (defined $self->{'collect_cfg'}->{'facetfields'}) {
	$self->{'buildproc'}->set_facetfields ($self->{'collect_cfg'}->{'facetfields'});
    }
    $self->{'buildproc'}->set_db_level($db_level);
    $self->{'buildproc'}->reset();

    print $handle "<update>\n";

    &plugin::read ($self->{'pluginfo'}, $self->{'source_dir'},
		   "", {}, {}, $self->{'buildproc'}, $self->{'maxdocs'}, 0, $self->{'gli'});
	
    print $handle "</update>\n";

    close ($handle) unless $self->{'debug'};

    $self->print_stats();

    # reinstate the main levels list - is it used for something else??
    $self->{'buildproc'}->set_levels ($store_levels);
    print STDERR "</Stage>\n" if $self->{'gli'};
    

    ##print STDERR "@@@@@ FINISHED PROCESSING INDEX: indexlevel $self->{'index_mapping'}->{$index}\n\n";

}


sub post_build_indexes {
    my $self = shift(@_);

    # if the user had specified 'metadata' as an index, there may
    # be new fields to add into thee map
    $self->make_final_field_list('indexfieldmap');
    my $solr_server = $self->{'solr_server'};

    # 1 Aug 2019: now we unload (remove) building-cores for each index during buildcol itself
    # instead of during activate.
    # Kathy described a problem that when calling buildcol.pl successively without succeeding each
    # with a call to activate, there were Win file lock issues when attempting to manually remove
    # the building folder. This was what activate was to solve, however, there's no reason to
    # call activate after buildcol in such cases where it is known the buildcol failed in some way.
    # (In such cases, the user building the collection would have to manually unload the building-
    # cores through the solr servlet interface).
    # Dr Bainbridge instructed that the building- cores should be unloaded again at the end
    # of buildcol. And any symmetrical step during pre-building, if any is found necessary.
    # I'm still not sure this won't break activate in some way, for some combination,
    # as that is meant to ensure building-cores exist whenever the building folder exists...
    # But I was asked not to take to long on this, so I can't test all the different combinations
    # (removeold/incremental/..., or even remote GS situation) in which building can happen and in
    # which buildcol can be combined or not with activate, or be sequenced with further calls to
    # buildcol, with or without -activate.
    # So to compensate, I've tried to keep the code changes as conservative as possible to keep the chances
    #for things going wrong to a minimum, by pinging for building-* cores before unloading them here
    # in solrbuilder.pm (note that unload doesn't delete the index directory associated with the core)
    # and then in activate.pl the building-* cores get pinged again to determine whether they exist
    # before attempting to unload them there as well, since I can no longer assume the cores exist
    # and can be unloaded. There is now the additional overhead of all the extra pinging going on,
    # but it helps ensure we only unload building-* cores when they exist.

    # Note that pre-build-indexes() was already creating the building- cores, so don't need to
    # worry about the needed symmetry at start and end of buildcol to create building- cores
    # in symmetry with unloading them here.

    # update: now that we can do subcollections with solr, we don't know what cores may have been there
    # lets just removes all building cores for the collection.
    my $site        = $self->{'site'};
    my $collection     = $self->{'collection'};
    my $core_prefix = (defined $site) ? "$site-$collection" : $collection;    
    my $build_dir = $self->{'build_dir'};
    
    $solr_server->admin_unload_all_cores_for_prefix("building-$core_prefix");

    # Also need to stop the Solr server (be it tomcat or jetty) if it was explicitly started
    # in pre_build_indexes()

    if ($solr_server->explicitly_started()) {
	$solr_server->stop();
    }

    $self->{'solr_server'} = undef;

}    

sub build_cfg_extra {
    my $self = shift (@_);
    my ($build_cfg) = @_;

    $self->lucenebuilder::build_cfg_extra($build_cfg);

    # need to add in facet stuff
    my @facetfields = ();
    my @facetfieldmap = ();

    foreach my $sf (@{$self->{'buildproc'}->{'facetfields'}}) {
	if ($self->{'buildproc'}->{'actualfacetfields'}->{$sf}) {
	    my $shortname = $self->{'buildproc'}->{'facetfieldnamemap'}->{$sf};
	    push(@facetfields, $shortname);
	    push (@facetfieldmap, "$sf\-\>$shortname");
	}
	
    }
    $build_cfg->{'indexfacetfields'} = \@facetfields;
    $build_cfg->{'indexfacetfieldmap'} = \@facetfieldmap;
    # store the core names in buildConfig, so that activate.pl can use them,
    $build_cfg->{'solrcores'} = $self->{'solrcores'};
    $build_cfg->{'solrcoreforretrieval'} = $self->{'solrcoreforretrieval'};

    }

#####################################
#The following functions are deliberately not object member functions.
#We call them from activate.pl and don't want to instantiate the builder + buildproc etc

sub get_collection_core_directory {
    my ($site, $collection) = @_;

    return &FileUtils::filenameConcatenate($ENV{'GSDL3DATAHOME'}, "ext", "solr9", "cores", $site, $collection);
}

# need incremental arg??
sub pre_activate {
    my ($buildcfg, $site, $collection, $incremental) = @_;

    if ($incremental) {
        # no need to mess around with renaming or moving cores as we just built into the current core.
        return;
    }
    my $core_basename = "$site-$collection";
	
    # If the Solr server is already running, then we unload the cores and
    # recreate them
    # if its not running, we just modify files on the file system
    my $solr_server = new solrserver();
    my $server_running = $solr_server->server_running();
    #    $solr_server->start();
    
#    $gsserver->print_task_msg("unloading all solr cores associated with $core_basename & building-$core_basename");

    if ($server_running) {
        #print STDERR "unloading all cores for $core_basename and building-$core_basename\n";
        $solr_server->admin_unload_all_cores_for_prefix($core_basename);
        $solr_server->admin_unload_all_cores_for_prefix("building-$core_basename");
    }
#    if ($solr_server->explicitly_started()) {
#	$solr_server->stop();
#    }

    my @corenames = @{$buildcfg->{'solrcores'}};

    my %valid_cores;
    foreach my $corename (@corenames) {
        
        $valid_cores{"building-$core_basename-$corename"} = 1;
        #print STDERR "adding to valid building-$core_basename-$corename\n";
    }
    # look through the data folder, and rename any folder with building- in front
    my $data_dir = &get_collection_core_directory($site, $collection);
    # only want directories
    my $options = { 'strict' => 1, 'exclude_files' => 1 };
    my ($ret_val, $contents) = &FileUtils::_readdirWithOptions($data_dir, $options);
    if (scalar(@$contents) == 0) {
        print STDERR "no cores found in $data_dir\n";
        return;
    }

    foreach my $dir (@$contents) {
        if (! ($valid_cores{$dir} == 1)) {
            #print STDERR "$dir in $data_dir is not valid, removing it\n";
            my $full_dir = &FileUtils::filenameConcatenate($data_dir, $dir);
            
            &FileUtils::removeFilesRecursive($full_dir);
            if (-e $full_dir) {
                print STDERR "Couldn't remove all of directory $full_dir\n";
                # should this stop the build??
            }
            #if (&FileUtils::isDirectoryEmpty($full_dir)) {
            #    rmdir($full_dir);
            #}
        }
    }
    foreach my $corename (@corenames) {
        my $new_dir = &FileUtils::filenameConcatenate($data_dir, "$core_basename-$corename");
        my $old_dir = &FileUtils::filenameConcatenate($data_dir, "building-$core_basename-$corename");
        #print STDERR "renaming $old_dir => $new_dir\n";
        &FileUtils::renameDirectory($old_dir, $new_dir); # how can this work if the server is running??? - we have unloaded the cores
        if (!$server_running) {
            my $data_dir = &FileUtils::filenameConcatenate($ENV{'GSDLCOLLECTDIR'}, "index", $corename);
            &generate_core_properties($new_dir, "$core_basename-$corename", $data_dir);
        }
    }
}

sub generate_core_properties {
    my ($dir, $corename, $datadir) = @_;

    my $text = "#Written by solrbuilder\nname=$corename\ndataDir=$datadir\n";
    my $prop_file = &FileUtils::filenameConcatenate($dir, "core.properties");

    &FileUtils::writeUTF8File($prop_file, \$text);
}

# if solr server is running, we need to create the new cores (unless incremental)
sub post_activate {
    my ($buildcfg, $site, $collection, $incremental) = @_;

    if ($incremental) {
        return;
    }
    # If the Solr server is not currently running, we don't need to do anything here
    my $solr_server = new solrserver();
    if ($solr_server->server_running()) {
        # Call CREATE action to get the old cores pointing to the index folder
        #  -- any building or index cores have been unloaded already
        #  -- load up the new one
        my $core_basename = "$site-$collection";
        my @corenames = @{$buildcfg->{'solrcores'}};
        
        foreach my $corename (@corenames) {
            # should be corename, site collection
            $solr_server->admin_create_core("$core_basename-$corename", $site, $collection);
        }
    }
}

# this is called if a collections indexes were solr originally, but have now
# changed to something else

sub cleanup_after_buildtype_change {
    my ($site, $collection, $index_dir, $verbosity) = @_;
    my $solr_server = new solrserver();
    my $server_running = $solr_server->server_running();
    $verbosity = 2 unless defined $verbosity;

    print STDERR "in solr cleanup after buildtype change\n" if ($verbosity >= 2);
    if ($server_running) {
        print STDERR "unloading cores\n" if ($verbosity >= 2);
        $solr_server->admin_unload_all_cores_for_prefix("building-$site-$collection");
        $solr_server->admin_unload_all_cores_for_prefix("$site-$collection");
    }

    my $data_dir = &get_collection_core_directory($site, $collection);

    if (&FileUtils::directoryExists($data_dir)) {
        # delete this directory
        print STDERR "deleting files in $data_dir\n" if ($verbosity >= 2);
        &FileUtils::removeFilesRecursive($data_dir);
    }

    # now remove solr-conf from etc - actually, maybe we shouldn't?? as the user may have edited these files, and might want to go back to solr?
    #my $collect_home = $ENV{'GSDLCOLLECTDIR'};
    #my $conf_dir = &FileUtils::filenameConcatenate($collect_home,"etc","solr-conf");
    #if (&FileUtils::directoryExists($conf_dir)) {
    #    &FileUtils::removeFilesRecursive($conf_dir);
    #}
}

    
1;