##########################################################################
#
# modmetadataaction.pm -- 
# A component of the Greenstone digital library software
# from the New Zealand Digital Library Project at the 
# University of Waikato, New Zealand.
#
# Copyright (C) 2009 New Zealand Digital Library Project
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
#
###########################################################################

# This is a submodule of metadataaction.pm. It is loaded conditionally
# upon having the permissions to modify (not just get) metadata.
# With sufficient permissions, these additional subroutines should be made
# available to metadataaction.pm

# See http://www.perlmonks.org/?node_id=881761 for splitting module into multiple files
# and how variables declared with 'our' are used there.

package metadataaction;

use strict;

use cgiactions::baseaction;
use cgiactions::metadataactioncommon;

use dbutil;
use ghtml;

use JSON;

# convenient to have shorter version around
my $FLDV_HISTORY_DIR = $metadataactioncommon::FLDV_HISTORY_DIR;

$metadataaction::modmeta_action_table = #OR: our $modmeta_action_table =
{
	#SET METHODS
	"set-live-metadata" => { 
		'compulsory-args' => [ "d", "metaname", "metavalue" ],
		'optional-args'   => [ ] },

	"set-metadata" => { # generic set-meta function. The 'where' param can be set to any combination of index|archives|import|live. docid d is still compulsory for setting index, archives and live meta
		'compulsory-args' => [ "metaname", "metavalue" ],
		'optional-args'   => [ "where", "metapos", "metamode", "prevmetavalue", "d", "dv", "f" ] }, 

	"set-index-metadata" => { 
		'compulsory-args' => [ "d", "metaname", "metavalue" ],
		'optional-args'   => [ "metapos", "metamode", "prevmetavalue" ] },

	"set-archives-metadata" => { 
		'compulsory-args' => [ "d", "metaname", "metavalue" ],
		'optional-args'   => [ "dv", "metapos", "metamode", "prevmetavalue" ] }, # metamode can be "accumulate", "override",

        ###### !!!!!
	# Should there not be a "set-archives-text" also ???!!!!??
	###### !!!!!

	"set-archives-assocfile" => {
	    'compulsory-args' => [ "d", "assocname" ],
	    'optional-args'   => [ "dv", "fileupload", "filedata" ] ,
	    'help-string'     => [ "Because this action uploads a file, this action needs to be POSTed. The 'fileupload' parameter/field is the uploaded file content; 'assocname' specifies the filename within the archives's document's area that it will be saved as" ]
	}, 
	    
	"set-import-metadata" => { 
		'compulsory-args' => [ "metaname", "metavalue" ],
		'optional-args'   => [ "d", "f", "metamode", "metapos", "prevmetavalue" ] }, # metamode can be "accumulate", "override", or "unique-id". Also need to add the ability to specify a previous metadata value to overwrite (because we can't use metapos). Metapos now supported, but assumes you are working with a Simple (instead of Complex) collection
				 
	#SET METHODS (ARRAY)
	"set-metadata-array" => { 
		'compulsory-args' => [ "where", "json" ],
		'optional-args'   => [ "metamode" ],
		'help-string' => [
	    'A simple example: metadata-server.pl?a=set-metadata-array&where=archives|index|import&c=demo&json=[{"docid":"HASHc5bce2d6d3e5b04e470ec9","metaname":"Title","metavalue":"Tralalala","metamode":"accumulate"},{"docid":"HASHbe483fa4df4e096335d1c8","metaname":"Title","metavalue":"Lala was here","metapos":0, "metamode":"override"}]', 
	    
	    'A more complex example: metadata-server.pl?a=set-metadata-array&where=archives|index&c=demo&json=[{"docid":"HASHc5bce2d6d3e5b04e470ec9.1","metatable":[{"metaname":"Title","metavals":["Transformers","Robots in disguise","Autobots"]}],"metamode":"override"},{"docid":"HASHbe483fa4df4e096335d1c8.2","metaname":"Title","metavalue":"Pinky was here","metamode":"accumulate"}]' ] },

# The same examples rewritten for when running the metadata-server.pl script from the commandline:

# the simple example: metadata-server.pl a="set-metadata-array" where="archives|index|import" c="demo" json="[{\"docid\":\"HASHc5bce2d6d3e5b04e470ec9\",\"metaname\":\"Title\",\"metavalue\":\"Tralalala\",\"metamode\":\"accumulate\"},{\"docid\":\"HASHbe483fa4df4e096335d1c8\",\"metaname\":\"Title\",\"metavalue\":\"Lala was here\",\"metapos\":0, \"metamode\":\"override\"}]",
	    
# the more complex example: metadata-server.pl a="set-metadata-array" where="archives|index" c="demo" json="[{\"docid\":\"HASHc5bce2d6d3e5b04e470ec9.1\",\"metatable\":[{\"metaname\":\"Title\",\"metavals\":[\"Transformers\",\"Robots in disguise\",\"Autobots\"]}],\"metamode\":\"override\"},{\"docid\":\"HASHbe483fa4df4e096335d1c8.2\",\"metaname\":\"Title\",\"metavalue\":\"Pinky was here\",\"metamode\":\"accumulate\"}]"
					 
	"set-archives-metadata-array" => { 
		'compulsory-args' => [ "json" ],
		'optional-args'   => [ "metamode" ] },
		
	"set-import-metadata-array" => {
		'compulsory-args' => [ "json" ],
		'optional-args'   => [ "metamode" ] },

	"set-index-metadata-array" => {
		'compulsory-args' => [ "json" ],
		'optional-args'   => [ "metamode" ] },
	
	"set-live-metadata-array" => {
		'compulsory-args' => [ "json" ],
		'optional-args'   => [ ] },
		
	#REMOVE METHODS
	"remove-import-metadata" => { 
		'compulsory-args' => [ "d", "metaname" ], #TODO: add f argument
		'optional-args'   => [ "metapos", "metavalue", "metamode" ] }, # only provide metapos arg for SIMPLE collections.
# Metavalue is now an optional arg for remove_import_metadata() based on what the implementation did, which allowed metavalue to be undefined, and if so, used metapos.
					 
	"remove-archives-metadata" => { 
		'compulsory-args' => [ "d", "metaname" ], #TODO: add f argument
		'optional-args'   => [ "dv", "metapos", "metavalue", "metamode" ] },

	"remove-live-metadata" => {
		'compulsory-args' => [ "d", "metaname" ],
		'optional-args'   => [ ] },

	"remove-index-metadata" => {
		'compulsory-args' => [ "d", "metaname" ],
		'optional-args'   => [ "metapos", "metavalue" ] },

	"remove-metadata" => { # generic remove-meta function. The 'where' param can be set to any combination of index|archives|import|live. docid d is still compulsory for setting index, archives and live meta
		'compulsory-args' => [ "d", "metaname" ],
		'optional-args'   => [ "where", "dv", "metapos", "metavalue", "metamode" ] }, # metamode is optional since remove-metadata can call any of remove_import_meta and remove_archives_meta, remove_index_meta, of which the first two accept metamode as an optional param

	#REMOVE METHODS (ARRAY)
	"remove-metadata-array" => { 
		'compulsory-args' => [ "where", "json" ],
		'optional-args'   => [ ],
		    'help-string' => [
			"No remove-metadata-array examples yet."
		    ] },
	"remove-archives-metadata-array" => { 
		'compulsory-args' => [ "json" ],
		'optional-args'   => [ ] },
		
	"remove-import-metadata-array" => {
		'compulsory-args' => [ "json" ],
		'optional-args'   => [ ] },

	"remove-index-metadata-array" => {
		'compulsory-args' => [ "json" ],
		'optional-args'   => [ ] },
	
	"remove-live-metadata-array" => {
		'compulsory-args' => [ "json" ],
		    'optional-args'   => [ ] },
	    
	#REMOVE MULTIPLE (REMOVE ALL MATCHING METANAME AT DOCID)
	"erase-import-metadata" => { 
	    'compulsory-args' => [ "d", "metaname" ] },

	"erase-archives-metadata" => { 
	    'compulsory-args' => [ "d", "metaname" ],
	    'optional-args'   => [ "dv" ] },

	"erase-index-metadata" => { 
	    'compulsory-args' => [ "d", "metaname" ] },
	
	"erase-live-metadata" => { 
	    'compulsory-args' => [ "d", "metaname" ] },
	
	"erase-metadata" => { # generic erase-meta function. The 'where' param can be set to any combination of index|archives|import|live. docid d is still compulsory for setting index, archives and live meta
		'compulsory-args' => [ "d", "metaname" ],
		'optional-args'   => [ "where" ] },
	
	
	#INSERT METHODS
	"insert-metadata" => {
	    'compulsory-args' => [ "d", "metaname", "metavalue" ],
	    'optional-args'   => [ ] },


	#INC METHODS
	"inc-fldv-nminus1" => {
	    'compulsory-args' => [ "d" ],
	    'optional-args'   => [ ] }	    
};

sub _set_live_metadata
{
    my $self = shift @_;

    my $collect   = $self->{'collect'};
    my $gsdl_cgi  = $self->{'gsdl_cgi'};
    my $infodbtype = $self->{'infodbtype'};

    # Obtain the collect dir
    my $site = $self->{'site'};
    my $collect_dir = $gsdl_cgi->get_collection_dir($site);


    # look up additional args
    my $docid     = $self->{'d'};
    if ((!defined $docid) || ($docid =~ m/^\s*$/)) {
      $gsdl_cgi->generate_error("No docid (d=...) specified."); # generates error and dies
    }
    my $metavalue = $self->{'metavalue'};

    # Generate the dbkey    
    my $metaname  = $self->{'metaname'};
    my $dbkey = "$docid.$metaname";

    # To people who know $collect_tail please add some comments
    # Obtain path to the database
    my $collect_tail = $collect;
    $collect_tail =~ s/^.*[\/|\\]//;
    my $index_text_directory = &util::filename_cat($collect_dir,$collect,"index","text");
    my $infodb_file_path = &dbutil::get_infodb_file_path($infodbtype, "live-$collect_tail", $index_text_directory);

    # Set the new value
    my $cmd = "gdbmset \"$infodb_file_path\" \"$dbkey\" \"$metavalue\"";
    my $status = system($cmd);
    if ($status != 0) {
        # Catch error if gdbmget failed
	my $mess = "Failed to set metadata key: $dbkey\n";

	$mess .= "PATH: $ENV{'PATH'}\n";
	$mess .= "cmd = $cmd\n";
	$mess .= "Exit status: $status\n";
	$mess .= "System Error Message: $!\n";

	$gsdl_cgi->generate_error($mess);
    }
    else {
	$gsdl_cgi->generate_ok_message("set-live-metadata successful: Key[$metaname]=$metavalue");
    }

    #return $status; # in case calling functions have any further use for this
}

sub set_live_metadata
{
    my $self = shift @_;

    my $username  = $self->{'username'};
    my $collect   = $self->{'collect'};
    my $gsdl_cgi  = $self->{'gsdl_cgi'};
  
    if ($baseaction::authentication_enabled) {
	# Ensure the user is allowed to edit this collection	
	$self->authenticate_user($username, $collect); #&authenticate_user($gsdl_cgi, $username, $collect);
    }

    # Make sure the collection isn't locked by someone else
    $self->lock_collection($username, $collect);

    $self->_set_live_metadata(@_);

    # Release the lock once it is done
    $self->unlock_collection($username, $collect);
}

sub set_index_metadata_entry
{
    my $self = shift @_;
    my ($collect_dir,$collect,$infodbtype,$docid,$metaname,$metapos,$metavalue,$metamode,$prevmetavalue) = @_;

    $metapos = undef if(defined $metapos && ($metapos =~ m/^\s*$/));
    $prevmetavalue = undef if(defined $prevmetavalue && ($prevmetavalue =~ m/^\s*$/));
    
    # To people who know $collect_tail please add some comments
    # Obtain path to the database
    my $collect_tail = $collect;
    $collect_tail =~ s/^.*[\/|\\]//;
    my $index_text_directory = &util::filename_cat($collect_dir,$collect,"index","text");
    my $infodb_file_path = &dbutil::get_infodb_file_path($infodbtype, $collect_tail, $index_text_directory);
	
#	print STDERR "**** infodb file path = $infodb_file_path\n";
#	print STDERR "***** infodb type = $infodbtype\n";
    
    # Read the docid entry
    my $doc_rec = &dbutil::read_infodb_entry($infodbtype, $infodb_file_path, $docid);
    
    # Set the metadata value
    if (defined $metapos) {
	# if metamode=accumulate AND metapos, warn user and then use metapos
	if (defined $metamode && $metamode eq "accumulate") {
	    print STDERR "**** Warning: metamode is set to accumulate yet metapos is also provided for $docid\n";
	    print STDERR "**** Proceeding by using metapos\n";
	}
	$doc_rec->{$metaname}->[$metapos] = $metavalue;
    }
    elsif (defined $prevmetavalue) {
	my $array = $doc_rec->{$metaname};
        my $length = @$array;

	my $found = 0;
        for (my $i = 0; $i < $length; $i++){
            if(defined $doc_rec->{$metaname}->[$i] && $doc_rec->{$metaname}->[$i] eq $prevmetavalue){
                $doc_rec->{$metaname}->[$i] = $metavalue;
                $found = 1;
                last;				
            }
        }

	if($found == 0){
	    print STDERR "**** Warning: could not find $prevmetavalue to replace. Appending (accumulating) replacement value.\n";
            ##$doc_rec->{$metaname} = [ $metavalue ]; # if prevmetavalue not found, don't overwrite all previous values.
	    # Accumulate is less destructive. But should the correct behaviour be to not do Anything?

	    if(defined $doc_rec->{$metaname}) {
		push(@{$doc_rec->{$metaname}}, $metavalue); # accumulate the value for that metaname
	    } else {
		$doc_rec->{$metaname} = [ $metavalue ];
	    }
        }
    }
    elsif (defined $metamode && $metamode eq "override") {
	$doc_rec->{$metaname} = [ $metavalue ];	
    }
    else { # default for index was to override, but because accumulate is less destructive, 
	# and because accumulate is the default for archives and import, that's the new default for index too
	if(defined $doc_rec->{$metaname}) {
	    push(@{$doc_rec->{$metaname}}, $metavalue); # accumulate the value for that metaname
	} else {
	    $doc_rec->{$metaname} = [ $metavalue ];
	}
    }
 
    my $status = &dbutil::set_infodb_entry($infodbtype, $infodb_file_path,$docid,$doc_rec);
	
    return $status;
	
}

sub _set_import_metadata
{
    my $self = shift @_;

    my $collect   = $self->{'collect'};
    my $gsdl_cgi  = $self->{'gsdl_cgi'};
    my $infodbtype = $self->{'infodbtype'};
     
    # Obtain the collect and archive dir    
    my $site = $self->{'site'};
    my $collect_dir = $gsdl_cgi->get_collection_dir($site);
    my $archive_dir = &util::filename_cat($collect_dir,$collect,"archives");
   
    # look up additional args
    # want either d= or f=
    my $docid  = $self->{'d'};
    my ($docid_root,$docid_secnum);
    if(defined $docid) {	
	($docid_root,$docid_secnum) = ($docid =~ m/^(.*?)(\..*)?$/);	
	# as yet no support for setting subsection metadata in metadata.xml
	if ((defined $docid_secnum) && ($docid_secnum !~ m/^\s*$/)) {
	    $gsdl_cgi->generate_message("*** No support yet for setting import metadata at subsections level.\n");
	    return;
	}
    }

    my $import_file  = $self->{'f'};
    if ((!defined $docid || $docid =~ m/^\s*$/) && (!defined $import_file || $import_file =~ m/^\s*$/)) {
	$gsdl_cgi->generate_error("No docid (d=...) or import file (f=) specified."); # at least d or f must be specified
    } 

    # Get the parameters and set default mode to "accumulate"
    my $metaname   = $self->{'metaname'};
    my $metavalue  = $self->{'metavalue'};
##    $metavalue =~ s/&amp;lt;(.*?)&amp;gt;/<$1>/g;
    $metavalue =~ s/&lt;(.*?)&gt;/<$1>/g;
    
    my $metamode   = $self->{'metamode'};
    if ((!defined $metamode) || ($metamode =~ m/^\s*$/)) {
	# make "accumulate" the default (less destructive, as it won't actually 
	# delete any existing values)
	$metamode = "accumulate";
    }

    # adding metapos and prevmetavalue support to import_metadata subroutines
    my $metapos   = $self->{'metapos'}; # don't force undef to 0. Undef has meaning when metamode=override
    my $prevmetavalue = $self->{'prevmetavalue'};
    $metapos = undef if(defined $metapos && ($metapos =~ m/^\s*$/));
    $prevmetavalue = undef if(defined $prevmetavalue && ($prevmetavalue =~ m/^\s*$/));

    my $arcinfo_doc_filename = &dbutil::get_infodb_file_path($infodbtype, "archiveinf-doc", $archive_dir);
    my $metadata_xml_filename = $self->set_import_metadata_entry($gsdl_cgi, $arcinfo_doc_filename, $infodbtype, $docid_root, $metaname, $metapos,$metavalue, $metamode,$prevmetavalue, $collect, $collect_dir); # at this point, docid_root = docid

    my $mess = "set-import-metadata successful: Key[$docid] -> $metadata_xml_filename\n";
    $mess .= "  $metaname";
    $mess .= " = $metavalue";
    $mess .= " ($metamode)\n";

    $gsdl_cgi->generate_ok_message($mess);

    #return $status; # in case calling functions have any further use for this
}

# the version of set_archives_meta that doesn't do authentication
sub _set_archives_metadata
{
    my $self = shift @_;

    my $collect   = $self->{'collect'};
    my $gsdl_cgi  = $self->{'gsdl_cgi'};
    my $infodbtype = $self->{'infodbtype'};
    
    # Obtain the collect and archive dir   
    my $site = $self->{'site'};
    my $collect_dir = $gsdl_cgi->get_collection_dir($site);	
    my $archive_dir = &util::filename_cat($collect_dir,$collect,"archives");

    # look up additional args
    my $docid  = $self->{'d'};
    my $dv     = $self->{'dv'};
    
    my $metaname   = $self->{'metaname'};
    my $metavalue  = $self->{'metavalue'};
    my $prevmetavalue = $self->{'prevmetavalue'};
    
    my $metapos    = $self->{'metapos'}; # don't force undef to 0. Undef has meaning when metamode=override
                                  # Don't append "|| undef", since if metapos=0 it will then be set to undef

    $metapos = undef if(defined $metapos && ($metapos =~ m/^\s*$/));
    $prevmetavalue = undef if(defined $prevmetavalue && ($prevmetavalue =~ m/^\s*$/));

    my $metamode   = $self->{'metamode'};
    if ((!defined $metamode) || ($metamode =~ m/^\s*$/)) {
	# make "accumulate" the default (less destructive, as it won't actually 
	# delete any existing values)
	$metamode = "accumulate";
    }

    my $status = $self->set_archives_metadata_entry($gsdl_cgi,$archive_dir, $collect_dir,$collect, $infodbtype,
						    $docid,$dv, $metaname,$metapos,$metavalue,$metamode,$prevmetavalue);
   
    if ($status == 0) {
	my $mess = "set-archives-metadata successful: Key[$docid]\n";
	$mess .= "  $metaname";
	$mess .= "->[$metapos]" if (defined $metapos);
	$mess .= " = $metavalue";
	$mess .= " ($metamode)\n";
	
	$gsdl_cgi->generate_ok_message($mess);	
    }
    else {
	my $mess .= "Failed to set archives metadata key: $docid\n";
	$mess .= "Exit status: $status\n";
	if(defined $self->{'error_msg'}) {
	    $mess .= "Error Message: $self->{'error_msg'}\n";
	} else {
	    $mess .= "System Error Message: $!\n";
	}
	$mess .= "-" x 20 . "\n";
	
	$gsdl_cgi->generate_error($mess);
    }

    #return $status; # in case calling functions have any further use for this
}


# the version of set_index_meta that doesn't do authentication
sub _set_index_metadata
{
    print STDERR "START SET INDEX METADATA\n";
    my $self = shift @_;

    my $collect   = $self->{'collect'};
    my $gsdl_cgi  = $self->{'gsdl_cgi'};

    # Obtain the collect dir
    my $site = $self->{'site'};
    my $collect_dir = $gsdl_cgi->get_collection_dir($site);

    # look up additional args
    my $docid     = $self->{'d'};
    my $metaname  = $self->{'metaname'};
    my $metapos   = $self->{'metapos'}; # undef has meaning
    my $metavalue = $self->{'metavalue'};
    my $infodbtype = $self->{'infodbtype'};
    my $metamode  = $self->{'metamode'};
    my $prevmetavalue = $self->{'prevmetavalue'};

    print STDERR "SETTING INDEX METADATA ENTRY\n";
    my $status = $self->set_index_metadata_entry($collect_dir,$collect,$infodbtype,$docid,$metaname,$metapos,$metavalue,$metamode,$prevmetavalue);
    print STDERR "DONE SETTING INDEX METADATA ENTRY\n";
    if ($status != 0) {
        # Catch error if set infodb entry failed
	my $mess = "Failed to set metadata key: $docid\n";
	
	$mess .= "PATH: $ENV{'PATH'}\n";
	$mess .= "Exit status: $status\n";
	$mess .= "System Error Message: $!\n";
	
	$gsdl_cgi->generate_error($mess);
    }
    else {
	my $mess = "set-index-metadata successful: Key[$docid]\n";
	$mess .= "  $metaname";
	$mess .= "->[$metapos]" if (defined $metapos);
	$mess .= " = $metavalue\n";
	
	$gsdl_cgi->generate_ok_message($mess);
    }

    print STDERR "END SET INDEX METADATA\n";
    #return $status; # in case calling functions have any further use for this
}

sub set_index_metadata
{
    my $self = shift @_;

    my $username  = $self->{'username'};
    my $collect   = $self->{'collect'};
    my $gsdl_cgi  = $self->{'gsdl_cgi'};

    if ($baseaction::authentication_enabled) {
	# Ensure the user is allowed to edit this collection	
	$self->authenticate_user($username, $collect); #&authenticate_user($gsdl_cgi, $username, $collect);
    }

    # Obtain the collect dir
    my $site = $self->{'site'};
    my $collect_dir = $gsdl_cgi->get_collection_dir($site);
	
    $gsdl_cgi->checked_chdir($collect_dir);

    # Make sure the collection isn't locked by someone else
    $self->lock_collection($username, $collect);

    $self->_set_index_metadata(@_);
    
    # Release the lock once it is done
    $self->unlock_collection($username, $collect);
}

# call this to set the metadata for a combination of dirs archives, import or index, or live
# if none specified, defaults to index which was the original behaviour of set_metadata.
sub set_metadata
{
    my $self = shift @_;

    # Testing that not defining a variable, setting it to "" or to "  " all return false
    # >perl -e 'my $whichdirs=""; if($whichdirs) {print "$whichdirs\n"};'

    my $where = $self->{'where'};
    if(!$where || ($where =~ m/^\s*$/)) {	
	$self->set_index_metadata(@_); # call the full version of set_index_meta for the default behaviour
	return;
    } 

    # authenticate and lock collection once, even if processing multiple dirs
    my $username  = $self->{'username'};
    my $collect   = $self->{'collect'};
    my $gsdl_cgi  = $self->{'gsdl_cgi'};
    
    if ($baseaction::authentication_enabled) {
	# Ensure the user is allowed to edit this collection	
	$self->authenticate_user($username, $collect); #&authenticate_user($gsdl_cgi, $username, $collect);
    }
    
    if($where =~ m/index/) {
	my $site = $self->{'site'};
	my $collect_dir = $gsdl_cgi->get_collection_dir($site);
	$gsdl_cgi->checked_chdir($collect_dir);
    }

    # Make sure the collection isn't locked by someone else
    $self->lock_collection($username, $collect);


    # now at last can set the metadata. $where can specify multiple
    # $where is of the form: import|archives|index, or a subset thereof

    #my @whichdirs = split('\|', $where);

    # just check whether $where contains import/archives/index/live in turn, and 
    # for each case, process it accordingly
    if($where =~ m/import/) {
	$self->_set_import_metadata(@_);	    
    }

    if($where =~ m/archives/) {

	# look up docID arg which is optional to set_metadata because it's optional 
	# to set_import, but which is compulsory to set_archives_metadata
	my $docid     = $self->{'d'};
	if ((!defined $docid) || ($docid =~ m/^\s*$/)) {
	    $gsdl_cgi->generate_error("No docid (d=...) specified."); # generates error and dies
	} 
	# we have a docid, so can set archives meta
	$self->_set_archives_metadata(@_);	
    }

    if($where =~ m/index/) {
	
	# look up docID arg which is optional to set_metadata because it's optional 
	# to set_import, but which is compulsory to set_archives_metadata
	my $docid     = $self->{'d'};
	if ((!defined $docid) || ($docid =~ m/^\s*$/)) {
	    $gsdl_cgi->generate_error("No docid (d=...) specified.");
	}
	# we have a docid, so can set index meta
	$self->_set_index_metadata(@_);	
    }	

    if($where =~ m/live/) {
	$self->_set_live_metadata(@_); # docid param, d, is compulsory, but is checked for in subroutine
    }

    # Release the lock once it is done
    $self->unlock_collection($username, $collect);
}

sub set_metadata_array
{
    my $self = shift @_;

    my $where = $self->{'where'};
    if(!$where || ($where =~ m/^\s*$/)) {	
	$self->set_index_metadata_array(@_); # default behaviour is the full version of set_index_meta_array
	return;
    }

    my $username  = $self->{'username'};
    my $collect   = $self->{'collect'};
    my $gsdl_cgi  = $self->{'gsdl_cgi'};

    if ($baseaction::authentication_enabled) {
	# Ensure the user is allowed to edit this collection
	$self->authenticate_user($username, $collect); #&authenticate_user($gsdl_cgi, $username, $collect);
    }
    
    # Not sure if the checked_chdir into the collect directory is necessary,
    # since lock_collection does a further chdir into the collection directory anyway
    # But including the stmt during this code reorganisation to preserve as-is what used to happen
    my $site = $self->{'site'};
    my $collect_dir = $gsdl_cgi->get_collection_dir($site);
    $gsdl_cgi->checked_chdir($collect_dir);

    # Make sure the collection isn't locked by someone else
    $self->lock_collection($username, $collect);

    if($where =~ m/import/) {
	$self->_set_import_metadata_array(@_);
    }
    if($where =~ m/archives/) {
	$self->_set_archives_metadata_array(@_);
    }
    if($where =~ m/index/) {
	$self->_set_index_metadata_array(@_);
    }
    if($where =~ m/live/) {
    	$self->_set_live_metadata_array(@_);
    }

    # Release the lock once it is done
    $self->unlock_collection($username, $collect);
}

sub _set_index_metadata_array
{
    my $self = shift @_;

    my $collect   = $self->{'collect'};
    my $gsdl_cgi  = $self->{'gsdl_cgi'};

    # Obtain the collect dir
    my $site = $self->{'site'};
    my $collect_dir = $gsdl_cgi->get_collection_dir($site);
    
    # look up additional args
    
    my $infodbtype = $self->{'infodbtype'};
    
    my $json_str      = $self->{'json'};
    my $doc_array = decode_json $json_str;
    
    
    my $global_status = 0;
    my $global_mess = "";
    
    my @all_docids = ();
    
    foreach my $doc_array_rec ( @$doc_array ) {
	
	my $status = -1;
	my $docid     = $doc_array_rec->{'docid'};
	
	push(@all_docids,$docid);

	my $metaname  = $doc_array_rec->{'metaname'};
	if(defined $metaname) {
	    my $metapos   = $doc_array_rec->{'metapos'}; # can legitimately be undef
	    my $metavalue = $doc_array_rec->{'metavalue'};
	    my $metamode = $doc_array_rec->{'metamode'} || $self->{'metamode'};

	    $status = $self->set_index_metadata_entry($collect_dir,$collect,$infodbtype,$docid,$metaname,$metapos,$metavalue,$metamode);
	} elsif (defined $doc_array_rec->{'metatable'}) { # if no metaname, we expect a metatable
	    my $metatable = $doc_array_rec->{'metatable'}; # a subarray, or need to generate an error saying JSON structure is wrong
	    
	    foreach my $metatable_rec ( @$metatable ) { # the subarray metatable is an array of hashmaps		
		$metaname  = $metatable_rec->{'metaname'};
		my $metamode  = $metatable_rec->{'metamode'} || $doc_array_rec->{'metamode'} || $self->{'metamode'};
		my $metapos = undef;
		my $prevmetaval = undef;
		if(defined $metatable_rec->{'metapos'}) {
		    $metapos = $metatable_rec->{'metapos'};
		    my $metavalue = $metatable_rec->{'metavalue'};
		    $prevmetaval = $metatable_rec->{'prevmetavalue'};
		    $status = $self->set_index_metadata_entry($collect_dir,$collect,$infodbtype,$docid,$metaname,$metapos,$metavalue,$metamode,$prevmetaval);
		} else {
		    my $metavals = $metatable_rec->{'metavals'}; # a sub-subarray
		    
		    foreach my $metavalue ( @$metavals ) { # metavals is an array
			$status = $self->set_index_metadata_entry($collect_dir,$collect,$infodbtype,$docid,$metaname,$metapos,$metavalue,$metamode,$prevmetaval); # how do we use metamode in set_meta_entry?
			if($metamode eq "override") { # now, having overridden the metavalue for the first, 
			    # need to accumulate subsequent metavals for this metaname, else the just-assigned
			    # metavalue for this metaname will be lost
			    $metamode = "accumulate";
			}
		    }
		}
	    }
	}
	
	if ($status != 0) {
	    # Catch error if set infodb entry failed
	    $global_status = $status;
	    $global_mess .= "Failed to set metadata key: $docid\n";
	    $global_mess .= "Exit status: $status\n";
	    $global_mess .= "System Error Message: $!\n";
	    $global_mess .= "-" x 20;
	}
    }
    
    if ($global_status != 0) {
	$global_mess .= "PATH: $ENV{'PATH'}\n";
	$gsdl_cgi->generate_error($global_mess);
    }
    else {
	my $mess = "set-metadata-array successful: Keys[ ".join(", ",@all_docids)."]\n";
	$gsdl_cgi->generate_ok_message($mess);
    }
}

sub set_index_metadata_array
{
    my $self = shift @_;

    my $username  = $self->{'username'};
    my $collect   = $self->{'collect'};
    my $gsdl_cgi  = $self->{'gsdl_cgi'};

    if ($baseaction::authentication_enabled) {
	# Ensure the user is allowed to edit this collection	
	$self->authenticate_user($username, $collect); #&authenticate_user($gsdl_cgi, $username, $collect);
    }

    # Obtain the collect dir
    my $site = $self->{'site'};
    my $collect_dir = $gsdl_cgi->get_collection_dir($site);
	
    $gsdl_cgi->checked_chdir($collect_dir);


    # Make sure the collection isn't locked by someone else
    $self->lock_collection($username, $collect);

    $self->_set_index_metadata_array(@_);

    # Release the lock once it is done
    $self->unlock_collection($username, $collect);
}

# experimental, newly added in and untested
sub _set_live_metadata_array
{
    my $self = shift @_;

    my $collect   = $self->{'collect'};
    my $gsdl_cgi  = $self->{'gsdl_cgi'};

    # Obtain the collect dir
    my $site = $self->{'site'};
    my $collect_dir = $gsdl_cgi->get_collection_dir($site);
    
    # look up additional args
    my $infodbtype = $self->{'infodbtype'};
    # To people who know $collect_tail please add some comments
    # Obtain path to the database
    my $collect_tail = $collect;
    $collect_tail =~ s/^.*[\/|\\]//;
    my $index_text_directory = &util::filename_cat($collect_dir,$collect,"index","text");
    my $infodb_file_path = &dbutil::get_infodb_file_path($infodbtype, "live-$collect_tail", $index_text_directory);

    
    my $json_str      = $self->{'json'};
    my $doc_array = decode_json $json_str;
    
    
    my $global_status = 0;
    my $global_mess = "";
    
    my @all_docids = ();


    foreach my $doc_array_rec ( @$doc_array ) {
	
	my $status = -1;
	my $docid     = $doc_array_rec->{'docid'};

	push(@all_docids,$docid);

	my $metaname  = $doc_array_rec->{'metaname'};
	if(defined $metaname) {
	    my $dbkey = "$docid.$metaname";
	    my $metavalue = $doc_array_rec->{'metavalue'};

	    # Set the new value
	    my $cmd = "gdbmset \"$infodb_file_path\" \"$dbkey\" \"$metavalue\"";
	    $status = system($cmd);

	} elsif (defined $doc_array_rec->{'metatable'}) { # if no metaname, we expect a metatable
	    my $metatable = $doc_array_rec->{'metatable'}; # a subarray, or need to generate an error saying JSON structure is wrong
	    foreach my $metatable_rec ( @$metatable ) {
		$metaname  = $metatable_rec->{'metaname'};
		my $dbkey = "$docid.$metaname";

		my $metavals = $metatable_rec->{'metavals'}; # a sub-subarray
		foreach my $metavalue ( @$metavals ) {
		     my $cmd = "gdbmset \"$infodb_file_path\" \"$dbkey\" \"$metavalue\"";
		     $status = system($cmd);
		}
	    }
	    
	}

	if ($status != 0) {
	    # Catch error if gdbmget failed
	    $global_status = $status;
	    $global_mess .= "Failed to set metadata key: $docid\n"; # $dbkey
	    $global_mess .= "Exit status: $status\n";
	    $global_mess .= "System Error Message: $!\n";
	    $global_mess .= "-" x 20;
	}
    }
    
    if ($global_status != 0) {
	$global_mess .= "PATH: $ENV{'PATH'}\n";
	$gsdl_cgi->generate_error($global_mess);
    }
    else {
	my $mess = "set-live-metadata-array successful: Keys[ ".join(", ",@all_docids)."]\n";
	$gsdl_cgi->generate_ok_message($mess);
    }
}

sub set_live_metadata_array
{
    my $self = shift @_;

    my $username  = $self->{'username'};
    my $collect   = $self->{'collect'};
    my $gsdl_cgi  = $self->{'gsdl_cgi'};

    if ($baseaction::authentication_enabled) {
	# Ensure the user is allowed to edit this collection	
	$self->authenticate_user($username, $collect); #&authenticate_user($gsdl_cgi, $username, $collect);
    }

    # Obtain the collect dir
    my $site = $self->{'site'};
    my $collect_dir = $gsdl_cgi->get_collection_dir($site);
	
    $gsdl_cgi->checked_chdir($collect_dir);

    # Make sure the collection isn't locked by someone else
    $self->lock_collection($username, $collect);

    $self->_set_live_metadata_array(@_);

    # Release the lock once it is done
    $self->unlock_collection($username, $collect);
}


sub dxml_metadata
{
    my ($tagname, $attrHash, $contextArray, $parentDataArray, $parser) = @_;
    my $metaname = $parser->{'parameters'}->{'metaname'};
    my $metamode = $parser->{'parameters'}->{'metamode'};
	
###!!!    print STDERR "**** Processing closing </Metadata> tag\n";
    
    my $opt_doc_secnum = $parser->{'parameters'}->{'secnum'};
    
    # Find the right metadata tag and checks if we are going to
    # override it
    #
    # Note: This over writes the first metadata block it
    # encountered. If there are multiple Sections in the doc.xml, it
    # might not behave as you would expect

    my $curr_secnum = $parser->{'parameters'}->{'curr_section_num'};
##    print STDERR "**** checking $opt_doc_secnum <=> $curr_secnum\n" if (defined $opt_doc_secnum);
##    print STDERR "**** metamode = $metamode\n";
    
    if ((!defined $opt_doc_secnum) || ($opt_doc_secnum eq $curr_secnum)) 
	{
	    my $name_attr = $attrHash->{'name'};
	    # print STDOUT "*** testing: $name_attr eq $metaname ?   and  $metamode eq override ?\n";
		if (($name_attr eq $metaname) && ($metamode eq "override")) 
		{
			if (!defined $parser->{'parameters'}->{'poscount'}) 
			{ 
				$parser->{'parameters'}->{'poscount'} = 0; 
			} 
			else 
			{ 
				$parser->{'parameters'}->{'poscount'}++; 
			} 
			
			if (defined $parser->{'parameters'}->{'metapos'}) { # if pos defined, ignore any prevmeta if also defined, and just wait for pos match

			    if ($parser->{'parameters'}->{'poscount'} == $parser->{'parameters'}->{'metapos'}) {
				###print STDERR "#### got pos match!!\n";
				# Get the value and override the current value
				my $metavalue = $parser->{'parameters'}->{'metavalue'};
				$attrHash->{'_content'} = $metavalue;
				
				# until we have clearly determined the role of metamode in doc.xml files (we know what it means in import/metadata.xml files)
				# we're not setting metamode to override here.
				
				# Don't want it to wipe out any other pieces of metadata
			    $parser->{'parameters'}->{'metamode'} = "done";
			    }
			}
			elsif (defined $parser->{'parameters'}->{'prevmetavalue'}) { # if no pos defined but prevmeta defined, then wait for prevmeta match
			    if ($parser->{'parameters'}->{'prevmetavalue'} eq $attrHash->{'_content'}) {
				###print STDERR "### prev meta value matches\n";
				my $metavalue = $parser->{'parameters'}->{'metavalue'};
				$attrHash->{'_content'} = $metavalue;
				
				# until we have clearly determined the role of metamode in doc.xml files (we know what it means in import/metadata.xml files)
				# we're not setting metamode to override here.
				
				$parser->{'parameters'}->{'metamode'} = "done";

			    }
			}
			# Note that before commit rev 32076, if neither metapos nor prevmeta defined, then code tried to set pos 0 for that metaname to the new value			
			# However that is not how the code works now (and couldn't get it to work that way yet after rev 32076).
			# What it does now, and what Dr Bainbridge approved of is that:
			# If no pos or prevmeta defined, then on override, the metavalue set for the specificed metaname becomes the ONLY instance of that metaname
			# So any earlier instances of that metaname is removed and replaced by the single new instance.			
		}
	}

    # RAW is [$tagname => $attrHash] not $tagname => $attrHash!!
    return [$tagname => $attrHash];
}

# This method exists purely for catching invalid section numbers that the client
# requested to edit. Once the parser has reached the end (the final </Archive> tag),
# we've seen all the Sections in the doc.xml, and none of their section nums matched
# if the metamode has not been set to 'done' by then.
sub dxml_archive
{
    my ($tagname, $attrHash, $contextArray, $parentDataArray, $parser) = @_;
    my $metamode = $parser->{'parameters'}->{'metamode'};
    
    my $curr_secnum = $parser->{'parameters'}->{'curr_section_num'};
    my $opt_doc_secnum = $parser->{'parameters'}->{'secnum'};
    
    # print STDERR "@@@ $tagname Processing a closing </Archive> tag [$curr_secnum|$opt_doc_secnum]\n";
    
    if ($metamode ne "done" && $curr_secnum ne $opt_doc_secnum) {
	#print STDERR "@@@ $tagname Finished processing FINAL Section.\n";

	my $metaname = $parser->{'parameters'}->{'metaname'};
	my $metavalue = $parser->{'parameters'}->{'metavalue'};
	
	print STDERR "@@@ Requested section number $opt_doc_secnum not found.\n";
	print STDERR "\t(last seen section number in document was $curr_secnum)\n";
	print STDERR "\tDiscarded metadata value '$metavalue' for meta '$metaname'\n";
	print STDERR "\tin section $opt_doc_secnum.\n";
	$parser->{'custom_err_msg'} = "Requested section number $opt_doc_secnum not found.";
    }
    
    # RAW is [$tagname => $attrHash] not $tagname => $attrHash!!
    return [$tagname => $attrHash];
}

sub dxml_description
{
	my ($tagname, $attrHash, $contextArray, $parentDataArray, $parser) = @_;
	my $metamode = $parser->{'parameters'}->{'metamode'};

	my $curr_secnum = $parser->{'parameters'}->{'curr_section_num'};
	my $opt_doc_secnum = $parser->{'parameters'}->{'secnum'} || "";
	
###!!!	print STDERR "**** Processing a closing </Description> tag \n";
	# print STDERR "@@@ $tagname Processing a closing </Description> tag [$curr_secnum|$opt_doc_secnum]\n";
	
	# Accumulate the metadata

	# We'll be accumulating metadata at this point if we haven't found and therefore
	# haven't processed the metadata yet. 
	# For subsections, this means that if we're at a matching subsection, but haven't 
	# found the correct metaname to override in that subsection, we accumulate it as new
	# meta in the subsection by adding it to the current description.
	# If there's no subsection info for the metadata, it will accumulate at the top level
	# section description if we hadn't found a matching metaname to override at this point.

	# Both curr_secnum and opt_doc_secnum can be "". In the former case, it means we're now
	# at the toplevel section. In the latter case, it means we want to process meta in the
	# toplevel section. So the eq check between the values below will work in all cases.
	
	# The only time this won't work is if an opt_doc_secnum beyond the section numbers of
	# this document has been provided. In that case, the metadata for that opt_doc_secnum
	# won't get attached/accumulated to any part of the doc, not even its top-level section.

	if ($curr_secnum eq $opt_doc_secnum 
	    && ($metamode eq "accumulate" || $metamode eq "override")) {
	    # note, metamode would have been set to done if a metadata operation had already succeeded. 
	    # If we get here, we still have the metadata to append.
	    if ($metamode eq "override") {
		print "Got to end of <Description> block. No metadata value to override.  Switching 'metamode' to accumulate\n";
	    }

	    # If we get to here and metamode is override, this means there 
	    # was no existing value to overide => treat as an append operation
	    
	    # Tack a new metadata tag on to the end of the <Metadata>+ block
	    my $metaname = $parser->{'parameters'}->{'metaname'};
	    my $metavalue = $parser->{'parameters'}->{'metavalue'};
	    
	    my $metadata_attr = { 
		'_content' => $metavalue, 
		'name' => $metaname, 
		'mode' => "accumulate" 
	    };
	    
	    my $append_metadata = [ "Metadata" => $metadata_attr ];
	    my $description_content = $attrHash->{'_content'};
	    print "Appending metadata to doc.xml\n";
	    
	    if (ref($description_content)) {
		# got some existing interesting nested content		
		push(@$description_content, "  ", $append_metadata, "\n  ");
	    }
	    else {
		#description_content is most likely a string such as "\n"
		$attrHash->{'_content'} = [$description_content, "  ", $append_metadata ,"\n" ];
	    }
	    
	    $parser->{'parameters'}->{'metamode'} = "done";
	}	    
	else {
	    # metamode most likely "done" signifying that it has already found a position to add the metadata to. 
##	    print STDERR "**** NOT ACCUMULATE?!? \n";
	}
    # RAW is [$tagname => $attrHash] not $tagname => $attrHash!!
    return [$tagname => $attrHash];
}

sub edit_xml_file
{
    my $self = shift @_;
    my ($gsdl_cgi, $filename, $start_rules, $rules, $options) = @_;

    # use XML::Rules to add it in (read in and out again)
    my $parser = XML::Rules->new(start_rules     => $start_rules,
				 rules           => $rules, 
				 style           => 'filter',
                                 output_encoding => 'utf8' );

    my $xml_in = "";
    if (!open(MIN,"<$filename")) {
	$gsdl_cgi->generate_error("Unable to read in $filename: $!");
    }
    else {
        # Read all the text in
	my $line;
	while (defined ($line=<MIN>)) {
		$xml_in .= $line if($line !~ m/^\s*$/); # preserve all but empty lines
	}
	close(MIN);
	
	my $MOUT;    
	if (!open($MOUT,">$filename")) {
	    $gsdl_cgi->generate_error("Unable to write out to $filename: $!");
	}
	else {
	    # Matched lines will get handled by the call backs
##	    my $xml_out = "";

	    binmode($MOUT,":utf8");
	    $parser->filter($xml_in,$MOUT, $options);

#	    binmode(MOUT,":utf8");
#	    print MOUT $xml_out;
	    close($MOUT);	    
	}
    }

    # copy across any custom error information that was stored during parsing
    $self->{'error_msg'} = $parser->{'custom_err_msg'} if(defined $parser->{'custom_err_msg'});    
}

sub edit_doc_xml
{
    my $self = shift @_;
    my ($gsdl_cgi, $doc_xml_filename, $metaname, $metavalue, $metapos, $metamode, $opt_secnum, $prevmetavalue) = @_;

    my $info_mess = <<RAWEND;
****************************
  edit_doc_xml()
****************************
RAWEND

    $info_mess .= " doc_xml_filename = $doc_xml_filename\n" if defined($doc_xml_filename);
    $info_mess .= " metaname    = $metaname\n"    if defined($metaname);
    $info_mess .= " metapos     = $metapos\n"     if defined($metapos);
    $info_mess .= " metavalue   = $metavalue\n"   if defined($metavalue);
    $info_mess .= " metamode    = $metamode\n"    if defined($metamode);
    $info_mess .= " opt_secnum  = $opt_secnum\n"  if defined($opt_secnum);
    $info_mess .= " prevmetaval = $prevmetavalue\n" if defined($prevmetavalue);
     
    $info_mess .= "****************************\n";

    $gsdl_cgi->generate_message($info_mess);
	
    # To monitor which section/subsection number we are in
    my @start_rules = 
	( 'Section'    => \&dxml_start_section );

    # use XML::Rules to add it in (read in and out again)
    # Set the call back functions
    my @rules = 
	( _default => 'raw',
	  'Metadata'    => \&dxml_metadata,
	  'Description' => \&dxml_description,
	  'Archive'     => \&dxml_archive); # just for catching errors at end
	  
    # Sets the parameters
    my $options = { 'metaname'  => $metaname,
		    'metapos'   => $metapos,
		    'metavalue' => $metavalue,
		    'metamode'  => $metamode,
			'prevmetavalue' => $prevmetavalue };
			
    if (defined $opt_secnum) {
	$options->{'secnum'} = $opt_secnum;
    }

    $self->edit_xml_file($gsdl_cgi,$doc_xml_filename,\@start_rules,\@rules,$options);
}

sub set_archives_metadata_entry
{
    my $self = shift @_;
    my ($gsdl_cgi, $archive_dir, $collect_dir, $collect, $infodbtype, $docid, $dv, $metaname, $metapos, $metavalue, $metamode, $prevmetavalue) = @_;

    my $info_mess = <<RAWEND;
****************************
  set_archives_metadata_entry()
****************************
RAWEND

    $info_mess .= " archive_dir = $archive_dir\n" if defined($archive_dir);
    $info_mess .= " collect_dir = $collect_dir\n" if defined($collect_dir);
    $info_mess .= " collect     = $collect\n"     if defined($collect);
    $info_mess .= " infodbtype  = $infodbtype\n"  if defined($infodbtype);
    $info_mess .= " docid       = $docid\n"       if defined($docid);
    $info_mess .= " dv          = $dv\n"          if defined($dv);
    $info_mess .= " metaname    = $metaname\n"    if defined($metaname);
    $info_mess .= " metapos     = $metapos\n"     if defined($metapos);
    $info_mess .= " metavalue   = $metavalue\n"   if defined($metavalue);
    $info_mess .= " metamode    = $metamode\n"    if defined($metamode);
    $info_mess .= " prevmetaval = $prevmetavalue\n" if defined($prevmetavalue);
     
    $info_mess .= "****************************\n";

    $gsdl_cgi->generate_message($info_mess);
	
    # Obtain the doc.xml path for the specified docID
    my ($docid_root,$docid_secnum) = ($docid =~ m/^(.*?)(\..*)?$/);

    my $arcinfo_doc_filename = &dbutil::get_infodb_file_path($infodbtype, "archiveinf-doc", $archive_dir);
    my $doc_rec = &dbutil::read_infodb_entry($infodbtype, $arcinfo_doc_filename, $docid_root);
    my $doc_xml_file = $doc_rec->{'doc-file'}->[0];
    
    # check if request if for file-level doc-version history 'nminus-<n>' version
    if (defined $dv && ($dv ne "")) {
	# Need to insert '_fldv_history/nminus-<n>' into doc_filename
	
	my ($doc_xml_tailname, $doc_xml_dirname) = File::Basename::fileparse($doc_xml_file);
	$doc_xml_file = &util::filename_cat($doc_xml_dirname,$FLDV_HISTORY_DIR,$dv,$doc_xml_tailname);
    }
	
    # The $doc_xml_file is relative to the archives, and now let's get the full path
    my $archives_dir = &util::filename_cat($collect_dir,$collect,"archives");

    my $doc_xml_filename = &util::filename_cat($archives_dir,$doc_xml_file);

    # If we're overriding everything, then $metamode=override combined with $metapos=undefined and $prevmetavalue=undefined
    # in which case, we need to remove all metavalues for the metaname at the given (sub)section
    # Thereafter, we will finally be setting the overriding metavalue for this metaname
    if (!defined $prevmetavalue && !defined $metapos && $metamode eq "override") {
	print "override mode with no pos or prev value set. Removing all existing values for $metaname, and setting mode to accumulate for the new value.\n";
	# remove all values of $metaname metadata 
	$self->remove_from_doc_xml($gsdl_cgi, &util::filename_cat($archive_dir, $doc_xml_file), $metaname, undef, undef, $docid_secnum, $metamode);
	$metamode = "accumulate";
    }
    # Edit the doc.xml file with the specified metadata name, value and position.
    # TODO: there is a potential problem here as this edit_doc_xml function 
    # is assuming the simple doc.xml situation where there is only one Section and no SubSections.
    # Running import.pl -groupsize will cause this to have multiple sections in one doc.xml
	
    # dxml_metadata method ignores metapos if metamode anything other than override
    $self->edit_doc_xml($gsdl_cgi,$doc_xml_filename,
			$metaname,$metavalue,$metapos,$metamode,$docid_secnum,$prevmetavalue);

    # return 0; # return 0 for now to indicate no error
    return (defined $self->{'error_msg'}) ? 1 : 0;
}


sub set_archives_metadata
{
    my $self = shift @_;

    my $username  = $self->{'username'};
    my $collect   = $self->{'collect'};
    my $gsdl_cgi  = $self->{'gsdl_cgi'};
	
    if ($baseaction::authentication_enabled) {
	# Ensure the user is allowed to edit this collection
	$self->authenticate_user($username, $collect);
    }

    # Make sure the collection isn't locked by someone else
    $self->lock_collection($username, $collect);

    $self->_set_archives_metadata(@_);

    # Release the lock once it is done
    $self->unlock_collection($username, $collect);
}

sub _set_archives_metadata_array
{
    my $self = shift @_;
    
    my $collect   = $self->{'collect'};
    my $gsdl_cgi  = $self->{'gsdl_cgi'};

    # Obtain the collect dir
    my $site = $self->{'site'};
    my $collect_dir = $gsdl_cgi->get_collection_dir($site);

    # look up additional args
	
    my $infodbtype = $self->{'infodbtype'};
    
    my $archive_dir = &util::filename_cat($collect_dir,$collect,"archives");
	
    my $json_str      = $self->{'json'};
    my $doc_array = decode_json $json_str;
    
    
    my $global_status = 0;
    my $global_mess = "";
    
    my @all_docids = ();
    
    foreach my $doc_array_rec ( @$doc_array ) {
	my $status    = -1;
	my $docid     = $doc_array_rec->{'docid'};
	my $dv        = $doc_array_rec->{'dv'} || undef;
	
	push(@all_docids,$docid);
	
	my $metaname  = $doc_array_rec->{'metaname'};
	if(defined $metaname) {
	    
	    my $metapos   = $doc_array_rec->{'metapos'}; # don't force undef to 0. Undef has meaning when metamode=override

	    my $metamode  = $doc_array_rec->{'metamode'} || $self->{'metamode'};
	    my $metavalue = $doc_array_rec->{'metavalue'};
	    my $prevmetavalue = $self->{'prevmetavalue'}; # to make this sub behave as _set_archives_metadata
	    
	    
	    if ((!defined $metamode) || ($metamode =~ m/^\s*$/)) {
		# make "accumulate" the default (less destructive, as it won't actually 
		# delete any existing values)
		$metamode = "accumulate";
	    }		
	    
	    $status = $self->set_archives_metadata_entry($gsdl_cgi,$archive_dir, $collect_dir,$collect, $infodbtype,
							 $docid,$dv, $metaname,$metapos,$metavalue,$metamode,$prevmetavalue);
	} elsif (defined $doc_array_rec->{'metatable'}) { # if no metaname, we expect a metatable
	    my $metatable = $doc_array_rec->{'metatable'}; # a subarray, or need to generate an error saying JSON structure is wrong
	    
	    foreach my $metatable_rec ( @$metatable ) {
		$metaname  = $metatable_rec->{'metaname'};
		my $metamode  = $metatable_rec->{'metamode'} || $doc_array_rec->{'metamode'} || $self->{'metamode'};
		my $metapos = undef;
		my $prevmetavalue = undef;
		if(defined $metatable_rec->{'metapos'}) {
		    $metapos = $metatable_rec->{'metapos'};
		    $prevmetavalue = $metatable_rec->{'prevmetavalue'}; # or still undef
		    my $metavalue = $metatable_rec->{'metavalue'};
		    $status = $self->set_archives_metadata_entry($gsdl_cgi,$archive_dir,
						 $collect_dir,$collect,$infodbtype,
								 $docid,$dv, $metaname,$metapos,
								 $metavalue,$metamode,$prevmetavalue);
		} else {
		    my $metavals = $metatable_rec->{'metavals'}; # a sub-subarray
		    
		    foreach my $metavalue ( @$metavals ) {
			$status = $self->set_archives_metadata_entry($gsdl_cgi,$archive_dir, $collect_dir,$collect,$infodbtype,
								     $docid,$dv, $metaname,$metapos,$metavalue,$metamode,$prevmetavalue);
			
			if($metamode eq "override") { # now, having overridden the metavalue for the first, 
			    # need to accumulate subsequent metavals for this metaname, else the just-assigned
			    # metavalue for this metaname will be lost
			    $metamode = "accumulate";
			}
		    }
		}
	    }
	}
	    
	if ($status != 0) {
	    # Catch error if set infodb entry failed
	    $global_status = $status;
	    $global_mess .= "Failed to set metadata key: $docid\n";
	    $global_mess .= "Exit status: $status\n";
	    $global_mess .= "System Error Message: $!\n";
	    $global_mess .= "-" x 20 . "\n";
	}
    }
    
    if ($global_status != 0) {
	$global_mess .= "PATH: $ENV{'PATH'}\n";
	$gsdl_cgi->generate_error($global_mess);
    }
    else {
	my $mess = "set-archives-metadata-array successful: Keys[ ".join(", ",@all_docids)."]\n";
	$gsdl_cgi->generate_ok_message($mess);
    }
}

sub set_archives_metadata_array
{
    my $self = shift @_;

    my $username  = $self->{'username'};
    my $collect   = $self->{'collect'};
    my $gsdl_cgi  = $self->{'gsdl_cgi'};

    if ($baseaction::authentication_enabled) {
	# Ensure the user is allowed to edit this collection	
	$self->authenticate_user($username, $collect); #&authenticate_user($gsdl_cgi, $username, $collect);
    }

    # Obtain the collect dir
    my $site = $self->{'site'};
    my $collect_dir = $gsdl_cgi->get_collection_dir($site);
	
    $gsdl_cgi->checked_chdir($collect_dir);

    # Make sure the collection isn't locked by someone else
    $self->lock_collection($username, $collect);

    $self->_set_archives_metadata_array(@_);
    
    # Release the lock once it is done
    $self->unlock_collection($username, $collect);
}



# the version of set_archives_meta that doesn't do authentication
sub _set_archives_assocfile
{
    my $self = shift @_;

    my $collect   = $self->{'collect'};
    my $gsdl_cgi  = $self->{'gsdl_cgi'};
    my $infodbtype = $self->{'infodbtype'};

    # Obtain the collect and archive dir   
    my $site = $self->{'site'};
    my $collect_dir = $gsdl_cgi->get_collection_dir($site);	
    my $archive_dir = &util::filename_cat($collect_dir,$collect,"archives");

    # look up additional args
    my $docid  = $self->{'d'};

    my $assocname = $self->{'assocname'};

    my $arcinfo_doc_filename = &dbutil::get_infodb_file_path($infodbtype, "archiveinf-doc", $archive_dir);
    my $doc_rec = &dbutil::read_infodb_entry($infodbtype, $arcinfo_doc_filename, $docid);
    
    my $doc_file = $doc_rec->{'doc-file'}->[0];
    
    # check if request if for file-level doc-version history 'nminus-<n>' version
    my $dv = $self->{'dv'};	
    
    my ($unused_tailname, $doc_dirname) = File::Basename::fileparse($doc_file);

    my $output_file = $assocname;
    
    if (defined $dv && ($dv ne "")) {
	# Need to insert '_fldv_history/nminus-<n>' into output_file
	
	$output_file = &util::filename_cat($doc_dirname,$FLDV_HISTORY_DIR,$dv,$output_file);
    }
    else {
	$output_file = &util::filename_cat($doc_dirname,$output_file);
    }
    
    my $output_filename = &util::filename_cat($archive_dir, $output_file);

    my $filedata = $self->{'filedata'};
    my $FIN = $self->{'fileupload'};
    
    if ((!defined $filedata) && (!defined $FIN)) {
	$gsdl_cgi->generate_error("set-archives-assocfile: either 'fileupload' or 'filedata' must be set");
    }
    else {

	my $had_error = 0;
	my $had_error_mess = undef;
	
	if (defined $filedata) {
	    if (open(FOUT,">$output_filename")) {		
		print FOUT $filedata;
		close(FOUT);    
	    }
	    else {
		$had_error = 1;
		$had_error_mess = "Failed to save filedata to: $output_filename";
		print STDERR "Error - $had_error_mess:\n$!\n";
	    }
	}
	else {
	    my $FIN = $self->{'fileupload'};
	    if (open(FOUT,">$output_filename")) {		
		binmode(FOUT, ":raw"); 
		
		while (1) {
		    my $buffer = "";
		    my $bytes_read = read($FIN, $buffer, 1024);
		    
		    if (defined $bytes_read) {
			if ($bytes_read>0) {
			    print FOUT $buffer;
			}
			last if $bytes_read < 1024;
		    }
		    else {
			$had_error = 1;
			$had_error_mess = "set-archives-assocfile: Failed to open uploaded file for reading: $!\n";
			print STDERR "Error - $had_error_mess";
			last;
		    }
		}
		
		close($FIN); 	
		close(FOUT);    
	    }
	    else {
		$had_error = 1;
		$had_error_mess = "Failed to save file uploaded file as: $output_filename";
		print STDERR "Error - $had_error_mess:\n$!\n";
	    }
	}

	if (!$had_error) {
	    my $mess = "set-archives-assocfile successful save uploaded content into 'archives' as: $output_file\n";
	    $gsdl_cgi->generate_ok_message($mess);
	}
	else {
	    $gsdl_cgi->generate_error($had_error_mess);
	}
    }
}


sub set_archives_assocfile
{
    my $self = shift @_;

    my $username  = $self->{'username'};
    my $collect   = $self->{'collect'};
    my $gsdl_cgi  = $self->{'gsdl_cgi'};

    if ($baseaction::authentication_enabled) {
	# Ensure the user is allowed to edit this collection
	$self->authenticate_user($username, $collect);
    }


    # Make sure the collection isn't locked by someone else
    $self->lock_collection($username, $collect);

    $self->_set_archives_assocfile(@_);

    # Release the lock once it is done
    $self->unlock_collection($username, $collect);
}


sub _remove_archives_metadata
{
	my $self = shift @_;

	my $collect   = $self->{'collect'};
	my $gsdl_cgi  = $self->{'gsdl_cgi'};
	my $infodbtype = $self->{'infodbtype'};


	# Obtain the collect and archive dir   
	my $site = $self->{'site'};		
	my $collect_dir = $gsdl_cgi->get_collection_dir($site);
	
	my $archive_dir = &util::filename_cat($collect_dir,$collect,"archives");

	# look up additional args
	my ($docid, $docid_secnum) = ($self->{'d'} =~ m/^(.*?)(\..*)?$/);
	
	my $metaname = $self->{'metaname'};
	my $metapos = $self->{'metapos'};
	my $metavalue = $self->{'metavalue'};

	my $metamode = $self->{'metamode'};
	
	my $dv = $self->{'dv'};	
	
	my $status = $self->remove_archives_metadata_entry($gsdl_cgi,$archive_dir, $infodbtype,
			$docid,$docid_secnum,$dv, $metaname,$metapos,$metavalue,$metamode);
	
	if ($status == 0) 
	{
		my $mess = "\nremove-archives-metadata successful: \nKey[$docid]\n";
		$mess .= "  $metaname";
		$mess .= "->[$metapos]" if (defined $metapos);
		$mess .= " ($metavalue)" if (defined $metavalue);
		$gsdl_cgi->generate_ok_message($mess);	
	}
	else 
	{
		my $mess .= "Failed to remove archives metadata key: $docid\n";
		$mess .= "Exit status: $status\n";
		$mess .= "System Error Message: $!\n";
		$mess .= "-" x 20 . "\n";
		
		$gsdl_cgi->generate_error($mess);
	}
	
	#return $status; # in case calling functions have a use for this
}

sub remove_archives_metadata_entry
{
    my $self = shift @_;
    my ($gsdl_cgi, $archive_dir, $infodbtype, $docid, $docid_secnum, $dv, $metaname, $metapos, $metavalue, $metamode) = @_;
    
	$metapos = undef if(defined $metapos && ($metapos =~ m/^\s*$/));
	$metavalue = undef if(defined $metavalue && ($metavalue =~ m/^\s*$/)); # necessary to force fallback to undef here

	# if the user hasn't told us what to delete, not having given a metavalue or metapos,
	# default to deleting the first metavalue for the given metaname
	# Beware that if both metapos AND metavalue are defined, both matches (if any) 
	# seem to get deleted in one single remove_archives_meta action invocation.
	# Similarly, if 2 identical metavalues for a metaname exist and that metavalue is being
	# deleted, both get deleted.
	if(!defined $metapos && !defined $metavalue) {
	    $metapos = 0;
	}
	
	$metamode = undef if(defined $metamode && ($metamode =~ m/^\s*$/));

	my $arcinfo_doc_filename = &dbutil::get_infodb_file_path($infodbtype, "archiveinf-doc", $archive_dir);
	my $doc_rec = &dbutil::read_infodb_entry($infodbtype, $arcinfo_doc_filename, $docid);

	# This now stores the full pathname
	my $doc_file = $doc_rec->{'doc-file'}->[0];	

	# check if request if for file-level doc-version history 'nminus-<n>' version	
	if (defined $dv && ($dv ne "")) {
	    # Need to insert '_fldv_history/nminus-<n>' into doc_filename
	    
	    my ($doc_tailname, $doc_dirname) = File::Basename::fileparse($doc_file);
	    $doc_file = &util::filename_cat($doc_dirname,$FLDV_HISTORY_DIR,$dv,$doc_tailname);
	}

	my $doc_filename = &util::filename_cat($archive_dir, $doc_file);
	    
	my $status = $self->remove_from_doc_xml($gsdl_cgi, $doc_filename, $metaname, $metapos, $metavalue, $docid_secnum, $metamode);
#	my $status = $self->remove_from_doc_xml($gsdl_cgi, $doc_filename, $metaname, $metapos, undef, $docid_secnum);

    return $status;
}

sub remove_archives_metadata
{
	my $self = shift @_;

	my $username  = $self->{'username'};
	my $collect   = $self->{'collect'};
	my $gsdl_cgi  = $self->{'gsdl_cgi'};
	
	if ($baseaction::authentication_enabled) 
	{
	    # Ensure the user is allowed to edit this collection		
	    $self->authenticate_user($username, $collect); #&authenticate_user($gsdl_cgi, $username, $collect);	
	}

	# Make sure the collection isn't locked by someone else
	$self->lock_collection($username, $collect);

	$self->_remove_archives_metadata(@_);

	# Release the lock once it is done
	$self->unlock_collection($username, $collect);
}

sub remove_from_doc_xml
{
	my $self = shift @_;
	my ($gsdl_cgi, $doc_xml_filename, $metaname, $metapos, $metavalue, $secid, $metamode) = @_;
	
	my @start_rules = ('Section' => \&dxml_start_section);
	
	# Set the call-back functions for the metadata tags
	my @rules = 
	( 
		_default => 'raw',
		'Metadata' => \&rfdxml_metadata
	);
	    
	my $parser = XML::Rules->new
	(
		start_rules => \@start_rules,
		rules => \@rules, 
		style => 'filter',
		output_encoding => 'utf8',
#	 normalisespaces => 1, # http://search.cpan.org/~jenda/XML-Rules-1.16/lib/XML/Rules.pm
	 	stripspaces => 2|0|0 # ineffectual
	);
	
	my $status = 0;
	my $xml_in = "";
	if (!open(MIN,"<$doc_xml_filename")) 
	{
		$gsdl_cgi->generate_error("Unable to read in $doc_xml_filename: $!");
		$status = 1;
	}
	else 
	{
		# Read them in
		my $line;
		while (defined ($line=<MIN>)) {
			$xml_in .= $line if($line !~ m/^\s*$/); # preserve all but empty lines
		}
		close(MIN);	

		# Filter with the call-back functions
		my $xml_out = "";

		my $MOUT;
		if (!open($MOUT,">$doc_xml_filename")) {
			$gsdl_cgi->generate_error("Unable to write out to $doc_xml_filename: $!");
			$status = 1;
		}
		else {
			binmode($MOUT,":utf8");
			$parser->filter($xml_in, $MOUT, {metaname => $metaname, metapos => $metapos, metavalue => $metavalue, secid => $secid, metamode => $metamode});
			close($MOUT);
		}
	}
	return $status;
}

sub rfdxml_metadata
{
	my ($tagname, $attrHash, $contextArray, $parentDataArray, $parser) = @_;

	# For comparisons, toplevel section is indicated by ""
	my $curr_sec_num = $parser->{'parameters'}->{'curr_section_num'} || "";
	my $secid = $parser->{'parameters'}->{'secid'} || "";

	if (!($secid eq $curr_sec_num))
	{
		# RAW is [$tagname => $attrHash] not $tagname => $attrHash!!
		return [$tagname => $attrHash];
	}

	if ($parser->{'parameters'}->{'metaname'} eq $attrHash->{'name'})
	{
		if (!defined $parser->{'parameters'}->{'poscount'})
		{
			$parser->{'parameters'}->{'poscount'} = 0;
		}
		else
		{
			$parser->{'parameters'}->{'poscount'}++;
		}
		
		# if overriding (for set-meta) but no metapos, then clear all the meta for this metaname
		if ((defined $parser->{'parameters'}->{'metamode'}) && ($parser->{'parameters'}->{'metamode'} eq "override") && (!defined $parser->{'parameters'}->{'metapos'}) &&(!defined $parser->{'parameters'}->{'metavalue'})) {		    
		    return [];
		}

		if ((defined $parser->{'parameters'}->{'metapos'}) && ($parser->{'parameters'}->{'poscount'} == $parser->{'parameters'}->{'metapos'}))
		{	
		    return [];
		}
		
		if ((defined $parser->{'parameters'}->{'metavalue'}) && ($parser->{'parameters'}->{'metavalue'} eq $attrHash->{'_content'}))
		{	
		    return [];
		}
	}
	
	# RAW is [$tagname => $attrHash] not $tagname => $attrHash!!
	return [$tagname => $attrHash];
}

sub mxml_metadata
{
    my ($tagname, $attrHash, $contextArray, $parentDataArray, $parser) = @_;
    my $metaname = $parser->{'parameters'}->{'metaname'};
    my $metamode = $parser->{'parameters'}->{'metamode'};

    # Report error if we don't see FileName tag before this
    die "Fatal Error: Unexpected metadata.xml structure. Undefined current_file, possibly encountered Description before FileName" if (!defined($parser->{'parameters'}->{'current_file'}));
    
    # Don't do anything if we are not in the right FileSet
    my $file_regexp = $parser->{'parameters'}->{'current_file'};
    if ($file_regexp =~ /\.\*/) {
	# Only interested in a file_regexp if it specifies precisely one
	# file.  
	# So, skip anything with a .* in it as it is too general
##	print STDERR "@@@@ Skipping entry in metadata.xml where FileName=.* as it is too general\n";
	return [$tagname => $attrHash];
    }
    my $src_file = $parser->{'parameters'}->{'src_file'};
    if (!($src_file =~ /$file_regexp/)) {
	return [$tagname => $attrHash];
    }
##    print STDERR "*** mxl metamode = $metamode\n";

    # Find the right metadata tag and checks if we are going to override it
    my $name_attr = $attrHash->{'name'};
    if (($name_attr eq $metaname) && ($metamode eq "override")) {

	# now metadata.xml functions need to keep track of metapos
	if (!defined $parser->{'parameters'}->{'poscount'}) 
	{ 
	    $parser->{'parameters'}->{'poscount'} = 0; 
	} 
	else 
	{ 
	    $parser->{'parameters'}->{'poscount'}++; 
	} 

	# If either the metapos or prevmetavalue is set,
        # get the value and override the current value
	my $metavalue = $parser->{'parameters'}->{'metavalue'};

	if(defined $parser->{'parameters'}->{'prevmetavalue'} && $parser->{'parameters'}->{'prevmetavalue'} eq $attrHash->{'_content'})
	{
	    $attrHash->{'_content'} = $metavalue;

	    ##	print STDERR "**** overriding metadata.xml\n";
	    
	    # Don't want it to wipe out any other pieces of metadata
	    $parser->{'parameters'}->{'metamode'} = "done";
	}
	elsif(defined $parser->{'parameters'}->{'metapos'} && $parser->{'parameters'}->{'poscount'} == $parser->{'parameters'}->{'metapos'})
	{
	    $attrHash->{'_content'} = $metavalue;
	    $parser->{'parameters'}->{'metamode'} = "done";
	}
    } 

    # mxml_description will process the metadata if metadata is accumulate, 
    # or if we haven't found the metadata to override

    # RAW is [$tagname => $attrHash] not $tagname => $attrHash!!
    return [$tagname => $attrHash];
}


sub mxml_description
{
    my ($tagname, $attrHash, $contextArray, $parentDataArray, $parser) = @_;
    my $metamode = $parser->{'parameters'}->{'metamode'};    

    # Failed... Report error if we don't see FileName tag before this
    die "Fatal Error: Unexpected metadata.xml structure. Undefind current_file, possiblely encountered Description before FileName" if (!defined($parser->{'parameters'}->{'current_file'}));

    # Don't do anything if we are not in the right FileSet
    my $file_regexp = $parser->{'parameters'}->{'current_file'};
    if ($file_regexp =~ m/\.\*/) {
	# Only interested in a file_regexp if it specifies precisely one
	# file.  
	# So, skip anything with a .* in it as it is too general
	return [$tagname => $attrHash];
    }
    my $src_file = $parser->{'parameters'}->{'src_file'};
	
    if (!($src_file =~ m/$file_regexp/)) {
	return [$tagname => $attrHash];
    }

    # Accumulate the metadata block to the end of the description block
    # Note: This adds metadata block to all description blocks, so if there are 
    # multiple FileSets, it will add to all of them
    if (($metamode eq "accumulate") || ($metamode eq "override")) {

	# if metamode was "override" but get to here then it failed to
	# find an item to override, in which case it should append its 
	# value to the end, just like the "accumulate" mode

	if ($metamode eq "override") {
	    print "No metadata value to override.  Switching 'metamode' to accumulate\n";
	}

	# tack a new metadata tag on to the end of the <Metadata>+ block
	my $metaname = $parser->{'parameters'}->{'metaname'};
	my $metavalue = $parser->{'parameters'}->{'metavalue'};
	
	my $metadata_attr = { '_content' => $metavalue, 
			      'name'     => $metaname, 
			      'mode'     => "accumulate" };

	my $append_metadata = [ "Metadata" => $metadata_attr ];
	my $description_content = $attrHash->{'_content'};
	
##	print STDERR "*** appending to metadata.xml\n";

	# append the new metadata element to the end of the current
	# content contained inside this tag
	if (ref($description_content) eq "") {
		# => string or numeric literal
		# this is caused by a <Description> block has no <Metadata> child elements
		# => set up an empty array in '_content'
		$attrHash->{'_content'} = [ "\n" ]; 
		$description_content = $attrHash->{'_content'};
	}

	push(@$description_content, "  ", $append_metadata, "\n  ");
	$parser->{'parameters'}->{'metamode'} = "done";
    }

    # RAW is [$tagname => $attrHash] not $tagname => $attrHash!!
    return [$tagname => $attrHash];
}


sub mxml_fileset
{
    my ($tagname, $attrHash, $contextArray, $parentDataArray, $parser) = @_;

    # Initilise the current_file
    # Note: According to http://greenstone.org/dtd/DirectoryMetadata/1.0/DirectoryMetadata.dtd
    # FileName tag must come before Description tag
    $parser->{'parameters'}->{'current_file'} = "";

    # RAW is [$tagname => $attrHash] not $tagname => $attrHash!!
    return [$tagname => $attrHash];
}

sub mxml_directorymetadata
{
    my ($tagname, $attrHash, $contextArray, $parentDataArray, $parser) = @_;

    # if we haven't processed the metadata when we reach the end of metadata.xml
    # it's because there's no particular FileSet element whose FileName matched
    # In which case, add a new FileSet for this FileName
    my $metamode = $parser->{'parameters'}->{'metamode'};
    if($metamode ne "done") {
	
	if ($metamode eq "override") {
	    print "No metadata value to override.  Switching 'metamode' to accumulate\n";
	}

	# If we get to here and metamode is override, this means there 
	# was no existing value to overide => treat as an append operation
	
	# Create a new FileSet element and append to DirectoryMetadata
	# <FileSet>
	# <FileName>src_file</FileName>
	# <Description>
	# <Metadata mode="" name="">metavalue</Metadata>
	# </Description>
	# </FileSet>
	my $src_file = $parser->{'parameters'}->{'src_file'};
	my $metaname = $parser->{'parameters'}->{'metaname'};
	my $metavalue = $parser->{'parameters'}->{'metavalue'};
	my $metadata_attr = { 
	    '_content' => $metavalue, 
	    'name' => $metaname, 
	    'mode' => "accumulate" 
	};
	my $append_metadata = [ "Metadata" => $metadata_attr ];
	my $description_attr = { '_content' => [ "\n\t\t   ", $append_metadata, "\n\t\t"] };
	my $description_element = [ "Description" => $description_attr ];

	#_content is not an attribute, it's special and holds the children of this element
	# including the textnode value embedded in this element if any.
	my $filename_attr = {'_content' => $src_file};
	my $filename_element = [ "FileName" => $filename_attr ];

	my $fileset_attr = {};
	$fileset_attr->{'_content'} = [ "\n\t\t", $filename_element,"\n\t\t",$description_element ,"\n\t" ];
	my $fileset = [ "FileSet" => $fileset_attr ]; #my $fileset = [ "FileSet" => {} ];
	
	
	# get children of dirmeta, and push the new FileSet element onto it
	print "Appending metadata to metadata.xml\n";
	my $dirmeta_content = $attrHash->{'_content'};
	if (ref($dirmeta_content)) {
	    # got some existing interesting nested content
	    #push(@$dirmeta_content, "    ", $fileset ,"\n        ");
	    push(@$dirmeta_content, "\t", $fileset ,"\n");
	}
	else {
	    #description_content is most likely a string such as "\n"
	    #$attrHash->{'_content'} = [$dirmeta_content, "    ", $fileset ,"\n" ];
	    $attrHash->{'_content'} = [$dirmeta_content, "\t", $fileset ,"\n" ];
	}	

	$parser->{'parameters'}->{'metamode'} = "done";
    }
    # RAW is [$tagname => $attrHash] not $tagname => $attrHash!!
    return [$tagname => $attrHash];
}


sub edit_metadata_xml
{
    my $self = shift @_;
    my ($gsdl_cgi, $metadata_xml_filename, $metaname, $metapos, $metavalue, $metamode, $src_file, $prevmetavalue) = @_;

    # Set the call-back functions for the metadata tags
    my @rules = 
	( _default => 'raw',
          'FileName' => \&mxml_filename,
	  'Metadata' => \&mxml_metadata,
	  'Description' => \&mxml_description,
          'FileSet' => \&mxml_fileset,
	  'DirectoryMetadata' => \&mxml_directorymetadata);

    # use XML::Rules to add it in (read in and out again)
    my $parser = XML::Rules->new(rules => \@rules, 
				 style => 'filter',
                                 output_encoding => 'utf8',
				 stripspaces => 2|0|0); # http://search.cpan.org/~jenda/XML-Rules-1.16/lib/XML/Rules.pm

	if (!-e $metadata_xml_filename) {
	
		if (open(MOUT,">$metadata_xml_filename")) {
			
			my $src_file_re = &util::filename_to_regex($src_file);
			# shouldn't the following also be in the above utility routine??
			# $src_file_re =~ s/\./\\./g;
		
			print MOUT "<?xml version=\"1.0\"?>\n";
			print MOUT "<DirectoryMetadata>\n";
			print MOUT " <FileSet>\n";
			print MOUT "  <FileName>$src_file_re</FileName>\n";
			print MOUT "  <Description>\n";
			print MOUT "  </Description>\n";
			print MOUT " </FileSet>\n";
			print MOUT "</DirectoryMetadata>\n";

			close(MOUT);
		}
		else {
			$gsdl_cgi->generate_error("Unable to create $metadata_xml_filename: $!");
		}
	}
	
	
	my $xml_in = "";
	if (!open(MIN,"<$metadata_xml_filename")) {
		$gsdl_cgi->generate_error("Unable to read in $metadata_xml_filename: $!");
	}
	else {
		# Read them in
		my $line;
		while (defined ($line=<MIN>)) {
			$xml_in .= $line if($line !~ m/^\s*$/); # preserve all but empty lines
		}
		close(MIN);	

		# Filter with the call-back functions
		my $xml_out = "";

		my $MOUT;
		if (!open($MOUT,">$metadata_xml_filename")) {
			$gsdl_cgi->generate_error("Unable to write out to $metadata_xml_filename: $!");
		}
		else {
			binmode($MOUT,":utf8");

			# Some wise person please find out how to keep the DTD and encode lines in after it gets filtered by this XML::Rules
			# At the moment, I will just hack it!
			#my $header_with_utf8_dtd = "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n";
			#$header_with_utf8_dtd .= "<!DOCTYPE DirectoryMetadata SYSTEM \"http://greenstone.org/dtd/DirectoryMetadata/1.0/DirectoryMetadata.dtd\">";
			#$xml_out =~ s/\<\?xml\sversion\=\"1.0\"\?\>/$header_with_utf8_dtd/;
			#print MOUT $xml_out;

			$parser->filter($xml_in, $MOUT, { metaname => $metaname,
							  metapos => $metapos,
				      metavalue => $metavalue,
				      metamode => $metamode,
				      src_file => $src_file,
				      prevmetavalue => $prevmetavalue,
				      current_file => undef} );
			close($MOUT);	    
		}
   }
}


sub set_import_metadata
{
    my $self = shift @_;
    
    my $username  = $self->{'username'};
    my $collect   = $self->{'collect'};
    my $gsdl_cgi  = $self->{'gsdl_cgi'};
	
    if ($baseaction::authentication_enabled) {
	# Ensure the user is allowed to edit this collection
	$self->authenticate_user($username, $collect);
    }

    # Make sure the collection isn't locked by someone else
    $self->lock_collection($username, $collect);
 
    $self->_set_import_metadata(@_);

    # Release the lock once it is done
    $self->unlock_collection($username, $collect);
    
}

sub set_import_metadata_array
{
    my $self = shift @_;

    my $username  = $self->{'username'};
    my $collect   = $self->{'collect'};
    my $gsdl_cgi  = $self->{'gsdl_cgi'};

    if ($baseaction::authentication_enabled) {
	# Ensure the user is allowed to edit this collection	
	$self->authenticate_user($username, $collect); #&authenticate_user($gsdl_cgi, $username, $collect);
    }

    # Obtain the collect dir
    my $site = $self->{'site'};
    my $collect_dir = $gsdl_cgi->get_collection_dir($site);
    
    $gsdl_cgi->checked_chdir($collect_dir);

    # Make sure the collection isn't locked by someone else
    $self->lock_collection($username, $collect);

    $self->_set_import_metadata_array(@_);

    # Release the lock once it is done
    $self->unlock_collection($username, $collect);

}


sub _set_import_metadata_array
{
    my $self = shift @_;

    my $collect   = $self->{'collect'};
    my $gsdl_cgi  = $self->{'gsdl_cgi'};

    # Obtain the collect dir
    my $site = $self->{'site'};
    my $collect_dir = $gsdl_cgi->get_collection_dir($site);
	
    # look up additional args
	
    my $infodbtype = $self->{'infodbtype'};
    
    my $archive_dir = &util::filename_cat($collect_dir,$collect,"archives");	
    my $arcinfo_doc_filename = &dbutil::get_infodb_file_path($infodbtype, "archiveinf-doc", $archive_dir);
    
    my $json_str = $self->{'json'};
    my $doc_array = decode_json $json_str;
    
    my $global_status = 0;
    my $global_mess = "";
    
    my @all_docids = ();
    
    foreach my $doc_array_rec ( @$doc_array ) 
    {
	my $status = -1;
	my $docid = $doc_array_rec->{'docid'};
	
	my ($docid_root,$docid_secnum);
	if(defined $docid) {	
	    ($docid_root,$docid_secnum) = ($docid =~ m/^(.*?)(\..*)?$/);	
	    # as yet no support for setting subsection metadata in metadata.xml
	    if ((defined $docid_secnum) && ($docid_secnum !~ m/^\s*$/)) {
		$gsdl_cgi->generate_message("*** docid: $docid. No support yet for setting import metadata at subsections level.\n");
		next; # skip this docid in for loop
	    }
	}

	push(@all_docids,$docid); # docid_root rather
	
	my $metaname = $doc_array_rec->{'metaname'};
	if (defined $metaname) {
	    my $metamode = $doc_array_rec->{'metamode'} || $self->{'metamode'};
	    my $metavalue = $doc_array_rec->{'metavalue'};
	    $metavalue =~ s/&lt;(.*?)&gt;/<$1>/g;

	    if ((!defined $metamode) || ($metamode =~ m/^\s*$/)) {
		# make "accumulate" the default (less destructive, as it won't actually 
		# delete any existing values)
		$metamode = "accumulate";
	    }

	    # adding metapos and prevmetavalue support to import_metadata subroutines
	    my $metapos   = $doc_array_rec->{'metapos'}; # don't force undef to 0. Undef has meaning when metamode=override
	    my $prevmetavalue = $self->{'prevmetavalue'};

	    $self->set_import_metadata_entry($gsdl_cgi, $arcinfo_doc_filename, $infodbtype, $docid_root, $metaname, $metapos, $metavalue, $metamode, $prevmetavalue, $collect, $collect_dir); # at this point, docid_root = docid
	    
	} elsif (defined $doc_array_rec->{'metatable'}) { # if no metaname, we expect a metatable
	    my $metatable = $doc_array_rec->{'metatable'}; # a subarray, or need to generate an error saying JSON structure is wrong
	    
	    foreach my $metatable_rec ( @$metatable ) {
		$metaname  = $metatable_rec->{'metaname'}; 
		my $metamode  = $metatable_rec->{'metamode'} || $doc_array_rec->{'metamode'} || $self->{'metamode'};
		if ((!defined $metamode) || ($metamode =~ m/^\s*$/)) {
		    # make "accumulate" the default (less destructive, as it won't actually 
		    # delete any existing values)
		    $metamode = "accumulate";
		}

		# No support for metapos and prevmetavalue in the JSON metatable substructure
		my $metapos = undef; 
		my $prevmetavalue = undef;
		if(defined $metatable_rec->{'metapos'}) {
		    $metapos = $metatable_rec->{'metapos'};
		    $prevmetavalue = $metatable_rec->{'prevmetavalue'}; # or still undef
		    my $metavalue = $metatable_rec->{'metavalue'};
		    #$metavalue =~ s/&lt;(.*?)&gt;/<$1>/g;
		    $self->set_import_metadata_entry($gsdl_cgi, $arcinfo_doc_filename, $infodbtype, $docid_root, $metaname, $metapos, $metavalue, $metamode, $prevmetavalue, $collect, $collect_dir); # at this point, docid_root = docid
		} else {
		    
		    my $metavals = $metatable_rec->{'metavals'}; # a sub-subarray
		    
		    foreach my $metavalue ( @$metavals ) {
			$metavalue =~ s/&lt;(.*?)&gt;/<$1>/g;
			
			$self->set_import_metadata_entry($gsdl_cgi, $arcinfo_doc_filename, $infodbtype, $docid_root, $metaname, $metapos, $metavalue, $metamode, $prevmetavalue, $collect, $collect_dir); # at this point, docid_root = docid
			if($metamode eq "override") { # now, having overridden the first metavalue of the metaname, 
			    # need to accumulate subsequent metavals for this metaname, else the just-assigned
			    # metavalue for this metaname will be lost
			    $metamode = "accumulate";
			}
		    }
		}
	    }
	}		
    }

    # always a success message
    my $mess = "set-archives-metadata-array successful: Keys[ ".join(", ",@all_docids)."]\n";
    $gsdl_cgi->generate_ok_message($mess);
}

# always returns true (1)
sub set_import_metadata_entry
{
    my $self = shift @_;
    my ($gsdl_cgi, $arcinfo_doc_filename, $infodbtype, $docid, $metaname, $metapos, $metavalue, $metamode, $prevmetavalue, $collect, $collect_dir) = @_;

    my $info_mess = <<RAWEND;
****************************
  set_import_metadata_entry()
****************************
RAWEND

    $info_mess .= " collect_dir = $collect_dir\n" if defined($collect_dir);
    $info_mess .= " collect     = $collect\n"     if defined($collect);
    $info_mess .= " infodbtype  = $infodbtype\n"  if defined($infodbtype);
    $info_mess .= " arcinfo_doc_filename  = $arcinfo_doc_filename\n"  if defined($arcinfo_doc_filename);
    $info_mess .= " docid       = $docid\n"       if defined($docid);
    $info_mess .= " metaname    = $metaname\n"    if defined($metaname);
    $info_mess .= " metapos     = $metapos\n"     if defined($metapos);
    $info_mess .= " metavalue   = $metavalue\n"   if defined($metavalue);
    $info_mess .= " metamode    = $metamode\n"    if defined($metamode);
    $info_mess .= " prevmetaval = $prevmetavalue\n" if defined($prevmetavalue);
     
    $info_mess .= "****************************\n";

    $gsdl_cgi->generate_message($info_mess);

    # import works with metadata.xml which can have inherited metadata
    # so setting or removing at a metapos can have unintended effects for a COMPLEX collection
    # (a collection that has or can have inherited metadata). Metapos has expected behaviour for
    # a SIMPLE collection, which is one that doesn't have inherited metadata. Assume caller knows 
    # what they're doing if they provide a metapos.
    if(defined $metapos) {
	print STDERR "@@@@ WARNING: metapos defined.\n";
	print STDERR "@@@@ Assuming SIMPLE collection and proceeding to modify the import meta at $metapos.\n";
    }

    # Obtain where the metadata.xml is from the archiveinfo-doc.gdb file
    # If the doc oid is not specified, we assume the metadata.xml is next to the specified "f"
    my $metadata_xml_file;
    my $import_filename = undef;
    
    if (defined $docid) {
	# my $arcinfo_doc_filename = &dbutil::get_infodb_file_path($infodbtype, "archiveinf-doc", $archive_dir);
	my $doc_rec = &dbutil::read_infodb_entry($infodbtype, $arcinfo_doc_filename, $docid);

	# This now stores the full pathname
	$import_filename = $doc_rec->{'src-file'}->[0];	
	$import_filename = &util::placeholders_to_abspath($import_filename);

    } else { # only for set_import_meta, not the case when calling method is set_import_metadata_array
	     # as the array version of the method doesn't support the -f parameter yet
	my $import_file  = $self->{'f'};
	$import_filename = &util::filename_cat($collect_dir,$collect,$import_file);
    }
    
    # figure out correct metadata.xml file [?]
    # Assuming the metadata.xml file is next to the source file
    # Note: This will not work if it is using the inherited metadata from the parent folder
    my ($import_tailname, $import_dirname) = File::Basename::fileparse($import_filename);
    my $metadata_xml_filename = &util::filename_cat($import_dirname,"metadata.xml");
    
    # If we're overriding everything, then $prevmetavalue=undefined and
    # $metamode=override combined with $metapos=undefined
    # in which case we need to remove all metavalues for the metaname at the given (sub)section
    # Thereafter, we will finally be able to set the overriding metavalue for this metaname
    if(!defined $prevmetavalue && !defined $metapos && $metamode eq "override") {
##	print STDERR "@@@ REMOVING all import metadata for $metaname\n";
	$self->remove_from_metadata_xml($gsdl_cgi, $metadata_xml_filename, $metaname, $metapos, undef, $import_tailname, $metamode); # we're removing all values, so metavalue=undef

    }

    # Edit the metadata.xml
    # Modified by Jeffrey from DL Consulting
    # Handle the case where there is one metadata.xml file for multiple FileSets
    # The XML filter needs to know whether it is in the right FileSet
    # TODO: This doesn't fix the problem where the metadata.xml is not next to the src file.
    # TODO: This doesn't handle the common metadata (where FileName doesn't point to a single file)
    $self->edit_metadata_xml($gsdl_cgi, $metadata_xml_filename, $metaname, 
			     $metapos, $metavalue, $metamode, $import_tailname, $prevmetavalue);
    #return 0;
    return $metadata_xml_filename;
}

sub _remove_import_metadata
{
	my $self = shift @_;

	my $collect   = $self->{'collect'};
	my $gsdl_cgi  = $self->{'gsdl_cgi'};
	my $infodbtype = $self->{'infodbtype'};

	# Obtain the collect and archive dir
	my $site = $self->{'site'};
	my $collect_dir = $gsdl_cgi->get_collection_dir($site);
	
	my $archive_dir = &util::filename_cat($collect_dir,$collect,"archives");

	# look up additional args
	my $docid = $self->{'d'};
	
	my $arcinfo_doc_filename = &dbutil::get_infodb_file_path($infodbtype, "archiveinf-doc", $archive_dir);	
	
	my $metaname = $self->{'metaname'};
	my $metapos = $self->{'metapos'};
	my $metavalue = $self->{'metavalue'};	

	my $metamode = $self->{'metamode'};

	my $metadata_xml_filename = $self->remove_import_metadata_entry($gsdl_cgi, $arcinfo_doc_filename, $infodbtype, $docid, $metaname, $metapos, $metavalue, $metamode);

	# any errors would have been reported inside the remove_import_metadata_entry
	
	my $mess = "remove-import-metadata successful: Key[$docid] -> $metadata_xml_filename\n";
	$mess .= "  $metaname";
	$mess .= " = $metavalue\n";
	
	$gsdl_cgi->generate_ok_message($mess);
	    
	#return $status; # in case calling functions have a use for this
}

sub remove_import_metadata_entry
{
    my $self = shift @_;
    my ($gsdl_cgi, $arcinfo_doc_filename, $infodbtype, $docid, $metaname, $metapos, $metavalue, $metamode) = @_;
    
    $metapos = undef if(defined $metapos && ($metapos =~ m/^\s*$/));
    $metavalue = undef if(defined $metavalue && ($metavalue =~ m/^\s*$/));
    
    if(defined $metavalue) { # metavalue is not a compulsory arg for remove_import_metadata()
	$metavalue =~ s/&lt;(.*?)&gt;/<$1>/g;
    } elsif (!defined $metapos) { # if given no metavalue or metapos to delete, default to deleting the 1st
	$metapos = 0;
    }
	
    $metamode = undef if(defined $metamode && ($metamode =~ m/^\s*$/));

	# import works with metadata.xml which can have inherited metadata
	# so setting or removing at a metapos can have unintended effects for a COMPLEX collection
	# (a collection that has or can have inherited metadata). Metapos has expected behaviour for
	# a SIMPLE collection, which is one that doesn't have inherited metadata. Assume caller knows 
	# what they're doing if they provide a metapos.
	if(defined $metapos) {
	    print STDERR "@@@@ WARNING: metapos defined.\n";
	    print STDERR "@@@@ Assuming SIMPLE collection and proceeding to modify the import meta at $metapos.\n";
	}
	
	# Obtain where the metadata.xml is from the archiveinfo-doc.gdb file
	# If the doc oid is not specified, we assume the metadata.xml is next to the specified "f"
	my $metadata_xml_file;
	my $import_filename = undef;
	if ((!defined $docid) || ($docid =~ m/^\s*$/)) 
	{
		$gsdl_cgi->generate_error("No docid (d=...) specified.\n");
	} else {		
		my $doc_rec = &dbutil::read_infodb_entry($infodbtype, $arcinfo_doc_filename, $docid);

		# This now stores the full pathname
		$import_filename = $doc_rec->{'src-file'}->[0];	
		$import_filename = &util::placeholders_to_abspath($import_filename);
	}

	if((!defined $import_filename) || ($import_filename =~ m/^\s*$/))
	{
		$gsdl_cgi->generate_error("There is no metadata\n");
	}
	
	# figure out correct metadata.xml file [?]
	# Assuming the metadata.xml file is next to the source file
	# Note: This will not work if it is using the inherited metadata from the parent folder
	my ($import_tailname, $import_dirname) = File::Basename::fileparse($import_filename);
	my $metadata_xml_filename = &util::filename_cat($import_dirname,"metadata.xml");
	
	$self->remove_from_metadata_xml($gsdl_cgi, $metadata_xml_filename, $metaname, $metapos, $metavalue, $import_tailname, $metamode); # metamode has no meaning for removing meta, but is used by set_meta when overriding All

    return $metadata_xml_filename;
}

sub remove_import_metadata
{
	my $self = shift @_;
	
	my $username = $self->{'username'};
	my $collect   = $self->{'collect'};
	my $gsdl_cgi  = $self->{'gsdl_cgi'};
	
	if ($baseaction::authentication_enabled) {
	    # Ensure the user is allowed to edit this collection		
	    $self->authenticate_user($username, $collect); #&authenticate_user($gsdl_cgi, $username, $collect);
	}

	# Make sure the collection isn't locked by someone else
	$self->lock_collection($username, $collect);
	
	$self->_remove_import_metadata(@_);

	# Release the lock once it is done
	$self->unlock_collection($username, $collect);

}

sub remove_from_metadata_xml
{
	my $self = shift @_;
	my ($gsdl_cgi, $metadata_xml_filename, $metaname, $metapos, $metavalue, $src_file, $metamode) = @_;
	# metamode generally has no meaning for removing meta, but is used by set_meta 
	# when overriding all metavals for a metaname, in which case remove_meta is called with metamode

	# Set the call-back functions for the metadata tags
	my @rules = 
	( 
		_default => 'raw',
		'Metadata' => \&rfmxml_metadata,
		'FileName' => \&mxml_filename
	);
	    
	my $parser = XML::Rules->new
	(
		rules => \@rules, 
		style => 'filter',
		output_encoding => 'utf8',
	 #normalisespaces => 1,
	        stripspaces => 2|0|0 # ineffectual
	);
	
	my $xml_in = "";
	if (!open(MIN,"<$metadata_xml_filename")) 
	{
		$gsdl_cgi->generate_error("Unable to read in $metadata_xml_filename: $!");
	}
	else 
	{
		# Read them in
		my $line;
		while (defined ($line=<MIN>)) {
			$xml_in .= $line if($line !~ m/^\s*$/); # preserve all but empty lines
		}
		close(MIN);	

		# Filter with the call-back functions
		my $xml_out = "";

		my $MOUT;
		if (!open($MOUT,">$metadata_xml_filename")) {
			$gsdl_cgi->generate_error("Unable to write out to $metadata_xml_filename: $!");
		}
		else {
			binmode($MOUT,":utf8");
			$parser->filter($xml_in, $MOUT, {metaname => $metaname, metapos => $metapos, metavalue => $metavalue, src_file => $src_file, metamode => $metamode, current_file => undef});
			close($MOUT);	    
		}
	}
}

sub rfmxml_metadata
{
	my ($tagname, $attrHash, $contextArray, $parentDataArray, $parser) = @_;

	# metadata.xml does not handle subsections

	# since metadata.xml now has to deal with metapos, we keep track of the metadata position
	if (($parser->{'parameters'}->{'src_file'} eq $parser->{'parameters'}->{'current_file'}) 
	    && $parser->{'parameters'}->{'metaname'} eq $attrHash->{'name'})
	{
		if (!defined $parser->{'parameters'}->{'poscount'})
		{
			$parser->{'parameters'}->{'poscount'} = 0;
		}
		else
		{
			$parser->{'parameters'}->{'poscount'}++;
		}

		# if overriding but no metapos, then clear all the meta for this metaname
		# This is used by set_import_metadata_entry, so don't change this behaviour
		if ((defined $parser->{'parameters'}->{'metamode'}) && ($parser->{'parameters'}->{'metamode'} eq "override") && (!defined $parser->{'parameters'}->{'metapos'}) && (!defined $parser->{'parameters'}->{'metavalue'})) {
		    return [];
		}
	
		if ((defined $parser->{'parameters'}->{'metapos'}) && ($parser->{'parameters'}->{'poscount'} == $parser->{'parameters'}->{'metapos'}))
		{
		    return [];
		}
		if ((defined $parser->{'parameters'}->{'metavalue'}) && ($attrHash->{'_content'} eq $parser->{'parameters'}->{'metavalue'}))
		{
		    return [];
		}		
	}

	# RAW is [$tagname => $attrHash] not $tagname => $attrHash!!
	return [$tagname => $attrHash];
}

sub _remove_live_metadata
{
    my $self = shift @_;

    my $collect   = $self->{'collect'};
    my $gsdl_cgi  = $self->{'gsdl_cgi'};
    my $infodbtype = $self->{'infodbtype'};

    # Obtain the collect dir
    my $site = $self->{'site'};
    my $collect_dir = $gsdl_cgi->get_collection_dir($site);

    # look up additional args
    my $docid     = $self->{'d'};
    if ((!defined $docid) || ($docid =~ m/^\s*$/)) {
      $gsdl_cgi->generate_error("No docid (d=...) specified.");
    }
    
    # Generate the dbkey
    my $metaname  = $self->{'metaname'};
    my $dbkey = "$docid.$metaname";

    # To people who know $collect_tail please add some comments
    # Obtain the live gdbm_db path 
    my $collect_tail = $collect;
    $collect_tail =~ s/^.*[\/|\\]//;
    my $index_text_directory = &util::filename_cat($collect_dir,$collect,"index","text");
    my $infodb_file_path = &dbutil::get_infodb_file_path($infodbtype, "live-$collect_tail", $index_text_directory);

    # Remove the key
    my $cmd = "gdbmdel \"$infodb_file_path\" \"$dbkey\"";
    my $status = system($cmd);
    if ($status != 0) {
        # Catch error if gdbmdel failed
	my $mess = "Failed to delete metadata key: $dbkey\n";
	
	$mess .= "PATH: $ENV{'PATH'}\n";
	$mess .= "cmd = $cmd\n";
	$mess .= "Exit status: $status\n";
	$mess .= "System Error Message: $!\n";

	$gsdl_cgi->generate_error($mess);
    }
    else {
	$gsdl_cgi->generate_ok_message("DB remove successful: Key[$metaname]");
    }

}

sub remove_live_metadata
{
    my $self = shift @_;

    my $username  = $self->{'username'};
    my $collect   = $self->{'collect'};
    my $gsdl_cgi  = $self->{'gsdl_cgi'};
	
    if ($baseaction::authentication_enabled) {
	# Ensure the user is allowed to edit this collection	
	$self->authenticate_user($username, $collect); #&authenticate_user($gsdl_cgi, $username, $collect);
    }

    # Make sure the collection isn't locked by someone else
    $self->lock_collection($username, $collect);

    $self->_remove_live_metadata(@_);

    $self->unlock_collection($username, $collect);
}

sub remove_metadata
{
    my $self = shift @_;

    my $where = $self->{'where'};
    if(!$where || ($where =~ m/^\s*$/)) {
	$self->remove_index_metadata(@_); # call the full version of set_index_meta for the default behaviour
	return;
    }

    my $username  = $self->{'username'};
    my $collect   = $self->{'collect'};
    my $gsdl_cgi  = $self->{'gsdl_cgi'};

    if ($baseaction::authentication_enabled) {
	# Ensure the user is allowed to edit this collection	
	$self->authenticate_user($username, $collect); #&authenticate_user($gsdl_cgi, $username, $collect);
    }

    # Make sure the collection isn't locked by someone else
    $self->lock_collection($username, $collect);

    # check which directories need to be processed, specified in $where as 
    # any combination of import|archives|index|live
    if($where =~ m/import/) {
	$self->_remove_import_metadata(@_);	    
    }
    if($where =~ m/archives/) {
	$self->_remove_archives_metadata(@_);	    
   } 
    if($where =~ m/index/) {
	$self->_remove_index_metadata(@_);	    
    }

    # Release the lock once it is done
    $self->unlock_collection($username, $collect);
}

# the internal version, without authentication
sub _remove_index_metadata
{    
    my $self = shift @_;

    my $collect   = $self->{'collect'};
    my $gsdl_cgi  = $self->{'gsdl_cgi'};
    my $infodbtype = $self->{'infodbtype'};
	
    # Obtain the collect dir
    my $site = $self->{'site'};
    my $collect_dir = $gsdl_cgi->get_collection_dir($site);

        # look up additional args
    my $docid     = $self->{'d'};
    if ((!defined $docid) || ($docid =~ m/^\s*$/)) {
      $gsdl_cgi->generate_error("No docid (d=...) specified.");
    }
    my $metaname  = $self->{'metaname'};
    my $metapos   = $self->{'metapos'};
    my $metavalue = $self->{'metavalue'};

    my $status = $self->remove_index_metadata_entry($collect_dir,$collect,$infodbtype,$docid,$metaname,$metapos,$metavalue);

    if ($status != 0) {
	my $mess = "Failed to remove metadata key: $docid\n";
	
	$mess .= "PATH: $ENV{'PATH'}\n";
	$mess .= "Exit status: $status\n";
	$mess .= "System Error Message: $!\n";

	$gsdl_cgi->generate_error($mess);
    }
    else {
	my $mess = "DB remove successful: Key[$docid]\n";
	$mess .= "  $metaname";
	$mess .= "->[$metapos]" if (defined $metapos);
	$mess .= " ($metavalue)" if (defined $metavalue);

	$gsdl_cgi->generate_ok_message($mess);
    }

    #return $status; # in case calling functions have a use for this
}

sub remove_index_metadata_entry
{
    my $self = shift @_;
    my ($collect_dir,$collect,$infodbtype,$docid,$metaname,$metapos,$metavalue) = @_;
    my $gsdl_cgi  = $self->{'gsdl_cgi'};
    
    $metapos = undef if(defined $metapos && ($metapos =~ m/^\s*$/));
    $metavalue = undef if(defined $metavalue && ($metavalue =~ m/^\s*$/)); # necessary to force fallback to undef here
    
    # To people who know $collect_tail please add some comments
    # -> In collection groups, I think collect_tailname is the subcollection name,
    # e.g. colgroup-name/col-tail-name
    # Obtain the path to the database
    my $collect_tail = $collect;
    $collect_tail =~ s/^.*[\/|\\]//;
    my $index_text_directory = &util::filename_cat($collect_dir,$collect,"index","text");
    my $infodb_file_path = &dbutil::get_infodb_file_path($infodbtype, $collect_tail, $index_text_directory);

    # Read the docid entry
    my $doc_rec = &dbutil::read_infodb_entry($infodbtype, $infodb_file_path, $docid);

    # Check to make sure the key does exist
    if (!defined ($doc_rec->{$metaname})) {
        $gsdl_cgi->generate_error("No metadata field \"" . $metaname . "\" in the specified document: [" . $docid . "]");
    }

    # Obtain the specified metadata pos
    # if no metavalue or metapos to delete, default to deleting the 1st value for the metaname
    if(!defined $metapos && !defined $metavalue) {
	    $metapos = 0;
    }
    

    # consider check key is defined before deleting?
    # Loop through the metadata array and ignore the specified position
    my $filtered_metadata = [];
    my $num_metadata_vals = scalar(@{$doc_rec->{$metaname}});    
    for (my $i=0; $i<$num_metadata_vals; $i++) {
	my $metaval = shift(@{$doc_rec->{$metaname}});

	if (!defined $metavalue && $i != $metapos) {
	    push(@$filtered_metadata,$metaval);
	}
	
	if(defined $metavalue && !($metavalue eq $metaval))
	{
	    push(@$filtered_metadata,$metaval);
	}
    }
    $doc_rec->{$metaname} = $filtered_metadata;

    ## Use the dbutil set_entry method instead of assuming the database is gdbm
    my $status = &dbutil::set_infodb_entry($infodbtype, $infodb_file_path, $docid, $doc_rec);    
    return $status;
}

sub remove_index_metadata
{
    my $self = shift @_;

    my $username  = $self->{'username'};
    my $collect   = $self->{'collect'};
    my $gsdl_cgi  = $self->{'gsdl_cgi'};
	
    if ($baseaction::authentication_enabled) {
	# Ensure the user is allowed to edit this collection	
	$self->authenticate_user($username, $collect); #&authenticate_user($gsdl_cgi, $username, $collect);
    }

    # Obtain the collect dir
    my $site = $self->{'site'};
    my $collect_dir = $gsdl_cgi->get_collection_dir($site);

    # Make sure the collection isn't locked by someone else
    $self->lock_collection($username, $collect);

    $self->_remove_index_metadata(@_);

    # Release the lock once it is done
    $self->unlock_collection($username, $collect);
}

#################################################################################
# ERASE META METHODS: ERASE ALL VALUES FOR MATCHING METANAME AT SPECIFIED DOCID #
#################################################################################

sub erase_metadata
{
    my $self = shift @_;

    my $where = $self->{'where'};
    # when $where is unspecified, following behaviour of sub remove_metadata by defaulting to erasing metadata from index
    if(!$where || ($where =~ m/^\s*$/)) {
	$self->erase_index_metadata(@_); # call the full version of set_index_meta for the default behaviour
	return;
    }

    my $username  = $self->{'username'};
    my $collect   = $self->{'collect'};
    my $gsdl_cgi  = $self->{'gsdl_cgi'};

    if ($baseaction::authentication_enabled) {
	# Ensure the user is allowed to edit this collection	
	$self->authenticate_user($username, $collect); #&authenticate_user($gsdl_cgi, $username, $collect);
    }

    # Make sure the collection isn't locked by someone else
    $self->lock_collection($username, $collect);

    # check which directories need to be processed, specified in $where as 
    # any combination of import|archives|index|live
    if($where =~ m/import/) {
	$self->_erase_import_metadata(@_);	    
    }
    if($where =~ m/archives/) {
	$self->_erase_archives_metadata(@_);	    
    } 
    if($where =~ m/index/) {
	$self->_erase_index_metadata(@_);	    
    }

    # Release the lock once it is done
    $self->unlock_collection($username, $collect);
}

# the internal version, without authentication
sub _erase_index_metadata
{    
    my $self = shift @_;

    my $collect   = $self->{'collect'};
    my $gsdl_cgi  = $self->{'gsdl_cgi'};
    my $infodbtype = $self->{'infodbtype'};
	
    # Obtain the collect dir
    my $site = $self->{'site'};
    my $collect_dir = $gsdl_cgi->get_collection_dir($site);

    # look up additional args
    my $docid     = $self->{'d'};
    if ((!defined $docid) || ($docid =~ m/^\s*$/)) {
      $gsdl_cgi->generate_error("No docid (d=...) specified.");
    }
    my $metaname  = $self->{'metaname'};
    my $metapos   = undef;
    my $metavalue = undef;

    # To people who know $collect_tail please add some comments
    # -> In collection groups, I think collect_tailname is the subcollection name,
    # e.g. colgroup-name/col-tail-name
    # Obtain the path to the database
    my $collect_tail = $collect;
    $collect_tail =~ s/^.*[\/|\\]//;
    my $index_text_directory = &util::filename_cat($collect_dir,$collect,"index","text");
    my $infodb_file_path = &dbutil::get_infodb_file_path($infodbtype, $collect_tail, $index_text_directory);

    # Read the docid entry
    my $doc_rec = &dbutil::read_infodb_entry($infodbtype, $infodb_file_path, $docid);

    # Check to make sure the key does exist
    if (defined ($doc_rec->{$metaname})) {        

	$doc_rec->{$metaname} = [];

	## Use the dbutil set_entry method instead of assuming the database is gdbm
	my $status = &dbutil::set_infodb_entry($infodbtype, $infodb_file_path, $docid, $doc_rec);

	if ($status != 0) {
	    my $mess = "Failed to erase metadata key: $docid\n";
	    
	    $mess .= "PATH: $ENV{'PATH'}\n";
	    $mess .= "Exit status: $status\n";
	    $mess .= "System Error Message: $!\n";
	    
	    $gsdl_cgi->generate_error($mess);
	}
	else {
	    my $mess = "DB set (with item deleted) successful: Key[$docid]\n";
	    $mess .= "  $metaname";
	    $mess .= "->[$metapos]" if (defined $metapos);
	    $mess .= " ($metavalue)" if (defined $metavalue);
	    
	    $gsdl_cgi->generate_ok_message($mess);
	}
    }
    else { # no such metaname. Is it really an error?
	$gsdl_cgi->generate_ok_message("Can't erase. No metadata field \"" . $metaname . "\" in the specified document: [" . $docid . "]");
    }
    #return $status; # in case calling functions have a use for this
}

sub erase_index_metadata
{
    my $self = shift @_;

    my $username  = $self->{'username'};
    my $collect   = $self->{'collect'};
    my $gsdl_cgi  = $self->{'gsdl_cgi'};
	
    if ($baseaction::authentication_enabled) {
	# Ensure the user is allowed to edit this collection	
	$self->authenticate_user($username, $collect); #&authenticate_user($gsdl_cgi, $username, $collect);
    }

    # Obtain the collect dir
    my $site = $self->{'site'};
    my $collect_dir = $gsdl_cgi->get_collection_dir($site);

    # Make sure the collection isn't locked by someone else
    $self->lock_collection($username, $collect);

    $self->_erase_index_metadata(@_);

    # Release the lock once it is done
    $self->unlock_collection($username, $collect);
}


# Given a metaname, removes ALL metadata with that metaname at the provided docid
sub _erase_archives_metadata
{
	my $self = shift @_;

	my $collect   = $self->{'collect'};
	my $gsdl_cgi  = $self->{'gsdl_cgi'};
	my $infodbtype = $self->{'infodbtype'};
			
	# Obtain the collect and archive dir   
	my $site = $self->{'site'};
	my $collect_dir = $gsdl_cgi->get_collection_dir($site);	
	
	my $archive_dir = &util::filename_cat($collect_dir,$collect,"archives");

	# look up additional args
	my ($docid, $docid_secnum) = ($self->{'d'} =~ m/^(.*?)(\..*)?$/);
	
	my $metaname = $self->{'metaname'};

	# metapos, metavalue and metamode are never provided and therefore all undefined for erase_archives_metadata
	my $metapos = undef;
	my $metavalue = undef;
	my $metamode = "override"; # when override and no metapos or metaval, all values for metaname will be affected

	my $arcinfo_doc_filename = &dbutil::get_infodb_file_path($infodbtype, "archiveinf-doc", $archive_dir);
	my $doc_rec = &dbutil::read_infodb_entry($infodbtype, $arcinfo_doc_filename, $docid);

	# This now stores the full pathname
	my $doc_file = $doc_rec->{'doc-file'}->[0];	

	# check if request if for file-level doc-version history 'nminus-<n>' version
	my $dv = $self->{'dv'};	
	if (defined $dv && ($dv ne "")) {
	    # Need to insert '_fldv_history/nminus-<n>' into doc_filename
	    
	    my ($doc_tailname, $doc_dirname) = File::Basename::fileparse($doc_file);
	    $doc_file = &util::filename_cat($doc_dirname,$FLDV_HISTORY_DIR,$dv,$doc_tailname);
	}

	my $doc_filename = &util::filename_cat($archive_dir, $doc_file);
	
	my $status = $self->remove_from_doc_xml($gsdl_cgi, $doc_filename, $metaname, $metapos, $metavalue, $docid_secnum, $metamode);
	
	if ($status == 0) 
	{
		my $mess = "\nerase-archives-metadata successful: \nKey[$docid]\n";
		$mess .= "  $metaname";
		$gsdl_cgi->generate_ok_message($mess);	
	}
	else 
	{
		my $mess .= "Failed to erase archives metadata key: $docid\n";
		$mess .= "Exit status: $status\n";
		$mess .= "System Error Message: $!\n";
		$mess .= "-" x 20 . "\n";
		
		$gsdl_cgi->generate_error($mess);
	}
	
	#return $status; # in case calling functions have a use for this
}

# practically identical to remove_archives_metadata
sub erase_archives_metadata
{
	my $self = shift @_;

	my $username  = $self->{'username'};
	my $collect   = $self->{'collect'};
	my $gsdl_cgi  = $self->{'gsdl_cgi'};
	
	if ($baseaction::authentication_enabled) 
	{
	    # Ensure the user is allowed to edit this collection		
	    $self->authenticate_user($username, $collect); #&authenticate_user($gsdl_cgi, $username, $collect);	
	}

	# Make sure the collection isn't locked by someone else
	$self->lock_collection($username, $collect);

	$self->_erase_archives_metadata(@_);

	# Release the lock once it is done
	$self->unlock_collection($username, $collect);
}

sub _erase_import_metadata
{
	my $self = shift @_;

	my $collect   = $self->{'collect'};
	my $gsdl_cgi  = $self->{'gsdl_cgi'};
	my $infodbtype = $self->{'infodbtype'};
	
	# Obtain the collect dir
	my $site = $self->{'site'};
	my $collect_dir = $gsdl_cgi->get_collection_dir($site);
	
	my $archive_dir = &util::filename_cat($collect_dir,$collect,"archives");

	# look up additional args
	my $docid = $self->{'d'};
	if ((!defined $docid) || ($docid =~ m/^\s*$/)) 
	{
		$gsdl_cgi->generate_error("No docid (d=...) specified.\n");
	}
	
	my $metaname = $self->{'metaname'};
	my $metapos = undef;
	my $metavalue = undef;
	my $metamode = "override";

	# Obtain where the metadata.xml is from the archiveinfo-doc.gdb file
	# If the doc oid is not specified, we assume the metadata.xml is next to the specified "f"
	my $metadata_xml_file;
	my $import_filename = undef;
	if (defined $docid) 
	{
		my $arcinfo_doc_filename = &dbutil::get_infodb_file_path($infodbtype, "archiveinf-doc", $archive_dir);
		my $doc_rec = &dbutil::read_infodb_entry($infodbtype, $arcinfo_doc_filename, $docid);

		# This now stores the full pathname
		$import_filename = $doc_rec->{'src-file'}->[0];	
		$import_filename = &util::placeholders_to_abspath($import_filename);
	}

	if((!defined $import_filename) || ($import_filename =~ m/^\s*$/))
	{
		$gsdl_cgi->generate_error("There is no metadata\n");
	}
	
	# figure out correct metadata.xml file [?]
	# Assuming the metadata.xml file is next to the source file
	# Note: This will not work if it is using the inherited metadata from the parent folder
	my ($import_tailname, $import_dirname) = File::Basename::fileparse($import_filename);
	my $metadata_xml_filename = &util::filename_cat($import_dirname,"metadata.xml");
	
	$self->remove_from_metadata_xml($gsdl_cgi, $metadata_xml_filename, $metaname, $metapos, $metavalue, $import_tailname, $metamode); # metamode=override, means all values of metaname are affected
	
	my $mess = "erase-import-metadata successful: Key[$docid] -> $metadata_xml_filename\n";
	$mess .= "  $metaname";
	$mess .= " = $metavalue\n";
	
	$gsdl_cgi->generate_ok_message($mess);

	#return $status; # in case calling functions have a use for this
}

sub erase_import_metadata
{
	my $self = shift @_;
	
	my $username = $self->{'username'};
	my $collect   = $self->{'collect'};
	my $gsdl_cgi  = $self->{'gsdl_cgi'};
	
	if ($baseaction::authentication_enabled) {
	    # Ensure the user is allowed to edit this collection		
	    $self->authenticate_user($username, $collect); #&authenticate_user($gsdl_cgi, $username, $collect);
	}

	# Make sure the collection isn't locked by someone else
	$self->lock_collection($username, $collect);
	
	$self->_erase_import_metadata(@_);

	# Release the lock once it is done
	$self->unlock_collection($username, $collect);

}

# removes all meta with matching metaname for the specified docid from the live metadata index
sub _erase_live_metadata
{
    my $self = shift @_;
    # At present, _remove_live_metadata doesn't take metaval or metapos either and therefore has the desired effect
    $self->_remove_live_metadata(@_);

}

sub erase_live_metadata
{
    my $self = shift @_;

    my $username  = $self->{'username'};
    my $collect   = $self->{'collect'};
    my $gsdl_cgi  = $self->{'gsdl_cgi'};
	
    if ($baseaction::authentication_enabled) {
	# Ensure the user is allowed to edit this collection	
	$self->authenticate_user($username, $collect); #&authenticate_user($gsdl_cgi, $username, $collect);
    }

    # Make sure the collection isn't locked by someone else
    $self->lock_collection($username, $collect);

    $self->_erase_live_metadata(@_);

    $self->unlock_collection($username, $collect);
}


# Was trying to reused the codes, but the functions need to be broken
# down more before they can be reused, otherwise there will be too
# much overhead and duplicate process...
sub insert_metadata
{
    my $self = shift @_;
    
    my $username  = $self->{'username'};
    my $collect   = $self->{'collect'};
    my $gsdl_cgi  = $self->{'gsdl_cgi'};
    my $infodbtype = $self->{'infodbtype'};
	
    # If the import metadata and gdbm database have been updated, we
    # need to insert some notification to warn user that the the text
    # they see at the moment is not indexed and require a rebuild.
    my $rebuild_pending_macro = "_rebuildpendingmessage_";

    if ($baseaction::authentication_enabled) {
	# Ensure the user is allowed to edit this collection
	$self->authenticate_user($username, $collect);
    }

    # Obtain the collect and archive dir   
    my $site = $self->{'site'};
    my $collect_dir = $gsdl_cgi->get_collection_dir($site);
    my $archive_dir = &util::filename_cat($collect_dir,$collect,"archives");

    # Make sure the collection isn't locked by someone else
    $self->lock_collection($username, $collect);
    
    # Check additional args
    my $docid = $self->{'d'};
    if (!defined($docid) || ($docid =~ m/^\s*$/)) {
	$gsdl_cgi->generate_error("No document id is specified: d=...");
    } 
    my $metaname = $self->{'metaname'};
    if (!defined($metaname) || ($metaname =~ m/^\s*$/)) {
	$gsdl_cgi->generate_error("No metaname is specified: metadataname=...");
    } 
    my $metavalue = $self->{'metavalue'};
    if (!defined($metavalue) || ($metavalue =~ m/^\s*$/)) {
	$gsdl_cgi->generate_error("No metavalue or empty metavalue is specified: metadataname=...");
    } 
    # make "accumulate" the default (less destructive, as it won't actually 
    # delete any existing values)
    my $metamode = "accumulate";

    # metapos/prevmetavalue were never before used in this subroutine, so set them to undefined
    my $metapos   = undef;
    my $prevmetavalue = undef;

    #=======================================================================#
    # set_import_metadata [START]
    #=======================================================================#
    # Obtain where the metadata.xml is from the archiveinfo-doc.gdb file
    # If the doc oid is not specified, we assume the metadata.xml is next to the specified "f"
    my $metadata_xml_file;
    my $arcinfo_doc_filename = &dbutil::get_infodb_file_path($infodbtype, "archiveinf-doc", $archive_dir);
    my $archive_doc_rec = &dbutil::read_infodb_entry($infodbtype, $arcinfo_doc_filename, $docid);
    
    # This now stores the full pathname
    my $import_filename = $archive_doc_rec->{'src-file'}->[0];
    $import_filename = &util::placeholders_to_abspath($import_filename);
    
    # figure out correct metadata.xml file [?]
    # Assuming the metadata.xml file is next to the source file
    # Note: This will not work if it is using the inherited metadata from the parent folder
    my ($import_tailname, $import_dirname) 
	= File::Basename::fileparse($import_filename);
    my $metadata_xml_filename = &util::filename_cat($import_dirname,"metadata.xml");

    # Shane's escape characters
    $metavalue = pack "U0C*", unpack "C*", $metavalue;
    $metavalue =~ s/\,/&#44;/g;
    $metavalue =~ s/\:/&#58;/g;
    $metavalue =~ s/\|/&#124;/g;
    $metavalue =~ s/\(/&#40;/g;
    $metavalue =~ s/\)/&#41;/g;
    $metavalue =~ s/\[/&#91;/g;
    $metavalue =~ s/\\/&#92;/g;
    $metavalue =~ s/\]/&#93;/g;
    $metavalue =~ s/\{/&#123;/g;
    $metavalue =~ s/\}/&#125;/g;
    $metavalue =~ s/\"/&#34;/g;
    $metavalue =~ s/\`/&#96;/g;
    $metavalue =~ s/\n/_newline_/g;

    # Edit the metadata.xml
    # Modified by Jeffrey from DL Consulting
    # Handle the case where there is one metadata.xml file for multiple FileSets
    # The XML filter needs to know whether it is in the right FileSet
    # TODO: This doesn't fix the problem where the metadata.xml is not next to the src file.
    # TODO: This doesn't handle the common metadata (where FileName doesn't point to a single file)
    $self->edit_metadata_xml($gsdl_cgi, $metadata_xml_filename, $metaname,
			     $metapos, $metavalue, $metamode, $import_tailname, $prevmetavalue);
    #=======================================================================#
    # set_import_metadata [END]
    #=======================================================================#


    #=======================================================================#
    # set_metadata (accumulate version) [START]
    #=======================================================================#
    # To people who know $collect_tail please add some comments
    # Obtain path to the database
    my $collect_tail = $collect;
    $collect_tail =~ s/^.*[\/|\\]//;
    my $index_text_directory = &util::filename_cat($collect_dir,$collect,"index","text");
    my $infodb_file_path = &dbutil::get_infodb_file_path($infodbtype, $collect_tail, $index_text_directory);

    # Read the docid entry
    my $doc_rec = &dbutil::read_infodb_entry($infodbtype, $infodb_file_path, $docid);

    # Protect the quotes
    $metavalue =~ s/\"/\\\"/g;

    # Adds the pending macro
    my $macro_metavalue = $rebuild_pending_macro . $metavalue;

    # If the metadata doesn't exist, create a new one
    if (!defined($doc_rec->{$metaname})){    
	$doc_rec->{$metaname} = [ $macro_metavalue ];
    }
    # Else, let's acculumate the values
    else {
        push(@{$doc_rec->{$metaname}},$macro_metavalue);
    }

    ## Use the dbutil set_entry method instead of assuming the database is gdbm
    my $status = &dbutil::set_infodb_entry($infodbtype, $infodb_file_path, $docid, $doc_rec);

    if ($status != 0) {
        # Catch error if gdbmget failed
	my $mess = "Failed to set metadata key: $docid\n";
	
	$mess .= "PATH: $ENV{'PATH'}\n";
	$mess .= "Exit status: $status\n";
	$mess .= "System Error Message: $!\n";

	$gsdl_cgi->generate_error($mess);
    }
    else {
	my $mess = "insert-metadata successful: Key[$docid]\n";
	$mess .= "  [In metadata.xml] $metaname";
	$mess .= " = $metavalue\n";
	$mess .= "  [In database] $metaname";
	$mess .= " = $macro_metavalue\n";
	$mess .= "  The new text has not been indexed, rebuilding collection is required\n";
        $gsdl_cgi->generate_ok_message($mess);
    }    
    #=======================================================================#
    # set_metadata (accumulate version) [END]
    #=======================================================================#

    # Release the lock once it is done
    $self->unlock_collection($username, $collect);
}




sub inc_fldv_nminus1
{
    my $self = shift @_;

    my $username  = $self->{'username'};
    my $collect   = $self->{'collect'};
    my $gsdl_cgi  = $self->{'gsdl_cgi'};
    my $infodbtype = $self->{'infodbtype'};
    
    # Obtain the collect dir
    my $site = $self->{'site'};
    my $collect_dir = $gsdl_cgi->get_collection_dir($site);
    
    my $archive_dir = &util::filename_cat($collect_dir, $collect, "archives");
    
    # look up additional args
    my $docid = $self->{'d'};
    
    
    my $arcinfo_doc_filename = &dbutil::get_infodb_file_path($infodbtype, "archiveinf-doc", $archive_dir);
    my $doc_rec = &dbutil::read_infodb_entry($infodbtype, $arcinfo_doc_filename, $docid);
    
	# This now stores the full pathname [is this still true??]
    my $doc_file = $doc_rec->{'doc-file'}->[0];	

    my ($unused_doc_tailname, $doc_dirname) = File::Basename::fileparse($doc_file);
    my $doc_full_dirname = &util::filename_cat($archive_dir,$doc_dirname);
    
    my $fldv_full_dirname = &util::filename_cat($doc_full_dirname,$FLDV_HISTORY_DIR);

    my $had_error = 0;
    my $had_error_mess = undef;
    
    if (-d $fldv_full_dirname) {
	my $fldv_filtered_dirs = &FileUtils::readDirectoryFiltered($fldv_full_dirname,undef,"^nminus-\\d+\$");
	
	my @sorted_fldv_filtered_dirs = sort {
	    my ($a_num) = ($a =~ m/(\d+)$/);
	    my ($b_num) = ($b =~ m/(\d+)$/);
	    
	    # sort into descending order
	    return $b_num <=> $a_num;
	} @$fldv_filtered_dirs;
	
	## shuffle all the nminus-<n> folders down by one
	
	foreach my $nminus_n (@sorted_fldv_filtered_dirs) {
	    
	    my $nminus_n_full_dir = &FileUtils::filenameConcatenate($fldv_full_dirname,$nminus_n);
	    if (-d $nminus_n_full_dir) {
		
		my ($n) = ($nminus_n =~ m/(\d+)$/);
		my $new_n = $n + 1;
		
		my $new_full_dir_plus1 = &FileUtils::filenameConcatenate($fldv_full_dirname,"nminus-$new_n");

		my $move_ok = &FileUtils::renameDirectory($nminus_n_full_dir,$new_full_dir_plus1,
							  { 'strict' => 1 } );
		
		if (!$move_ok) {
		    $had_error_mess = "Error: Failed to move '$nminus_n' to nminus-$new_n in $fldv_full_dirname\n";
		    $had_error = 1;
		    last;
		}
		
	    }
	    else {
		print STDERR "Warning: skipping $nminus_n_full_dir as it is not a directory\n";
	    }
	}
    }
    else {
	# first time asked to perform the inc-nminus-1 action for this doc,
	#  and there is not yet even a _fldv directory!
	# => make the directory (and that's all that's needed at this stage)
	print STDERR "First time file-level document-version history has been applied to this document\n";
	print STDERR "Creating $fldv_full_dirname\n";
	
	&FileUtils::makeDirectory($fldv_full_dirname);
    }
    
    if (!$had_error) {
	### Now need to copy everything top level in doc dir into nminus-1

	my $nminus_1_full_dir = &FileUtils::filenameConcatenate($fldv_full_dirname,"nminus-1");
	
	my $copy_ok = &FileUtils::copyFilesRefRecursive([$doc_full_dirname],$nminus_1_full_dir,
							{ 'strict' => 1, 'exclude_filter_re' => "^$FLDV_HISTORY_DIR\$" } );
	
	if (!$copy_ok) {
	    $had_error_mess = "Error: Failed to clone document $docid to form 'nminus-1' directory in $fldv_full_dirname";
	    $had_error = 1;
	}
    }
    
    if ($had_error) {
	$gsdl_cgi->generate_error($had_error_mess);
    }
    else {
	$gsdl_cgi->generate_ok_message("Successfully incremented the File-Level Document-Version History for $docid");
    }
}


############################# REMOVE METADATA ARRAY METHODS ############################


sub remove_metadata_array
{
    my $self = shift @_;

    my $where = $self->{'where'};
    if(!$where || ($where =~ m/^\s*$/)) {	
	$self->remove_index_metadata_array(@_); # mimic set_metadata_array for default behaviour
	return;
    }

    my $username  = $self->{'username'};
    my $collect   = $self->{'collect'};
    my $gsdl_cgi  = $self->{'gsdl_cgi'};

    if ($baseaction::authentication_enabled) {
	# Ensure the user is allowed to edit this collection
	$self->authenticate_user($username, $collect); #&authenticate_user($gsdl_cgi, $username, $collect);
    }

    # Not sure if the checked_chdir is necessary, since lock_collection also does a chdir
    # But copied code from set_metadata_array
    my $site = $self->{'site'};
    my $collect_dir = $gsdl_cgi->get_collection_dir($site);
    $gsdl_cgi->checked_chdir($collect_dir);

    # Make sure the collection isn't locked by someone else
    $self->lock_collection($username, $collect);

    if($where =~ m/import/) {
	$self->_remove_import_metadata_array(@_);
    }
    if($where =~ m/archives/) {
	$self->_remove_archives_metadata_array(@_);
    }
    if($where =~ m/index/) {
	$self->_remove_index_metadata_array(@_);
    }
    if($where =~ m/live/) {
    	$self->_remove_live_metadata_array(@_);
    }

    # Release the lock once it is done
    $self->unlock_collection($username, $collect);
}

sub _remove_index_metadata_array
{
    my $self = shift @_;

    my $collect   = $self->{'collect'};
    my $gsdl_cgi  = $self->{'gsdl_cgi'};

    # Obtain the collect dir
    my $site = $self->{'site'};
    my $collect_dir = $gsdl_cgi->get_collection_dir($site);
    
    # look up additional args
    
    my $infodbtype = $self->{'infodbtype'};
    
    my $json_str      = $self->{'json'};
    my $doc_array = decode_json $json_str;
    
    
    my $global_status = 0;
    my $global_mess = "";
    
    my @all_docids = ();
    
    foreach my $doc_array_rec ( @$doc_array ) {
	
	my $status = -1;
	my $docid     = $doc_array_rec->{'docid'};
	
	push(@all_docids,$docid);

	my $metaname  = $doc_array_rec->{'metaname'};
	if(defined $metaname) {
	    # metapos can be undef and if metaval is also undef, metapos=0 will be used
	    my $metapos   = $doc_array_rec->{'metapos'};
	    my $metavalue = $doc_array_rec->{'metavalue'};
	    # remove-import-metadata and remove-archives-metadata recognise metamode option
	    # but not remove-index-metadata (nor remove-live-metadata)
	    #my $metamode = $doc_array_rec->{'metamode'} || $self->{'metamode'};

	    $status = $self->remove_index_metadata_entry($collect_dir,$collect,$infodbtype,$docid,$metaname,$metapos,$metavalue);
	} elsif (defined $doc_array_rec->{'metatable'}) { # if no metaname, we expect a metatable
	    my $metatable = $doc_array_rec->{'metatable'}; # a subarray, or need to generate an error saying JSON structure is wrong
	    
	    foreach my $metatable_rec ( @$metatable ) { # the subarray metatable is an array of hashmaps
		$metaname  = $metatable_rec->{'metaname'};
		#my $metamode  = $metatable_rec->{'metamode'} || $doc_array_rec->{'metamode'} || $self->{'metamode'}; # metamode not used by remove_index_metadata
		
		#my $metapos = $metatable_rec->{'metapositions'} || $doc_array_rec->{'metapos'} || $self->{'metapos'} || undef; # try most-to-least specific metapos setting, and fallback on undef


		my $metapositions = $metatable_rec->{'metapositions'} || undef; # a sub-subarray
		if(defined $metapositions) {
		    
		    my $metavalue = undef;
		    # Need to have metapositions sorted in *descending* order for
		    # remove_metadata_array to work. Start deleting from end of metadata list,
		    # so we don't have to recalculate metapos after each delete at a metapos
		    @$metapositions = reverse(sort(@$metapositions)); # https://perldoc.perl.org/functions/reverse
		    
		    foreach my $metapos ( @$metapositions ) { # metapositions is an array
			$status = $self->remove_index_metadata_entry($collect_dir,$collect,$infodbtype,$docid,$metaname,$metapos,$metavalue); # how do we use metamode in set_meta_entry?
		    
		   } 
		} else {		    
		    my $metavals = $metatable_rec->{'metavals'}; # a sub-subarray
		    next unless defined $metavals;
		    my $metapos = undef;
		    
		    foreach my $metavalue ( @$metavals ) { # metavals is an array
			$status = $self->remove_index_metadata_entry($collect_dir,$collect,$infodbtype,$docid,$metaname,$metapos,$metavalue); # how do we use metamode in set_meta_entry?	
		    }
		}
	    }
	}
	
	if ($status != 0) {
	    # Catch error if set infodb entry failed
	    $global_status = $status;
	    $global_mess .= "Failed to remove metadata key from index: $docid\n";
	    $global_mess .= "Exit status: $status\n";
	    # Need to check the error message $! immediately after an IO call goes wrong
	    # not after returning from many nested calls
	    # https://stackoverflow.com/questions/1605195/inappropriate-ioctl-for-device
	    # $global_mess .= "System Error Message: $!\n";
	    $global_mess .= "-" x 20;
	}
    }
    
    if ($global_status != 0) {
	$global_mess .= "PATH: $ENV{'PATH'}\n";
	$gsdl_cgi->generate_error($global_mess);
    }
    else {
	my $mess = "remove-metadata-array successful: Keys[ ".join(", ",@all_docids)."]\n";
	$gsdl_cgi->generate_ok_message($mess);
    }
}

sub remove_index_metadata_array
{
    my $self = shift @_;

    my $username  = $self->{'username'};
    my $collect   = $self->{'collect'};
    my $gsdl_cgi  = $self->{'gsdl_cgi'};

    if ($baseaction::authentication_enabled) {
	# Ensure the user is allowed to edit this collection	
	$self->authenticate_user($username, $collect); #&authenticate_user($gsdl_cgi, $username, $collect);
    }

    # Obtain the collect dir
    my $site = $self->{'site'};
    my $collect_dir = $gsdl_cgi->get_collection_dir($site);

    # Skip this step as lock_collection does a chdir into final destination directory anyway?
    $gsdl_cgi->checked_chdir($collect_dir);


    # Make sure the collection isn't locked by someone else
    $self->lock_collection($username, $collect);

    $self->_remove_index_metadata_array(@_);

    # Release the lock once it is done
    $self->unlock_collection($username, $collect);
}

# Untested, based on also (probably) untested, experimental _set_live_metadata_array
# TODO: How can we delete at a dbkey with a particular value or at a particular metapos?
# Same question for sub remove_live_metadata
sub _remove_live_metadata_array
{
    my $self = shift @_;

    my $collect   = $self->{'collect'};
    my $gsdl_cgi  = $self->{'gsdl_cgi'};

    # Obtain the collect dir
    my $site = $self->{'site'};
    my $collect_dir = $gsdl_cgi->get_collection_dir($site);
    
    # look up additional args
    my $infodbtype = $self->{'infodbtype'};
    # To people who know $collect_tail please add some comments
    # Obtain path to the database
    my $collect_tail = $collect;
    $collect_tail =~ s/^.*[\/|\\]//;
    my $index_text_directory = &util::filename_cat($collect_dir,$collect,"index","text");
    my $infodb_file_path = &dbutil::get_infodb_file_path($infodbtype, "live-$collect_tail", $index_text_directory);

    
    my $json_str      = $self->{'json'};
    my $doc_array = decode_json $json_str;
    
    
    my $global_status = 0;
    my $global_mess = "";
    
    my @all_docids = ();


    foreach my $doc_array_rec ( @$doc_array ) {
	
	my $status = -1;
	my $docid     = $doc_array_rec->{'docid'};

	push(@all_docids,$docid);

	my $dbkey = "";
	my $metaname  = $doc_array_rec->{'metaname'};
	if(defined $metaname) {
	    $dbkey = "$docid.$metaname";

	    # Remove the key
	    #my $metavalue = $doc_array_rec->{'metavalue'};
	    #my $cmd = "gdbmdel \"$infodb_file_path\" \"$dbkey\" \"$metavalue\"";
	    my $cmd = "gdbmdel \"$infodb_file_path\" \"$dbkey\"";
	    $status = system($cmd);

	} elsif (defined $doc_array_rec->{'metatable'}) { # if no metaname, we expect a metatable
	    my $metatable = $doc_array_rec->{'metatable'}; # a subarray, or need to generate an error saying JSON structure is wrong
	    foreach my $metatable_rec ( @$metatable ) {
		$metaname  = $metatable_rec->{'metaname'};
		$dbkey = "$docid.$metaname";
		
		my $cmd = "gdbmdel \"$infodb_file_path\" \"$dbkey\"";
		$status = system($cmd);
		#my $metavals = $metatable_rec->{'metavals'}; # a sub-subarray
		#foreach my $metavalue ( @$metavals ) {
		     #my $cmd = "gdbmdel \"$infodb_file_path\" \"$dbkey\" \"$metavalue\"";
		     #$status = system($cmd);
		#}
	    }
	    
	}

	if ($status != 0) {
	    # Catch error if gdbmdel failed
	    $global_status = $status;
	    $global_mess .= "Failed to delete metadata key: $dbkey\n"; # $dbkey
	    $global_mess .= "Exit status: $status\n";
	    $global_mess .= "System Error Message: $!\n";
	    $global_mess .= "-" x 20;
	}
    }
    
    if ($global_status != 0) {
	$global_mess .= "PATH: $ENV{'PATH'}\n";
	$gsdl_cgi->generate_error($global_mess);
    }
    else {
	my $mess = "remove-live-metadata-array successful: Keys[ ".join(", ",@all_docids)."]\n";
	$gsdl_cgi->generate_ok_message($mess);
    }
}

sub remove_live_metadata_array
{
    my $self = shift @_;

    my $username  = $self->{'username'};
    my $collect   = $self->{'collect'};
    my $gsdl_cgi  = $self->{'gsdl_cgi'};

    if ($baseaction::authentication_enabled) {
	# Ensure the user is allowed to edit this collection	
	$self->authenticate_user($username, $collect); #&authenticate_user($gsdl_cgi, $username, $collect);
    }

    # Obtain the collect dir
    my $site = $self->{'site'};
    my $collect_dir = $gsdl_cgi->get_collection_dir($site);
	
    $gsdl_cgi->checked_chdir($collect_dir);

    # Make sure the collection isn't locked by someone else
    $self->lock_collection($username, $collect);

    $self->_remove_live_metadata_array(@_);

    # Release the lock once it is done
    $self->unlock_collection($username, $collect);
}


sub remove_import_metadata_array
{
    my $self = shift @_;

    my $username  = $self->{'username'};
    my $collect   = $self->{'collect'};
    my $gsdl_cgi  = $self->{'gsdl_cgi'};

    if ($baseaction::authentication_enabled) {
	# Ensure the user is allowed to edit this collection	
	$self->authenticate_user($username, $collect); #&authenticate_user($gsdl_cgi, $username, $collect);
    }

    # Obtain the collect dir
    my $site = $self->{'site'};
    my $collect_dir = $gsdl_cgi->get_collection_dir($site);
    
    $gsdl_cgi->checked_chdir($collect_dir);

    # Make sure the collection isn't locked by someone else
    $self->lock_collection($username, $collect);

    $self->_remove_import_metadata_array(@_);

    # Release the lock once it is done
    $self->unlock_collection($username, $collect);

}


sub _remove_import_metadata_array
{
    my $self = shift @_;

    my $collect   = $self->{'collect'};
    my $gsdl_cgi  = $self->{'gsdl_cgi'};

    # Obtain the collect dir
    my $site = $self->{'site'};
    my $collect_dir = $gsdl_cgi->get_collection_dir($site);
    
    # look up additional args
	
    my $infodbtype = $self->{'infodbtype'};
    
    my $archive_dir = &util::filename_cat($collect_dir,$collect,"archives");	
    my $arcinfo_doc_filename = &dbutil::get_infodb_file_path($infodbtype, "archiveinf-doc", $archive_dir);
    
    my $json_str = $self->{'json'};
    my $doc_array = decode_json $json_str;
    
    my $global_status = 0;
    my $global_mess = "";

#    $gsdl_cgi->generate_error("*** Deliberately causing error to test errorResponse function\n");
    
    my @all_docids = ();
    
    foreach my $doc_array_rec ( @$doc_array ) 
    {
	my $status = -1;
	my $docid = $doc_array_rec->{'docid'};

	my $metadata_xml_filename = undef;
	
	my ($docid_root,$docid_secnum);
	if(defined $docid) {	
	    ($docid_root,$docid_secnum) = ($docid =~ m/^(.*?)(\..*)?$/);	
	    # as yet no support for setting or removing subsection metadata in metadata.xml
	    if ((defined $docid_secnum) && ($docid_secnum !~ m/^\s*$/)) {
		$gsdl_cgi->generate_message("*** docid: $docid. No support yet for deleting import metadata at subsections' level.\n");
		next; # skip this docid in for loop
	    }
	}

	push(@all_docids,$docid); # docid_root rather
	
	my $metaname = $doc_array_rec->{'metaname'};
	if (defined $metaname) {
	    my $metapos   = $doc_array_rec->{'metapos'} || $self->{'metapos'};	    
	    my $metavalue = $doc_array_rec->{'metavalue'}; #|| $self->{'metavalue'};
	    my $metamode = $doc_array_rec->{'metamode'} || $self->{'metamode'} || undef; # see set_import_meta_array for fallback
	    
	    $metadata_xml_filename = $self->remove_import_metadata_entry($gsdl_cgi, $arcinfo_doc_filename, $infodbtype, $docid_root, $metaname, $metapos, $metavalue, $metamode); # at this point, docid_root = docid
	    
	} elsif (defined $doc_array_rec->{'metatable'}) { # if no metaname, we expect a metatable
	    my $metatable = $doc_array_rec->{'metatable'}; # a subarray, or need to generate an error saying JSON structure is wrong
	    
	    foreach my $metatable_rec ( @$metatable ) {
		$metaname  = $metatable_rec->{'metaname'}; 
		my $metamode  = $metatable_rec->{'metamode'} || $doc_array_rec->{'metamode'} || $self->{'metamode'} || undef;
		
		my $metapositions = $metatable_rec->{'metapositions'}; # a sub-subarray
		if(defined $metapositions) {
		    
		    my $metavalue = undef;
		    
		    # need metapositions sorted in *descending* order
		    @$metapositions = reverse(sort(@$metapositions));
		    
		    foreach my $metapos ( @$metapositions ) {
			
			$metadata_xml_filename = $self->remove_import_metadata_entry($gsdl_cgi, $arcinfo_doc_filename, $infodbtype, $docid_root, $metaname, $metapos, $metavalue, $metamode); # at this point, docid_root = docid
			
			#if($metamode eq "override") { # now, having overridden the first metavalue of the metaname, 
			    # need to accumulate subsequent metavals for this metaname, else the just-assigned
			    # metavalue for this metaname will be lost
			    #$metamode = "accumulate";
			#}
		    }		    
		}
		else {
		    my $metavals = $metatable_rec->{'metavals'}; # a sub-subarray
		    next unless defined $metavals;
		    foreach my $metavalue ( @$metavals ) {
			$metavalue =~ s/&lt;(.*?)&gt;/<$1>/g;
			my $metapos = undef;
			
			$self->remove_import_metadata_entry($gsdl_cgi, $arcinfo_doc_filename, $infodbtype, $docid_root, $metaname, $metapos, $metavalue, $metamode); # at this point, docid_root = docid
			
			#if($metamode eq "override") { # now, having overridden the first metavalue of the metaname, 
			    # need to accumulate subsequent metavals for this metaname, else the just-assigned
			    # metavalue for this metaname will be lost
			    #$metamode = "accumulate";
			#}
		    }
		}
	    }
	}

	# errors would have caused a call to die, so can't report those
	if (defined $metadata_xml_filename) {
	   
	    $global_mess .= "remove-import-metadata successful: Key[$docid] -> $metadata_xml_filename\n";
	}
    }


    # always a success message, as any failure would have caused a call to die within remove_import_metadata_entry
    my $mess = "remove-import-metadata-array successful: Keys[ ".join(", ",@all_docids)."]\n";
    $mess .= $global_mess . "\n";
    $gsdl_cgi->generate_ok_message($mess);
}


sub _remove_archives_metadata_array
{
    my $self = shift @_;
    
    my $collect   = $self->{'collect'};
    my $gsdl_cgi  = $self->{'gsdl_cgi'};

    # Obtain the collect dir
    my $site = $self->{'site'};
    my $collect_dir = $gsdl_cgi->get_collection_dir($site);

    # look up additional args
	
    my $infodbtype = $self->{'infodbtype'};
    
    my $archive_dir = &util::filename_cat($collect_dir,$collect,"archives");
	
    my $json_str      = $self->{'json'};
    my $doc_array = decode_json $json_str;
    
    
    my $global_status = 0;
    my $global_mess = "";
    
    my @all_docids = ();
    
    foreach my $doc_array_rec ( @$doc_array ) {
	my $status    = -1;
	my $full_docid     = $doc_array_rec->{'docid'};
	my ($docid, $docid_secnum) = ($full_docid =~ m/^(.*?)(\..*)?$/);
	# usercomments meta are at root and so docid_secnum will be undef which is acceptable
	my $dv        = $doc_array_rec->{'dv'};
	
	push(@all_docids,$full_docid);
	
	my $metaname  = $doc_array_rec->{'metaname'};
	if(defined $metaname) {
	    
	    my $metapos   = $doc_array_rec->{'metapos'};
	    my $metavalue = $doc_array_rec->{'metavalue'};
	    my $metamode  = $doc_array_rec->{'metamode'} || $self->{'metamode'};
	    
	    $status = $self->remove_archives_metadata_entry($gsdl_cgi,$archive_dir,$infodbtype,
			 $docid, $docid_secnum, $dv, $metaname,$metapos,$metavalue,$metamode);

	} elsif (defined $doc_array_rec->{'metatable'}) { # if no metaname, we expect a metatable
	    my $metatable = $doc_array_rec->{'metatable'}; # a subarray, or need to generate an error saying JSON structure is wrong
	    
	    foreach my $metatable_rec ( @$metatable ) {
		$metaname  = $metatable_rec->{'metaname'};
		my $metamode  = $metatable_rec->{'metamode'} || $doc_array_rec->{'metamode'} || $self->{'metamode'};
		
		my $metapositions = $metatable_rec->{'metapositions'}; # a sub-subarray
		if(defined $metapositions) {
		    
		    my $metavalue = undef;
		    
		    # need metapositions sorted in *descending* order
		    @$metapositions = reverse(sort(@$metapositions));
		    
		    foreach my $metapos ( @$metapositions ) {
			
			$status = $self->remove_archives_metadata_entry($gsdl_cgi, $archive_dir, $infodbtype,
			      $docid,$docid_secnum,$dv, $metaname,$metapos,$metavalue,$metamode);
			
			#if($metamode eq "override") { # now, having overridden the first metavalue of the metaname, 
			    # need to accumulate subsequent metavals for this metaname, else the just-assigned
			    # metavalue for this metaname will be lost
			    #$metamode = "accumulate";
			#}
		    }		    
		}
		else {
		    my $metavals = $metatable_rec->{'metavals'}; # a sub-subarray
		    next unless defined $metavals;
		    foreach my $metavalue ( @$metavals ) {
			$metavalue =~ s/&lt;(.*?)&gt;/<$1>/g;
			my $metapos = undef;

			$status = $self->remove_archives_metadata_entry($gsdl_cgi,$archive_dir,$infodbtype,
					$docid,$docid_secnum, $dv, $metaname,$metapos,$metavalue,$metamode);			
			#if($metamode eq "override") { # now, having overridden the first metavalue of the metaname, 
			    # need to accumulate subsequent metavals for this metaname, else the just-assigned
			    # metavalue for this metaname will be lost
			    #$metamode = "accumulate";
			#}
		    }
		}		
	    }		
	}
	    
	if ($status != 0) {
	    # Catch error if set infodb entry failed
	    $global_status = $status;
	    $global_mess .= "Failed to remove archives metadata key: $full_docid\n";
	    $global_mess .= "Exit status: $status\n";
	    $global_mess .= "System Error Message: $!\n";
	    $global_mess .= "-" x 20 . "\n";
	}
    }
    
    if ($global_status != 0) {
	$global_mess .= "PATH: $ENV{'PATH'}\n";
	$gsdl_cgi->generate_error($global_mess);
    }
    else {
	my $mess = "remove-archives-metadata-array successful: Keys[ ".join(", ",@all_docids)."]\n";
	$gsdl_cgi->generate_ok_message($mess);
    }
}

sub remove_archives_metadata_array
{
    my $self = shift @_;

    my $username  = $self->{'username'};
    my $collect   = $self->{'collect'};
    my $gsdl_cgi  = $self->{'gsdl_cgi'};

    if ($baseaction::authentication_enabled) {
	# Ensure the user is allowed to edit this collection	
	$self->authenticate_user($username, $collect); #&authenticate_user($gsdl_cgi, $username, $collect);
    }

    # Obtain the collect dir
    my $site = $self->{'site'};
    my $collect_dir = $gsdl_cgi->get_collection_dir($site);
	
    $gsdl_cgi->checked_chdir($collect_dir);

    # Make sure the collection isn't locked by someone else
    $self->lock_collection($username, $collect);

    $self->_remove_archives_metadata_array(@_);
    
    # Release the lock once it is done
    $self->unlock_collection($username, $collect);
}


##################################### END REMOVE ARRAY METHODS #################

# not returning 1; here since this file is conditionally included by metadataction.pm
# and not otherwise meant to be used on its own