###########################################################################
#
# NPPlug.pm -- Plugin for the niupepa collection
# A component of the Greenstone digital library software
# from the New Zealand Digital Library Project at the 
# University of Waikato, New Zealand.
#
# Copyright (C) 1999 New Zealand Digital Library Project
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
#
###########################################################################

# The niupepa collection has a file structure as follows:
# Each niupepa series has its own directory containing some/all 
# of the following:

# meta.txt - file contains metadata to be associated with all documents
# in series.
# *.issue - each issue should have a .issue file which may or may not
# contain metadata to associate with the issue. also contains the list
# of filenames that make up the issue (i.e. one for each page). meta.txt
# is read before *.issue so metadata in .issue files will override that in
# meta.txt
# *.commentary - the commentary of the niupepa series (1 per series)
# text/*.txt/htm - text/html files of issue pages (1 per page) -
# text files are expected to be either .htm or .txt (lower case).
# images/*.gif - image files of issue pages (1 per page)
# abstracts/*.abstract - html files of issue abstracts (1 per issue) 


package NPPlug;

use BasPlug;
use util;

sub BEGIN {
    @ISA = ('BasPlug');
}

use strict;

sub print_usage {
    print STDERR "\nIncorrect options passed to NPPlug, check your collect.cfg configuration file\n";

    print STDERR "\n  usage: plugin NPPlug [options]\n\n";
    print STDERR "  options:\n";
    print STDERR "   -create_log            Creates a log file containing info about which portions\n";
    print STDERR "                          of papers are missing\n";
    print STDERR "   -logfile               Path of logfile (defaults to ./log.txt)\n\n";
}

sub new {
    my ($class) = @_;
    my $self = new BasPlug ();

    if (!parsargv::parse(\@_, 
			 q^create_log^, \$self->{'create_log'},
			 q^logfile/.*/./log.txt^, \$self->{'logfile'})) {
	&print_usage();
	die "\n";
    }

    $self->{'commentaries'} = {};
    $self->{'num_issues'} = 0;
    $self->{'num_text_pages'} = 0;
    $self->{'num_image_pages'} = 0;
    $self->{'num_abstracts'} = 0;
    return bless $self, $class;
}

sub is_recursive {
    my $self = shift (@_);

    return 0; # this is not a recursive plugin
}

sub begin {
    my $self = shift (@_);
    my ($pluginfo, $base_dir, $processor, $maxdocs) = @_;

    # open up logfile
    # note that we append to logfile as building will otherwise
    # overwrite a file generated at import time
    if ($self->{'create_log'}) {
	open (LOGFILE, ">>$self->{'logfile'}") || die 
	    "NPPlug.pm: Couldn't open log file $self->{'logfile'}\n";

	my @time = localtime (time);

	print LOGFILE "------------------------------------------------------------\n";
	print LOGFILE "Log start $time[3]/$time[4]/" . (1900 + $time[5]) . "\n";
	print LOGFILE "------------------------------------------------------------\n";
    }
}

sub end {
    my $self = shift (@_);

    if ($self->{'create_log'}) {
	my $numseries = 0;
	my $numcommentaries = 0;

	# record missing commentaries in logfile
	foreach my $key (keys %{$self->{'commentaries'}}) {
	    $numseries ++;
	    if (!$self->{'commentaries'}->{$key}) {
		print LOGFILE "Commentary missing for series $key\n";
	    } else {
		$numcommentaries ++;
	    }
	}
	
	print LOGFILE "\n\nStatistics:\n";
	print LOGFILE "series: $numseries\n";
	print LOGFILE "commentaries: $numcommentaries\n";
	print LOGFILE "issues: $self->{'num_issues'}\n";
	print LOGFILE "abstracts: $self->{'num_abstracts'}\n";
	print LOGFILE "text pages: $self->{'num_text_pages'}\n";
	print LOGFILE "image pages: $self->{'num_image_pages'}\n";

	# close logfile
	close LOGFILE;
    }
}

# return number of files processed, undef if can't process
# Note that $base_dir might be "" and that $file might 
# include directories
sub read {
    my $self = shift (@_);
    my ($pluginfo, $base_dir, $file, $metadata, $processor) = @_;

    my $filename = &util::filename_cat($base_dir, $file);

    # we don't want RecPlug to go recursing into the text, images or 
    # abstracts directories
    return 0 if (-d $filename && $filename =~ /(abstracts|images|text)/);

    return 0 if $filename =~ /meta\.txt$/i;

    return undef unless ($filename =~ /\.(issue|commentary)$/i && (-e $filename));

    $self->{'verbosity'} = $processor->{'verbosity'};
    print STDERR "NPPlug: processing $filename\n" if $self->{'verbosity'};

    my ($dir);
    ($dir, $file) = $filename =~ /^(.*?)([^\/\\]*)$/;
    my ($issuekey) = $file =~ /^([^\.\_]*)/;

    if ($filename =~ /\.commentary$/i) {
	# commentary
	return $self->process_commentary ($filename, $issuekey, $dir, $file, $processor);
    }

    my $numprocessed = 0;
    $self->{'commentaries'}->{$issuekey} = 0 unless defined $self->{'commentaries'}->{$issuekey};

    my ($abstractfile) = $file =~ /^([^\.]*)\.issue/i;
    my $abstractOID = $abstractfile . "abstract";
    $abstractfile .= ".abstract";
    my $afile = &util::filename_cat($dir, "abstracts", $abstractfile);
    my $hasabstract = 0;
    if (-e $afile) {$hasabstract = 1;}
    else {$abstractOID = undef;}

    # process the .issue file
    my %meta = ();
    $numprocessed += $self->process_issue ($filename, $issuekey, $dir, $file, 
					   $abstractOID, $processor, \%meta);

    # process abstract of this issue
    if ($hasabstract) {
	$numprocessed += $self->process_abstract ($afile, $issuekey, $dir, 
						  $abstractfile, $processor, \%meta);
    }

    return $numprocessed;
}

sub process_issue {
    my $self = shift (@_);
    my ($filename, $issuekey, $dir, $file, $abstract, $processor, $meta) = @_;

    $self->{'num_issues'} ++;
    my $doc_obj = new doc ($file, "indexed_doc");
    my $topsection = $doc_obj->get_top_section();
    my $cursection = $topsection;
    $self->associate_cover_images ($doc_obj, $dir, $issuekey);
    $doc_obj->set_metadata_element ($topsection, 'Title', $self->get_title_string($file));
    $doc_obj->set_metadata_element ($topsection, 'abstract', $abstract) if defined $abstract;
    $self->set_main_metadata ($doc_obj, $dir);

    open (ISSUEFILE, $filename) || die "couldn't open $filename\n";
    my $line = "";
    while (defined ($line = <ISSUEFILE>)) {
	next unless $line =~ /\w/;
	chomp $line;
	if ($line =~ /^<([^>]*)>(.*?)\s*$/) {
	    $doc_obj->set_metadata_element ($topsection, $1, $2);
	    $meta->{$1} = $2;
	} else {
	    # should be a section name
	    $line =~ s/^\s+//;
	    $line =~ s/\s+$//;
	    $cursection = $doc_obj->insert_section($doc_obj->get_end_child($topsection));
	    my ($pagenum) = $line =~ /(\d+)$/;
	    $doc_obj->set_metadata_element($cursection, 'Title', $pagenum);
	    $self->process_text ($dir, $line, $doc_obj, $cursection);
	    $self->process_images ($dir, $line, $doc_obj, $cursection);
	}
    }
    $file =~ s/\.issue//i;
    $doc_obj->set_OID ($file);
    $processor->process ($doc_obj);
    return 1;
}

sub process_images {
    my $self = shift (@_);
    my ($dir, $page, $doc_obj, $cursection) = @_;

    my $filename = &util::filename_cat ($dir, "images", $page);

    if (-e "$filename.gif") {
	$self->{'num_image_pages'} ++;
	$doc_obj->set_metadata_element ($cursection, "hasimg", "1");
	$doc_obj->set_metadata_element ($cursection, "Source", $page);
	$doc_obj->associate_file("$filename.gif", "$page.gif", "image/gif");	
    } elsif ($self->{'create_log'}) {
	$doc_obj->set_metadata_element ($cursection, "hasimg", "0");
	print LOGFILE "no fullsize image file for $page\n";
    }

    if (-e "${filename}_p.gif") {
	$doc_obj->set_metadata_element ($cursection, "hasprevimg", "1");
	$doc_obj->set_metadata_element ($cursection, "Source", $page);
	$doc_obj->associate_file("${filename}_p.gif", "${page}_p.gif", "image/gif");	
    } elsif ($self->{'create_log'}) {
	$doc_obj->set_metadata_element ($cursection, "hasprevimg", "0");
	print LOGFILE "no preview image file for $page\n";
    }
}

sub process_text {
    my $self = shift (@_);
    my ($dir, $page, $doc_obj, $cursection) = @_;
    my ($text);

    my $filename = &util::filename_cat ($dir, "text", $page);
    if (-e "$filename.htm") {
	$text = $self->get_text ("$filename.htm");
    } elsif (-e "$filename.txt") {
	$text = $self->get_text ("$filename.txt");
    }

    if (defined $text) {
	$self->{'num_text_pages'} ++;
	$doc_obj->add_text ($cursection, $text);
    } elsif ($self->{'create_log'}) {
	print LOGFILE "no txt or htm file for $page\n";
    }
}

sub process_abstract {
    my $self = shift (@_);
    my ($filename, $issuekey, $dir, $file, $processor, $meta) = @_;

    my $text = $self->get_text ($filename);
    if (defined $text) {
	# remove html header and stuff (assumes all future abstracts
	# are same format as those for niupepa 01)
	$text =~ s/^.*?<b>.*?<\/b>//is;
	
	$self->{'num_abstracts'} ++;
	my $doc_obj = new doc ($file, "indexed_doc");
	my $cursection = $doc_obj->get_top_section();
	$self->associate_cover_images ($doc_obj, $dir, $issuekey);
	$doc_obj->set_metadata_element ($cursection, 'Title', "_abstract_ " . 
					$self->get_title_string($file));
	$self->set_main_metadata ($doc_obj, $dir);
	map { $doc_obj->set_metadata_element ($cursection, $_, $meta->{$_}); } keys %$meta;
	$doc_obj->set_metadata_element ($cursection, "doctype", "Description");
	$doc_obj->add_text ($cursection, $text);
	$file =~ s/\.abstract//i;
	$doc_obj->set_OID ($file . "abstract");
	$processor->process ($doc_obj);
	return 1;
    }

    if ($self->{'create_log'}) {
	print LOGFILE "abstract file $filename doesn't exist\n";
    }
    return 0;
}

sub process_commentary { 
    my $self = shift (@_);
    my ($filename, $issuekey, $dir, $file, $processor) = @_;

    my $text = $self->get_text ($filename);

    return 0 unless defined $text;

    $self->{'commentaries'}->{$issuekey} = 1;
    my $doc_obj = new doc ($file, "indexed_doc");
    my $cursection = $doc_obj->get_top_section();
    $self->associate_cover_images ($doc_obj, $dir, $issuekey);
    $doc_obj->set_metadata_element ($cursection, 'Title', "_commentary_");
    $self->set_main_metadata ($doc_obj, $dir);
    $doc_obj->set_metadata_element ($cursection, "doctype", "Commentary");
    $doc_obj->add_text ($cursection, $text);
    $doc_obj->set_OID ($issuekey . "commentary");
    $processor->process ($doc_obj);
    return 1;
}

sub associate_cover_images {
    my $self = shift (@_);
    my ($doc_obj, $dir, $issuekey) = @_;

    my $cover = &util::filename_cat ($dir, $issuekey);
    $doc_obj->associate_file("${cover}on.gif", "${issuekey}/coveron.gif", "image/gif");
    $doc_obj->associate_file("${cover}of.gif", "${issuekey}/coverof.gif", "image/gif");
}

# reads in the meta.txt file and sets metadata
sub set_main_metadata {
    my $self = shift (@_);
    my ($doc_obj, $dir) = @_;

    my $metafile = &util::filename_cat ($dir, "meta.txt");
    return unless (-e $metafile);

    if (!open (METAFILE, $metafile)) {
	print STDERR "NPPlug: Couldn't read $metafile\n" if $self->{'verbosity'};
	return;
    }

    my $cursection = $doc_obj->get_top_section();
    my $line = "";
    while (defined ($line = <METAFILE>)) {
	next unless $line =~ /\w/;
	chomp $line;
	if ($line =~ /<([^>]*)>(.*)$/) {
	    # note we're using set_metadata_element (not add_metadata_element)
	    # this will override any previously set metadata of the same name
	    $doc_obj->set_metadata_element ($cursection, $1, $2);
	} elsif ($self->{'verbosity'}) {
	    print STDERR "NPPlug: Badly formatted line in $metafile\n";
	    print STDERR "meta.txt lines should be formatted '<metaname>metavalue'\n";
	}
    }
}

sub get_text {
    my $self = shift (@_);
    my ($filename) = @_;

    if (open (FILE, $filename)) {
	my $text = "";
	my $line = "";
	if ($filename =~ /\.(htm|commentary|abstract)$/i) {
	    my $savedtext = "";
	    my $foundbody = 0;
	    while (defined ($line = <FILE>)) {
		if ($line =~ s/.*?<body[^>]*>//i) {
		    $foundbody = 1;
		}
		$line =~ s/(<\/?html[^>]*>|<\/?head[^>]*>|<\/p>|<\/?font[^>]*>|<\/?body[^>]*>)//ig;
		if ($foundbody) {
		    $text .= $line; 
		} else {
		    $savedtext .= $line;
		}
	    }
	    close FILE;
	    if ($foundbody) {return $text;}
	    else {return $savedtext;}

	} else {
	    while (defined ($line = <FILE>)) {
		$line = "<p>\n" unless $line =~ /\w/;
		$text .= $line;
	    }
	    close FILE;
	    return $text;
	}

    } else {
	print STDERR "NPPlug: Warning: get_text() couldn't open $filename\n" 
	    if $self->{'verbosity'};
	return undef;
    }
}

sub get_title_string {
    my $self = shift (@_);
    my ($filename) = @_;

    $filename =~ s/\.(issue|abstract)$//i;
    my ($series, $vol, $num) = split /\_/, $filename;
    my $title = "";
    $title .= "_vol_ $vol" if defined $vol && $vol =~ /\w/;
    if (defined $num && $num =~ /\w/) {
	$title .= ": " if defined $vol && $vol =~ /\w/;
	$title .= "_num_ $num";
    }
    return $title;
}

1;
