###########################################################################
#
# PDFBoxConverter - helper plugin that does pdf document conversion with PDFBox
#
# A component of the Greenstone digital library software
# from the New Zealand Digital Library Project at the 
# University of Waikato, New Zealand.
#
# Copyright (C) 2010 New Zealand Digital Library Project
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
#
###########################################################################
package PDFBoxConverter;

use BaseMediaConverter;

use strict;
no strict 'refs'; # allow filehandles to be variables and viceversa
no strict 'subs'; # allow barewords (eg STDERR) as function arguments

#use HTML::Entities; # for encoding characters into their HTML entities when PDFBox converts to text

use gsprintf 'gsprintf';
use FileUtils;

# these two variables mustn't be initialised here or they will get stuck 
# at those values.
our $pdfbox_conversion_available; 
our $no_pdfbox_conversion_reason; 

BEGIN {
    @PDFBoxConverter::ISA = ('BaseMediaConverter');

    # Check that PDFBox is installed and available on the path 
    $pdfbox_conversion_available = 1;
    $no_pdfbox_conversion_reason = "";
    
    if (!defined $ENV{'GEXT_PDFBOX'}) {
	$pdfbox_conversion_available = 0;
	$no_pdfbox_conversion_reason = "gextpdfboxnotinstalled";
    }
    else {
	my $gextpb_home = $ENV{'GEXT_PDFBOX'};
	my $pbajar = &FileUtils::filenameConcatenate($gextpb_home,"lib","java","pdfbox-app.jar");

	if (!-e $pbajar) {
	    &gsprintf(STDERR,"**** Failed to find $pbajar\n");
	    $pdfbox_conversion_available = 0;
	    $no_pdfbox_conversion_reason = "gextpdfboxjarnotinstalled";
	}
	else {
	    # test to see if java is in path
	    # Need to run java -version instead of just java, since the %ERRORLEVEL% returned
	    # for `java` (which is checked below for failure of the command) is 0 for JDK 1.6*
	    # while %ERRORLEVEL% is 1 for JDK 1.7*
	    # If `java -version` is run however, %ERRORLEVEL% returned is 0 if java is 
	    # installed, regardless of whether the JDK version is 1.6* or 1.7*.
	    my $java = &util::get_java_command();
			
	    my $cmd = "$java -version";
	    if ($ENV{'GSDLOS'} =~ /^windows/i) {
		$cmd .= " >nul 2>&1"; # java 2>&1 >null or java >null 2>&1 both work (%ERRORLEVEL% is 0)
	    }
	    else {
		# On Ubuntu, java >/dev/null 2>&1 works, 
		# but java 2>&1 >/dev/null doesn't work: output goes to screen anyway
		$cmd .= " >/dev/null 2>&1"; # " >/dev/null 2>&1 &" - don't need & at end for Linux Centos anymore (Ubuntu was already fine without it)
	    }

	    my $status = system($cmd);

	    if ($status != 0) {
	       
		my $error_message =  "**** Testing for java\n";
		$error_message .= "Failed to run: $cmd\n";
		$error_message .=  "Error variable: |$!| and status: $status\n";

		&gsprintf(STDERR, "PDFBoxConverter: $error_message");

		$pdfbox_conversion_available = 0;
		$no_pdfbox_conversion_reason = "couldnotrunjava";
	    } 
	}
    }

}

my $arguments = [ ];

my $options = { 'name' => "PDFBoxConverter",
		'desc' => "{PDFBoxConverter.desc}",
		'abstract' => "yes",
		'inherits' => "yes",
		'args' => $arguments };

sub new {
    my ($class) = shift (@_);
    my ($pluginlist,$inputargs,$hashArgOptLists,$auxilary) = @_;
    push(@$pluginlist, $class);

    push(@{$hashArgOptLists->{"ArgList"}},@{$arguments});
    push(@{$hashArgOptLists->{"OptList"}},$options);


    my $self = new BaseMediaConverter($pluginlist, $inputargs, 
				      $hashArgOptLists, $auxilary);

    if ($self->{'info_only'}) {
	# don't worry about any options etc
	return bless $self, $class;
    }
    if ($pdfbox_conversion_available) {
	my $gextpb_home = $ENV{'GEXT_PDFBOX'};
	my $pbajar = &FileUtils::filenameConcatenate($gextpb_home,"lib","java","pdfbox-app.jar");
	my $pb_jbig2_jar = &FileUtils::filenameConcatenate($gextpb_home,"lib","java","jbig2-imageio-3.0.1.jar");

	my $pb_jaicore_jar = &FileUtils::filenameConcatenate($gextpb_home,"lib","java","jai-imageio-core-1.4.0.jar");
	my $pb_jaijpeg2000_jar = &FileUtils::filenameConcatenate($gextpb_home,"lib","java","jai-imageio-jpeg2000-1.4.0.jar");
		
	# Not including the following JPEG2000 jar, as it is under commercial license:
	# https://github.com/jai-imageio/jai-imageio-jpeg2000 leading to https://bintray.com/jai-imageio/maven/jai-imageio-jpeg2000# (Files tab)
	# my $pbjp2jar = &FileUtils::filenameConcatenate($gextpb_home,"lib","java","jai-imageio-jpeg2000-1.3.0.jar"); # jpeg2000
	
	my $java = &util::get_java_command();
	$self->{'pdfbox_txt_launch_cmd'} = "$java -cp \"$pbajar\" org.apache.pdfbox.tools.ExtractText";
	$self->{'pdfbox_html_launch_cmd'} = "$java -cp \"$pbajar\" -Dline.separator=\"<br />\" org.apache.pdfbox.tools.ExtractText";

	# We use this next cmd to launch our new custom PDFBox class (PDFBoxToImagesAndText.java) to convert each PDF page into an image (gif, jpg, png)
	# AND its extracted text. Or just each page's extracted text. An item file is still generated,
	# but this time referring to txtfiles too, not just the images. Result: searchable paged output.
	# Our new custom class PDFBoxToImagesAndText.java lives in the new build folder, so add that to the classpath for the launch cmd
	my $pdfbox_build = &FileUtils::filenameConcatenate($gextpb_home,"build");
	# put the pdfbox jar, the jbig2-imageio library (Apache Software License 2.0)
	# and our build folder containing our custom PDFBox class on the classpath
	my $classpath = &util::pathname_cat($pbajar, $pb_jbig2_jar);

	if(&FileUtils::fileExists($pb_jaicore_jar)) {
	    $classpath = &util::pathname_cat($classpath, $pb_jaicore_jar);
	}
	if(&FileUtils::fileExists($pb_jaijpeg2000_jar)) {
	    $classpath = &util::pathname_cat($classpath, $pb_jaijpeg2000_jar);
	}
	$classpath = &util::pathname_cat($classpath, $pdfbox_build);
	
#	$self->{'pdfbox_img_launch_cmd'} = "java -cp \"$classpath\" org.apache.pdfbox.tools.PDFToImage"; # pdfbox 2.09 cmd for converting each PDF page to an image (jpg, png)	
	$self->{'pdfbox_imgtxt_launch_cmd'} = "java -cp \"$classpath\" org.greenstone.pdfbox.PDFBoxToImagesAndText";
    }
    else {       
	$self->{'no_pdfbox_conversion_reason'} = $no_pdfbox_conversion_reason;

	my $outhandle = $self->{'outhandle'};
	&gsprintf($outhandle, "PDFBoxConverter: {PDFBoxConverter.noconversionavailable} ({PDFBoxConverter.$no_pdfbox_conversion_reason})\n");
    }  

    $self->{'pdfbox_conversion_available'} = $pdfbox_conversion_available;
    
    return bless $self, $class;

}

sub init {
    my $self = shift(@_);
    my ($verbosity, $outhandle, $failhandle) = @_;

    $self->{'pbtmp_file_paths'} = ();
}

sub deinit {
    my $self = shift(@_);

    $self->clean_up_temporary_files();
}


sub convert {
    my $self = shift(@_);
    my ($source_file_full_path, $target_file_type) = @_;

    return 0 unless $pdfbox_conversion_available;
    # check the filename
    return 0 if ( !-f $source_file_full_path);

    # Although PDFBoxConverter inherits from AutoLoadConverters and therefore
    # doesn't go through gsConvert.pl, still set the -pdf_tool flag in convert_options
    # in case in future PDFBoxConverter no longer inherits from AutoLoadConverters
    # and ends up going through gsConvert.pl
    $self->{'convert_options'} .= " -pdf_tool pdfbox";

    my $img_output_mode = 0;
    
    my $convert_to = $self->{'convert_to'};
    my $paged_txt_output_mode = ($convert_to =~ /(pagedimgtxt|paged_text)/) ? 1 : 0;
    
    # the following line is necessary to avoid 'uninitialised variable' error
    # messages concerning the converted_to member variable when PDFPlugin's 
    # use_sections option is checked.
    # PDFBox plugin now processes use_sections option, when working with v1.5.0
    # of the PDFBox jar file (which embeds each page in special <div> tags).
    if ($target_file_type eq "html") {
	$self->{'converted_to'} = "HTML";
    } elsif ($target_file_type eq "jpg" || $target_file_type eq "png") { # || $target_file_type eq "gif"
	# GIF not supported by PDFBox at present, see https://pdfbox.apache.org/1.8/commandline.html#pdftoimage
	$self->{'converted_to'} = $target_file_type;	
	$img_output_mode = 1;
    } else {
	$self->{'converted_to'} = "text";
    }

    my $outhandle = $self->{'outhandle'};
    my $verbosity = $self->{'verbosity'};

    my $source_file_no_path = &File::Basename::basename($source_file_full_path);
    # Determine the full name and path of the output file
    my $target_file_path;
    
    if ($self->{'enable_cache'}) {
	$self->init_cache_for_file($source_file_full_path);
	my $cache_dir = $self->{'cached_dir'}; # full path to cache dir and file_root subdir inside it
	my $file_root = $self->{'cached_file_root'}; # just the name of file_root subdir
	#$file_root .= "_$convert_id" if ($convert_id ne "");

	# append the output filetype suffix only for non-image output formats, since for
	# images we can be outputting multiple image files per single PDF input file
	#my $target_file = ($img_output_mode || $paged_txt_output_mode) ? "" : "$file_root.$target_file_type";
	my $target_file = ($img_output_mode || $paged_txt_output_mode) ? "pages" : "$file_root.$target_file_type";	
	
	$target_file_path = &FileUtils::filenameConcatenate($cache_dir,$target_file);
	
	if($img_output_mode || $paged_txt_output_mode) {
	    if(!&FileUtils::directoryExists($target_file_path)) {
		mkdir($target_file_path);
	    }
	}
	
	# In this branch of the if-statement, we *don't* want this stored as part of 'pbtmp_file_path'	
    }
    else {
	# this is in gsdl/tmp. get a tmp filename in collection instead???
	$target_file_path = &util::get_tmp_filename($target_file_type);
	
	# for image files, remove the suffix, since we can have many output image files
	# per input PDF (one img for each page of the PDF, for example)
	if($img_output_mode || $paged_txt_output_mode) {
	    $target_file_path =~ s/\.[^.]*$//g;
	    
	    if(!&FileUtils::directoryExists($target_file_path)) {		
		mkdir($target_file_path);
	    }
	    
	    # once the item file for the imgs has been created, need to adjust target_file_path

	    # below, we'll store the dir just created to pbtmp_file_paths, so all imgs and the
	    # item file generated in it can be deleted in one go on clean_up
	}
	
	push(@{$self->{'pbtmp_file_paths'}}, $target_file_path);
    }

    # Generate and run the convert command
    my $convert_cmd = "";

    # want the filename without extension, because any images 
    # are to be generated with the same filename as the PDF
    my ($tailname, $dirname, $suffix) = &File::Basename::fileparse($source_file_full_path, "\\.[^\\.]+\$");

    if($img_output_mode || $paged_txt_output_mode) { # converting each page to image and/or text
	my $output_prefix = &FileUtils::filenameConcatenate($target_file_path, $tailname);

	# Our custom class does renaming of the pages (simplified to just numbers) for PagedImagePlugin
	#$convert_cmd = $paged_txt_output_mode ? $self->{'pdfbox_imgtxt_launch_cmd'} : $self->{'pdfbox_img_launch_cmd'};
	$convert_cmd = $self->{'pdfbox_imgtxt_launch_cmd'};
	$convert_cmd .= " -imagesOnly" unless($paged_txt_output_mode); # set to images only unless there's text too
	if($img_output_mode) { # whether images-only or images-and-text mode
	    $convert_cmd .= " -imageType $target_file_type";
	    $convert_cmd .= " -dpi ". $self->{"dpi"} if defined $self->{"dpi"};
	} else { # img_output_mode off, so paged txt only and no images
	    $convert_cmd .= " -textOnly";	    
	}
	
	$convert_cmd .= " -outputPrefix \"$output_prefix\"";
	$convert_cmd .= " \"$source_file_full_path\"";
	
    } else { # single stream of text or html
	
	if ($target_file_type eq "html") {
	    $convert_cmd = $self->{'pdfbox_html_launch_cmd'};
	    $convert_cmd .= " -html" if ($target_file_type eq "html");
	} else {
	    $convert_cmd = $self->{'pdfbox_txt_launch_cmd'};
	}
	$convert_cmd .= " \"$source_file_full_path\" \"$target_file_path\"";
    }

    if ($verbosity>2) {
	&gsprintf($outhandle,"Convert command: $convert_cmd\n");
    }

    my $print_info = { 'message_prefix' => "PDFBox Conversion",
		       'message' => "Converting $source_file_no_path to: $target_file_type" };
    # $print_info->{'cache_mode'} = $cache_mode if ($cache_mode ne "");

    my ($regenerated,$result,$had_error) 
	= $self->autorun_general_cmd($convert_cmd,$source_file_full_path, $target_file_path,$print_info);

    if($img_output_mode || $paged_txt_output_mode) {
	# now the images have been generated, generate the "$target_file_path/tailname.item" 
	# item file for them, which is also the target_file_path that needs to be returned
	$target_file_path = &util::create_itemfile($target_file_path, $tailname, $target_file_type);
	#print STDERR "**** item file: $target_file_path\n";
    }
    elsif ($self->{'converted_to'} eq "text") {
	# ensure html entities are doubly escaped for pdfbox to text conversion: &amp; -> &amp;amp;
	# conversion to html does it automatically, but conversion to text doesn't
	# and this results in illegal characters in doc.xml

	my $fulltext = &FileUtils::readUTF8File($target_file_path);
	if(defined $fulltext) {
	    #$fulltext = &HTML::Entities::encode($fulltext); # doesn't seem to help
	    $fulltext =~ s@&@&amp;@sg; # Kathy's fix to ensure doc contents don't break XML
	    &FileUtils::writeUTF8File($target_file_path, \$fulltext);
	} else {
	    print STDERR "PDFBoxConverter::convert(): Unable to read from converted file\n";
	    $had_error = 1;
	}
    }

    if ($had_error) {
	return (0, $result,$target_file_path);
    }
    return (1, $result,$target_file_path);
}

sub convert_without_result {
    my $self = shift(@_);

    my $source_file_path = shift(@_);
    my $target_file_type = shift(@_);
    my $convert_options  = shift(@_) || "";
    my $convert_id       = shift(@_) || "";

    return $self->convert($source_file_path,$target_file_type,
			  $convert_options,$convert_id,"without_result");
}

sub clean_up_temporary_files {
    my $self = shift(@_);

    foreach my $pbtmp_file_path (@{$self->{'pbtmp_file_paths'}}) {
	if (-d $pbtmp_file_path) {
	    #print STDERR "@@@@@@ cleanup called on $pbtmp_file_path\n";
	    &FileUtils::removeFilesRecursive($pbtmp_file_path);
	}
	elsif (-e $pbtmp_file_path) {
	    &FileUtils::removeFiles($pbtmp_file_path);
	}
    }

    $self->{'pbtmp_file_paths'} = ();
}


1;