package flatdbdiff;

BEGIN {
    die "GSDLHOME not set\n" unless defined $ENV{'GSDLHOME'};
    die "GSDLOS not set\n" unless defined $ENV{'GSDLOS'};
    unshift (@INC, "$ENV{'GSDLHOME'}/perllib");
    unshift (@INC, "$ENV{'GSDLHOME'}/perllib/cpan");
}

use util;
use diffutil;
use Text::Diff;
use Cwd;

if ($^O =~ m/mswin/i) {
    require Win32; # for working out Windows Long Filenames from Win 8.3 short filenames
}

sub readin_flat_db
{
    my ($cmd) = @_;

    open(PIN,"$cmd|") 
	|| die "Unable to open pipe to $cmd: $!\n";

    my $text_content = "";

    while (defined (my $line = <PIN>)) {
	$text_content .= $line;
    }

    close(PIN);
    return $text_content;
}

sub read_flat_db 
{
    # need to sort text output of both test and model col database files, to normalise them for the comparison
    # the -sort option to db2txt was added specifically to support diffcol
    my($db_file) = @_;
    my $db_cmd = "db2txt -sort $db_file 2>&1";
    if($db_file =~ m/\.jdb$/) {
	my $runperl = ($ENV{'GSDLOS'} =~ m/windows/) ? "perl -S" : "";
	$db_cmd = "$runperl jdb2txt.pl -sort $db_file";
    }

    my $db_text = readin_flat_db($db_cmd);
    return $db_text;
}

sub text_to_db_to_text
{
    my($db_text, $db_filename) = @_;

    # http://stackoverflow.com/questions/1909262/how-can-i-pipe-input-into-a-java-command-from-perl
    
    if($db_filename =~ m/\.jdb$/) {
	my $runperl = ($ENV{'GSDLOS'} =~ m/windows/) ? "perl -S" : "";
	open PIPE, "| $runperl txt2jdb.pl $db_filename";
	print PIPE "$db_text";
	close(PIPE);
    } else {
	open PIPE, "| txt2db $db_filename";
	print PIPE "$db_text";
	close(PIPE);
    }
    return &read_flat_db("$db_filename");
}

# for debugging. Prints txt contents of db to file
sub print_string_to_file 
{
    my ($text, $outfile) = @_;
	
    open(FOUT, ">$outfile") or die "ERROR failed to write to $outfile: $!\n";
    print FOUT $text;
    close(FOUT);
}

sub test_flat_db
{
    my ($full_modeldb, $full_testdb, $strColName, $test_os, $model_os, $strTestCol, $strModelCol, $debugging) = @_;

   # print "Now is testing database\n";
	my ($dbname, $dirname, $suffix)= &File::Basename::fileparse($full_testdb, "\\.[^\\.]+\$");
   
    my $model_text = read_flat_db($full_modeldb);
    my $test_text = read_flat_db($full_testdb);

    my $savepath = &getcwd."/../"; # TASK_HOME env var does not exist at this stage, but it's one level up from current directory
    if($debugging) {
	print_string_to_file($test_text, $savepath.$dbname."_test.out1");	
	print_string_to_file($model_text, $savepath.$dbname."_model.out1");
    }

    # filter out the fields that can be ignored in the two database files
	# The total_numbytes field can vary depending on how many backslashes exist in the urls in the main body text, as each
	# of these windows slashes get escaped with another backslash, and the resulting string is used as key into rel link db
    my $ignore_line_re = "\n<(FileSize|lastmodified|lastmodifieddate|oailastmodified|oailastmodifieddate|ex.File.FileModifyDate|ex.File.FilePermissions|ex.File.FileAccessDate|ex.File.FileInodeChangeDate|ex.File.FileCreateDate|total_numbytes|ex.Composite.LightValue)>([^\n])*";
    $model_text =~ s/$ignore_line_re//g;
    $test_text =~ s/$ignore_line_re//g;

    # tmp dirs have subdirs with random numbers in name, remove subdir
	# these tmpdirs are located inside the collection directory
    $model_text =~ s@(tmp[\\\/])(\d*[\\\/])@$1@g;
    $test_text =~ s@(tmp[\\\/])(\d*[\\\/])@$1@g;


#print STDERR "@@@@ DEBUGGING: $debugging\n";
#print STDERR "******** full_modeldb: $full_modeldb\n$model_text\n\n";
#print STDERR "******** full_testdb: $full_testdb\n$test_text\n\n";
	
	# if the OS doesn't match and one of them is windows, extra work needs to be done to bring the db files
	# in test and model collection to an even base for comparison
	
	my $testIsWin = ($test_os ne "compute") ? ($test_os eq "windows") : &isDBWindowsSensitive($dbname, $test_text);
	my $modelIsWin = ($model_os ne "compute") ? ($model_os eq "windows") : &isDBWindowsSensitive($dbname, $model_text);
	
	if($testIsWin == $modelIsWin) {		
	# both unix or both windows, do the basic test we did on linux machines:
		    # ignore absolute path prefixes in modelcol and testcol (necessary for archiveinf-doc and -src.gdb files)

			# Remember the original model col on SVN could have been built anywhere, 
			# and in the gdb files, absolute paths are stored to the collection location. 
			# Crop these paths to the collect/<colname> point.
			
			# Entries are of the form [Entry] or <Entry>. In order to do a sensible diff,
			# need to remove the prefix to the collect/colname folder in any (absolute) path that occurs in Entry
			# E.g. [/full/path/collect/colname/import/file.ext] should become [collect/colname/import/file.ext]
			# Better regex is of the form /BEGIN((?:(?!BEGIN).)*)END/, see http://docstore.mik.ua/orelly/perl/cookbook/ch06_16.htm

			$model_text =~ s@^([^\\/]*(//)*).*(\\|/)(collect(\\|/)$strColName)(.*)$@$1$4$6@mg;
			$test_text =~ s@^([^\\/]*(//)*).*(\\|/)(collect(\\|/)$strColName)(.*)$@$1$4$6@mg;
			#$model_text =~ s@^([^\\//]*).*(\\|/)(collect(\\|/)$strColName)(.*)$@$1$3$5@mg;			
			#$test_text =~ s@^([^\\//]*).*(\\|/)(collect(\\|/)$strColName)(.*)$@$1$3$5@mg;

			# File extensions on darwin may be listed in archiveinf-src|doc as CAPS, make them lowercase. This problem popped up in CDS-ISIS collection
			if($test_os =~ m/darwin/ && $dbname =~ m/archiveinf-/) {
			    
			    my $collection_path = $strTestCol; # full path to a windows collection
			    my $tmp = ""; # rebuild darwin test file's set of lines after processing them one by one

			    for my $line (split /^/, $test_text) { # split the string into newlines
				if($line =~ m/\@THISCOLLECTPATH\@[^\.]*\.([^\.]+)/s) {

				    my $file_suffix = $1;

				    if(!defined $file_suffix || $file_suffix eq "") {
					next;
				    }
				    # there can be a mixture of lowercase and uppercase suffixes if test_os is darwin, so don't skip processing any line
				    #if($file_suffix !~ m/[A-Z]+/) { # if there are all caps suffixes, it will be consistently so in the DB file, if not skip lowercasing them
					#last;
				    #}
				    
				    my $lc_suffix = lc($file_suffix); # lowercase it

				    $line =~ s/$file_suffix$/$lc_suffix/; #$line =~ s/(\@THISCOLLECTPATH\@[^\.]*\.)($file_suffix)/$1$lc_suffix/;
				}
				$tmp .= $line; # don't need to add a newline at end
			    }

			    if($tmp ne "") {
				$test_text = $tmp; 
				#print STDERR "NEW FILE:\n************\n$test_text\n***********\n";

				# need to re- sort the keys, now that the absolute paths to tmp locations has been removed
				# so that we get the files in the same order as the model collection			    
				
				$test_text = text_to_db_to_text($test_text, "test.gdb");
			    }
			}
	}
	
	else { # one of the collections was built on windows
		# handling slashes and other differences between a model coll built on one OS (e.g. linux) 
		# and a test collection built and diffed on another OS (windows)
	
		my ($win_text, $lin_text); # references
		my $collection_path = $strTestCol; # full path to a windows collection
		
		if($testIsWin) {
			$collection_path = $strTestCol; # test collection path is windows 
			$win_text = \$test_text;
			$lin_text = \$model_text;
		} else {
			$collection_path = $strModelCol; # model collection path is windows
			$win_text = \$model_text;
			$lin_text = \$test_text;
		}
		
		if($dbname =~ m/archiveinf-doc/) {
			
			(my $collection_path_re = $collection_path) =~ s@\\@\\\\@g;
			
			my $tmp = ""; # rebuild windows file's set of lines after processing them one by one
	
			# convert short filenames to long perl: 
			# http://www.mombu.com/programming/perl/t-convert-dos-83-filenames-to-win32-long-filenames-using-perl-525448.html
			for my $line (split /^/, $$win_text) { # split the string into newlines
				
				# assoc-file and meta-file contain filepaths, ensure these are long windows file paths now (will later convert to linux slashes)	
				if($line =~ m@^<(assoc-file|meta-file|src-file)>(.*)(\s+)@s) {
					my ($field, $value, $suffix) = ($1, $2, $3);
					$line = $value; # it may be a short file name	
					
					if($line !~ m/^\@/) { # if the path doesn't use a "relative" @GSPATH@ placeholder string, but is an absolute path instead						
						# perhaps test here if it is a shortfilename? should match /CAPS....~number(.ext)/						
						$line = "<$field>".&Win32::GetLongPathName($line)."$suffix"; # make it a long file name and prefix assoc-file/meta-file tagname to it again					
					} 
					else { # if $line contains @THISCOLLECTIONPATH@, still need to deal with DOS filenames suffixes: 
						# replace placeholder with absolute path and expand to long filename, then insert placeholder in its original place again						
						$line =~ s/\@THISCOLLECTPATH\@/$collection_path/;
						$line = &Win32::GetLongPathName($line);
						$line =~ s/^$collection_path_re/\@THISCOLLECTPATH\@/;
						$line = "<$field>".$line."$suffix";
					}
				}
				$tmp .= $line;
			}
			$$win_text = $tmp;			
		}
		
		
		# index gdb file
		if($dbname =~ m/$strColName/) {
			my $tmp = ""; # rebuild windows file's set of lines after processing them one by one
			for my $line (split /^/, $$win_text) { # split the string into newlines			

				# In the following regex, add any .gdb fieldnames that represent a path and so would contain double backslashes 
				# on Windows (to escape the single backlash of win filepaths). They will be turned into single-backslashes here, 
				# and converted into single forward slashes futher below when the txt version of the win gdb file is normalised 
				# to compare it with the linux version. 
				# E.g. On windows, the Word-PDF collection(s) contains double backslashes in the ex.File.Directory field
				# the MARC-Exploded collection contains double backslashes in the null_file entry field of the .gdb file				
				if($line =~ m@^<(ex.File.Directory|null_file)>(.*)@s) {
					my ($fieldname, $escaped_path) = ($1, $2);
					$escaped_path =~ s@\\\\@/@g; #(my $escaped_path = $2) =~ s@\\\\@\\@g;
					$line = "<$fieldname>$escaped_path";
				} 
				elsif($line =~ m@^<Title>(.*)@s) {
#					print STDERR "***** TITLE: |$1|\n";
				
					# word-pdf collection: Title of ps files contain new lines at end when
					# GreenstoneXMLPlugin::xml_end_tag() writes the Title back out after utf8 decode
					# if($metadata_name eq "Title") { $metadata_value =~ s/[\n\r]*$//; }
				
					(my $title = $1) =~ s@(\r|\n|\\n)*$@@; # get rid of trailing newlines/carriage returns
					$line = "<Title>$title\n"; # add single newline					
				}
				$tmp .= $line;
			}
			$$win_text = $tmp;
		
			# slashes in windows metadata text need to be turned into linux style slashes. 
			# index\col.gdb uses double backslashes, and single for \n,\t
			#$$win_text =~ s@\\\\@/@g;
		}
		else { # archiveinf gdb file
		
			# slashes in windows metadata text need to be turned into linux style slashes. 
			# In the two archivesinf gdb files, filepaths may use single backslashes
			$$win_text =~ s@\\@/@g; #$$win_text =~ s@\\([^n|r|\|"])@/$1@g; # filepath something\rtf remains something\rtf
		}
		
		# cut down absolute paths to files to just collect/colname/.../file, same as before
		$$lin_text =~ s@^([^\\/]*(//)*).*(\\|/)(collect(\\|/)$strColName)(.*)$@$1$4$6@mg; # $$lin_text =~ s@^([^\\\/]*(//)?).*(\\|/)(collect(\\|/)$strColName)(.*)$@$1$4$6@mg;
		$$win_text =~ s@^([^\\/]*(//)*).*(\\|/)(collect(\\|/)$strColName)(.*)$@$1$4$6@mg;		
		
		# for the windows text, need to further get rid of the driveletter after [ or <meta>
		$$win_text =~ s@^(\[|<[^>]*>)[a-zA-Z]:collect@$1collect@mg;		
		
	} # end of equalising differences between a windows collection's db file and linux coll's db file
	
	
	# Windows or linux: if index is a flat db file, then ensure the docIDs listed in <contains> field of
	# both test and model flat db file are alphabetically sorted. So too the numbers in <mdoffset> field.
	# Despite PERL_PERTURB_KEYS envvar being set to 0 on both machine when generating model collections
	# and when test collections were generated on test machine, still collections like Images-GPS and some
	# other colls list items in <contains> and <mdoffset> in different orders. So reordering alphabetically.	
	#if($dbname =~ m/$strColName/) {	
		# regex modifiers mge: multi-line, global (replace as many as match), e allows function call in substitution
		##$model_text =~ s@^<contains>(.*)@sort_contains_field($1, "MODEL", $debugging)@mge;
		##$test_text =~ s@^<contains>(.*)@sort_contains_field($1, "TEST", $debugging)@mge;
	#	$model_text =~ s@^<(contains|mdoffset)>(.*)@sort_field_value($1, $2, "MODEL", $debugging)@mge;
	#	$test_text =~ s@^<(contains|mdoffset)>(.*)@sort_field_value($1, $2, "TEST", $debugging)@mge;		
	#}
	
	# The following block of code is necessary to deal with tmp (html) source files generated when using PDFBox
	# These tmpdirs are located inside the toplevel *greenstone* directory
	(my $gsdlhome_re = $ENV{'GSDLHOME'}) =~ s@\\@\/@g;		
	$gsdlhome_re = ".*" unless $$ENV{'GSDLHOME'};
	my $tmpfile_regex = "<URL>http://$gsdlhome_re/tmp/([^\.]*?)(\..{3,4})"; # $gsdlhome/tmp/randomfilename.html, file ext can be 3 or 4 chars long			
	if($test_text =~ m@$tmpfile_regex@g) {			
		# found a match, replace the tmp file name with "random", keeping the original file extension 
		# in <OrigSource|URL|UTF8URL|gsdlconvertedfilename>
	
		# This code is slightly different from doc.xml because each document has its own doc.xml, so this needs to be done
		# only once for doc.xml, but multiple times in index/col.gdb since it contains the random filenames of all docs in the col	
		#my ($old_tmp_filename, $ext) = ($1, $2);
		
		my $new_tmp_filename = "random";			
	
		
		$tmpfile_regex = "(<(URL|UTF8URL|gsdlconvertedfilename|OrigSource)>(http://)?)($gsdlhome_re)?(/tmp/)?.*?(\..{3,4})";
		if($5) { 
			$test_text =~ s@$tmpfile_regex@$1$5$new_tmp_filename$6@mg;
		} else { # OrigSource contains only the filename
			$test_text =~ s@$tmpfile_regex@$1$new_tmp_filename$6@mg;
		}
		
		# modelcol used a different gsdlhome, but also a tmp dir, so make the same changes to its random filename			
		$tmpfile_regex = "(<(URL|UTF8URL|gsdlconvertedfilename|OrigSource)>(http://)?)(.*)?(/tmp/)?.*?(\..{3,4})";
		if($5) { 
			$model_text =~ s@$tmpfile_regex@$1$5$new_tmp_filename$6@mg;
		} else { # OrigSource contains only the filename
			$model_text =~ s@$tmpfile_regex@$1$new_tmp_filename$6@mg;
		}
		
		# index/col.gdb also has entries for the random tmp file names in the form: [http://research/ak19/GS2bin_5July2013/tmp/F639.html]
		# need to equalise these also. Sadly, when there are multiple intermediate files, their random tmp filenames are not 
		# guaranteed to be generated in the same (alphabetical/numerical) order between model and test collection, so the 
		# HASH OIDs, although all of them accounted for, appear in a different order. So we have to remove the Hash OIDs.
		#$test_text =~ s@\[http://.*/tmp/.*(\..{3,4})\]@tmp/random$1@mg; # HASH OIDs can appear in different order
		#$model_text =~ s@\[http://.*/tmp/.*(\..{3,4})\]@tmp/random$1@mg;
		
		$test_text  =~ s@\[http://[^\n]*?/tmp/.*?(\..{3,4})\]\n<section>([^\n]*?)\n@[tmp/random$1\n<section>$2]\n@sg;
		$model_text =~ s@\[http://[^\n]*?/tmp/.*?(\..{3,4})\]\n<section>([^\n]*?)\n@[tmp/random$1\n<section>$2]\n@sg;
		
		# need to re- sort the keys, now that the absolute paths to tmp locations has been removed
		# so that we get the tmp files in the same order in both model and test collections
		
		$model_text = text_to_db_to_text($model_text, "model.gdb");
		$test_text = text_to_db_to_text($test_text, "test.gdb");
	}	

    # now can go back to using $model_text and $test_text

    if($debugging) {
	print_string_to_file($test_text, $savepath.$dbname."_test.out");	
	print_string_to_file($model_text, $savepath.$dbname."_model.out");
    }

    my $report_type = "OldStyle"; # Can not change this type.
    my $diff_gdb = diff \$model_text, \$test_text, { STYLE => $report_type };
    
    # leaving the ignore regex as it used to be in the following, in case it helps with single line comparisons
    $diff_gdb = &diffutil::GenerateOutput($diff_gdb,"^<(lastmodified|lastmodifieddate|oailastmodified|oailastmodifieddate|ImageSize)>.*");

    if($diff_gdb eq "")
    {
	return "";
    }
    else
    {
	return "Difference Report: Differences found in the Database file: \n$diff_gdb";
    }
    # Call diff?
}

# Unused, but may come in handy when debugging again: regex substitution helper function
sub sort_field_value {
	my($fieldname, $fieldvalue, $displayStr, $debugging) = @_;
	
	print STDERR "\n$displayStr BEFORE sort: <$fieldname>$fieldvalue\n" if($debugging);
	
	$fieldvalue =~ s@(\r|\n|\\n)*$@@; # get rid of trailing newlines/carriage returns
	my @values_list = split(';', $fieldvalue);
	@values_list = sort @values_list;
	$fieldvalue = "<$fieldname>".join(';', @values_list). "\n";
	
	print STDERR "$displayStr AFTER  sort: $fieldvalue\n" if($debugging);
	
	return $fieldvalue;
}


# returns true if the contents are windows AND it matters for the diffing on the db that it's windows
# For col.gdb it does not seem to matter so far, if it is generated on a windows machine and to be compared to a linux-generated col.gdb
sub isDBWindowsSensitive 
{
	my ($dbtailname, $db_contents) = @_; # db filename without suffix
	
#	return ($db_contents =~ m/\\/) ? 1 : 0; # windows slashes detected. Better test would be: [Something\something] OR <tag>something\something	
	
	if($dbtailname =~ m/^archiveinf-doc/) {
		return ($db_contents =~ m@<src-file>[a-zA-Z]:\\@) ? 1 : 0; # <src-file>C:\path
	} 
	elsif($dbtailname =~ m/^archiveinf-src/) { # <src-file>C:\path
		return ($db_contents =~ m@\[[a-zA-Z]:\\@) ? 1 : 0; # [C:\path]
	} 
	else { # index/col.gdb file
		if ($db_contents =~ m@<URL>http://[a-zA-Z]:/@) { # <URL>http://C:/path
			return 1;
		}		
		elsif ($db_contents =~ m@^(<URL>http://[a-zA-Z]:/)|(<null_file>[^\\]*\\)@m) { # <URL>http://C:/path OR <null_file>CMSwp-all.00000001\\00000035.nul			
			return 1;
		}
		elsif ($db_contents =~ m@^(<ex.File.Directory>[a-zA-Z]:\\\\)@m) { # <ex.File.Directory>C:\\path\\path for OAI collection
			return 1;
		}
		return 0;		
	}	
}

1;
