/**
 *#########################################################################
 *
 * A component of the Gatherer application, part of the Greenstone digital
 * library suite from the New Zealand Digital Library Project at the
 * University of Waikato, New Zealand.
 *
 * Copyright (C) 1999 New Zealand Digital Library Project
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
 *########################################################################
 */

package org.greenstone.gatherer.gui;

import org.greenstone.gatherer.Configuration;
import org.greenstone.gatherer.Dictionary;
import org.greenstone.gatherer.Gatherer;
import org.greenstone.gatherer.cdm.CollectionConfigXMLReadWrite;
import org.greenstone.gatherer.util.Codec;
import org.greenstone.gatherer.util.SafeProcess;
import org.greenstone.gatherer.util.StaticStrings;
import org.greenstone.gatherer.util.XMLTools;

import org.w3c.dom.*;

import java.io.File;
import java.io.IOException;

import java.awt.*;
import java.awt.event.*;
import java.util.*;
import javax.swing.*;
import javax.swing.border.*;
import javax.swing.event.*;


// TO DO:
// + StreamGobblers: http://www.javaworld.com/article/2071275/core-java/when-runtime-exec---won-t.html
// Need to test on Windows: the Ctrl-D/Ctrl-Z sent to formatconverter.exe. May not need to do anything special for Windows now that this code is using StreamGobblers to process the in/out streams of the process
// + Need to use dictionary for labels
// X convertFormat() for remote GS. Remote does not display Format Conversion wizard
// + HTML tidy (put all "<br/>" back to "<br>" in luce/etc/collectionConfig.xml and run again)
// + Help tooltips on buttons
// + Undo, Redo buttons
// X Split class into dialog/widgets and data processing?
// http://www.java2s.com/Tutorial/Java/0240__Swing/SettingJOptionPanebuttonlabelstoFrench.htm

public class FormatConversionDialog extends ModalDialog
{
    public static final String GSF_FORMAT_GS2_TAG = "gsf:format-gs2";
    private static final String GSF_GS3_ROOT_TAG = "gsf:gs3-root";

    // The cmdline programs launched by this dialog
    private static final int XMLTIDY = 0;
    private static final int FORMATCONVERTER = 1;
    private static final int FORMATCONVERTER_DOCUMENTNODE = 2;
    private static final int FORMATCONVERTER_CLASSIFIERNODE = 3;

    // Online HTML tidy for learning usage: http://infohound.net/tidy/
    private static final String[] xmltidy_cmd_args = {"tidy", "-config", Configuration.getGS3BinPath() + "tidyconfig.cfg", "-utf8", "-wrap", "0", "-raw", "-q"}; // {"tidy", "-xml", "-utf8"};
    private static final String[] formatconverter_cmd_base_args = {Configuration.getGS3BinOSPath() + "formatconverter", "--silent"};

    private static final Dimension SIZE = new Dimension(640,480);

    // the important member variables
    private File collect_cfg_file = null;
    private Document xml_file_doc = null;
    private NodeList gsf_format_gs2_list = null;    

    private int current_index = -1;
    private int dlgResult = OpenCollectionDialog.OK_OPTION;
    private int process_exitValue = -1;
    
    // widgets
    final String WARNING_TITLE = Dictionary.get("General.Warning") 
	+ ": " + Dictionary.get("FormatConversionDialog.Invalid_XML")
	+ " " + Dictionary.get("FormatConversionDialog.Invalid_XML_Warning_Title");

    private static final Border EMPTYBORDER = new EmptyBorder(5, 5, 5, 5);
    private JLabel section_label = null;
    private GLIButton cancel_button = null;
    private GLIButton next_button = null;
    private GLIButton accept_all_button = null;
    private GLIButton htmltidy_button = null;
    private GLIButton xmltidy_button = null;
    // the 2 NumberedJTextAreas. Each provide their own undo and redo buttons already hooked up to listeners
    private NumberedJTextArea gs2_textarea = null;
    private NumberedJTextArea gs3_textarea = null;
    private JLabel count_label = null;
    private JLabel statusbar = null;

    public FormatConversionDialog (File collect_cfg_file, Document xml_file_doc, NodeList gsf_format_gs2_list) {
	super(Gatherer.g_man, "", true);

	this.collect_cfg_file = collect_cfg_file;	
	this.xml_file_doc = xml_file_doc;
	this.gsf_format_gs2_list = gsf_format_gs2_list;

	this.setComponentOrientation(Dictionary.getOrientation());
	setSize(SIZE);
	setTitle(Dictionary.get("FormatConversionDialog.Title"));
	this.addWindowListener(new WindowClosingListener()); // the dialog's top right close button should behave as a Cancel press
	setDefaultCloseOperation(DISPOSE_ON_CLOSE);
	
	// The NumberedJTextArea provides its own undo and redo buttons, already hooked up to listeners
	// Just need to place these buttons in the dialog
	gs2_textarea = new NumberedJTextArea("gs2", Dictionary.get("FormatConversionDialog.GS2_Text_Tooltip"));
	gs3_textarea = new NumberedJTextArea("gs3", Dictionary.get("FormatConversionDialog.GS3_Text_Tooltip"));


	JPanel midbutton_panel = new JPanel(); // FlowLayout by default in a JPanel
	midbutton_panel.setComponentOrientation(Dictionary.getOrientation());

	JButton reconvert_button = new GLIButton(Dictionary.get("FormatConversionDialog.Reconvert"), Dictionary.get("FormatConversionDialog.Reconvert_Tooltip"));

	midbutton_panel.add(gs2_textarea.undoButton);
	midbutton_panel.add(gs2_textarea.redoButton);
	midbutton_panel.add(reconvert_button);
	reconvert_button.addActionListener(new ReconvertListener());
	reconvert_button.setAlignmentY(Component.CENTER_ALIGNMENT);

	section_label = new JLabel("Section Label Goes Here"); // title will be replaced, don't need to translate
	section_label.setBorder(EMPTYBORDER);
	section_label.setComponentOrientation(Dictionary.getOrientation());

	JPanel button1_panel = new JPanel();
	button1_panel.setComponentOrientation(Dictionary.getOrientation());

	xmltidy_button = new GLIButton(Dictionary.get("FormatConversionDialog.XHTML_Tidy"), Dictionary.get("FormatConversionDialog.XHTML_Tidy_Tooltip"));
	xmltidy_button.addActionListener(new XMLTidyButtonListener());

	button1_panel.add(gs3_textarea.undoButton);
	button1_panel.add(gs3_textarea.redoButton);
	button1_panel.add(xmltidy_button);

	JPanel button_panel = new JPanel(new FlowLayout());
	button_panel.setComponentOrientation(Dictionary.getOrientation());
	count_label = new JLabel("<count>"); // placeholder text for numbers, no need to translate
	cancel_button = new GLIButton(Dictionary.get("General.Cancel"), Dictionary.get("FormatConversionDialog.Cancel_Tooltip"));
	next_button = new GLIButton(Dictionary.get("FormatConversionDialog.Next"), Dictionary.get("FormatConversionDialog.Next_Tooltip"));
	accept_all_button = new GLIButton(Dictionary.get("FormatConversionDialog.Accept_All"), Dictionary.get("FormatConversionDialog.Accept_All_Tooltip"));
	cancel_button.addActionListener(new CancelButtonListener());
	next_button.addActionListener(new NextButtonListener());
	accept_all_button.addActionListener(new AcceptAllButtonListener());
	button_panel.add(cancel_button);
	button_panel.add(next_button);
	button_panel.add(accept_all_button);
	button_panel.add(count_label);


	JPanel centre_panel = new JPanel();
	centre_panel.setLayout(new BoxLayout(centre_panel, BoxLayout.Y_AXIS));
	centre_panel.setComponentOrientation(Dictionary.getOrientation());
	centre_panel.setBorder(EMPTYBORDER);
	centre_panel.add(new JScrollPane(gs2_textarea));
	centre_panel.add(midbutton_panel);
	centre_panel.add(new JScrollPane(gs3_textarea));

	JPanel bottom_panel = new JPanel();
	bottom_panel.setLayout(new BoxLayout(bottom_panel, BoxLayout.Y_AXIS));
	bottom_panel.setComponentOrientation(Dictionary.getOrientation());
	bottom_panel.setBorder(EMPTYBORDER);

	bottom_panel.add(button1_panel);
	bottom_panel.add(button_panel);
	statusbar = new JLabel("");
	statusbar.setBorder(EMPTYBORDER);
	// http://stackoverflow.com/questions/2560784/how-to-center-elements-in-the-boxlayout-using-center-of-the-element
	statusbar.setAlignmentX(Component.CENTER_ALIGNMENT);
	bottom_panel.add(statusbar);

	// add all the widgets to the contentpane
	JPanel content_pane = (JPanel) getContentPane();
	content_pane.setComponentOrientation(Dictionary.getOrientation());
	content_pane.setLayout(new BorderLayout());
	//content_pane.setBorder(EMPTYBORDER);

	content_pane.add(section_label, BorderLayout.NORTH);
	content_pane.add(centre_panel, BorderLayout.CENTER);
	content_pane.add(bottom_panel, BorderLayout.SOUTH);
	

	// Final dialog setup & positioning.
	getRootPane().setDefaultButton(next_button);
	Dimension screen_size = Configuration.screen_size;
	setLocation((screen_size.width - SIZE.width) / 2, (screen_size.height - SIZE.height) / 2);
	screen_size = null;
	//setVisible(true);
    }


    public int getDialogResult() {
	return dlgResult; // OK_OPTION or CANCEL_OPTION
    }
    
    //*************************PROCESSING FUNCTIONS***************************//
    public static int checkForGS2FormatStatements(File collect_cfg_file) {

	if(Gatherer.GS3 && collect_cfg_file.getAbsolutePath().endsWith(".xml")) {

	    //System.err.println("*** Opening an xml config file");
		
	    Document xml_file_doc = XMLTools.parseXMLFile(collect_cfg_file);
	    Element root = xml_file_doc.getDocumentElement();

	    // check if there are any <gsf:format-gs2 /> elements. If there are, then may need to process them
	    
	    //NodeList gsf_format_gs2_list = root.getElementsByTagNameNS("gsf", "format-gs2");
	    NodeList gsf_format_gs2_list = root.getElementsByTagName(FormatConversionDialog.GSF_FORMAT_GS2_TAG);
	    if(gsf_format_gs2_list != null && gsf_format_gs2_list.getLength() > 0) {

		// Sample the first of the <gsf:gs2-format/> elements to
		// check we don't have any CDataSections in the <gsf:gs2-format/> elements
		// If the first <gsf:gs2-format/> has a CDataSection, it means we've already 
		// converted it to GS3 format statements during an earlier GLI session.

		Node gs2format = gsf_format_gs2_list.item(0);
		//gs2format.normalize();
		NodeList children = gs2format.getChildNodes();

		for(int i = 0; i < children.getLength(); i++) {
		    Node child = children.item(i);
		    if(child.getNodeType() == Node.CDATA_SECTION_NODE) {	
			// there are GS2 statements in col config, but they've already been converted to GS3
			// can open the collection without going through the FormatConversionDialog
			return OpenCollectionDialog.OK_OPTION; 
		    }
		}
		System.err.println("*** Found GS2 format statements in config file to be converted to GS3.");

		
		// if remote GS3, do we open the collection with the html-encoded GS2 format statements
		// or do we not allow the remote user to open such a collection at all?
		// For now we allow them to open it, but print a warning that conversions are not possible.
		
		if(Gatherer.isGsdlRemote) { // remote GS3
		    System.err.println("*** Cannot convert GS2 collections from a remote GS3 server.");	
		    return OpenCollectionDialog.OK_OPTION;
		}

		// If we get here, it means there were no CDataSections in the first (any) <gsf:gs2-format/>
		// This means it's the first time the GS2 collection is being opened in GLI 
		// Open the FormatConversionDialog and convert the gs2 statements to gs3:
		FormatConversionDialog formatconversionDlg = new FormatConversionDialog(collect_cfg_file, xml_file_doc, gsf_format_gs2_list);
		formatconversionDlg.convertGS2FormatStatements();
		return formatconversionDlg.getDialogResult();
		
	    }
	}	
	return OpenCollectionDialog.OK_OPTION; // no GS2 statements in col config, and can open the collection
    }


    /** 
     * runInteractiveProgram() runs a cmdline program that reads from stdinput 
     * until Ctrl-D is encountered. It outputs the result to stdout.
     *
     * The cmdline programs HTML Tidy and FormatConverter both behave the same way:
     * When the formatconverter binary is run in silent mode, it expects input
     * followed by a newline and then EOF (or if no newline, then 2 EOFs) 
     * which is Ctrl-D on Linux/Mac and Ctrl-Z on Windows. 
     * Then the cmdline program exits by printing the result of the conversion.
     *
     * Explicitly sending EOFs from java is no longer necessary, as the OutputStreamGobbler 
     * Thread takes care of closing the process stream.
     *
     * HTMLTidy returns 0 if no warnings or errors, 1 if just Warnings, 2 if Errors (failure)
     * http://sourceforge.net/p/tidy/mailman/tidy-develop/thread/20020811204058.GD1137@shaw.ca/
     *
     * This code uses the StreamGobbler classes based on
     * http://www.javaworld.com/article/2071275/core-java/when-runtime-exec---won-t.html?page=2
    */    
    public String runInteractiveProgram(int program, String inputstr) {
	String outputstr = "";
	String[] command_args;
	process_exitValue = -1;

	switch(program) {
	case XMLTIDY:
	    command_args = xmltidy_cmd_args;
	    break;
	case FORMATCONVERTER:
	    command_args = formatconverter_cmd_base_args;
	    break;
	case FORMATCONVERTER_DOCUMENTNODE: 
	case FORMATCONVERTER_CLASSIFIERNODE:
	    command_args = new String[formatconverter_cmd_base_args.length+1]; 
	    System.arraycopy(formatconverter_cmd_base_args, 0, command_args, 0, formatconverter_cmd_base_args.length);
	    if(program == FORMATCONVERTER_DOCUMENTNODE) {
		command_args[command_args.length-1] = "--documentNode";		
	    } else if(program == FORMATCONVERTER_CLASSIFIERNODE) {
		command_args[command_args.length-1] = "--classifierNode";		
	    } 
	    break;
	default:
	    System.err.println("*** Unrecognised program code: " + program);
	    return outputstr;
	}

	// Generate the formatconverter command		
	/*if (Gatherer.isGsdlRemote) {
	  
	  }*/

	SafeProcess prcs = new SafeProcess(command_args);
	prcs.setInputString(inputstr);
	prcs.setSplitStdErrorNewLines(true);
	prcs.runProcess();
	// process done, now get process output
	outputstr = prcs.getStdOutput();
	String errmsg = prcs.getStdError();
	if(!errmsg.equals("")) {
	    System.err.println("*** Process errorstream: \n" + errmsg + "\n****");
	}

	///System.err.println("#### Got output: " + outputstr);

	/*
	try {	    
	    
	    Runtime rt = Runtime.getRuntime();
	    Process prcs = null;
	    
	    prcs = rt.exec(command_args);

	    // send inputstr to process
	    OutputStreamGobbler inputGobbler = new OutputStreamGobbler(prcs.getOutputStream(), inputstr);
	    
	    // monitor for any error messages
            InputStreamGobbler errorGobbler = new InputStreamGobbler(prcs.getErrorStream(), true);
            
            // monitor for the expected output line(s)
            InputStreamGobbler outputGobbler = new InputStreamGobbler(prcs.getInputStream());

            // kick them off
            inputGobbler.start();
            errorGobbler.start();
            outputGobbler.start();
                                    
            // any error???
            process_exitValue = prcs.waitFor();
            //System.out.println("ExitValue: " + exitVal); 

	    // From the comments of 
	    // http://www.javaworld.com/article/2071275/core-java/when-runtime-exec---won-t.html?page=2
	    // To avoid running into nondeterministic failures to get the process output
	    // if there's no waiting for the threads, call join() on each Thread (StreamGobbler) object:
	    outputGobbler.join();
	    errorGobbler.join();
	    inputGobbler.join();
	    
	    outputstr = outputGobbler.getOutput();
	    String errmsg = errorGobbler.getOutput();
	    if(!errmsg.equals("")) {
		System.err.println("*** Process errorstream: \n" + errmsg + "\n****");
	    }
       
	} catch(IOException ioe) {
	    System.err.println("IOexception " + ioe.getMessage());
	    //ioe.printStackTrace();
	} catch(InterruptedException ie) {
	    System.err.println("Process InterruptedException " + ie.getMessage());
	    //ie.printStackTrace();
	}
	*/
	return outputstr;
    }

    public void convertGS2FormatStatements() {
	    		
	// at this point, we know there are one or more <gsf:format-gs2 /> elements
	// process each of them as follows: unescape, then call formatconverter, then call html tidy on it
	//NodeList gsf_format_gs2_list = root.getElementsByTagNameNS("gsf", "format-gs2");

	int len = gsf_format_gs2_list.getLength();

	for(int i = 0; i < len; i++) {
	        
	    Element gs2format = (Element)gsf_format_gs2_list.item(i);
	    String gs2formatstr = XMLTools.getElementTextValue(gs2format); // seems to already unescape the html entities
	    //gs2formatstr = Codec.transform(gs2formatstr, Codec.ESCAPEDHTML_TO_UNESCAPED);
	    
	    processFormatStatement(i, gs2formatstr);
	} 

	increment(); // modifies the textareas to initialise them
	setVisible(true);
    }


    private String processFormatStatement(int i, String gs2formatstr) {
	String errorMsg = "";

	boolean startsWithTableCell = (gs2formatstr.toLowerCase().startsWith("<td")) ? true : false;

	//System.err.println("*** Found: " + gs2formatstr);

	// Running formatconverter. Decide on whether to pass in option --documentNode|--classifierNode
	int formatConverterProgramMode = formatConverterMode(i);
	String gs3formatstr = runInteractiveProgram(formatConverterProgramMode, gs2formatstr);
	gs3formatstr = gs3formatstr.replaceAll(">\\s+<", "><");

	//System.err.println("*** Format is now: " + gs3formatstr);

	String gs3formatstr_notags = gs3formatstr;
	gs3formatstr = addSurroundingTags(gs3formatstr);


	String validationMsg = XMLTools.parseDOM(gs3formatstr);
	if(!validationMsg.startsWith(XMLTools.WELLFORMED)) {
	    // Run Html Tidy in XML mode
	    //System.err.println("*** Needing to run HTML Tidy on: ");
	    //System.err.println(gs3formatstr_notags);	    


	    // HTMLTidy returns 0 if no warnings or errors, 1 if just Warnings, 2 if Errors (failure)
	    // http://sourceforge.net/p/tidy/mailman/tidy-develop/thread/20020811204058.GD1137@shaw.ca/
	    String htmltidy_string = runInteractiveProgram(XMLTIDY, gs3formatstr_notags);

	    if(process_exitValue >= 2) {
		// System.err.println("@@@ Process exit value: " + process_exitValue);
		errorMsg = Dictionary.get("FormatConversionDialog.Tidy_Failed") 
		    + " " + Dictionary.get("FormatConversionDialog.XML_Still_Invalid");
	    } else {
		errorMsg = "";

		gs3formatstr_notags = htmltidy_string;

		gs3formatstr_notags = removeHTMLTags(i, gs3formatstr_notags, startsWithTableCell);
		gs3formatstr = addSurroundingTags(gs3formatstr_notags); 
		
		// Having applied html-tidy, setGS3Format() will return true if it finally parsed
	    }
	}
	
	// For now, assume HTML Tidy has worked and that the gs3format has now been parsed successfully

	boolean parsed = setGS3Format(i, gs3formatstr); // will parse the gs3formatstr into a DOM object
	if(!parsed && errorMsg.equals("")) {
	    errorMsg = Dictionary.get("FormatConversionDialog.Tidy_Done")
		+ " " + Dictionary.get("FormatConversionDialog.XML_Still_Invalid");
	}	
	return errorMsg;	
	//return gs3formatstr;
    }

    // HTML tidy adds entire HTML tags around a single format statement. This method removes it.
    private String removeHTMLTags(int i, String gs3formatstr_notags, boolean startsWithTD) {

	// if it's a VList classifier <gsf:template match="documentNode|classifierNode"/>
	// and the gs2 format statement starts with a <td>, 
	// then remove up to and including the outermost <tr>, 
	// else remove up to and including the <body> tag
	
	String removeOuterTag = "body>";
	Element parent = (Element)getParentNode(i); // gets parent of GS2format: <gsf:template>
	if(parent.hasAttribute("match") && (!parent.hasAttribute("mode") || !parent.getAttribute("mode").equals("horizontal"))) {
	    if(startsWithTD) {
		removeOuterTag = "tr>"; // remove the outermost <tr> cell HTML tidy added around the tablecell
	    }
	}

	// <unwanted>
	// <removeThisOuterTag>
	// lines we want
	// </removeThisOuterTag>
	// <unwanted>
	
	int end = gs3formatstr_notags.indexOf(removeOuterTag);
	if(end != -1) {
	    gs3formatstr_notags = gs3formatstr_notags.substring(end+removeOuterTag.length());
	}
	int start = gs3formatstr_notags.lastIndexOf("</"+removeOuterTag); //closing tag
	if(start != -1) {
	    gs3formatstr_notags = gs3formatstr_notags.substring(0, start);
	}

	//System.err.println("@@@@ " + startsWithTD + " - TAG: " + removeOuterTag + " - AFTER REMOVING TAGS:\n" + gs3formatstr_notags);

	return gs3formatstr_notags;	
    }


    //**************** ACCESS FUNCTIONS ***************//

    // gs2 format text is the text string that goes into <gsf:format-gs2/>: <gsf:format-gs2>text</gsf:format-gs2>
    private void setGS2Format(int i, String text) {
	XMLTools.setElementTextValue(getGS2Format(i), text);
    }
    
    // gs3FormatStr represents DOM, and must be parsed and appended as sibling to the gs2format element
    // as <gsf:gs3-root/>. If it fails to parse, nest it in a CDATA element of <gsf:gs3-root/>
    private boolean setGS3Format(int i, String gs3formatstr) {

	Document ownerDoc = getGS2Format(i).getOwnerDocument();
	Node gs3format = null;
	boolean parse_success = false;

	
	String validationMsg = XMLTools.parseDOM(gs3formatstr);
	if(!validationMsg.startsWith(XMLTools.WELLFORMED)) {	    
	    // silently add the gs3formatstr in a CDATA block into the root

	    gs3formatstr = removeSurroundingTags(gs3formatstr);
	    gs3format = ownerDoc.createElement(GSF_GS3_ROOT_TAG);	    
	    Node cdataSection = ownerDoc.createCDATASection(gs3formatstr);
	    gs3format.appendChild(cdataSection);
	    parse_success = false;
	}
	else {

	    // parse DOM into XML
	    Document doc = XMLTools.getDOM(gs3formatstr);
	    Element root = doc.getDocumentElement();
	    gs3format = ownerDoc.importNode(root, true); // an element
	    parse_success = true;

	    //	System.err.println("@@@@ ROOT:\n" + XMLTools.xmlNodeToString(root));
	}

	// add gs3format element as sibling to gs2format element
	Element oldgs3format = getGS3Format(i);
	if(oldgs3format == null) {	    
	    getParentNode(i).appendChild(gs3format);
	    //	    System.err.println("@@@ APPEND");
	} else {
	    // replace the existing
	    getParentNode(i).replaceChild(gs3format, oldgs3format);
	    //	    System.err.println("@@@ REPLACE");
	}

	//http://stackoverflow.com/questions/12524727/remove-empty-nodes-from-a-xml-recursively

	return parse_success;
    }

    private Node getParentNode(int i) {
	return gsf_format_gs2_list.item(i).getParentNode();
    }

    private Element getGS2Format(int i) {
	return (Element)gsf_format_gs2_list.item(i);
    }

    private String getGS2FormatString(int i) {
	return XMLTools.getElementTextValue(getGS2Format(i));
    }

    private Element getGS3Format(int i) {
	Element parent  = (Element)getParentNode(i);
	NodeList nl = parent.getElementsByTagName(GSF_GS3_ROOT_TAG);
	if(nl.getLength() > 0) {
	    return (Element)nl.item(0);
	}
	return null;
    }

    private String getGS3FormatString(int i) {	
	Element gs3format = getGS3Format(i);
	if(gs3format == null) {
	    return "";
	} else {
	    
	    // if any child is a CData section, return its text as-is. There will be no indenting
	    NodeList children = gs3format.getChildNodes();
	    for(int j = 0 ; j < children.getLength(); j++) {
		if(children.item(j).getNodeType() == Node.CDATA_SECTION_NODE) {		    
		    return children.item(j).getNodeValue(); // content of CDataSection
		}
	    }
	    
	    // else we have proper nodes, return indented string
	    StringBuffer sb = new StringBuffer();
	    XMLTools.xmlNodeToString(sb, gs3format, true, "  ", 0);
	    return sb.toString();
	}
    }

    private int formatConverterMode(int i) {
	String docOrClassNodeType = "";

	// Given XML of the form:
	// <browse|search>
	//   <format>
	//     <gsf:template match="documentNode|classifierNode" [mode=horizontal]>
	//       <gsf-format:gs2 />
	//       <gs3format/>
	//   </format>
	// </browse|search>

	Element parent = (Element)getParentNode(i); // gets parent of GS2format: <gsf:template>
	String nodeType = parent.getAttribute("match"); //e.g. documentNode, classifierNode, or "" if no match attr
	
	if(nodeType.equals("documentNode")) {
	    return FORMATCONVERTER_DOCUMENTNODE; // convert just the part of the format stmt applicable to documentNodes
	} else if(nodeType.equals("classifierNode")) {
	    return FORMATCONVERTER_CLASSIFIERNODE; // convert just the part of the format stmt applicable to classifierNodes
	}
	return FORMATCONVERTER;	// convert the entire format statement
    }

    private String getLabel(int i) {
	String label = "";

	// Given XML of the form:
	// <browse|search>
	//   <format>
	//     <gsf:template match="documentNode|classifierNode" [mode=horizontal]>
	//       <gsf-format:gs2 />
	//       <gs3format/>
	//   </format>
	// </browse|search>

	// Want to return the label: "browse|search > documentNode|classifierNode [> horizontal]"

	Element parent = (Element)getParentNode(i); // gets parent of GS2format: <gsf:template>
	label = parent.getAttribute("match"); //e.g. documentNode, classifierNode

	if(parent.hasAttribute("mode")) { // e.g. classifierNode mode=horizontal
	    label = label + " > " + parent.getAttribute("mode");
	}

	Element ancestor = (Element) parent.getParentNode().getParentNode(); // ancestor: <browse>|<search>
	label = ancestor.getTagName() + " > " + label;

	return label;
    }

    // gs2 format statements have spaces instead of newlines. For display, replace with newlines
    private String makeLines(String gs2formatstr) {
	return gs2formatstr.replaceAll(">\\s+<", ">\n<");
    }
   
    private String singleLine(String gs2formatstr) {
	return gs2formatstr.replace(">\n<", "> <"); // put the spaces back
    }

    private String removeSurroundingTags(String xml)
    {
	return xml.replaceAll("<"+GSF_GS3_ROOT_TAG+" xmlns:gsf=(\"|\')http://www.greenstone.org/greenstone3/schema/ConfigFormat(\"|\') xmlns:gslib=(\"|\')http://www.greenstone.org/skinning(\"|\') xmlns:xsl=(\"|\')http://www.w3.org/1999/XSL/Transform(\"|\')>\n?", "").replaceAll("\n?</"+GSF_GS3_ROOT_TAG+">", "");//.trim();
    }

    private String addSurroundingTags(String gs3formatstr) {
	return "<"+GSF_GS3_ROOT_TAG+" xmlns:gsf='http://www.greenstone.org/greenstone3/schema/ConfigFormat' xmlns:gslib='http://www.greenstone.org/skinning' xmlns:xsl='http://www.w3.org/1999/XSL/Transform'>"+gs3formatstr+"</"+GSF_GS3_ROOT_TAG+">";
    }

    //*************************FUNCTIONS THAT INTERACT WITH WIDGETS************************//
    // increment() loads the next values into the dialog
    private boolean increment() {
	current_index++;
	section_label.setText ( getLabel(current_index) );
	gs2_textarea.setText( makeLines(getGS2FormatString(current_index)) );
	gs3_textarea.setText( removeSurroundingTags(getGS3FormatString(current_index)) );
	statusbar.setText("");

	// as we're on a new screen of dialog, need to clear all undo/redo history
	gs2_textarea.discardAllEdits(); // will disable redo/undo buttons
	gs3_textarea.discardAllEdits();

	int len = gsf_format_gs2_list.getLength();
	count_label.setText((current_index+1) + " / " + len);
	if((current_index+1) == len) {
	    return false;
	} else {
	    return true;
	}	
    }


    private void setStatus(String msg) {
	statusbar.setText(msg);
    }
    private void setErrorStatus(String msg) {
	statusbar.setBackground(Color.red);
	statusbar.setText(msg);
    }


    public void dispose() {
	//System.err.println("@@@@ DIALOG CLOSING!");
	if(dlgResult != OpenCollectionDialog.CANCEL_OPTION) {
	    // Need to remove the <gsf:gs3-root/> siblings of all <gsf:format-gs2/>
	    // Then, get the children of each <gsf:gs3-root/> and add these as siblings of <gsf:format-gs2/>
	    
	    
	    int len = gsf_format_gs2_list.getLength();
	    for(int k=len-1; k >= 0; k--) {
		Element parent = (Element)getParentNode(k);		
		
		//parent.normalize();

		NodeList children = parent.getChildNodes();

		// now have to loop/remove nodes in reverse order, since the following loop 
		// modifies the very nodelist we're looping over by removing nodes from it

		int numChildren = children.getLength()-1;

		for(int i=numChildren; i >= 0; i--) {
		//for(int i = 0; i < children.getLength(); i++) {
		    Node child = children.item(i);

		    if(child.getNodeName().equals(GSF_FORMAT_GS2_TAG)) { 
			// if we're dealing with gs2-format-stmts, put their textnode contents in CData sections
			// http://www.w3schools.com/xml/xml_cdata.asp
			// This avoids having to look at html-encoded gs2-format tags in the Format pane

			Element gs2format = (Element)child;
			String gs2formatstr = XMLTools.getElementTextValue(gs2format);
			gs2formatstr = Codec.transform(gs2formatstr, Codec.ESCAPEDHTML_TO_UNESCAPED);
			
			Node textNode = XMLTools.getNodeTextNode(gs2format);
			Node cdataSection = gs2format.getOwnerDocument().createCDATASection(gs2formatstr); 
			gs2format.replaceChild(cdataSection, textNode);
		    } 
		    else if(child.getNodeName().equals(GSF_GS3_ROOT_TAG)) { 

			// remove GS3 node and append its children to the parent in its place
			// the <gsf:gs3-root /> elements wouldn't be in the xml_file_doc DOM tree
			// unless they were valid XML, so don't need to check for validity here
			
			Node gs3root = child;
			NodeList gs3_format_lines = gs3root.getChildNodes();

			for(int j = 0; j < gs3_format_lines.getLength(); j++) {
			    Node duplicate = gs3_format_lines.item(j).cloneNode(true);
			    parent.appendChild(duplicate);			    
			}
			gs3root = parent.removeChild(gs3root);
			gs3root = null; // finished processing

		    } // else - skip all nodes other than <gsf:format-gs2/> and <gsf:gs3-root/>
		}
	    }
 
	    Element root = xml_file_doc.getDocumentElement();
	    //System.err.println("### XML file to write out:\n" + XMLTools.xmlNodeToString(root));
	    
	    // Finally, write out the collection xml file
	    String[] nonEscapingTagNames = { StaticStrings.FORMAT_STR, StaticStrings.DISPLAYITEM_STR };
	    XMLTools.writeXMLFile(collect_cfg_file, xml_file_doc, nonEscapingTagNames);
	    
	}
	super.dispose();
	
    }

    //******************INNER CLASSES including LISTENERS and STREAMGOBBLERS****************//

    // windowClosing() is called when the user presses the top-right close button of the dialog
    // this means the user wanted to cancel out of the entire Format Conversion Wizard.
    private class WindowClosingListener extends WindowAdapter {
	public void windowClosing(WindowEvent e) {	    
	    dlgResult = OpenCollectionDialog.CANCEL_OPTION;
	}
    }

    private class ReconvertListener implements ActionListener {
	public void actionPerformed(ActionEvent e) {
	    String gs2formatstr = singleLine(gs2_textarea.getText());
	    String errorMsg = processFormatStatement(current_index, gs2formatstr);
	    gs3_textarea.setText( removeSurroundingTags(getGS3FormatString(current_index)) );
	    if(!errorMsg.equals("")) {
		setErrorStatus(errorMsg);
	    } else {
		statusbar.setText("");
	    }
	}
    }

    
    private class NextButtonListener implements ActionListener {
	public void actionPerformed(ActionEvent e) {
	    //statusbar.setText("");

	    // check if the GS3 format statement is valid XML before storing. If not, let user to decide 
	    // whether they want to re-edit it or if they want to keep it as-is, in which case it needs
	    // to be stored as CDATA, which will make it an inactive format statement. 
	    // See http://www.w3schools.com/xml/xml_cdata.asp
	    // setGS3Format() already stores invalidXML as CDATA.
	    
	    // Check if the GS3 format statement is valid XML before storing. If not, let the
	    // user to decide whether they want to re-edit it or store it as-is and continue	    

	    // user okay-ed the lines currently displayed, store them
	    setGS2Format( current_index, singleLine(gs2_textarea.getText()) );
	    boolean parse_success = setGS3Format( current_index, addSurroundingTags(gs3_textarea.getText()) );

	    if(!parse_success) { // invalid XML, warn the user
		String message = Dictionary.get("FormatConversionDialog.Invalid_XML") + " " + Dictionary.get("FormatConversionDialog.Cancel_Or_Continue_Next");
		int user_choice = JOptionPane.showConfirmDialog(FormatConversionDialog.this, message, WARNING_TITLE, JOptionPane.OK_CANCEL_OPTION, JOptionPane.WARNING_MESSAGE);

		if(user_choice == JOptionPane.CANCEL_OPTION) {
		    return; // do nothing on this NextButton press. Don't increment. Let user re-adjust invalid XML for GS3 statement.
		}
	    }
	    
	    if(increment()) {
		repaint();
	    } else {
		next_button.setEnabled(false);
		getRootPane().setDefaultButton(accept_all_button); 

	    }
	    
	}
    }

    private class CancelButtonListener implements ActionListener {
	public void actionPerformed(ActionEvent e) {
	     dlgResult = OpenCollectionDialog.CANCEL_OPTION;
	     FormatConversionDialog.this.dispose(); // close dialog
	}
    }


    private class AcceptAllButtonListener implements ActionListener {
	public void actionPerformed(ActionEvent e) {

	    // user okay-ed the lines, store them
	    setGS2Format(current_index, gs2_textarea.getText());
	    String gs3formatstr = gs3_textarea.getText();
	    boolean parse_success = setGS3Format(current_index, addSurroundingTags(gs3formatstr));
	    String message = "";

	    if(!parse_success) { // invalid XML for current format statement, warn the user
		setErrorStatus(Dictionary.get("FormatConversionDialog.Invalid_XML"));

		message = Dictionary.get("FormatConversionDialog.Invalid_XML") + " " + Dictionary.get("FormatConversionDialog.Cancel_Or_Continue_Next");
	    }

	    // Even if the current GS3 format statement is valid XML, the user could have pressed 
	    // Accept All at the very start of the FormatConversion dialog too. Check all the 
	    // subsequent format statements, and if any have invalid XML, warn user.
	    for(int i = current_index+1; parse_success && i < gsf_format_gs2_list.getLength(); i++) {		
		gs3formatstr = getGS3FormatString(i);
		String validationMsg = XMLTools.parseDOM(gs3formatstr);
		if(!validationMsg.startsWith(XMLTools.WELLFORMED)) {
		    parse_success = false;
		    message = Dictionary.get("FormatConversionDialog.Cancel_Or_Accept_All");
		}
	    }

	    if(!parse_success) {
		int user_choice = JOptionPane.showConfirmDialog(FormatConversionDialog.this, message, WARNING_TITLE, JOptionPane.OK_CANCEL_OPTION, JOptionPane.WARNING_MESSAGE);
		if(user_choice == JOptionPane.CANCEL_OPTION) {
		    return; // Don't close the dialog. Let the user continue looking at this or subsequent GS3 format statements.
		}
	    }

	    // If we're here, then either the format statements parsed, or the user accepted them anyway
	    FormatConversionDialog.this.dispose(); // close dialog
	}
    }

    private class XMLTidyButtonListener implements ActionListener {
	// run HTML tidy taking input from stdin
	// http://www.w3.org/People/Raggett/tidy/
	public void actionPerformed(ActionEvent e) {
	    String gs3formatstr_notags = gs3_textarea.getText();

	    // Flag to determine which tags that HTML tidy adds need to be removed
	    boolean startsWithTableCell = (gs3formatstr_notags.trim().toLowerCase().startsWith("<td")) ? true : false;
	    // run HTML tidy on the GS3 format statement
	    String htmltidy_string = runInteractiveProgram(XMLTIDY, gs3formatstr_notags);

	    if(process_exitValue >= 2) {
		//System.err.println("@@@ Process exit value: " + process_exitValue);
		setErrorStatus(Dictionary.get("FormatConversionDialog.Tidy_Failed"));
	    } else {
		gs3formatstr_notags = htmltidy_string;
		
		// HTML tidy adds extra HTML markup around the formatstring, so will need to remove it:
		gs3formatstr_notags = removeHTMLTags(current_index, gs3formatstr_notags, startsWithTableCell);
		
		// put the XML tags around the gs3 format string again so we can convert it to a DOM object
		String gs3formatstr = addSurroundingTags(gs3formatstr_notags);
		
		boolean parse_success = setGS3Format(current_index, gs3formatstr); // converts to DOM object
		
		// Get indented GS3 format string from DOM object and display it in the text area
		gs3_textarea.setText( removeSurroundingTags(getGS3FormatString(current_index)) );

		if(parse_success) {
		    statusbar.setText(Dictionary.get("FormatConversionDialog.Tidy_Done"));
		} else {
		    setErrorStatus(Dictionary.get("FormatConversionDialog.Tidy_Done") 
				   + " " + Dictionary.get("FormatConversionDialog.Error_GS3_Format") 
				   + " " + Dictionary.get("FormatConversionDialog.Invalid_XML"));
		}
	    }
	}
    }

}
