/**
 *#########################################################################
 *
 * A component of the Gatherer application, part of the Greenstone digital
 * library suite from the New Zealand Digital Library Project at the
 * University of Waikato, New Zealand.
 *
 * Author: Michael Dewsnip, NZDL Project, University of Waikato
 *
 * Copyright (C) 2005 New Zealand Digital Library Project
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
 *########################################################################
 */

package org.greenstone.gatherer.remote;

import java.io.*;
import java.net.*;
import java.util.*;
import java.util.zip.*;
import javax.swing.*;
import java.io.ByteArrayOutputStream;
import org.greenstone.gatherer.Configuration;
import org.greenstone.gatherer.DebugStream;
import org.greenstone.gatherer.Dictionary;
import org.greenstone.gatherer.FedoraInfo;
import org.greenstone.gatherer.GAuthenticator;
import org.greenstone.gatherer.Gatherer;
import org.greenstone.gatherer.collection.CollectionManager;
import org.greenstone.gatherer.shell.GShell;
import org.greenstone.gatherer.util.UnzipTools;
import org.greenstone.gatherer.util.Utility;
import org.greenstone.gatherer.util.XMLTools;
import org.apache.commons.httpclient.HttpClient;
import org.apache.commons.httpclient.methods.PostMethod;
import org.apache.commons.httpclient.methods.GetMethod;
import org.apache.commons.httpclient.HttpException;
import org.apache.commons.httpclient.methods.multipart.FilePart;
import org.apache.commons.httpclient.methods.multipart.MultipartRequestEntity;
import org.apache.commons.httpclient.methods.multipart.Part;
import org.apache.commons.httpclient.methods.multipart.*;
import org.apache.commons.httpclient.params.*;
import org.apache.commons.httpclient.HttpStatus;


public class RemoteGreenstoneServer
{
    // A PasswordAuthentication object is created whenever it is required
    static private PasswordAuthentication remote_greenstone_server_authentication = null;
    // static private PasswordAuthentication remote_greenstone_server_authentication = new PasswordAuthentication(System.getProperty("user.name"), new char[] { });

    // the language and region environment variables (in "lang_REGION" form)
    // this is necessary in order for the client and server sides to zip and unzip
    // using the same settings
    public final String lang_region;

    private ActionQueue remote_greenstone_server_action_queue;
    private RemoteGreenstoneServer.ProgressBar progress_bar;

    public RemoteGreenstoneServer() {
	// Create the progress_bar first since ActionQueue uses it in its thread 
	// (the thread will start immediately).
	progress_bar = new RemoteGreenstoneServer.ProgressBar();
	remote_greenstone_server_action_queue = new ActionQueue();

	String langReg = System.getenv("LANG");
	lang_region = (langReg == null) ? "" : langReg;	
    }

    // ----------------------------------------------------------------------------------------------------
    //   PUBLIC LAYER
    // ----------------------------------------------------------------------------------------------------
    public String deleteCollection(String collection_name)
    {
	return performAction(new RemoteGreenstoneServerAction.DeleteCollectionAction(collection_name));
    }


    public String deleteCollectionFile(String collection_name, File collection_file)
    {
	return performAction(new RemoteGreenstoneServerAction.DeleteCollectionFileAction(collection_name, collection_file));
    }


    public String downloadCollection(String collection_name)
    {
	return performAction(new RemoteGreenstoneServerAction.DownloadCollectionAction(collection_name));
    }


    public String downloadCollectionArchives(String collection_name)
    {
	return performAction(new RemoteGreenstoneServerAction.DownloadCollectionArchivesAction(collection_name));
    }


    public String downloadCollectionConfigurations()
    {
	return performAction(new RemoteGreenstoneServerAction.DownloadCollectionConfigurationsAction());
    }


    public String downloadCollectionFile(String collection_name, File collection_file)
    {
	return performAction(new RemoteGreenstoneServerAction.DownloadCollectionFileAction(collection_name, collection_file));
    }
    
    // get web.xml from the server -- for a remote gli of GS3
    public String downloadWebXMLFile()
    {
	return performAction(new RemoteGreenstoneServerAction.DownloadWebXMLFileAction());
    }

    public String getScriptOptions(String script_name, String script_arguments)
    {
	return performAction(new RemoteGreenstoneServerAction.GetScriptOptionsAction(script_name, script_arguments));
    }
    
    //get all available site names from the server -- for a remote gli of GS3
    public String getSiteNames()
    {
	return performAction(new RemoteGreenstoneServerAction.GetSiteNamesAction());
    }

    // get the default servlet path from the server -- for a remote GLI of GS3
    public String getDefaultServletPath() {
	return performAction(new RemoteGreenstoneServerAction.GetDefaultServletPathAction());
    }
    public String moveCollectionFile(String collection_name, File source_collection_file, File target_collection_file)
    {
	return performAction(new RemoteGreenstoneServerAction.MoveCollectionFileAction(
                             collection_name, source_collection_file, target_collection_file));
    }


    public String newCollectionDirectory(String collection_name, File new_collection_directory)
    {
	return performAction(new RemoteGreenstoneServerAction.NewCollectionDirectoryAction(
                             collection_name, new_collection_directory));
    }


    public String runScript(String collection_name, String script_name, String script_arguments, GShell shell)
    {
	return performAction(new RemoteGreenstoneServerAction.RunScriptAction(
                             collection_name, script_name, script_arguments, shell));
    }


    public String uploadCollectionFile(String collection_name, File collection_file)
    {
	return performAction(new RemoteGreenstoneServerAction.UploadCollectionFilesAction(
                             collection_name, new File[] { collection_file }));
    }


    public String uploadCollectionFiles(String collection_name, File[] collection_files)
    {
	return performAction(new RemoteGreenstoneServerAction.UploadCollectionFilesAction(collection_name, collection_files));
    }


    public String uploadFilesIntoCollection(String collection_name, File[] source_files, File target_collection_directory)
    {
	return performAction(new RemoteGreenstoneServerAction.UploadFilesIntoCollectionAction(
                             collection_name, source_files, target_collection_directory));
    }

    public boolean exists(String collection_name, File collection_file)
    {
	String result = performAction(new RemoteGreenstoneServerAction.ExistsAction(collection_name, collection_file));
	if(result.indexOf("exists") != -1) {
	    return true;
	}
	else if(result.indexOf("does not exist") != -1) {
	    return false;
	}
	return false;
    }

    public int getGreenstoneVersion()
    {
	// returns message "Greenstone version is: <version number of the Greenstone remote server>"
	String result = performAction(new RemoteGreenstoneServerAction.VersionAction());
	int index = result.indexOf(":"); 
	if(index != -1) {
	    result = result.substring(index+1).trim(); // skip the space after colon, must remove surrounding spaces
	}
	// if space is returned, then the request failed (the server may not have been running)
	int greenstoneVersion = result.equals("") ? -1 : Integer.parseInt(result);
	return greenstoneVersion;
    }

    /** For constructing the preview command (the library URL) with */
    public String getLibraryURL(String serverHomeURL)
    {
	// returns message "Greenstone library URL suffix is: <e.g. /greenstone3/library or /gsdl/cgi-bin/library>"
	String libSuffix = performAction(new RemoteGreenstoneServerAction.LibraryURLSuffixAction());
	int index = libSuffix.indexOf(":"); 
	if(index != -1) {
	    libSuffix = libSuffix.substring(index+1).trim(); // skip the space after colon and remove surrounding spaces
	}
	
	// serverHomeURL is of the form, http://domain/other/stuff. We want the prefix upto & including domain
	// and prepend that to the libraryURLSuffix
	index = -1;
	for(int i = 0; i < 3; i++) {
	    index = serverHomeURL.indexOf("/", index+1);
	    if(index == -1) { // shouldn't happen, but if it does, we'll be in an infinite loop
		break; 
	    }
	}
	serverHomeURL = serverHomeURL.substring(0, index);
	return serverHomeURL + libSuffix;
    }

    // ----------------------------------------------------------------------------------------------------


    public void exit()
    {
	System.err.println("Exiting, number of jobs on queue: " + remote_greenstone_server_action_queue.size());

	// If there are still jobs on the queue we must wait for the jobs to finish
	while (remote_greenstone_server_action_queue.size() > 0) {
	    synchronized (remote_greenstone_server_action_queue) {
		try {
		    DebugStream.println("Waiting for queue to become empty...");
		    remote_greenstone_server_action_queue.wait(500);
		}
		catch (InterruptedException exception) {}
	    }
	}
    }
    

    // ----------------------------------------------------------------------------------------------------
    //   QUEUE LAYER
    // ----------------------------------------------------------------------------------------------------


    /** Returns null if we cannot wait for the action to finish, "" if the action failed, or the action output. */
    private String performAction(RemoteGreenstoneServerAction remote_greenstone_server_action)
    {
	// Check for whether the queue thread stopped running because 
	// the user cancelled out. If so, exit GLI.
	if(remote_greenstone_server_action_queue.hasExited()) {
	    remote_greenstone_server_action_queue.clear();
	    //remote_greenstone_server_action_queue = null;
	    Gatherer.exit();
	    return "";
	}

	// Add the action to the queue
	remote_greenstone_server_action_queue.addAction(remote_greenstone_server_action);
	String threadName = Thread.currentThread().getName();

	/*
	// If we're running in the GUI thread we must return immediately
	// We cannot wait for the action to complete because this will block any GUI updates
	if (SwingUtilities.isEventDispatchThread()) {
	    System.err.println(threadName 
			       + "\tACTION QUEUED: In event dispatch thread, returning immediately...\n\t" 
			       + remote_greenstone_server_action);
	    return null;
	}
	*/

	// Otherwise wait until the action is processed
	try {
	    synchronized (remote_greenstone_server_action) {
		while(!remote_greenstone_server_action.processed) {
		    //System.err.println("Waiting for action to complete...: " + remote_greenstone_server_action);
		    DebugStream.println("Waiting for action to complete...");
		    remote_greenstone_server_action.wait(); // wait to be notified when the action has been processed 
		}
	    }
	} catch (Exception e) {
	    System.err.println("RemoteGreenstoneServer.performAction() - exception: " + e);
	    e.printStackTrace();
	}

	// Return "" if the action failed
	if (!remote_greenstone_server_action.processed_successfully) {
	    return "";
	}
	// Otherwise return the action output
	return remote_greenstone_server_action.action_output;
    }


    // ----------------------------------------------------------------------------------------------------
    //   PROGRESS BAR
    // ----------------------------------------------------------------------------------------------------

    static class ProgressBar
	extends JProgressBar
    {
	public ProgressBar()
	{
	    setBackground(Configuration.getColor("coloring.collection_tree_background", false));
	    setForeground(Configuration.getColor("coloring.collection_tree_foreground", false));
	    setString(Dictionary.get("FileActions.No_Activity"));
	    setStringPainted(true);
	}

	/** synchronized to avoid conflicts since several threads access this */
	synchronized public void setAction(String action)
	{
	    if (action != null) {
		DebugStream.println(action);
	    }

	    // We cannot call this from the GUI thread otherwise the progress bar won't start
	    if (SwingUtilities.isEventDispatchThread()) {
		System.err.println("ERROR: RemoteGreenstoneServerProgressBar.setAction() called from event dispatch thread!");
		return;
	    }

	    // Set the string on the progress bar, and start or stop it
	    if (action == null) {
		setString(Dictionary.get("FileActions.No_Activity"));
		setIndeterminate(false);
	    }
	    else {
		setString(action);
		setIndeterminate(true);
	    }
	}
    }
    
    /** synchronized to avoid conflicts since several threads access this */
    synchronized public RemoteGreenstoneServer.ProgressBar getProgressBar()
    {
	return progress_bar;
    }


    // ----------------------------------------------------------------------------------------------------
    //   AUTHENTICATION LAYER
    // ----------------------------------------------------------------------------------------------------

    static private class RemoteGreenstoneServerAuthenticateTask
	extends Thread
    {
	public void run()
	{
	    remote_greenstone_server_authentication = new RemoteGreenstoneServerAuthenticator().getAuthentication();
	}
    

	static private class RemoteGreenstoneServerAuthenticator
	    extends GAuthenticator
	{
	    public PasswordAuthentication getAuthentication(String username, String password)
	    {
		return getPasswordAuthentication(username,password);
	    }

	    public PasswordAuthentication getAuthentication()
	    {
		return getPasswordAuthentication();
	    }

	    protected String getMessageString()
	    {
		if (Gatherer.GS3){
		    return (Dictionary.get("RemoteGreenstoneServer.Authentication_Message_gs3") + " " + Configuration.site_name);
		}
		return Dictionary.get("RemoteGreenstoneServer.Authentication_Message");
	    }
 	}
    }
    
    public void set_remote_greenstone_server_authentication_to_null() {
	remote_greenstone_server_authentication = null;
    }

    private void authenticateUser()
	throws RemoteGreenstoneServerAction.ActionCancelledException
    {
	// If we don't have any authentication information then ask for it now
	if (remote_greenstone_server_authentication == null) {
	    try {
		// We have to do this on the GUI thread
		SwingUtilities.invokeAndWait(new RemoteGreenstoneServerAuthenticateTask());
	    }
	    catch (Exception exception) {
		System.err.println("Exception occurred when authenticating the user: " + exception);
		DebugStream.printStackTrace(exception);
	    }

	    // If it is still null then the user has cancelled the authentication, so the action is cancelled
	    if (remote_greenstone_server_authentication == null) {
		throw new RemoteGreenstoneServerAction.ActionCancelledException();
	    }
	}
    }


    public String getUsername()
    {
	if (remote_greenstone_server_authentication != null) {
	    return remote_greenstone_server_authentication.getUserName();
	}

	return null;
    }


    // ----------------------------------------------------------------------------------------------------
    //   REQUEST LAYER
    // ----------------------------------------------------------------------------------------------------


    /** Returns the command output if the action completed, throws some kind of exception otherwise. */
    String downloadFile(String gliserver_args, String file_path)
	throws Exception
    {
	while (true) {
	    // Check that Configuration.gliserver_url is set
	    if (Configuration.gliserver_url == null) {
		throw new Exception("Empty gliserver URL: please set this in Preferences before continuing.");
	    }

	    // Ask for authentication information (if necessary), then perform the action
	    authenticateUser();
	    String gliserver_url_string = Configuration.gliserver_url.toString();
	    String command_output = downloadFileInternal(gliserver_url_string, gliserver_args, file_path);

	    // Debugging - print any ok messages to stderr
	    //System.err.println("**** RECEIVED (sendCommandToServer()): " + command_output);

	    // Check the first line to see if authentication has failed; if so, go around the loop again
	    if (command_output.startsWith("ERROR: Authentication failed:")) {
		JOptionPane.showMessageDialog(Gatherer.g_man, Dictionary.get("RemoteGreenstoneServer.Error", command_output.substring("ERROR: ".length())), Dictionary.get("RemoteGreenstoneServer.Error_Title"), JOptionPane.ERROR_MESSAGE);
		remote_greenstone_server_authentication = null;
		continue;
	    }
	    // Check if the collection is locked by someone else; if so, see if the user wants to steal the lock
	    else if (command_output.startsWith("ERROR: Collection is locked by: ")) {
		if (JOptionPane.showConfirmDialog(Gatherer.g_man, Dictionary.get("RemoteGreenstoneServer.Steal_Lock_Message", command_output.substring("ERROR: Collection is locked by: ".length())), Dictionary.get("RemoteGreenstoneServer.Error_Title"), JOptionPane.YES_NO_OPTION) == JOptionPane.NO_OPTION) {
		    // The user has decided to cancel the action
		    throw new RemoteGreenstoneServerAction.ActionCancelledException();
		}

		// The user has decided to steal the lock... rerun the command with "&steal_lock="
		gliserver_args += "&steal_lock=";
		continue;
	    }
	    // Handle other types of errors by throwing an exception
	    else if (command_output.startsWith("ERROR: ")) {
		throw new Exception(command_output.substring("ERROR: ".length()));
	    }

	    // There were no exceptions thrown so the action must have succeeded
	    return command_output;
	}
    }

    /** Returns true or false depending on whether authentication is required for the cmd 
     * string embedded in the given gliserver_args. No authentication is required for either 
     * of the commands greenstone-server-version and get-library-url-suffix. */
    private boolean isAuthenticationRequired(String gliserver_args) {
	return ((gliserver_args.indexOf("greenstone-server-version") == -1) 
		&& (gliserver_args.indexOf("get-library-url-suffix") == -1));
    }


    /** Returns the command output if the action completed, throws some kind of exception otherwise. 
     *  Package access, so that RemoteGreenstoneServerAction.java can call this.
     */
    String sendCommandToServer(String gliserver_args, GShell shell)
	throws Exception
    {
	while (true) {
	    
	    // Check that Configuration.gliserver_url is set
	    if (Configuration.gliserver_url == null) {
		throw new Exception("Empty gliserver URL: please set this in Preferences before continuing.");
	    }

	    // Ask for authentication information (if necessary), then perform the action
	    if(isAuthenticationRequired(gliserver_args)) {
		try {
		    authenticateUser();
		} catch (RemoteGreenstoneServerAction.ActionCancelledException e) {
		    // Authentication popup only appears at the start. If the user cancelled
		    // out of it, then another remote action always remains on the queue,
		    // preventing a clean exit. Need to clear queue before exit.
		    synchronized (remote_greenstone_server_action_queue) {			
			remote_greenstone_server_action_queue.clear();			
		    }
		    Gatherer.exit();
		}
	    }
	    String gliserver_url_string = Configuration.gliserver_url.toString();
	    String command_output = sendCommandToServerInternal(gliserver_url_string, gliserver_args, shell);
	    	    
	    // Debugging - print any ok messages to stderr
	    //if(!(command_output.trim().startsWith("<"))) {
	    //System.err.println("**** RECEIVED (sendCommandToServer()): " + command_output);
	    //}
		
	    // Check the first line to see if authentication has failed; if so, go around the loop again
	    if (command_output.startsWith("ERROR: Authentication failed:")) {
		JOptionPane.showMessageDialog(Gatherer.g_man, Dictionary.get("RemoteGreenstoneServer.Error", command_output.substring("ERROR: ".length())), Dictionary.get("RemoteGreenstoneServer.Error_Title"), JOptionPane.ERROR_MESSAGE);
		remote_greenstone_server_authentication = null;
		continue;
	    }
	    // Check if the collection is locked by someone else; if so, see if the user wants to steal the lock
	    else if (command_output.startsWith("ERROR: Collection is locked by: ")) {
		if (JOptionPane.showConfirmDialog(Gatherer.g_man, Dictionary.get("RemoteGreenstoneServer.Steal_Lock_Message", command_output.substring("ERROR: Collection is locked by: ".length())), Dictionary.get("RemoteGreenstoneServer.Error_Title"), JOptionPane.YES_NO_OPTION) == JOptionPane.NO_OPTION) {
		    // The user has decided to cancel the action
		    throw new RemoteGreenstoneServerAction.ActionCancelledException();
		}

		// The user has decided to steal the lock... rerun the command with "&steal_lock="
		gliserver_args += "&steal_lock=";
		continue;
	    }
	    // Handle other types of errors by throwing an exception
	    else if (command_output.startsWith("ERROR: ")) { 
		throw new Exception(command_output.substring("ERROR: ".length()));
	    } else if (command_output.indexOf("ERROR: ") != -1) { // check if ERROR occurs anywhere else in the output
		throw new Exception(command_output);
	    }

	    
	    // There were no exceptions thrown so the action must have succeeded
	    return command_output;
	}
    }


    /** Returns the command output if the action completed, throws some kind of exception otherwise. */
    String uploadFile(String gliserver_args, String file_path)
	throws Exception
    {
	while (true) {
	    // Check that Configuration.gliserver_url is set
	    if (Configuration.gliserver_url == null) {
		throw new Exception("Empty gliserver URL: please set this in Preferences before continuing.");
	    }

	    // Ask for authentication information (if necessary), then perform the action
	    authenticateUser();
	    String gliserver_url_string = Configuration.gliserver_url.toString();
	    String command_output = uploadFileInternal(gliserver_url_string, gliserver_args, file_path);
	    // System.err.println("Command output: " + command_output);

	    // Check the first line to see if authentication has failed; if so, go around the loop again
	    if (command_output.startsWith("ERROR: Authentication failed:")) {
		JOptionPane.showMessageDialog(Gatherer.g_man, Dictionary.get("RemoteGreenstoneServer.Error", command_output.substring("ERROR: ".length())), Dictionary.get("RemoteGreenstoneServer.Error_Title"), JOptionPane.ERROR_MESSAGE);
		remote_greenstone_server_authentication = null;
		continue;
	    }
	    // Check if the collection is locked by someone else; if so, see if the user wants to steal the lock
	    else if (command_output.startsWith("ERROR: Collection is locked by: ")) {
		if (JOptionPane.showConfirmDialog(Gatherer.g_man, Dictionary.get("RemoteGreenstoneServer.Steal_Lock_Message", command_output.substring("ERROR: Collection is locked by: ".length())), Dictionary.get("RemoteGreenstoneServer.Error_Title"), JOptionPane.YES_NO_OPTION) == JOptionPane.NO_OPTION) {
		    // The user has decided to cancel the action
		    throw new RemoteGreenstoneServerAction.ActionCancelledException();
		}

		// The user has decided to steal the lock... rerun the command with "&steal_lock="
		gliserver_args += "&steal_lock=";
		continue;
	    }
	    // Handle other types of errors by throwing an exception
	    else if (command_output.startsWith("ERROR: ")) {
		throw new Exception(command_output.substring("ERROR: ".length()));
	    }

	    // There were no exceptions thrown so the action must have succeeded
	    return command_output;
	}
    }


    // ----------------------------------------------------------------------------------------------------
    //   NETWORK LAYER
    // ----------------------------------------------------------------------------------------------------


    /** Returns the command output if the action completed, throws some kind of exception otherwise. */
    private String downloadFileInternal(String download_cgi, String cgi_args, String file_path)
	throws Exception
    {
	DebugStream.println("gliserver URL: " + download_cgi);
	System.err.println("gliserver args: " + cgi_args);

	// Add username and password, and a timestamp
	cgi_args += "&un=" + remote_greenstone_server_authentication.getUserName();
	cgi_args += "&pw=" + new String(remote_greenstone_server_authentication.getPassword());
	cgi_args += "&ts=" + System.currentTimeMillis();
	if (Gatherer.GS3){
	    String site_name = Configuration.site_name;
	    if (site_name != null && !site_name.equals("")) { 
		cgi_args += "&site=" + Configuration.site_name;
	    }
	    String library_servlet_name = Configuration.servlet_path;
	    if (library_servlet_name != null && !library_servlet_name.equals("")) {
		if (library_servlet_name.charAt(0) == '/') {
		    library_servlet_name = library_servlet_name.substring(1);
		}
		cgi_args += "&servlet=" + library_servlet_name;
	    }
	}

	URL download_url = new URL(download_cgi);
	URLConnection dl_connection = download_url.openConnection();
	dl_connection.setDoOutput(true);
	OutputStream dl_os = dl_connection.getOutputStream();

	PrintWriter dl_out = new PrintWriter(dl_os);
	dl_out.println(cgi_args);
	dl_out.close();

	// Download result from running cgi script
	InputStream dl_is = dl_connection.getInputStream();
	BufferedInputStream dl_bis = new BufferedInputStream(dl_is);
	DataInputStream dl_dbis = new DataInputStream(dl_bis);

	String first_line = "";
	byte[] buf = new byte[1024];
	int len = dl_dbis.read(buf);
	if (len >= 0) {
	    String first_chunk = new String(buf, 0, len);
	    // first_line = first_chunk.substring(0, ((first_chunk.indexOf("\n") != -1) ? first_chunk.indexOf("\n") : len));
	    first_line = first_chunk.substring(0, ((first_chunk.indexOf("\n") != -1) ? first_chunk.indexOf("\n") : ((first_chunk.length()<len) ? first_chunk.length():len)));
	    // Save the data to file
	    FileOutputStream zip_fos = new FileOutputStream(file_path);
	    BufferedOutputStream zip_bfos = new BufferedOutputStream(zip_fos);

	    while (len >= 0) {
		zip_bfos.write(buf, 0, len);
		len = dl_dbis.read(buf);
	    }

	    zip_bfos.close();
	    zip_fos.close();
	}

	dl_dbis.close();
	dl_bis.close();
	dl_is.close();
	return first_line;
    }


    /** Returns the command output if the action completed, throws some kind of exception otherwise. */
    private String sendCommandToServerInternal(String gliserver_url_string, String cgi_args, GShell shell)
	throws Exception
    {
	DebugStream.println("gliserver URL: " + gliserver_url_string);
	System.err.println("gliserver args: " + cgi_args);

	// Add username and password, and a timestamp
	if(isAuthenticationRequired(cgi_args)) {
	    cgi_args += "&un=" + remote_greenstone_server_authentication.getUserName();
	    cgi_args += "&pw=" + new String(remote_greenstone_server_authentication.getPassword());
	}
	cgi_args += "&ts=" + System.currentTimeMillis();
	if (Gatherer.GS3){
	    cgi_args += "&site=" + Configuration.site_name;
	}

	URL gliserver_url = new URL(gliserver_url_string + "?" + cgi_args);
	URLConnection gliserver_connection = gliserver_url.openConnection();

	// Read the output of the command from the server, and return it
	StringBuffer command_output_buffer = new StringBuffer(2048);
	InputStream gliserver_is = gliserver_connection.getInputStream();
	BufferedReader gliserver_in = new BufferedReader(new InputStreamReader(gliserver_is, "UTF-8"));
	String gliserver_output_line = gliserver_in.readLine();
	while (gliserver_output_line != null) {
	    if (shell != null) {
		shell.fireMessage(gliserver_output_line);
		if (shell.hasSignalledStop()) {
		    throw new RemoteGreenstoneServerAction.ActionCancelledException();
		}
	    }
	    command_output_buffer.append(gliserver_output_line + "\n");
	    gliserver_output_line = gliserver_in.readLine();
	}
	gliserver_in.close();

	
	int startIndex = command_output_buffer.indexOf("<?xml");
	if(startIndex > 0) { // not -1, so "<?xml" is present, and not 0, so not at start

	    // iff dealing with XML, make sure to parse out any lines preceding the XML content
	    command_output_buffer = XMLTools.readXMLStream(command_output_buffer.toString());
	}
	
	return command_output_buffer.toString();
    }

	       
    /** Returns the command output if the action completed, throws some kind of exception otherwise. */
    private String uploadFileInternal(String upload_cgi, String cgi_args, String file_path)
	throws Exception
    {
	System.err.println("gliserver URL: " + upload_cgi);
	System.err.println("upload file gliserver args: " + cgi_args);
        System.err.println("file path = "+file_path);
	//For a remote GS3
	//GS3 is running on Tomcat, and Tomcat requires a connection timeout to be set up at the client
	//side while uploading files. As HttpURLConnection couldn't set the connection timeout, HttpClient.jar 
	//from Jakarta is applied to solve this problem only for uploading files. 
	if (Gatherer.GS3) {

	    // Setup the POST method
	    PostMethod httppost = new PostMethod(upload_cgi+"?"+cgi_args); // cgi_args: QUERY_STRING on perl server side
	    		    
	    // construct the multipartrequest form
	    String[] cgi_array=cgi_args.split("&");// get parameter-value pairs from cgi_args string
	    Part[] parts=new Part[cgi_array.length+5];
	    
	    // The FilePart: consisting of the (cgi-arg) name and (optional) filename in the Content-Disposition 
	    // of the POST request Header (see CGI.pm), and the actual zip file itself. It uses the defaults:
	    // Content-Type: application/octet-stream; charset=ISO-8859-1,Content-Transfer-Encoding: binary
	    parts[0]= new FilePart("uploaded_file", "zipFile", new File(file_path));

	    parts[1]= new StringPart("un", remote_greenstone_server_authentication.getUserName());
	    parts[2]= new StringPart("pw", new String(remote_greenstone_server_authentication.getPassword()));
	    parts[3]= new StringPart("ts", String.valueOf(System.currentTimeMillis()));
	    parts[4]= new StringPart("site", Configuration.site_name);
	    // find all parameters of cgi-args and add them into Part[]
	    for (int i=0; i<cgi_array.length;i++){
		parts[5+i]=new StringPart(cgi_array[i].substring(0,cgi_array[i].indexOf("=")),cgi_array[i].substring(cgi_array[i].indexOf("=")+1,cgi_array[i].length()));
	    }

	    // set MultipartRequestEntity on the POST method
	    httppost.setRequestEntity(new MultipartRequestEntity(parts, httppost.getParams()));

	    // See file gli/request.txt for the multipart request that's been generated:
	    //httppost.getRequestEntity().writeRequest(new FileOutputStream("request.txt", true)); // true: appends
	    
	    //set up the HttpClient connection
	    HttpClient client=new HttpClient();	
	    client.getParams().setBooleanParameter(HttpMethodParams.USE_EXPECT_CONTINUE, false);
	    client.getParams().setConnectionManagerTimeout(200000); //set the connection timeout
	    // get the output of the command from the server
	    String command_output = ""; 
	    try{
		client.executeMethod(httppost);
		if (httppost.getStatusCode() == HttpStatus.SC_OK) {
		    command_output = httppost.getStatusLine().toString();
		} else {
		    command_output = httppost.getStatusLine().toString();
		    System.out.println("Unexpected failure: " + httppost.getStatusLine().toString());
		}
	    }catch(IOException e){
		e.printStackTrace();
	    }finally{
		httppost.releaseConnection();
	    }
	    return command_output;
	}

	//For a remote GS2
	// Add username and password, and a timestamp
	cgi_args += "&un=" + remote_greenstone_server_authentication.getUserName();
	cgi_args += "&pw=" + new String(remote_greenstone_server_authentication.getPassword());
	cgi_args += "&ts=" + System.currentTimeMillis();

	// Open a HTTP connection to the URL
	URL url = new URL(upload_cgi);
	HttpURLConnection gliserver_connection = (HttpURLConnection) url.openConnection();

	gliserver_connection.setDoInput(true);         // Allow Inputs
	gliserver_connection.setDoOutput(true);        // Allow Outputs
	gliserver_connection.setUseCaches(false);      // Don't use a cached copy.

	gliserver_connection.setRequestProperty("Connection", "Keep-Alive");	    

	// Send zip file to server
	File file = new File(file_path);
	FileInputStream fileInputStream = new FileInputStream(file);

	// Add file size argument, because IIS 6 needs a lot of help
	int file_size = fileInputStream.available();
	cgi_args += "&fs=" + file_size;

	DataOutputStream dos = new DataOutputStream(gliserver_connection.getOutputStream());
	dos.writeBytes(cgi_args + "\n");

	// create a buffer of maximum size
	final int maxBufferSize = 1024;
	int bytesAvailable = file_size;
	int bufferSize = Math.min(bytesAvailable, maxBufferSize);
	byte[] buffer = new byte[bufferSize];

	// read file and write it into form...
	// !! This uses a lot of memory when the file being uploaded is big -- Java seems to need to keep
	//   the entire file in the DataOutputStream? (Use Runtime.getRuntime().totalMemory() to see)
	int bytesRead = fileInputStream.read(buffer, 0, bufferSize);
	while (bytesRead > 0) {
	    dos.write(buffer, 0, bytesRead);
	    bytesAvailable = fileInputStream.available();
	    bufferSize = Math.min(bytesAvailable, maxBufferSize);
	    bytesRead = fileInputStream.read(buffer, 0, bufferSize);
	}

	// close streams
	fileInputStream.close();
	dos.flush();
	dos.close();

	// Read the output of the command from the server, and return it
	String command_output = "";
	InputStream gliserver_is = gliserver_connection.getInputStream();
	BufferedReader gliserver_in = new BufferedReader(new InputStreamReader(gliserver_is, "UTF-8"));
	String gliserver_output_line = gliserver_in.readLine();
	while (gliserver_output_line != null) {
	    command_output += gliserver_output_line + "\n";
	    gliserver_output_line = gliserver_in.readLine();
	}
	gliserver_in.close();

	return command_output;
    }


    // ----------------------------------------------------------------------------------------------------
    //   UTILITIES
    // ----------------------------------------------------------------------------------------------------


    public String getPathRelativeToDirectory(File file, String directory_path)
    {
	String file_path = file.getAbsolutePath();

	// Special case: if file path is longer by File separator,
	// still dealing with practcally the same folder path, return relative path as "."
	if(directory_path.equals(file_path + File.separator)) {
	    return ".";
	}
	if (!file_path.startsWith(directory_path)) {
	    System.err.println("ERROR: File path " + file_path + " is not a child of " + directory_path);
	    return file_path;
	}

	String relative_file_path = file_path.substring(directory_path.length());
	if (relative_file_path.startsWith(File.separator)) {
	    relative_file_path = relative_file_path.substring(File.separator.length());
	}
	return relative_file_path;
    }
}