/*
 *    AbstractGS2FieldSearch.java
 *    Copyright (C) 2006 New Zealand Digital Library, http://www.nzdl.org
 *
 *    This program is free software; you can redistribute it and/or modify
 *   the Free Software Foundation; either version 2 of the License, or
 *    (at your option) any later version.
 *
 *    This program is distributed in the hope that it will be useful,
 *    but WITHOUT ANY WARRANTY; without even the implied warranty of
 *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *    GNU General Public License for more details.
 *
 *    You should have received a copy of the GNU General Public License
 *    along with this program; if not, write to the Free Software
 *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
 */
package org.greenstone.gsdl3.service;

// Greenstone classes
import java.io.Serializable;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

import org.apache.log4j.Logger;
import org.greenstone.gsdl3.util.FacetWrapper;
import org.greenstone.gsdl3.util.GSXML;
import org.greenstone.gsdl3.util.XMLConverter;
import org.w3c.dom.Document;
import org.w3c.dom.Element;
import org.w3c.dom.NodeList;

abstract public class AbstractGS2FieldSearch extends AbstractGS2TextSearch
{
	// extra services offered by mgpp collections
	protected static final String FIELD_QUERY_SERVICE = "FieldQuery";
	protected static final String ADVANCED_FIELD_QUERY_SERVICE = "AdvancedFieldQuery";
	protected static final String RAW_QUERY_SERVICE = "RawQuery";

	// extra parameters used
	protected static final String LEVEL_PARAM = "level";
	protected static final String RANK_PARAM = "sortBy";
	protected static final String RANK_PARAM_RANK = "1";
	protected static final String RANK_PARAM_NONE = "0";
	protected static final String SIMPLE_FIELD_PARAM = "simpleField";
	protected static final String ADVANCED_FIELD_PARAM = "complexField";

	protected static final String RAW_PARAM = "rawquery";

	// more params for field query
	protected static final String FIELD_QUERY_PARAM = "fqv";
	protected static final String FIELD_STEM_PARAM = "fqs";
	protected static final String FIELD_CASE_PARAM = "fqc";
	protected static final String FIELD_ACCENT_PARAM = "fqa";
	protected static final String FIELD_FIELD_PARAM = "fqf";
	protected static final String FIELD_COMBINE_PARAM = "fqk";
	protected static final String FIELD_COMBINE_PARAM_AND = "0";
	protected static final String FIELD_COMBINE_PARAM_OR = "1";
	protected static final String FIELD_COMBINE_PARAM_NOT = "2";

	// some stuff for config files
	protected static final String SEARCH_TYPE_ELEM = "searchType";
	protected static final String SEARCH_TYPE_PLAIN = "plain";
	protected static final String SEARCH_TYPE_SIMPLE_FORM = "simpleform";
	protected static final String SEARCH_TYPE_ADVANCED_FORM = "advancedform";
	protected static final String SEARCH_TYPE_RAW = "raw";

	protected static final String DEFAULT_LEVEL_ELEM = "defaultLevel";
	protected static final String DEFAULT_DB_LEVEL_ELEM = "defaultDBLevel";
	protected static final String LEVEL_ELEM = "level";
	protected static final String FIELD_ATT = "field";

	protected static final int TEXT_QUERY = 0;
	protected static final int SIMPLE_QUERY = 1;
	protected static final int ADVANCED_QUERY = 2;
	protected static final int RAW_QUERY = 3;

	protected String AND_OPERATOR = "&";
	protected String OR_OPERATOR = "|";
	protected String NOT_OPERATOR = "!";

        // what levels we support
        protected ArrayList<String> level_ids = null;
	// the default level for searching
	protected String default_level = null;
	// default level for collection db
	protected String default_db_level = null;
        // metadata for service, such as does_paging = true
        protected Element service_metadata_list = null;
	// which search services will we offer??
	protected boolean plain_search = false;
	protected boolean simple_form_search = false;
	protected boolean advanced_form_search = false;
	protected boolean raw_search = false;
	//Parameters to get full field with highlighted terms
	protected String hldocOID = null;
	//index Field for highlighting
	protected String indexField = null;

	static Logger logger = Logger.getLogger(org.greenstone.gsdl3.service.AbstractGS2FieldSearch.class.getName());

	/** constructor */
	public AbstractGS2FieldSearch()
	{
	  super();
	  paramDefaults.put(RANK_PARAM, RANK_PARAM_RANK);
	  paramDefaults.put(FIELD_COMBINE_PARAM, FIELD_COMBINE_PARAM_AND);
	  // paramDefaults.put(FIELD_CASE_PARAM, BOOLEAN_PARAM_ON);
	  // paramDefaults.put(FIELD_STEM_PARAM, BOOLEAN_PARAM_OFF);
	  // paramDefaults.put(FIELD_ACCENT_PARAM, BOOLEAN_PARAM_ON);
	}

	public void cleanUp()
	{
		super.cleanUp();
	}

	/** configure this service */
	public boolean configure(Element info, Element extra_info)
	{
		if (!super.configure(info, extra_info))
		{
			return false;
		}

		// set up which params to save
		this.save_params.add(LEVEL_PARAM);
		this.save_params.add(RANK_PARAM);
		this.save_params.add(RAW_PARAM);
		this.save_params.add(FIELD_QUERY_PARAM);
		this.save_params.add(FIELD_STEM_PARAM);
		this.save_params.add(FIELD_CASE_PARAM);
		this.save_params.add(FIELD_ACCENT_PARAM);
		this.save_params.add(FIELD_FIELD_PARAM);
		this.save_params.add(FIELD_COMBINE_PARAM);
		
		// Get the default level out of <defaultLevel> (buildConfig.xml)
		Element def = (Element) GSXML.getChildByTagName(info, DEFAULT_LEVEL_ELEM);
		if (def != null)
		{
			this.default_level = def.getAttribute(GSXML.SHORTNAME_ATT);
		}
		if (this.default_level == null || this.default_level.equals(""))
		{
			logger.error("default level not specified!, assuming Doc");
			this.default_level = "Doc";
		}

		// Get the default DB level
		def = (Element) GSXML.getChildByTagName(info, DEFAULT_DB_LEVEL_ELEM);
		if (def != null)
		{
			this.default_db_level = def.getAttribute(GSXML.SHORTNAME_ATT);
		}
		if (this.default_db_level == null || this.default_db_level.equals(""))
		{
			logger.error("default database level (defaultDBLevel) not specified!, assuming Sec");
			this.default_db_level = "Sec";
		}

		// get stuff from from extra info (which is collectionConfig.xml)
		if (extra_info != null)
		{

			// the search element
			Element config_search = (Element) GSXML.getChildByTagName(extra_info, GSXML.SEARCH_ELEM);

			NodeList search_types = config_search.getElementsByTagName(SEARCH_TYPE_ELEM);
			if (search_types == null)
			{
				// none specified, assume plain only
				this.plain_search = true;
			}
			else
			{
				for (int i = 0; i < search_types.getLength(); i++)
				{
					Element t = (Element) search_types.item(i);

					String type_name = t.getAttribute(GSXML.NAME_ATT);
					if (type_name.equals(SEARCH_TYPE_PLAIN))
					{
						this.plain_search = true;
					}
					else if (type_name.equals(SEARCH_TYPE_SIMPLE_FORM))
					{
						this.simple_form_search = true;
					}
					else if (type_name.equals(SEARCH_TYPE_ADVANCED_FORM))
					{
						this.advanced_form_search = true;
					}
					else if (type_name.equals(SEARCH_TYPE_RAW))
					{
						this.raw_search = true;
					}
				}
			}

			// AbstractGS2TextSearch has set up the TextQuery service, but we may not want it
			if (!this.plain_search)
			{
				// need to remove the TextQuery service
				Element tq_service = GSXML.getNamedElement(short_service_info, GSXML.SERVICE_ELEM, GSXML.NAME_ATT, QUERY_SERVICE);
				short_service_info.removeChild(tq_service);
			}

			// lets add displayinfo from collectionConfig into the level elements from buildConfig -
			// so we can easily find them later.
			// will also store a list of level ids so we can easily see which levels we have
			Document owner = info.getOwnerDocument();

			level_ids = new ArrayList<String>();
			NodeList levels = info.getElementsByTagName(GSXML.LEVEL_ELEM);
			
			for (int i = 0; i < levels.getLength(); i++)
			  {
			        Element lev = (Element) levels.item(i);
			        String shortname = lev.getAttribute(GSXML.SHORTNAME_ATT);
				if (shortname.equals(""))
				  {
				    continue;
				  }
				level_ids.add(shortname);
				
				String name = lev.getAttribute(GSXML.NAME_ATT);
				Element node_extra = GSXML.getNamedElement(config_search, GSXML.LEVEL_ELEM, GSXML.NAME_ATT, name);
				if (node_extra == null)
				{
					logger.error("haven't found extra info for level named " + name);
					continue;
				}

				// get the display elements if any - displayName
				NodeList display_names = node_extra.getElementsByTagName(GSXML.DISPLAY_TEXT_ELEM);
				if (display_names != null)
				{
					for (int j = 0; j < display_names.getLength(); j++)
					{
						Element e = (Element) display_names.item(j);
						lev.appendChild(owner.importNode(e, true));
					}
				}
			  }// for each level
		}
		else
		{
			// for some reason we don't have the collectionConfig file. assume plain only
			this.plain_search = true;
		}

		// the format info is the same for all services
		Element format_info = (Element) format_info_map.get(QUERY_SERVICE);

		// set up the extra services which are available for this collection
		if (this.simple_form_search)
		{
			// set up short_service_info_ - for now just has id and type - name will be added in on the fly
			Element fq_service = this.desc_doc.createElement(GSXML.SERVICE_ELEM);
			fq_service.setAttribute(GSXML.TYPE_ATT, GSXML.SERVICE_TYPE_QUERY);
			fq_service.setAttribute(GSXML.NAME_ATT, FIELD_QUERY_SERVICE);
			this.short_service_info.appendChild(fq_service);

			if (format_info != null)
			{
				this.format_info_map.put(FIELD_QUERY_SERVICE, format_info);
			}
		}

		if (this.advanced_form_search)
		{
			Element afq_service = this.desc_doc.createElement(GSXML.SERVICE_ELEM);
			afq_service.setAttribute(GSXML.TYPE_ATT, GSXML.SERVICE_TYPE_QUERY);
			afq_service.setAttribute(GSXML.NAME_ATT, ADVANCED_FIELD_QUERY_SERVICE);
			this.short_service_info.appendChild(afq_service);

			if (format_info != null)
			{
				this.format_info_map.put(ADVANCED_FIELD_QUERY_SERVICE, format_info);
			}
		}

		if (this.raw_search)
		{
			Element rq_service = this.desc_doc.createElement(GSXML.SERVICE_ELEM);
			rq_service.setAttribute(GSXML.TYPE_ATT, GSXML.SERVICE_TYPE_QUERY);
			rq_service.setAttribute(GSXML.NAME_ATT, RAW_QUERY_SERVICE);
			this.short_service_info.appendChild(rq_service);

			if (format_info != null)
			{
				this.format_info_map.put(RAW_QUERY_SERVICE, format_info);
			}
		}

		return true;
	}

  // if field case/stem/accent defaults are not specified explicitly, we should use case/stem/accent defaults
  protected void getSearchParamDefaults(Element search_elem) {
    
    boolean found_f_stem = false;
    boolean found_f_case = false;
    boolean found_f_accent = false;
    NodeList param_defaults_list = search_elem.getElementsByTagName(GSXML.PARAM_DEFAULT_ELEM); //GSXML.getChildrenByTagName(search_elem, GSXML.PARAM_DEFAULT_ELEM);
    for (int i=0; i<param_defaults_list.getLength(); i++) {
      Element paramdef = (Element)param_defaults_list.item(i);
      String name = paramdef.getAttribute(GSXML.NAME_ATT);
      String val = paramdef.getAttribute(GSXML.VALUE_ATT);
      if (!name.equals("") && !val.equals("")) {
	paramDefaults.put(name, val);
	if (name.equals(FIELD_STEM_PARAM)) {
	  found_f_stem = true;
	} else if (name.equals(FIELD_CASE_PARAM)) {
	  found_f_case = true;
	} else if (name.equals(FIELD_ACCENT_PARAM)) {
	  found_f_accent = true;
	} 
	if (!found_f_stem) {
	  paramDefaults.put (FIELD_STEM_PARAM, paramDefaults.get(STEM_PARAM));
	}
	if (!found_f_case) {
	  paramDefaults.put (FIELD_CASE_PARAM, paramDefaults.get(CASE_PARAM));
	}
	if (!found_f_accent) {
	  paramDefaults.put (FIELD_ACCENT_PARAM, paramDefaults.get(ACCENT_PARAM));
	}

      }
    }
  }
  protected Element getServiceDescription(Document doc, String service_id, String lang, String subset)
	{
		// should we check that the service is actually on offer? presumably we wont get asked for services that we haven't advertised previously.

		if (!service_id.equals(FIELD_QUERY_SERVICE) && !service_id.equals(ADVANCED_FIELD_QUERY_SERVICE) && !service_id.equals(RAW_QUERY_SERVICE))
		{
		  return super.getServiceDescription(doc, service_id, lang, subset);
		}

		Element service = doc.createElement(GSXML.SERVICE_ELEM);
		service.setAttribute(GSXML.TYPE_ATT, GSXML.SERVICE_TYPE_QUERY);
		service.setAttribute(GSXML.NAME_ATT, service_id);
		if (subset == null || subset.equals(GSXML.DISPLAY_TEXT_ELEM + GSXML.LIST_MODIFIER))
		{
			service.appendChild(GSXML.createDisplayTextElement(doc, GSXML.DISPLAY_TEXT_NAME, getTextString(service_id + ".name", lang)));
			service.appendChild(GSXML.createDisplayTextElement(doc, GSXML.DISPLAY_TEXT_SUBMIT, getTextString(service_id + ".submit", lang)));
			service.appendChild(GSXML.createDisplayTextElement(doc, GSXML.DISPLAY_TEXT_DESCRIPTION, getTextString(service_id + ".description", lang)));

		}
		if (subset == null || subset.equals(GSXML.PARAM_ELEM + GSXML.LIST_MODIFIER))
		{
			Element param_list = doc.createElement(GSXML.PARAM_ELEM + GSXML.LIST_MODIFIER);
			service.appendChild(param_list);
			if (service_id.equals(FIELD_QUERY_SERVICE))
			{

				addCustomQueryParams(param_list, lang);
				if (!default_index_subcollection.equals(""))
				{
					createParameter(INDEX_SUBCOLLECTION_PARAM, param_list, lang);
				}
				if (!default_index_language.equals(""))
				{
					createParameter(INDEX_LANGUAGE_PARAM, param_list, lang);
				}

				if (does_chunking) {
				  createParameter(MAXDOCS_PARAM, param_list, lang);
				}
				if (does_paging)
				{
					createParameter(HITS_PER_PAGE_PARAM, param_list, lang);
					createParameter(START_PAGE_PARAM, param_list, lang);
				}

				// create a multi param for the fields etc
				// text box, field
				Element multiparam = null;
				Element param = null;
				multiparam = GSXML.createParameterDescription(doc, SIMPLE_FIELD_PARAM, "", GSXML.PARAM_TYPE_MULTI, null, null, null);
				multiparam.setAttribute("occurs", "4");
				param_list.appendChild(multiparam);

				// the components

				createParameter(FIELD_QUERY_PARAM, multiparam, lang);
				createParameter(FIELD_FIELD_PARAM, multiparam, lang);

			}
			else if (service_id.equals(ADVANCED_FIELD_QUERY_SERVICE))
			{
				addCustomQueryParamsAdvField(param_list, lang);
				if (!default_index_subcollection.equals(""))
				{
					createParameter(INDEX_SUBCOLLECTION_PARAM, param_list, lang);
				}
				if (!default_index_language.equals(""))
				{
					createParameter(INDEX_LANGUAGE_PARAM, param_list, lang);
				}

				// create a multi param for the fields etc
				// text box, stem, case, field

				Element multiparam = null;
				Element param = null;

				multiparam = GSXML.createParameterDescription(doc, ADVANCED_FIELD_PARAM, "", GSXML.PARAM_TYPE_MULTI, null, null, null);
				multiparam.setAttribute("occurs", "4");
				param_list.appendChild(multiparam);

				createParameter(FIELD_COMBINE_PARAM, multiparam, lang);
				createParameter(FIELD_QUERY_PARAM, multiparam, lang);
				if (this.does_case)
				{
					createParameter(FIELD_CASE_PARAM, multiparam, lang);
				}
				if (this.does_stem)
				{
					createParameter(FIELD_STEM_PARAM, multiparam, lang);
				}
				if (this.does_accent)
				{
					createParameter(FIELD_ACCENT_PARAM, multiparam, lang);
				}
				if (does_chunking) {
				  createParameter(MAXDOCS_PARAM, param_list, lang);
				}
				if (does_paging)
				{
					createParameter(HITS_PER_PAGE_PARAM, param_list, lang);
					createParameter(START_PAGE_PARAM, param_list, lang);
				}
				createParameter(FIELD_FIELD_PARAM, multiparam, lang);

			}
			else if (service_id.equals(RAW_QUERY_SERVICE))
			{
				if (does_chunking)
				{
					createParameter(MAXDOCS_PARAM, param_list, lang);
				}
				if (does_paging)
				{
					createParameter(HITS_PER_PAGE_PARAM, param_list, lang);
					createParameter(START_PAGE_PARAM, param_list, lang);
				}
				createParameter(RAW_PARAM, param_list, lang);
				service.appendChild(param_list);
			}
		}
		if (subset == null || subset.equals(GSXML.METADATA_ELEM + GSXML.LIST_MODIFIER)) {
		  if (service_metadata_list == null) {
		    Document ml_doc = XMLConverter.newDOM();
		    service_metadata_list = ml_doc.createElement(GSXML.METADATA_ELEM+GSXML.LIST_MODIFIER);
		    if (does_paging) {
		      service_metadata_list.appendChild(GSXML.createMetadataElement(ml_doc, "does_paging", "true"));
		    }
		    if (does_faceting) {
		      service_metadata_list.appendChild(GSXML.createMetadataElement(ml_doc, "does_facetsearch", "true"));
		    }
	    
		  }
		  service.appendChild(doc.importNode(service_metadata_list, true));
		}
		return service;

	}

	/** add in the level params to TextQuery */
	protected void addCustomQueryParams(Element param_list, String lang)
	{
		createParameter(LEVEL_PARAM, param_list, lang);
		super.addCustomQueryParams(param_list, lang);
	}
	/** add in the level params to TextQuery */
	protected void addCustomQueryParamsAdvField(Element param_list, String lang)
	{
		createParameter(LEVEL_PARAM, param_list, lang);
	}

	/** create a param and add to the list */
	protected void createParameter(String name, Element param_list, String lang)
	{
	  Document doc = param_list.getOwnerDocument();
		Element param = null;
		String param_default = paramDefaults.get(name);
		if (name.equals(LEVEL_PARAM))
		{
			ArrayList<String> level_names = new ArrayList<String>();
			getLevelData(level_names, lang);
			if (level_ids.size() > 1)
			{
				// the first one is the default
				//param = GSXML.createParameterDescription2(doc, LEVEL_PARAM, getTextString("param."+LEVEL_PARAM, lang), GSXML.PARAM_TYPE_ENUM_SINGLE, (String)level_ids.get(0), level_ids, level_names);
				param = GSXML.createParameterDescription2(doc, LEVEL_PARAM, getTextString("param." + LEVEL_PARAM, lang), GSXML.PARAM_TYPE_ENUM_SINGLE, this.default_level, level_ids, level_names);
			}
			else
			{
				// we need to set the level, but hidden, in case there is an invalid level saved
				//param = GSXML.createParameterDescription(doc, LEVEL_PARAM, "", GSXML.PARAM_TYPE_INVISIBLE, (String)level_ids.get(0), null, null);
				param = GSXML.createParameterDescription(doc, LEVEL_PARAM, "", GSXML.PARAM_TYPE_INVISIBLE, this.default_level, null, null);
			}
		}
		else if (name.equals(RANK_PARAM))
		{
			String[] vals1 = { RANK_PARAM_RANK, RANK_PARAM_NONE };
			String[] vals1_texts = { getTextString("param." + RANK_PARAM + "." + RANK_PARAM_RANK, lang), getTextString("param." + RANK_PARAM + "." + RANK_PARAM_NONE, lang) };

			param = GSXML.createParameterDescription(doc, RANK_PARAM, getTextString("param." + RANK_PARAM, lang), GSXML.PARAM_TYPE_ENUM_SINGLE, param_default, vals1, vals1_texts);

		}
		else if (name.equals(FIELD_QUERY_PARAM))
		{
			param = GSXML.createParameterDescription(doc, FIELD_QUERY_PARAM, getTextString("param." + FIELD_QUERY_PARAM, lang), GSXML.PARAM_TYPE_STRING, null, null, null);

		}
		else if (name.equals(FIELD_CASE_PARAM) || name.equals(FIELD_STEM_PARAM) || name.equals(FIELD_ACCENT_PARAM))
		{
			String[] bool_ops = { "0", "1" };
			String[] bool_texts = { getTextString("param.boolean.off", lang), getTextString("param.boolean.on", lang) };
			param = GSXML.createParameterDescription(doc, name, getTextString("param." + name, lang), GSXML.PARAM_TYPE_BOOLEAN, param_default, bool_ops, bool_texts);

		}
		else if (name.equals(FIELD_FIELD_PARAM))
		{
			ArrayList<String> fields = new ArrayList<String>();
			ArrayList<String> field_names = new ArrayList<String>();
			getIndexData(fields, field_names, lang);
			// the field list -  read from config file

			// Fix for http://trac.greenstone.org/ticket/245 "java crash, index out of bounds"
			// org.greenstone.gsdl3.service.AbstractGS2FieldSearch.createParameter(AbstractGS2FieldSearch.java:362)
			// Changed from:
			// param = GSXML.createParameterDescription2(doc, name, getTextString("param."+name, lang), GSXML.PARAM_TYPE_ENUM_SINGLE, (String)fields.get(0), fields, field_names );
			String default_value = (fields.size() > 0) ? fields.get(0) : null;
			// don't want to access element 0 if fields.size()==0, and
			// GSXML.createParameterDescription2 checks for default_value==null condition
			param = GSXML.createParameterDescription2(doc, name, getTextString("param." + name, lang), GSXML.PARAM_TYPE_ENUM_SINGLE, default_value, fields, field_names);

		}
		else if (name.equals(FIELD_COMBINE_PARAM))
		{

			String[] vals = { FIELD_COMBINE_PARAM_AND, FIELD_COMBINE_PARAM_OR, FIELD_COMBINE_PARAM_NOT };
			String[] val_texts = { getTextString("param." + FIELD_COMBINE_PARAM + "." + FIELD_COMBINE_PARAM_AND, lang), getTextString("param." + FIELD_COMBINE_PARAM + "." + FIELD_COMBINE_PARAM_OR, lang), getTextString("param." + FIELD_COMBINE_PARAM + "." + FIELD_COMBINE_PARAM_NOT, lang) };

			param = GSXML.createParameterDescription(doc, FIELD_COMBINE_PARAM, "", GSXML.PARAM_TYPE_ENUM_SINGLE, param_default, vals, val_texts);
			param.setAttribute(GSXML.PARAM_IGNORE_POS_ATT, "0");
		}

		if (param != null)
		{
			param_list.appendChild(param);
		}
		else
		{
			super.createParameter(name, param_list, lang);
		}
	}

	// should cache some of this
  protected void getLevelData(ArrayList<String> level_names, String lang)
	{
		Element level_list = (Element) GSXML.getChildByTagName(this.config_info, LEVEL_ELEM + GSXML.LIST_MODIFIER);
		for (int i=0; i<level_ids.size(); i++) {
		  Element level = GSXML.getNamedElement(level_list, GSXML.LEVEL_ELEM, GSXML.SHORTNAME_ATT, level_ids.get(i));
			String display_name = getDisplayText(level, GSXML.DISPLAY_TEXT_NAME, lang, "en");
			if (display_name.equals(""))
			{
				// we'll use the name, and look it up in the dictionary
				display_name = level.getAttribute(GSXML.NAME_ATT);
				if (display_name.equals(""))
				{
				  display_name = level_ids.get(i); // shortname
				}
				else
				{
					display_name = getTextString("level." + display_name, lang);
				}
			}
			level_names.add(display_name);
		}
	}


	// the following three functions are needed so the base class can 
	// call the process+SERVICE_NAME methods
	/** process a text query */
	protected Element processTextQuery(Element request)
	{
		return processAnyQuery(request, TEXT_QUERY);
	}

	/** process a field query */
	protected Element processFieldQuery(Element request)
	{
		return processAnyQuery(request, SIMPLE_QUERY);
	}

	/** process an advanced field query */
	protected Element processAdvancedFieldQuery(Element request)
	{
		return processAnyQuery(request, ADVANCED_QUERY);
	}

	protected Element processRawQuery(Element request)
	{
		return processAnyQuery(request, RAW_QUERY);
	}

	/** process a query */
	protected Element processAnyQuery(Element request, int query_type)
	{
				
		String service_name = null;
		String empty_query_test_param = null;
		// set up the type specific bits
		switch (query_type)
		{
		case TEXT_QUERY:
			service_name = QUERY_SERVICE;
			empty_query_test_param = QUERY_PARAM;
			break;
		case SIMPLE_QUERY:
			service_name = FIELD_QUERY_SERVICE;
			empty_query_test_param = FIELD_QUERY_PARAM;
			break;
		case ADVANCED_QUERY:
			service_name = ADVANCED_FIELD_QUERY_SERVICE;
			empty_query_test_param = FIELD_QUERY_PARAM;
			break;
		case RAW_QUERY:
			service_name = RAW_QUERY_SERVICE;
			empty_query_test_param = RAW_PARAM;
			break;
		default:
			// should never get here
			logger.error("wrong query type!!");
			return null;
		}

		// Create a new (empty) result message
		Document result_doc = XMLConverter.newDOM();
		Element result = result_doc.createElement(GSXML.RESPONSE_ELEM);
		result.setAttribute(GSXML.FROM_ATT, service_name);
		result.setAttribute(GSXML.TYPE_ATT, GSXML.REQUEST_TYPE_PROCESS);

		// Get the parameters of the request
		Element param_list = (Element) GSXML.getChildByTagName(request, GSXML.PARAM_ELEM + GSXML.LIST_MODIFIER);
		if (param_list == null)
		{
			logger.error("TextQuery request had no paramList.");
			return result; // Return the empty result
		}

		// Process the request parameters
		
		HashMap<String, Serializable> params = GSXML.extractParams(param_list, false);
		//Save as variable to make it accessable from GS2SolrSearch
		hldocOID = (String) params.get("hldocOID");
		//System.out.println("@@@@@Extracted hldocOID is " + hldocOID);
		// Make sure a query has been specified
		String query = (String) params.get(empty_query_test_param);
		if (query == null || query.equals(""))
		{
			return result; // Return the empty result
		}

		// If a field hasn't been specified, use the default - for textQuery
		String field = (String) params.get(INDEX_PARAM);
		//Save as variable to make it accessable from GS2SolrSearch
		
		if (field == null)
		{
			field = default_index;
		}
		indexField = field;
		// set up the appropriate query system
		Object queryObject = setUpQueryer(params);
		if (queryObject == null)
		{			
			return result;
		}

		// if field search, create the query string
		switch (query_type)
		{
		case TEXT_QUERY:
			query = addFieldInfo(query, field);
			break;
		case SIMPLE_QUERY:
			query = parseFieldQueryParams(params);
			break;
		case ADVANCED_QUERY:
			query = parseAdvancedFieldQueryParams(params);
			break;
		}		
		
		// run the query		
		Object query_result = runQuery(queryObject, query);
		
		
		// We want highlighted text to be returned right now!
		if (hldocOID != null && does_full_field_highlighting)
		{
			Element nodeContentElement = result_doc.createElement(GSXML.NODE_CONTENT_ELEM);
			nodeContentElement.setTextContent((String) query_result);
			result.appendChild(nodeContentElement);
			return result;
		}
		
			
		// build up the response

		// Create a metadata list to store information about the query results
		// should we be using metadataList? or something else?
		Element metadata_list = result_doc.createElement(GSXML.METADATA_ELEM + GSXML.LIST_MODIFIER);
		result.appendChild(metadata_list);

		// Add a metadata element specifying the number of matching documents
		long totalDocs = numDocsMatched(query_result);

		GSXML.addMetadata(metadata_list, "numDocsMatched", "" + totalDocs);

		// Create a document list to store the matching documents, and add them
		String[] docs = getDocIDs(query_result);
		String[] doc_ranks = getDocRanks(query_result);

		// add a metadata item to specify docs returned
		if (does_chunking) // this means we have a max docs param, and might ask for only a subset of results
		{
		  logger.error("does_chunking = true");
		        int docs_returned = docs.length;
			String maxdocs_str = (String) params.get(MAXDOCS_PARAM);
			if (maxdocs_str != null)
			{
				int maxdocs = Integer.parseInt(maxdocs_str);
				if (maxdocs > 0) { // maxdocs==-1 means return all
				  docs_returned = (maxdocs < (int) totalDocs ? maxdocs : (int) totalDocs);
				}
			}
			GSXML.addMetadata(metadata_list, "numDocsReturned", "" + docs_returned);
		}
		
		Map<String, Map<String, List<String>>> hlResults = null;
		if (does_highlight_snippets)
		{
			hlResults = getHighlightSnippets(query_result);
		}
		
		// add a metadata item to specify what actual query was done - eg if stuff was stripped out etc. and then we can use the query later, cos we don't know which parameter was the query
		GSXML.addMetadata(metadata_list, "query", query);
		if (docs.length > 0)
		{
			Element document_list = result_doc.createElement(GSXML.DOC_NODE_ELEM + GSXML.LIST_MODIFIER);
			result.appendChild(document_list);
			Element snippet_list = result_doc.createElement(GSXML.HL_SNIPPET_ELEM + GSXML.LIST_MODIFIER);
			result.appendChild(snippet_list);
			for (int d = 0; d < docs.length; d++)
			{
				String doc_id = internalNum2OID(docs[d]);
				Element doc_node = createDocNode(result_doc, doc_id, doc_ranks[d]);
				if (hlResults != null && hlResults.get(docs[d]) != null && hlResults.get(docs[d]).get(indexField) != null) {
					for (String snippet : hlResults.get(docs[d]).get(indexField)){
						//remove html tags
						snippet = snippet.replaceAll("\\<.*?>", "");
						//remove truncated tags
						snippet = snippet.replaceAll(".*>", "");
						snippet = snippet.replaceAll("\\<.*", "");
						//remove unwanted symbols at start of line
						snippet = snippet.replaceAll("^[ .,»):;-–]+", "");
						//highlighting tags transformation
						snippet = snippet.replaceAll("&lt;", "<");
						snippet = snippet.replaceAll("&gt;", ">");
						Element snippet_node = result_doc.createElement(GSXML.HL_SNIPPET_ELEM);
						snippet_node.setAttribute(GSXML.NODE_ID_ATT, doc_id);
						snippet_node.setTextContent(snippet);
						snippet_list.appendChild(snippet_node);
					}
				}
				document_list.appendChild(doc_node);
			}
		}

		// Create a term list to store the term information, and add it
		Element term_list = result_doc.createElement(GSXML.TERM_ELEM + GSXML.LIST_MODIFIER);
		result.appendChild(term_list);
		addTermInfo(term_list, params, query_result);

		if(does_faceting)
		{
		  String lang = request.getAttribute(GSXML.LANG_ATT);
		  ArrayList<FacetWrapper> facets = getFacets(query_result, lang);
			if(facets != null)
			{
				Element facet_list = result_doc.createElement(GSXML.FACET_ELEM + GSXML.LIST_MODIFIER);
				result.appendChild(facet_list);

				for(FacetWrapper currentFacet : facets)
				{
					Element facet_elem = result_doc.createElement(GSXML.FACET_ELEM);
					facet_elem.setAttribute(GSXML.NAME_ATT, currentFacet.getName());
					String display_name = currentFacet.getDisplayName();
					if (display_name != null && !display_name.equals("")) {
					  facet_elem.appendChild(GSXML.createDisplayTextElement(result_doc, GSXML.DISPLAY_TEXT_NAME, display_name));
					}
					facet_list.appendChild(facet_elem);
					
					HashMap<String, Long> countMap = currentFacet.getCounts();
					
					for(String countName : countMap.keySet())
					{
						long countValue = countMap.get(countName);
						if(countValue > 0)
						{
							Element count_elem = result_doc.createElement(GSXML.COUNT_ELEM);
							count_elem.setAttribute(GSXML.NAME_ATT, countName);
							count_elem.setTextContent("" + countValue);

							facet_elem.appendChild(count_elem);
						}
					}
				}
			}
		}
		
		queryObject = null;
		return result;

	}

	/** methods to handle actually doing the query */
	/** do any initialisation of the query object. Call before runQuery()
	  * @return the queryObject (e.g. GS2LuceneQuery)
	*/	
	abstract protected Object setUpQueryer(HashMap<String, Serializable> params);

	/** do the query 
	  * The queryObject parameter is the return value of setUpQueryer.
	*/
	abstract protected Object runQuery(Object queryObject, String query);

	/** get the total number of docs that match */
	abstract protected long numDocsMatched(Object query_result);

	/** get the list of doc ids */
	abstract protected String[] getDocIDs(Object query_result);

	/** get the list of doc ranks */
	abstract protected String[] getDocRanks(Object query_result);
	
	/** get the list of facets */
  abstract protected ArrayList<FacetWrapper> getFacets(Object query_result, String lang);
	
	/** get the map of highlighting snippets */
	abstract protected Map<String, Map<String, List<String>>> getHighlightSnippets(Object query_result);

	/** add in term info if available */
	abstract protected boolean addTermInfo(Element term_list, HashMap<String, Serializable> params, Object query_result);

	/**
	 * combines all the field params into a single query - for simple field
	 * query
	 */
	/** We assume the combination (AND/OR) is done by the match param */
	protected String parseFieldQueryParams(HashMap<String, Serializable> params)
	{

		StringBuffer final_query = new StringBuffer(256);
		String text_line = (String) params.get(FIELD_QUERY_PARAM);
		String[] texts = text_line.split(",", -1);
		String field_line = (String) params.get(FIELD_FIELD_PARAM);
		String[] fields = field_line.split(",", -1);

		for (int i = 0; i < texts.length; i++)
		{
			String q = texts[i].trim();
			if (!q.equals(""))
			{
				final_query.append(" " + addFieldInfo(q, fields[i]));
			}
		}

		return final_query.toString();
	}

	abstract protected String addFieldInfo(String query, String field);

	/**
	 * combines all the field params into a single query - for advanced field
	 * query
	 */
	protected String parseAdvancedFieldQueryParams(HashMap<String, Serializable> params)
	{

		StringBuffer final_query = new StringBuffer(256);
		String text_line = (String) params.get(FIELD_QUERY_PARAM);
		String[] texts = text_line.split(",", -1);
		String field_line = (String) params.get(FIELD_FIELD_PARAM);
		String[] fields = field_line.split(",", -1);
		String[] cases = null;
		String[] stems = null;
		String[] accents = null;

		// Enforce an explicit Boolean "Sum of Products"
		// precedence by bracketing the term (which might be
		// compound) before and after the OR operation

		// This explicit bracket is primarily done to avoid the following
		// undesirable situation with Lucene/Solr, where its query parser
		// "over optimises" things (if considering the advanced query form
		// Greenstone uses as implying AND having higher precedence than OR
		
		// For example, the following:
		//   TE:(music) AND OB:(Article) OR TE:(music) AND OB:(Photograph)
		// Gets turned into:
		//   +TE:music +OB:article +TE:music +OB:photograph
		//
		// The "+" in Lucene/Solr syntax means "required"
		//
		// Whereas we want:
		//   (TE:(music) AND OB:(Article)) OR TE:((music) AND OB:(Photograph))
		//
		// Here we force the issue by wrapping up the two terms (which might be compound)
		// of an OR operation in explicit brackets
		
		
		final_query.append("(");
		
		if (does_case)
		{
			String case_line = (String) params.get(FIELD_CASE_PARAM);
			if (case_line != null)
				cases = case_line.split(",", -1);
		}
		if (does_stem)
		{
			String stem_line = (String) params.get(FIELD_STEM_PARAM);
			if (stem_line != null)
				stems = stem_line.split(",", -1);
		}
		if (does_accent)
		{
			String accent_line = (String) params.get(FIELD_ACCENT_PARAM);
			if (accent_line != null)
				accents = accent_line.split(",", -1);
		}
		String combine_line = (String) params.get(FIELD_COMBINE_PARAM);
		String[] combines = combine_line.split(",", -1);
		String combine = "";
		for (int i = 0; i < texts.length; i++)
		{
			if (i == 0)
			{// assume first one is blank
				combine = "";
			}
			else
			{
				String x = combines[i];
				if (x.equals(FIELD_COMBINE_PARAM_AND))
				{
					combine = AND_OPERATOR;
				}
				else if (x.equals(FIELD_COMBINE_PARAM_OR))
				{
					combine = OR_OPERATOR;
				}
				else if (x.equals(FIELD_COMBINE_PARAM_NOT))
				{
					combine = NOT_OPERATOR;
				}

			}

			String q = texts[i].trim();
			boolean modified = false;
			if (!q.equals(""))
			{
				String c = null;
				String s = null;
				String a = null;
				if (does_case)
				{
					modified = true;
					c = cases[i];
				}
				if (does_stem)
				{
					modified = true;
					s = stems[i];
				}
				if (does_accent)
				{
					modified = true;
					a = accents[i];
				}
				if (modified)
				{
					q = addStemOptions(q, s, c, a);
				}

				addQueryElem(final_query, q, fields[i], combine);
				
			}
		}

		final_query.append(")");
		//System.err.println("**** AbstractGS2FieldSearch final_query = " + final_query.toString());
		
		return final_query.toString();
	}

	abstract protected void addQueryElem(StringBuffer final_query, String query, String field, String combine);

	abstract protected String addStemOptions(String query, String stem, String casef, String accent);

}