/*
 *    GS2SolrSearch.java
 *    Copyright (C) 2006 New Zealand Digital Library, http://www.nzdl.org
 *
 *    This program is free software; you can redistribute it and/or modify
 *   the Free Software Foundation; either version 2 of the License, or
 *    (at your option) any later version.
 *
 *    This program is distributed in the hope that it will be useful,
 *    but WITHOUT ANY WARRANTY; without even the implied warranty of
 *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *    GNU General Public License for more details.
 *
 *    You should have received a copy of the GNU General Public License
 *    along with this program; if not, write to the Free Software
 *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
 */

package org.greenstone.gsdl3.service;

import java.io.File;
import java.io.IOException;
// Greenstone classes
import java.util.ArrayList;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import java.util.Set;
import java.util.Vector;
import java.util.concurrent.TimeUnit;

import org.apache.log4j.Logger;
import org.apache.solr.client.solrj.SolrClient;
import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.impl.HttpJdkSolrClient;
//import org.apache.solr.client.solrj.impl.HttpSolrClient.RemoteSolrException;
import org.apache.solr.client.solrj.request.CoreAdminRequest;
import org.apache.solr.client.solrj.request.CoreStatus;
//import org.apache.solr.client.solrj.response.CoreAdminResponse;
import org.apache.solr.client.solrj.response.FacetField;
import org.apache.solr.common.params.CoreAdminParams.CoreAdminAction;
import org.apache.solr.common.util.NamedList;
import org.greenstone.LuceneWrapper4.SharedSoleneQueryResult;
import org.greenstone.gsdl3.util.FacetWrapper;
import org.greenstone.gsdl3.util.GSFile;
import org.greenstone.gsdl3.util.GSXML;
import org.greenstone.gsdl3.util.SolrFacetWrapper;
import org.greenstone.gsdl3.util.SolrQueryResult;
import org.greenstone.gsdl3.util.SolrQueryWrapper;
import org.greenstone.util.GlobalProperties;
import org.greenstone.util.ProtocolPortProperties;
import org.w3c.dom.Document;
import org.w3c.dom.Element;
import org.w3c.dom.NodeList;

//import org.apache.solr.client.solrj.impl.HttpSolrServer.RemoteSolrException;
//import org.apache.solr.client.solrj.request.CoreAdminRequest;
//import org.apache.solr.client.solrj.response.CoreAdminResponse;
//import org.apache.solr.common.params.CoreAdminParams.CoreAdminAction;
//import org.apache.solr.common.util.NamedList;

public class GS2SolrSearch extends SharedSoleneGS2FieldSearch
{

  protected static final String SORT_ORDER_PARAM = "sortOrder";
  protected static final String SORT_ORDER_DESCENDING = "1";
  protected static final String SORT_ORDER_ASCENDING = "0";

    protected static final String FACET_QUERIES_PARAM = "facetQueries";
    protected static final String DOC_FILTER_PARAM = "docFilter";
    
	static Logger logger = Logger.getLogger(org.greenstone.gsdl3.service.GS2SolrSearch.class.getName());

        protected String solr_server_base_url;
	protected HashMap<String, SolrClient> solr_core_cache;
	protected SolrQueryWrapper solr_src = null;

	protected ArrayList<String> _facets = new ArrayList<String>();

  protected HashMap<String, Element> _facet_display_names = new HashMap<String, Element>();
    protected ArrayList<String> _corenames = new ArrayList<String>();

  protected String solr_core_location = null;
	public GS2SolrSearch()
	{
                this.paramDefaults.put(SORT_ORDER_PARAM, SORT_ORDER_DESCENDING);
		this.does_faceting = true;
		this.does_highlight_snippets = true;
		this.does_full_field_highlighting = true;
		// Used to store the solr cores that match the required 'level' 
		// of search (e.g. either document-level=>didx, or 
		// section-level=>sidx.  The hashmap is filled out on demand
		// based on 'level' parameter passed in to 'setUpQueryer()'

		this.solr_core_cache = new HashMap<String, SolrClient>();

		this.solr_src = new SolrQueryWrapper();		

                String solr_host = System.getenv("SOLR_HOST");
                if (solr_host == null) {
                  logger.warn("SOLR_HOST env var is null, defaulting to 127.0.0.1");
                  solr_host = "127.0.0.1";
                }
                String solr_port = System.getenv("SOLR_PORT");
                if (solr_port == null) {
                  logger.warn("SOLR_PORT env var is null, defaulting to 7983");
                  solr_port = "7983";
                }
                this.solr_server_base_url = "http://"+solr_host+":"+ solr_port +"/solr";
                this.solr_core_location = GlobalProperties.getGSDL3DataHome()+File.separator+"ext"+File.separator + "solr9"+File.separator+"cores"+File.separator;
	}
  
	/** configure this service */
	public boolean configure(Element info, Element extra_info)
	{
		boolean success = super.configure(info, extra_info);

		// clear the map of solr cores for this collection added to the map upon querying
		solr_core_cache.clear(); 

		if(!success) {
		    return false;
		}
		
	       
		// set up which params to save
		this.save_params.add(LEVEL_PARAM);

		// Setting up facets

		// the search element from collectionConfig
		Element searchElem = (Element) GSXML.getChildByTagName(extra_info, GSXML.SEARCH_ELEM);

		Document owner = info.getOwnerDocument();
		// for each facet in buildConfig
		NodeList facet_list = info.getElementsByTagName("facet");
		for (int i=0; i<facet_list.getLength(); i++) {
		  Element facet = (Element)facet_list.item(i);
		  String shortname = facet.getAttribute(GSXML.SHORTNAME_ATT);
		  _facets.add(shortname);

		  // now add any displayItems into the facet element
		  // (which is stored as part of info), then we can add to
		  // the result if needed
		  String longname = facet.getAttribute(GSXML.NAME_ATT);
		  Element config_facet = GSXML.getNamedElement(searchElem, "facet", GSXML.NAME_ATT, longname);
		  if (config_facet != null) {
		    NodeList display_items = config_facet.getElementsByTagName(GSXML.DISPLAY_TEXT_ELEM);
		    for (int j=0; j<display_items.getLength(); j++) {
		      Element e = (Element) display_items.item(j);
		      facet.appendChild(owner.importNode(e, true));
		    }
		    _facet_display_names.put(shortname, facet);

		  }
		      
		}
		NodeList core_list = info.getElementsByTagName("solrcore");
		if (core_list.getLength()>0) {
		    for (int i=0; i<core_list.getLength(); i++) {
			Element core = (Element)core_list.item(i);
			String corename = core.getAttribute(GSXML.NAME_ATT);
			_corenames.add(corename);
		    }
		} else {
		    // old style coll, desn't have solrcores listed in buildConfig
		    // check levels, and use didx and sidx depending on which
		    // levels are specified
		    for (int i=0; i<level_ids.size(); i++) {
			String level = level_ids.get(i);
			if (level.toUpperCase().equals("SEC")) {
			    _corenames.add("sidx");
			} else if (level.toUpperCase().equals("DOC")){
			    _corenames.add("didx");
			}
		    }
		}

		// check if cores loaded
		// if (!loadSolrCores()) {
		// 	logger.error("Collection: couldn't configure collection: " + this.cluster_name + ", "
		// 			+ "Couldn't activate Solr cores");
		// 	 return false;
		// }
		// return true;

                // why should we have to load a core here?? lets try just checking them
                if (!checkSolrCores()) {
                  return false;
                }
                return true;
        }

  private boolean checkSolrCores() {


    SolrClient solrClient = new HttpJdkSolrClient.Builder(this.solr_server_base_url)
      .withConnectionTimeout(3, TimeUnit.SECONDS)
      .build();
    CoreAdminRequest adminRequest = new CoreAdminRequest();
    for (int i=0; i<_corenames.size(); i++) {
      String this_core = getCollectionCoreNamePrefix()+"-"+_corenames.get(i);
    
      try {
        CoreStatus rsp = adminRequest.getCoreStatus(this_core, solrClient);
        System.out.println("Core "+ this_core+" exists, status: " + rsp.toString());
        
      } catch (SolrServerException e) {
        logger.warn("Solr core " + this_core + " for collection " + cluster_name + " does not exist: " + e.getMessage());
        
        return false;
      } catch (IOException e) {
        e.printStackTrace();
        return false;
      }
    }

    // if we get here, all cores are active
    return true;
  }
                  
                

	public void cleanUp()
	{
		super.cleanUp();
		this.solr_src.cleanUp();

		// clear the map keeping track of the SolrClients in this collection
		solr_core_cache.clear();
	}

	/** add in the SOLR specific params to TextQuery */
	protected void addCustomQueryParams(Element param_list, String lang)
	{
		super.addCustomQueryParams(param_list, lang);
		/** Add in the sort order asc/desc param */
		createParameter(SORT_ORDER_PARAM, param_list, lang);
	}
  /** add in SOLR specific params for AdvancedFieldQuery */
  protected void addCustomQueryParamsAdvField(Element param_list, String lang)
	{
		super.addCustomQueryParamsAdvField(param_list, lang);
		createParameter(SORT_ORDER_PARAM, param_list, lang);
		
	}
	/** create a param and add to the list */
  protected void createParameter(String name, Element param_list, String lang)
  {
    Document doc = param_list.getOwnerDocument();
    Element param = null;
    String param_default = paramDefaults.get(name);
    if (name.equals(SORT_ORDER_PARAM)) {
        String[] vals = { SORT_ORDER_ASCENDING, SORT_ORDER_DESCENDING }; 
	String[] vals_texts = { getTextString("param." + SORT_ORDER_PARAM + "." + SORT_ORDER_ASCENDING, lang), getTextString("param." + SORT_ORDER_PARAM + "." + SORT_ORDER_DESCENDING, lang) }; 	    
	
	param = GSXML.createParameterDescription(doc, SORT_ORDER_PARAM, getTextString("param." + SORT_ORDER_PARAM, lang), GSXML.PARAM_TYPE_ENUM_SINGLE, param_default, vals, vals_texts);
    }

    if (param != null)
      {
	param_list.appendChild(param);
      }
    else
      {
	super.createParameter(name, param_list, lang);
      }
    
  }
  
	/** methods to handle actually doing the query */

	/** do any initialisation of the query object */
	protected Object setUpQueryer(HashMap params)
	{
		this.solr_src.clearFacets();
		this.solr_src.clearFacetQueries();

		for (int i = 0; i < _facets.size(); i++)
		{
			this.solr_src.addFacet(_facets.get(i));
		}

		String index = "didx";
		if (this.default_level.toUpperCase().equals("SEC")) {
		  index = "sidx";
		}
		String physical_index_language_name = null;
		String physical_sub_index_name = null;
		String docFilter = null;
		int maxdocs = 100;
		int hits_per_page = 20;
		int start_page = 1;
		// set up the query params
		Set entries = params.entrySet();
		Iterator i = entries.iterator();
		while (i.hasNext())
		{
			Map.Entry m = (Map.Entry) i.next();
			String name = (String) m.getKey();
			String value = (String) m.getValue();

			///System.err.println("### GS2SolrSearch.java: name " + name + " - value " + value);

			if (name.equals(MAXDOCS_PARAM) && !value.equals(""))
			{
				maxdocs = Integer.parseInt(value);
			}
			else if (name.equals(HITS_PER_PAGE_PARAM))
			{
				hits_per_page = Integer.parseInt(value);
			}
			else if (name.equals(START_PAGE_PARAM))
			{
				start_page = Integer.parseInt(value);
			}
			else if (name.equals(MATCH_PARAM))
			{
				if (value.equals(MATCH_PARAM_ALL))
				{
					this.solr_src.setDefaultConjunctionOperator("AND");
				}
				else
				{
					this.solr_src.setDefaultConjunctionOperator("OR");
				}
			}
			else if (name.equals(RANK_PARAM))
			{
				if (value.equals(RANK_PARAM_RANK))
				{
				  value = SolrQueryWrapper.SORT_BY_RANK;
				} else if (value.equals(RANK_PARAM_NONE)) {
				    value = SolrQueryWrapper.SORT_BY_INDEX_ORDER;
				  }
		       
				this.solr_src.setSortField(value);
			}
			else if (name.equals(SORT_ORDER_PARAM)) {
			    if (value.equals(SORT_ORDER_DESCENDING)) {
			      this.solr_src.setSortOrder(SolrQueryWrapper.SORT_DESCENDING);
			    } else {
			      this.solr_src.setSortOrder(SolrQueryWrapper.SORT_ASCENDING);
			    }
			  }
			else if (name.equals(LEVEL_PARAM))
			{
				if (value.toUpperCase().equals("SEC"))
				{
					index = "sidx";
				}
				else
				{
					index = "didx";
				}
			}
			// Would facets ever come in through params???
			else if (name.equals("facets") && value.length() > 0)
			{
				String[] facets = value.split(",");

				for (String facet : facets)
				{
					this.solr_src.addFacet(facet);
				}
			}
			else if (name.equals(FACET_QUERIES_PARAM) && value.length() > 0)
			{
			    //logger.info("@@@ SOLR FACET VALUE FOUND: " + value);
				this.solr_src.addFacetQuery(value);
			}
			else if (name.equals(INDEX_SUBCOLLECTION_PARAM))
			{
				physical_sub_index_name = value;
			}
			else if (name.equals(INDEX_LANGUAGE_PARAM))
			{
				physical_index_language_name = value;
			} // ignore any others
			else if (name.equals(DOC_FILTER_PARAM))
			{
				docFilter = value;
				docFilter = docFilter.replaceAll("[^A-Za-z0-9.]", "");
				this.solr_src.setDocFilter(value);
			}
		}
		// set up start and end results if necessary
		int start_results = 0;
		if (start_page != 1)
		{
			start_results = ((start_page - 1) * hits_per_page) ;
		}
		int end_results = hits_per_page * start_page;
		this.solr_src.setStartResults(start_results);
		this.solr_src.setEndResults(end_results);
		this.solr_src.setMaxDocs(maxdocs);

		if (index.equals(SECTION_INDEX) || index.equals(DOCUMENT_INDEX))
		{
			if (physical_sub_index_name != null)
			{
				index += physical_sub_index_name;
			}
			if (physical_index_language_name != null)
			{
				index += physical_index_language_name;
			}
		}

		// now we know the index level, we can dig out the required
		// solr-core, (caching the result in 'solr_core_cache')
		String core_name = getCollectionCoreNamePrefix() + "-" + index;
		
		SolrClient solr_core = null;
		//CHECK HERE
		if (!solr_core_cache.containsKey(core_name))
		{		    
		    solr_core = new HttpJdkSolrClient.Builder(this.solr_server_base_url)
                      .withDefaultCollection(core_name)
                      .build();
		    solr_core_cache.put(core_name, solr_core);		    
		}
		else
		{
		    solr_core = solr_core_cache.get(core_name);
		}

		this.solr_src.setSolrCore(solr_core);
		this.solr_src.setCollectionCoreNamePrefix(getCollectionCoreNamePrefix());
		this.solr_src.initialise();
		return this.solr_src; // return true
	}

	/** do the query */
	protected Object runQuery(Object queryObject, String query)
	{
		try
		{
			//if it is a Highlighting Query - execute it
			this.solr_src.setHighlightField(indexField);
			if(hldocOID != null)
			{
				String rslt = this.solr_src.runHighlightingQuery(query,hldocOID);
				// Check result
				if (rslt != null)
				{
				return rslt;
				}
				//Highlighting request failed. Do standard request.
				hldocOID = null;
			}
			//logger.info("@@@@ Query is now: " + query);
			SharedSoleneQueryResult sqr = this.solr_src.runQuery(query);

			return sqr;
		}
		catch (Exception e)
		{
			logger.error("Exception happened in run query: ", e);
		}

		return null;
	}
	
	
	/** get the total number of docs that match */
	protected long numDocsMatched(Object query_result)
	{
		return ((SharedSoleneQueryResult) query_result).getTotalDocs();

	}

	/** get the list of doc ids */
	protected String[] getDocIDs(Object query_result)
	{
		Vector docs = ((SharedSoleneQueryResult) query_result).getDocs();
		String[] doc_nums = new String[docs.size()];
		for (int d = 0; d < docs.size(); d++)
		{
			String doc_num = ((SharedSoleneQueryResult.DocInfo) docs.elementAt(d)).id_;
			doc_nums[d] = doc_num;
		}
		return doc_nums;
	}

	/** get the list of doc ranks */
	protected String[] getDocRanks(Object query_result)
	{
		Vector docs = ((SharedSoleneQueryResult) query_result).getDocs();
		String[] doc_ranks = new String[docs.size()];
		for (int d = 0; d < docs.size(); d++)
		{
			doc_ranks[d] = Float.toString(((SharedSoleneQueryResult.DocInfo) docs.elementAt(d)).rank_);
		}
		return doc_ranks;
	}

	/** add in term info if available */
	protected boolean addTermInfo(Element term_list, HashMap params, Object query_result)
	{
	  Document doc = term_list.getOwnerDocument();
		String query_level = (String) params.get(LEVEL_PARAM); // the current query level

		Vector terms = ((SharedSoleneQueryResult) query_result).getTerms();
		for (int t = 0; t < terms.size(); t++)
		{
			SharedSoleneQueryResult.TermInfo term_info = (SharedSoleneQueryResult.TermInfo) terms.get(t);

			Element term_elem = doc.createElement(GSXML.TERM_ELEM);
			term_elem.setAttribute(GSXML.NAME_ATT, term_info.term_);
			term_elem.setAttribute(FREQ_ATT, "" + term_info.term_freq_);
			term_elem.setAttribute(NUM_DOCS_MATCH_ATT, "" + term_info.match_docs_);
			term_elem.setAttribute(FIELD_ATT, term_info.field_);
			term_list.appendChild(term_elem);
		}

		Vector stopwords = ((SharedSoleneQueryResult) query_result).getStopWords();
		for (int t = 0; t < stopwords.size(); t++)
		{
			String stopword = (String) stopwords.get(t);

			Element stopword_elem = doc.createElement(GSXML.STOPWORD_ELEM);
			stopword_elem.setAttribute(GSXML.NAME_ATT, stopword);
			term_list.appendChild(stopword_elem);
		}

		return true;
	}

  protected ArrayList<FacetWrapper> getFacets(Object query_result, String lang)
	{
		if (!(query_result instanceof SolrQueryResult))
		{
			return null;
		}

		SolrQueryResult result = (SolrQueryResult) query_result;
		List<FacetField> facets = result.getFacetResults();

		if (facets == null)
		{
			return null;
		}

		ArrayList<FacetWrapper> newFacetList = new ArrayList<FacetWrapper>();

		for (FacetField facet : facets)
		{
		  SolrFacetWrapper wrap = new SolrFacetWrapper(facet);
		  String fname = wrap.getName();
		  String dname = getDisplayText(_facet_display_names.get(fname), GSXML.DISPLAY_TEXT_NAME, lang, "en", "metadata_names");
		  wrap.setDisplayName(dname);		    
		  newFacetList.add(wrap);
		}

		return newFacetList;
	}
	@Override
	protected Map<String, Map<String, List<String>>> getHighlightSnippets(Object query_result)
	{
		if (!(query_result instanceof SolrQueryResult))
		{
			return null;
		}

		SolrQueryResult result = (SolrQueryResult) query_result;
		
		return result.getHighlightResults();
	}


    protected String getCollectionCoreNamePrefix() {
	String site_name = this.router.getSiteName();
	String coll_name = this.cluster_name;
	String collection_core_name_prefix = site_name + "-" + coll_name;
	return collection_core_name_prefix;
    }

  //??? do we need this??
	private boolean loadSolrCores() {
		
          SolrClient solrClient = new HttpJdkSolrClient.Builder(this.solr_server_base_url)
            .withConnectionTimeout(3, TimeUnit.SECONDS)
            .build();
		// Max retries
		//solrServer.setMaxRetries(1); // retries not availabel anymore
		// Connection Timeout
		//solrServer.setConnectionTimeout(3000);
		//Cores
		for (int i=0; i<_corenames.size(); i++) {
		    String this_core = getCollectionCoreNamePrefix()+"-"+_corenames.get(i);
		    
		    if (!checkSolrCore(this_core, solrClient)){
		      if (!activateSolrCore(this_core, solrClient)){
			logger.error("Couldn't activate Solr core " + this_core + " for collection " + cluster_name);
			return false;
		      }
		    }
		}
		return true;
	}

  // not used??
	private boolean checkSolrCore(String coreName, SolrClient solrClient) {
		CoreAdminRequest adminRequest = new CoreAdminRequest();
		//adminRequest.setAction(CoreAdminAction.STATUS);
		//adminRequest.setCoreName(coreName);

		try {
                  CoreStatus rsp = adminRequest.getCoreStatus(coreName, solrClient);
                  System.out.println("Core "+coreName+" exists, status: " + rsp.toString());
                

               
			// CoreAdminResponse adminResponse = adminRequest.process(solrServer);
			// NamedList<NamedList<Object>> coreStatus = adminResponse.getCoreStatus();
			// NamedList<Object> coreList = coreStatus.getVal(0);
			// if (coreList != null) {
			// 	if (coreList.get("name") == null) {
			// 		logger.warn("Solr core " + coreName + " for collection " + cluster_name + " not exists.");
			// 		return false;
			// 	} 
			// }
			
		} catch (SolrServerException e) {
                  logger.warn("Solr core " + coreName + " for collection " + cluster_name + " does not exist: " + e.getMessage());
                 
			return false;
		} catch (IOException e) {
			e.printStackTrace();
			return false;
		} // catch (RemoteSolrException e1){
		// 	logger.error("Check solr core " + coreName + " for collection " + cluster_name + " failed.");
		// 	e1.printStackTrace();
		// 	return false;
		// }
		return true;
	}

  // not used??
  private String getSolrDataDirForColl(String site_name, String cluster_name, String core_name) {
    return this.solr_core_location + site_name+File.separator+cluster_name+File.separator+core_name;
  }
  // not used?
	private boolean activateSolrCore(String coreName, SolrClient solrClient) {
          
          //String dataDir = GSFile.collectionIndexDir(site_home, cluster_name) + File.separator + coreName.substring(coreName.length() - 4);
          //String instanceDir = GSFile.collectionEtcDir(site_home, cluster_name);
          String instanceDir = getSolrDataDirForColl(this.router.getSiteName(), this.cluster_name, coreName);
          try {
            CoreAdminRequest.createCore(coreName, instanceDir, solrClient);
            logger.warn("Solr core " + coreName + " for collection " + cluster_name + " activated.");
          } catch (SolrServerException e1) {
            logger.error("exception happened when trying to create core "+coreName+": "+e1.getMessage());
            e1.printStackTrace();
            return false;
          } catch (IOException e1) {
            e1.printStackTrace();
            return false;
          } // catch (RemoteSolrException e1){
          //       logger.error("Activation solr core " + coreName + " for collection " + cluster_name + " failed.");
          //       e1.printStackTrace();
          //       return false;
          //}
  
          return true;
	}
	
}