/*
 *    LibraryServlet.java
 *    Copyright (C) 2008 New Zealand Digital Library, http://www.nzdl.org
 *
 *    This program is free software; you can redistribute it and/or modify
 *    it under the terms of the GNU General Public License as published by
 *    the Free Software Foundation; either version 2 of the License, or
 *    (at your option) any later version.
 *
 *    This program is distributed in the hope that it will be useful,
 *    but WITHOUT ANY WARRANTY; without even the implied warranty of
 *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *    GNU General Public License for more details.
 *
 *    You should have received a copy of the GNU General Public License
 *    along with this program; if not, write to the Free Software
 *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
 */

package org.greenstone.gsdl3.servlet;

import java.io.File;
import java.io.IOException;
import java.io.PrintWriter;
import java.io.Serializable;
import java.nio.charset.StandardCharsets;
import java.lang.reflect.Type;
import java.sql.Driver;
import java.sql.DriverManager;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Enumeration;
import java.util.HashMap;
import java.util.Hashtable;
import java.util.Iterator;
import java.util.List;
import java.util.Map;

import jakarta.servlet.ServletConfig;
import jakarta.servlet.ServletContext;
import jakarta.servlet.ServletException;
import jakarta.servlet.http.Cookie;
import jakarta.servlet.http.HttpServletRequest;
import jakarta.servlet.http.HttpServletResponse;
import jakarta.servlet.http.HttpSession;
import jakarta.servlet.http.HttpSessionBindingEvent;
import jakarta.servlet.http.HttpSessionBindingListener;

import org.apache.commons.fileupload2.core.DiskFileItem;
import org.apache.commons.fileupload2.core.DiskFileItemFactory;
import org.apache.commons.fileupload2.jakarta.servlet6.JakartaServletDiskFileUpload;

import org.apache.commons.lang3.StringUtils;
import org.apache.log4j.Logger;
import org.greenstone.catalina.GoogleSigninJDBCRealm;
import org.greenstone.gsdl3.action.PageAction;
import org.greenstone.gsdl3.comms.Communicator;
import org.greenstone.gsdl3.core.DefaultReceptionist;
import org.greenstone.gsdl3.core.MessageRouter;
import org.greenstone.gsdl3.core.Receptionist;
import org.greenstone.gsdl3.service.Authentication;
import org.greenstone.gsdl3.util.Dictionary;
import org.greenstone.gsdl3.util.HttpHeaderUtil;
import org.greenstone.gsdl3.util.GroupsUtil;
import org.greenstone.gsdl3.util.GSConstants;
import org.greenstone.gsdl3.util.GSParams;
import org.greenstone.gsdl3.util.GSXML;
import org.greenstone.gsdl3.util.UserContext;
import org.greenstone.gsdl3.util.XMLConverter;
import org.greenstone.util.GlobalProperties;
import org.json.JSONObject;
import org.w3c.dom.Document;
import org.w3c.dom.Element;
import org.w3c.dom.Node;
import org.w3c.dom.NodeList;

import com.google.gson.Gson;
import com.google.gson.reflect.TypeToken;


//import org.greenstone.gsdl3.auth.oidc.*;

//import jakarta.servlet.http.*;
//import jakarta.servlet.annotation.WebServlet;
//import java.io.IOException;
//import java.io.InputStream;
//import java.util.*;

/**
 * a servlet to serve the greenstone library - we are using servlets instead of
 * cgi 
 * the init method is called only once - the first time the servlet classes
 * are loaded. Each time a request comes in to the servlet, the session() method
 * is called in a new thread (calls doGet/doPut etc) takes the a=p&p=home type
 * args and builds a simple request to send to its receptionist, which returns a
 * result in html, cos output=html is set in the request
 * 
 * @see Receptionist
 */
public class LibraryServlet extends BaseGreenstoneServlet
{
  protected static final String LOGIN_MESSAGE_PARAM = "loginMessage";
  protected static final String REDIRECT_URL_PARAM = "redirectURL";

    /** have we run the lazy init yet? */
    protected volatile boolean initialized = false;
    
  /** the receptionist to send messages to */
  protected Receptionist recept = null;

  /** We record the library name for later */
  protected String library_name = null;
  /** Whether or not client-side XSLT support should be exposed */
  protected boolean supports_client_xslt = false;

  /**
   * the default language - is specified by setting a servlet param, otherwise
   * DEFAULT_LANG is used
   */
  protected String default_lang = null;
  /**
   * The default default - used if a default lang is not specified in the
   * servlet params
   */
  protected final String DEFAULT_LANG = "en";

  /**
   * the cgi stuff - the Receptionist can add new args to this
   * 
   * its used by the servlet to determine what args to save,
   * and to set default values
   */
  protected GSParams gs_params = null;


  /**
   * a hash that contains all the active session IDs mapped to the cached
   * items It is updated whenever the whole site or a particular collection is
   * reconfigured using the command a=s&sa=c or a=s&sa=c&c=xxx It is in the
   * form: sid -> (UserSessionCache object)
   */
  protected Hashtable<String, UserSessionCache> session_ids_table = new Hashtable<String, UserSessionCache>();

  /**
   * the maximum interval that the cached info remains in session_ids_table
   * (in seconds) This is set in web.xml
   */
  protected int session_expiration = 1800;

  static Logger logger = Logger.getLogger(org.greenstone.gsdl3.servlet.LibraryServlet.class.getName());

  /** initialise the servlet */
  public void init(ServletConfig config) throws ServletException
  {
    // always call super.init;
    // this calls GlobalProperties.loadGlobalProperties()
    super.init(config);
    // disable preferences - does this work anyway??
    //System.setProperty("java.util.prefs.PreferencesFactory", "org.greenstone.gsdl3.util.DisabledPreferencesFactory");

    // Look through all the init params
    // these are the ones we are expecting
    String interface_name = null;
    String site_name = null;
    String useXslt = null;
    String sess_expire = null;


    // some class names
    String recept_classname = null;
    String params_classname = null;
    String mr_classname = null;
    
    // may have these instead of site_name
    String remote_site_name = null;
    String remote_site_type = null;
    String remote_site_address = null;

    String servlet_properties_filename = null;
    // the stored config params
    HashMap<String, Object> config_params = new HashMap<String, Object>();
    Enumeration<String> params = config.getInitParameterNames();
    while (params.hasMoreElements()) {
      String name = params.nextElement();
      String value = config.getInitParameter(name);
      if (name.equals(GSConstants.LIBRARY_NAME)) {
	this.library_name = value;
      } else if (name.equals(GSConstants.INTERFACE_NAME)) {
	interface_name = value;
      } else if (name.equals(GSConstants.USE_CLIENT_SIDE_XSLT)) {
	useXslt = value;
      } else if (name.equals(GSConstants.DEFAULT_LANG)) {
	this.default_lang = value;
      } else if (name.equals(GSXML.SESSION_EXPIRATION)) {
	sess_expire = value;
      } else if (name.equals(GSConstants.SITE_NAME)) {
	site_name = value;
      } else if (name.equals(GSConstants.REMOTE_SITE_NAME)) {
	remote_site_name = value;
      } else if (name.equals(GSConstants.REMOTE_SITE_TYPE)) {
	remote_site_type = value;
      } else if (name.equals(GSConstants.REMOTE_SITE_ADDRESS)) {
	remote_site_address = value;
      } else if (name.equals(GSConstants.RECEPTIONIST_CLASS)) {
	recept_classname = value;
      } else if (name.equals(GSConstants.PARAMS_CLASS)) {
	params_classname = value;
      } else if (name.equals(GSConstants.MR_CLASS)) {
	mr_classname = value;
      }else if (name.equals(GSConstants.SERVLET_PROPERTIES_FILENAME)) {
        servlet_properties_filename = value;
      } else {
	// If there is a param we are not expecting, just add to the list. That way users can add new servlet params which will go through to the XML source without modifying code.
        // shouldn't need this anymore, now that we have servlet.properties
        // but not all current params have been moved yet.
	config_params.put(name, value);
      }
    }

    if (servlet_properties_filename != null && !servlet_properties_filename.equals("")) {
      GlobalProperties.loadServletProperties(servlet_properties_filename);
      Map<String, String> s_props = GlobalProperties.getServletProperties();
      config_params.putAll(s_props);
    }
    // if our init params were not set, check in servlet.properties
    if (this.library_name == null) this.library_name = (String)config_params.get(GSConstants.LIBRARY_NAME);
    if (interface_name == null) interface_name = (String)config_params.get(GSConstants.INTERFACE_NAME);
    if (useXslt == null) useXslt = (String)config_params.get(GSConstants.USE_CLIENT_SIDE_XSLT);
    if (this.default_lang == null) this.default_lang = (String)config_params.get(GSConstants.DEFAULT_LANG);
    if (sess_expire == null) sess_expire = (String)config_params.get(GSXML.SESSION_EXPIRATION);
    if (site_name == null) site_name = (String)config_params.get(GSConstants.SITE_NAME);
    if (recept_classname == null) recept_classname = (String)config_params.get(GSConstants.RECEPTIONIST_CLASS);
    if (params_classname == null) params_classname = (String)config_params.get(GSConstants.PARAMS_CLASS);
    if (mr_classname == null) mr_classname = (String)config_params.get(GSConstants.MR_CLASS);
    

    // now check whether we have valid properties
    if (this.library_name == null || interface_name == null) {
		  
      // must have these
      System.err.println("initialisation parameters not all set!");
      System.err.println(" you must have libraryname and interfacename");
      throw new ServletException("Error, library_name and interface_name must be set!");
    }

    // If we don't have a site, then we must be using a Communicator, and all the remote params must be set
    if (site_name == null) {
      // check remote site info in config_params
      if (remote_site_name == null) remote_site_name = (String)config_params.get(GSConstants.REMOTE_SITE_NAME);
      if (remote_site_type == null) remote_site_type = (String)config_params.get(GSConstants.REMOTE_SITE_TYPE);
      if (remote_site_address  == null) remote_site_address = (String)config_params.get(GSConstants.REMOTE_SITE_ADDRESS);
      
      if (remote_site_name == null || remote_site_type == null || remote_site_address == null) {
		  
        System.err.println("initialisation paramters not all set!");
        System.err.println("if site_name is not set, then you must have remote_site_name, remote_site_type and remote_site_address set");
        throw new ServletException("Error, if site_name is not set, then you must have remote_site_name, remote_site_type and remote_site_address set");
      }
    }

    supports_client_xslt = useXslt != null && useXslt.equals("true");

    if (sess_expire != null && !sess_expire.equals(""))
      {
	this.session_expiration = Integer.parseInt(sess_expire);
      }
    if (this.default_lang == null)
      {
	// choose english
	this.default_lang = DEFAULT_LANG;
      }

    config_params.put(GSConstants.LIBRARY_NAME, library_name);
    config_params.put(GSConstants.INTERFACE_NAME, interface_name);
    config_params.put(GSConstants.USE_CLIENT_SIDE_XSLT, supports_client_xslt);

    if (site_name != null)
      {
	config_params.put(GSConstants.SITE_NAME, site_name);
      }

    
    String webswing_context = GlobalProperties.getProperty("webswing.context");
    if (!webswing_context.equals("")) {
      config_params.put(GSConstants.WEBSWING_CONTEXT, webswing_context);
    }

    
    // the receptionist -the servlet will talk to this
    if (recept_classname == null)
      {
	this.recept = new DefaultReceptionist();
      }
    else
      {
	try
	  {
	    this.recept = (Receptionist) Class.forName("org.greenstone.gsdl3.core." + recept_classname).getDeclaredConstructor().newInstance();
	  }
	catch (Exception e)
	  { // cant use this new one, so use normal one
	    System.err.println("LibraryServlet configure exception when trying to use a new Receptionist " + recept_classname + ": " + e.getMessage());
	    e.printStackTrace();
	    this.recept = new DefaultReceptionist();
	  }
      }
    this.recept.setConfigParams(config_params);

    // the params arg thingy

    if (params_classname == null)
      {
	this.gs_params = new GSParams();
      }
    else
      {
	try
	  {
	    this.gs_params = (GSParams) Class.forName("org.greenstone.gsdl3.util." + params_classname).getDeclaredConstructor().newInstance();
	  }
	catch (Exception e)
	  {
	    System.err.println("LibraryServlet configure exception when trying to use a new params thing " + params_classname + ": " + e.getMessage());
	    e.printStackTrace();
	    this.gs_params = new GSParams();
	  }
      }
		
    // the receptionist uses a MessageRouter or Communicator to send its requests to. We either create a MessageRouter here for the designated site (if site_name set), or we create a Communicator for a remote site. The is given to the Receptionist, and the servlet never talks to it again directly.
    if (site_name != null)
      {
        MessageRouter message_router = null;
	if (mr_classname == null)
	  { // just use the normal MR
	    message_router = new MessageRouter();
	  }
	else
	  { // try the specified one
	    try
	      {
		message_router = (MessageRouter) Class.forName("org.greenstone.gsdl3.core." + mr_classname).getDeclaredConstructor().newInstance();
	      }
	    catch (Exception e)
	      { // cant use this new one, so use normal one
		System.err.println("LibraryServlet configure exception when trying to use a new MessageRouter " + mr_classname + ": " + e.getMessage());
		e.printStackTrace();
		message_router = new MessageRouter();
	      }
	  }

	message_router.setSiteName(site_name);
	message_router.setLibraryName(library_name);
	message_router.setParams(this.gs_params);
	//message_router.configure(); // move to lazyInit
	this.recept.setMessageRouter(message_router);
      }
    else
      {
	// TODO: what do we do about params here?
	// talking to a remote site, create a communicator
	Communicator communicator = null;
	// we need to create the XML to configure the communicator
	Document doc = XMLConverter.newDOM();
	Element site_elem = doc.createElement(GSXML.SITE_ELEM);
	site_elem.setAttribute(GSXML.TYPE_ATT, remote_site_type);
	site_elem.setAttribute(GSXML.NAME_ATT, remote_site_name);
	site_elem.setAttribute(GSXML.ADDRESS_ATT, remote_site_address);

	if (remote_site_type.equals(GSXML.COMM_TYPE_SOAP_JAVA))
	  {
	    //communicator = new SOAPCommunicator();
            try {
              communicator = (Communicator) Class.forName("org.greenstone.gsdl3.comms.SOAPCommunicator").getDeclaredConstructor().newInstance();
            } catch (Exception e) {
              System.err.println("Couldn't instantiate a SOAPCommunicator, have you installed the SOAP extension??");
              throw new ServletException("Couldn't instantiate a SOAPCommunicator, have you installed the SOAP extension??");
            }
	  }
	else
	  {
	    System.err.println("LibraryServlet.init Error: invalid Communicator type: " + remote_site_type);
	    throw new ServletException("LibraryServlet.init Error: invalid Communicator type: " + remote_site_type);
	  }

	if (!communicator.configure(site_elem))
	  {
	    System.err.println("LibraryServlet.init Error: Couldn't configure communicator");
	   throw new ServletException("LibraryServlet.init Error: Couldn't configure communicator");
	  }
	this.recept.setMessageRouter(communicator);
      }

    // pass params to the receptionist
    this.recept.setParams(this.gs_params);
    //this.recept.configure(); // move to lazyInit

    //Allow the message router and the document to be accessed from anywhere in this servlet context
    this.getServletContext().setAttribute(library_name+"Router", this.recept.getMessageRouter());
  }

    private void lazyInit() {
	if (!initialized) {
            synchronized(this) {
                if (!initialized) {
                    MessageRouter mr = (MessageRouter)this.recept.getMessageRouter();
		    mr.configure();  // heavy lifting
		    this.recept.configure();
                    initialized = true;
                }
            }
        }
    }
    


    //private GS3OidcProvider oidc;

    /*
  public void initOIDC() {
      / *   // close-up C-style comments

    Properties p = new Properties();
    try (InputStream in = getServletContext().getResourceAsStream("/WEB-INF/global.properties")) {
      if (in == null) throw new IllegalStateException("Missing /WEB-INF/global.properties");
      p.load(in);
    } catch (IOException e) {
      throw new RuntimeException("Failed to load global.properties", e);
    }
      * /

    // **** OIDC
    OidcConfig cfg = OidcConfig.from(/ * uses properties in GlobalProperties * /); // Close-up C-style comments

    Map<String,String> clients = new HashMap<String,String>();
    clients.put(
        GlobalProperties.getProperty("auth.oidc.client.wp.id", "wp-client-id"),
        GlobalProperties.getProperty("auth.oidc.client.wp.redirect",
            "https://your-wordpress.example/wp-admin/admin-ajax.php?action=openid-connect-authorize")
    );

    GS3OidcProvider.KeyManager keyMgr = cfg.useKeystore()
        ? new CxfKeystoreKeyManager(cfg)
        : new CxfKeyManager();

    oidc = new GS3OidcProvider.Builder()
        .issuer(cfg.issuer)
        .basePath("/greenstone3")
        .clients(clients)
        .keyManager(keyMgr)
        .allowedScopes(cfg.scopes)
        .build();
  }

*/
    
  private void logUsageInfo(HttpServletRequest request, Map<String, String[]> queryMap)
  {
    String sessionInfo = "";

    //session-info: get params stored in the session
    HttpSession session = request.getSession(true);
    Enumeration attributeNames = session.getAttributeNames();
    while (attributeNames.hasMoreElements())
      {
	String name = (String) attributeNames.nextElement();
	sessionInfo += name + "=" + session.getAttribute(name) + " ";
      }

    String queryParamStr = "";
    if (queryMap != null)
      {
	Iterator<String> queryIter = queryMap.keySet().iterator();
	while (queryIter.hasNext()) {
	  String q = queryIter.next();
	  String[] vals = queryMap.get(q);
	  queryParamStr += q +"="+StringUtils.join(vals, ",")+" ";
	}
      }
    //logged info = general-info + session-info
    String usageInfo = request.getServletPath() + " " +      //serlvet
      "queryParams:[" + queryParamStr.trim()+"]"+" " +       // query map params 
      "sessionId="+request.getRequestedSessionId() + " " +   //session id
      "sessionParams:[" + sessionInfo.trim() + "]" + " - " + // params stored in a session
      request.getRemoteAddr() + " " +        //remote address
      request.getHeader("user-agent") + " "; //the remote brower info

    logger.info(usageInfo);

  }

  public class UserSessionCache implements HttpSessionBindingListener
  {
    String session_id = "";

    /**
     * a hash that maps the session ID to a hashtable that maps the
     * coll_name to its parameters coll_name -> Hashtable (param_name ->
     * param_value)
     */
    protected Hashtable<String, Hashtable<String, String>> coll_name_params_table = null;

    public UserSessionCache(String id, Hashtable<String, Hashtable<String, String>> table)
    {
      session_id = id;
      coll_name_params_table = (table == null) ? new Hashtable() : table;
    }

    protected void cleanupCache(String coll_name)
    {
      if (coll_name_params_table.containsKey(coll_name))
	{
	  coll_name_params_table.remove(coll_name);
	}
    }

    protected Hashtable<String, Hashtable<String, String>> getParamsTable()
    {
      return coll_name_params_table;
    }

    public void valueBound(HttpSessionBindingEvent event)
    {
      // Do nothing
    }

    // if session cache id removed from session, this triggers this valueUnbound method on the value=this object
    public void valueUnbound(HttpSessionBindingEvent event)
    {
      if (session_ids_table.containsKey(session_id))
	{
	  session_ids_table.remove(session_id);
	}
    }

    public int tableSize()
    {
      return (coll_name_params_table == null) ? 0 : coll_name_params_table.size();
    }
  }

  public void destroy()
  {
    recept.cleanUp();

    // Deregister any of our drivers
       ClassLoader cl = Thread.currentThread().getContextClassLoader();
       // Loop through all drivers
       Enumeration<Driver> drivers = DriverManager.getDrivers();
       while (drivers.hasMoreElements()) {
 	  Driver driver = drivers.nextElement();
 	  if (driver.getClass().getClassLoader() == cl) {
 	      // This driver was registered by the webapp's ClassLoader, so deregister it:
 	      try {
 		  System.err.println("LibraryServlet.destroy: Deregistering JDBC driver: " + driver); 
 		  DriverManager.deregisterDriver(driver);
 	      } catch (SQLException ex) {
 		  System.err.println("LibraryServlet.destroy: Error deregistering JDBC driver: "+ driver + ": "+ ex);
 	      }
 	  } else {
 	      
 	      // driver was not registered by the webapp's ClassLoader and may be in use elsewhere
 	      System.err.println("LibraryServlet.destroy: Not deregistering JDBC driver as it does not belong to this webapp's ClassLoader: "+ driver); 
 	  }
       }

  }

    
    
    
  public void doGetOrPost(HttpServletRequest request, HttpServletResponse response, Map<String, String[]> queryMap) throws ServletException, IOException
  {
      lazyInit(); //initialize on first request
      
    logUsageInfo(request, queryMap);
    
    if (processRedirectRequest(request, response, queryMap)) {
	// this method will do the redirect if needed and return true if it has
	// done so. 
	// e.g. el=direct/framed&rl=0&href=http://newurl.com
	return;
    }

    // **** ODIC
    /*
    String p = normalisePathForOIDC(req);

    if ("/oidc/jwks.json".equals(p))    { oidc.handleJwks(resp); return; }
    if ("/oidc/authorize".equals(p))    {
      if ("1".equals(req.getParameter("__dev_login"))) { handleDevLogin(req, resp); return; }
      oidc.handleAuthorize(req, resp);
      return;
    }
    if ("/oidc/userinfo".equals(p))     { oidc.handleUserInfo(req, resp); return; }

    resp.setStatus(404);
    */
    
    // Nested Diagnostic Configurator to identify the client for
    HttpSession session = request.getSession(true);
    session.setMaxInactiveInterval(session_expiration);
    String uid = session.getId();
    request.setCharacterEncoding("UTF-8");
    response.setContentType("text/html;charset=UTF-8");

    /* Uncomment the following if using web browser-based WorkerThread */
    /* An example extension that needs these headers set the tartan-ify part of 'web-audio' */
    // TODO: allow such headers to be optionally set in build.properties file

    //response.setHeader("Cross-Origin-Opener-Policy", "same-origin");
    //response.setHeader("Cross-Origin-Embedder-Policy", "require-corp");
	
    PrintWriter out = response.getWriter();

    String lang = getFirstParam(GSParams.LANGUAGE, queryMap);
    if (lang == null || lang.equals(""))
      {
	// try the session cached lang
	lang = (String) session.getAttribute(GSParams.LANGUAGE);
	if (lang == null || lang.equals(""))
	  {
	    // still not set, use the default
	    lang = this.default_lang;
	  }
      }

    // set the lang in the session
    session.setAttribute(GSParams.LANGUAGE, lang);

    String output = getFirstParam(GSParams.OUTPUT, queryMap);
    if (output == null || output.equals(""))
      {
	output = "html"; // uses html by default
      }

    /*
    // Code that helps show what header values come through with the request
    // Useful information to consult when working with a reverse-proxy setup

    String  rScheme      = request.getScheme();
    boolean rIsSecure    = request.isSecure();
    String  rServerName  = request.getServerName();    
    int     rServerPort  = request.getServerPort();
    String  rContextPath = request.getContextPath();
    
    String hForwardedProto  = request.getHeader("X-Forwarded-Proto");
    String hForwardedPrefix = request.getHeader("X-Forwarded-Prefix");

    
    System.err.println("**** rScheme      = " + rScheme);
    System.err.println("**** rIsSecure    = " + rIsSecure);
    System.err.println("**** rServerName  = " + rServerName);
    System.err.println("**** rServerPort  = " + rServerPort);
    System.err.println("**** rContextPath = " + rContextPath);

    System.err.println("**** hForwardedProto  = " + hForwardedProto);
    System.err.println("**** hForwardedPrefix = " + hForwardedPrefix);
    */

    //
    // The following is a direct replacement for using:
    //   'request.getRequestURL().toString()'
    // to get the requestedURL. The former way of doing this works
    // correctly for a localhost setup, but breaks down when trying
    // to do this when Greenstone3 is configured through a reverse-proxy
    // (esp. if the Greenstone3 context has been changed)
    //
    String requestedURL = HttpHeaderUtil.externalRequestedURL(request);
    
    String baseURL = "";
    //logger.error("requested URL = "+requestedURL);
    if (requestedURL.indexOf(library_name) != -1)
    {
      // we need to work out the baseURL and set it
      baseURL = requestedURL;
      int protocol_boundary_pos = baseURL.indexOf("://");
      if (protocol_boundary_pos>=1) {
        baseURL = baseURL.substring(protocol_boundary_pos+1); // change things like http:// or https:// to //
      }
      // The baseURL is everything up to the library_name.
      // e.g. https://community.greenstone.org/greenstone3/library/collection/twso/page/about
      // baseURL is //community.greenstone.org/greenstone3
      // Issues: the library_name may occur in more than one place - eg if its part of the domain name mylibrary.heritage.nz/greenstone3/library
      // or if a collection name is the same as the library name eg localhost:8585/greenstone3/twso/collection/twso/page/about.
      // So can't just use teh first or last occurrence. Look for /library-name, and check its not the first position (eg //library.nzdl.org/greenstone3/library)
      int library_name_index = baseURL.indexOf("/"+library_name);
      if (library_name_index == 1) {
        // we have library name at the start of the url - need to use the second one
        library_name_index = baseURL.indexOf("/"+library_name, 2);
      }
      baseURL = baseURL.substring(0, library_name_index+1);
      //logger.error("new base url = "+baseURL);
		
    }

    String fullURL;
    if (request.getQueryString() != null)
    {
	fullURL = requestedURL + "?" + request.getQueryString();
    }
    else
    {
	fullURL = requestedURL;
    }

    UserContext userContext = new UserContext();
    userContext.setLanguage(lang);
    userContext.setUserID(uid);

    // If GoogleSignin is operational (due to googlesignin_client_id specified in servlet.xml)
    // AND a Google Client ID Token is provided via googlesignin_id_token then
    // initiate authentication via Google Signin, which is then tied into Greenstone3 through
    // the customized Realm written for Greenstone3.
    //
    // This customized Realm, GoogleSigninJDBCRealm, works by overriding 'authenticate()'
    // in Realm.

    // 1.  If the username 'googlesign' is provided, then the credentials
    //     passed in needs to be a valid verifyable Google Client Id Token.
    // 1a. From this verified token, the email address that is registered with
    //     Google for that user account is then used to find a match in the
    //     Greenstone3 JDBC-specified Username table.
    // 1b. Assuming that an email address match is found, then the customized Realm
    //     completes the authentication process by making the found username, the
    //     one that is autheticated.
    // 2.  If the username is anything but 'googlesignin' then the authentication process
    //     continues as with the regular JDBCRealm process
        
    //String googlesignin_id_token = getFirstParam("googlesignin_id_token",queryMap);
    //if ((googlesignin_id_token != null) && (!googlesignin_id_token.equals(""))) {

    String googleidentity_signin = getFirstParam(GSParams.GOOGLE_SIGNIN,queryMap);
    if ((googleidentity_signin != null) && (googleidentity_signin.equals("1"))) {
	queryMap.put(GSParams.USERNAME, new String[] { GoogleSigninJDBCRealm.GOOGLESIGNIN_USERNAME_BRIDGE });

	String googlesignin_credential = getFirstParam("credential",queryMap);
	queryMap.put(GSParams.PASSWORD, new String[] { googlesignin_credential });
	// logger.info("**** googlesignin_credenital (aka id_token) = '" + googlesignin_credential +"'");
    }

    
    if (!processLoginChanges(request, userContext, out, baseURL, queryMap, lang, output)) {
	// any invalid login attempt will redirect to a new login page and return false
      return; 
    }

 
    if (request.getAuthType() != null)
    {
	// sets username, groups etc into usercontext
	updateUserContextWithAuthenticatedInfo(request, userContext);
    }

    // If server output, force a switch to traditional interface
    //output = (output.equals("server")) ? "html" : output;

    // Force change the output mode if client-side XSLT is supported - server vs. client
    // BUT only if the library allows client-side transforms	
    if (supports_client_xslt)
      {
	// MUST be done before the xml_message is built
	Cookie[] cookies = request.getCookies();
	Cookie xsltCookie = null;

	// The client has cookies enabled and a value set - use it!
	if (cookies != null)
	  {
	    for (Cookie c : cookies)
	      {
		if (c.getName().equals("supportsXSLT"))
		  {
		    xsltCookie = c;
		    break;
		  }
	      }
	    output = (xsltCookie != null && xsltCookie.getValue().equals("true") && output.equals("html")) ? "xsltclient" : output;
	  }
      }


    String action = getFirstParam(GSParams.ACTION, queryMap);
    String subaction = getFirstParam(GSParams.SUBACTION, queryMap);
    String collection = getFirstParam(GSParams.COLLECTION, queryMap);
    String document = getFirstParam(GSParams.DOCUMENT, queryMap);
    String service = getFirstParam(GSParams.SERVICE, queryMap);
    String specified_cache_key = getFirstParam(GSParams.CACHE_KEY, queryMap);

    if (collection != null && !collection.equals("")) {
	//is this user allowed to access this collection/document? 
	if (!runCollectionSecurityCheck(request, userContext, out, baseURL, collection, document, lang, output)) {
	    return;
	}
    }
    // We clean up the cache session_ids_table if system
    // commands are issued, and also don't need to do caching for these request requests
    boolean should_cache = true;
    if (action != null && action.equals(GSParams.SYSTEM_ACTION) 
	&& !subaction.equals(GSXML.SYSTEM_TYPE_PING) 
	&& !subaction.equals(GSXML.SYSTEM_TYPE_AUTHENTICATED_PING)) // don't 'clean' anything on a mere ping
      {
	should_cache = false;

	// System commands now need to be logged in/have permissions
	// like configuring, activation and deactivation
	if(!configureSecurityCheck(request, userContext, out, baseURL, lang, output, queryMap)) {
	    return;
	}
	
	// we may want to remove all collection cache info, or just a specific collection
	boolean clean_all = true;
	String clean_collection = null;
	// system commands are to activate/deactivate stuff
	// collection param is in the sc parameter.
	// don't like the fact that it is hard coded here
	String coll = getFirstParam(GSParams.SYSTEM_CLUSTER, queryMap);
	if (coll != null && !coll.equals(""))
	  {
	    clean_all = false;
	    clean_collection = coll;
	  }
	else
	  {	      
	    // check other system types
	    if (subaction.equals("a") || subaction.equals("d"))
	      {
		String module_name = getFirstParam("sn", queryMap);
		if (module_name != null && !module_name.equals(""))
		  {
		    clean_all = false;
		    clean_collection = module_name;
		  }
	      }
	  }
	if (clean_all)
	  {
	    // TODO
	    session_ids_table = new Hashtable<String, UserSessionCache>();
	    session.removeAttribute(GSParams.USER_SESSION_CACHE); // triggers valueUnbound(), which removes the session id from the session_ids_table
	  }
	else
	  {
	    // just clean up info for clean_collection
	    ArrayList<UserSessionCache> cache_list = new ArrayList<UserSessionCache>(session_ids_table.values());
	    for (int i = 0; i < cache_list.size(); i++)
	      {
		UserSessionCache cache = cache_list.get(i);
		cache.cleanupCache(clean_collection);
	      }

	  }
      }

    // cache_key is the collection name, or service name
    String cache_key = specified_cache_key;
    if (cache_key == null || cache_key.equals(""))
      {
	cache_key = collection;
      }
    if (cache_key == null || cache_key.equals(""))
      {
	cache_key = service;
      }
		
    //clear the collection-specific cache in the session, since we have no way to know whether this page is 
    //about the same collection as the last page or not.
    Enumeration attributeNames = session.getAttributeNames();
    while (attributeNames.hasMoreElements())
      {
	String name = (String) attributeNames.nextElement();
	if (!name.equals(GSParams.USER_SESSION_CACHE) && !name.equals(GSParams.LANGUAGE) && !name.equals(GSXML.USER_ID_ATT) && !this.gs_params.isGlobal(name))
	  {
	    session.removeAttribute(name);
	  } 
      }

    UserSessionCache session_cache = null;
    Hashtable<String, Hashtable<String, String>> param_table = null;
    Hashtable<String, String> table = null;
    boolean new_table = false;
    String sid = session.getId();
    boolean new_session = false;
    if (!session_ids_table.containsKey(sid)) {
      new_session = true;
    }
    if (should_cache == true && cache_key != null && !cache_key.equals(""))
      {
	if (!new_session)
	  {
	    session_cache = session_ids_table.get(sid);
	    param_table = session_cache.getParamsTable();
	    if (param_table.containsKey(cache_key))
	      {
		table = param_table.get(cache_key);
	      }
	    else
	      {
		table = new Hashtable<String, String>();
		param_table.put(cache_key, table);
		new_table = true;
	      }
	  }
	else
	  {
			  
	    param_table = new Hashtable<String, Hashtable<String, String>>();
	    table = new Hashtable<String, String>();
	    param_table.put(cache_key, table);
	    session_cache = new UserSessionCache(sid, param_table);
	    session_ids_table.put(sid, session_cache);
	    session.setAttribute(GSParams.USER_SESSION_CACHE, session_cache);
	    new_table = true;
	  }

      }
		
    // here we add in default params values if need be - if we have a new session, or if we have a new table
    // in an existing session
    // don't override any existing values
    if (new_session || new_table) {

      Iterator i = this.gs_params.getParamsWithDefaults().iterator();
      while (i.hasNext()) {
	String p = (String)i.next();
	String v = this.gs_params.getParamDefault(p);
	if (this.gs_params.isGlobal(p)) {
	  //need to add in to session unless its already there
	  if (session.getAttribute(p) == null) {
	    session.setAttribute(p, v);
	  }
	} else {
	  // add to table unless its already there
	  if (new_table && !table.containsKey(p)) {
	    table.put(p,v);
	  }
	}
		    
      }
    }

    // the request to the receptionist
    Document msg_doc = XMLConverter.newDOM();
    Element xml_message = msg_doc.createElement(GSXML.MESSAGE_ELEM);
    Element xml_request = GSXML.createBasicRequest(msg_doc, GSXML.REQUEST_TYPE_PAGE, "", userContext);
    xml_request.setAttribute(GSXML.OUTPUT_ATT, output);

    xml_message.appendChild(xml_request);

    if (request.getAuthType() != null) {
	// lots of classes are using the <userInformation> element. But all its info is now in userContext, so maybe we can get rid of this one day?
	appendUserInformation(xml_request, userContext);
    }
    
    if (action == null || action.equals(""))
    {
	// use page action as the default
	xml_request.setAttribute(GSXML.ACTION_ATT, "p");

    }
    else
    {
	xml_request.setAttribute(GSXML.ACTION_ATT, action);
	if (subaction != null)
	{
	    xml_request.setAttribute(GSXML.SUBACTION_ATT, subaction);
	}
    }
		
    //  create the param list for the greenstone request - includes
    // the params from the current request and any others from the saved session
    Element xml_param_list = msg_doc.createElement(GSXML.PARAM_ELEM + GSXML.LIST_MODIFIER);
    xml_request.appendChild(xml_param_list);
			
    if (queryMap.containsKey("s1.collection") || queryMap.containsKey("s1.group")){
      table.remove("s1.collection");
      table.remove("s1.group");
    }
    for (String name : queryMap.keySet())
      {
	if (!name.equals(GSParams.ACTION) && !name.equals(GSParams.SUBACTION) && !name.equals(GSParams.LANGUAGE) && !name.equals(GSParams.OUTPUT))
	  {// we have already dealt with these

	    String value = "";
	    String[] values = queryMap.get(name);
	    value = values[0];
	    if (values.length > 1)
	      {
		for (int i = 1; i < values.length; i++)
		  {
		    value += "," + values[i];
		  }
	      }
	    // if we need to save the value, then add it to the session/cache table.
	    // otherwise add directly to the paramList
	    if (this.gs_params.shouldSave(name)) {
	      if (this.gs_params.isGlobal(name) || table == null) {
		session.setAttribute(name, value);
	      } else {
		table.put(name, value);
	      } 
	    }
	    else
	      {
		  // If logging out as the very next step after logging in, then
		  // the 'password' param was found to be null
		  //
		  // This then causes GSXML.xsmSafe(value) to throw an exception
		  // For now, the coding decision is to test for null, and skip
		  // adding the param if it is null
		  //
		  // Could be that it is more meaningful to store the values as
		  // the empty string.  In which case updating GSXML.xsmlSafe to
		  // test for null and treat it as an empty string would be
		  // a better way to go
		 
		  if (value != null) {
		      Element param = msg_doc.createElement(GSXML.PARAM_ELEM);
		      param.setAttribute(GSXML.NAME_ATT, name);
		      param.setAttribute(GSXML.VALUE_ATT, GSXML.xmlSafe(value));
		      if (this.gs_params.isSensitive(name)) {
			  param.setAttribute(GSXML.SENSITIVE_ATT, "true");
		      }
		      xml_param_list.appendChild(param);
		  }
						
	      }
	  }
      }
    //put everything in the table into the session
    if (table != null)
      {
	Enumeration<String> keys = table.keys();
	while (keys.hasMoreElements())
	  {
	    String name = keys.nextElement();
	    session.setAttribute(name, table.get(name));
	  }
      }

    // put in all the params from the session cache
    Enumeration params = session.getAttributeNames();
    while (params.hasMoreElements())
      {
	String name = (String) params.nextElement();
	if (!name.equals(GSParams.USER_SESSION_CACHE) && !name.equals(GSParams.LANGUAGE) && !name.equals(GSXML.USER_ID_ATT))
	  {

	    // lang and uid are stored but we dont want it in the param list cos its already in the request
	    Element param = msg_doc.createElement(GSXML.PARAM_ELEM);
	    param.setAttribute(GSXML.NAME_ATT, name);
	    String value = GSXML.xmlSafe((String) session.getAttribute(name));

	    // ugly hack to undo : escaping
	    value = StringUtils.replace(value, "%3A", "\\:");
	    param.setAttribute(GSXML.VALUE_ATT, value);
	    xml_param_list.appendChild(param);
	  }
      }
	

    if (output.equals("json"))
      {
	response.setContentType("application/json");
      }
    else if (!output.equals("html") && !output.equals("server") && !output.equals("xsltclient"))
      {
	response.setContentType("text/xml"); // for now use text
      }

    //Add custom HTTP headers if requested
    String httpHeadersParam = getFirstParam(GSParams.HTTP_HEADER_FIELDS, queryMap);
    if (httpHeadersParam != null && httpHeadersParam.length() > 0)
      {
	Gson gson = new Gson();
	Type type = new TypeToken<List<Map<String, String>>>()
		    {
	  }.getType();
	List<Map<String, String>> httpHeaders = gson.fromJson(httpHeadersParam, type);
	if (httpHeaders != null && httpHeaders.size() > 0)
	  {

	    for (int j = 0; j < httpHeaders.size(); j++)
	      {
		Map nameValueMap = httpHeaders.get(j);
		String name = (String) nameValueMap.get("name");
		String value = (String) nameValueMap.get("value");

		if (name != null && value != null)
		  {
		    response.setHeader(name, value);
		  }
	      }
	  }
      }


    xml_request.setAttribute("remoteAddress", request.getRemoteAddr());
    xml_request.setAttribute("fullURL", fullURL.replace("&", "&amp;"));
    xml_request.setAttribute(GSXML.BASE_URL, baseURL);

    // logger.error("about to process this message");
    // logger.error(XMLConverter.getPrettyString(xml_message));
    Node xml_result = this.recept.process(xml_message);
    encodeURLs(xml_result, response);

    String xml_string = XMLConverter.getPrettyString(xml_result);
		
    if (output.equals("json"))
      {
	try
	  {
	    JSONObject json_obj = org.json.XML.toJSONObject(xml_string);

	    out.println(json_obj.toString());
	  }
	catch (Exception e)
	  {
	    e.printStackTrace();
	    out.println("Error: failed to convert output XML to JSON format");
	  }
      }
    else
      {
	out.println(xml_string);
      }

    displaySize(session_ids_table);
  }

  public void doGet(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException
  {
    doGetOrPost(request, response, request.getParameterMap());
  } //end of doGet(HttpServletRequest, HttpServletResponse)

    private boolean processRedirectRequest(HttpServletRequest request, HttpServletResponse response, Map<String, String[]> queryMap) throws IOException
    {
    if (queryMap != null)
    {
	Iterator<String> queryIter = queryMap.keySet().iterator();
	boolean redirect = false;
	String href = null;
	String rl = null;
	String el = null;
	String collection = null;

	while (queryIter.hasNext())
	{
	    String q = queryIter.next();
	    if (q.equals(GSParams.COLLECTION))
	    {
		collection = queryMap.get(q)[0];
	    }
	    else if (q.equals(GSParams.EXTERNAL_LINK_TYPE))
	    {
		el = queryMap.get(q)[0];
	    }
	    else if (q.equals(GSParams.HREF))
	    {
		href = queryMap.get(q)[0];
		href = StringUtils.replace(href, "%2f", "/");
		href = StringUtils.replace(href, "%7e", "~");
		href = StringUtils.replace(href, "%3f", "?");
		href = StringUtils.replace(href, "%3A", "\\:");
	    }
	    else if (q.equals(GSParams.RELATIVE_LINK))
	    {
		rl = queryMap.get(q)[0];
	    }
	}

	//if query_string contains "el=direct", an href is specified, and its not a relative link, then the web page will be redirected to the external URl, otherwise a greenstone page with an external URL will be displayed
	//"rl=0" this is an external link
	//"rl=1" this is an internal link
	if ((href != null) && (rl.equals("0")))
	{// This is an external link, 

	    if (el.equals("framed"))
	    {
		// framed means we are linking to an external page inside our greenstone page
		HttpSession session = request.getSession();
		ServletContext context = session.getServletContext();
		String new_url = context.getContextPath()+"/"+ library_name+"?a=p&sa=html&url="+href;
		if (collection != null && !collection.equals("")) {
		    new_url += "&c="+collection;
		}
		response.sendRedirect(new_url);
	    }
	    else
	    {
		// el = '' or direct
		//the web page is re-directed to the external URL (&el=&rl=0&href="http://...")
		//response.sendRedirect(href);

		// To live in the Greenstone URL, the href= arg must be encoded
		// => so need to decode before using ...

		// Possible 'href' needs to be %XX decoded as well?
		//String href_decoded = java.net.URLDecoder.decode(href, "UTF-8");
		//
		// However, 'href' definitely needs its '&amp;' entries decoded (which is not done by URLDecoder)
		String href_decoded = href.replace("&amp;","&");
		response.sendRedirect(href_decoded);
	    }
	    return true;
	  }
      }
    return false;
    }
    
    private void generateLoginPage(String redirect_url_string, String error_message, UserContext userContext, PrintWriter out, String baseURL, String output) {
	
	Document loginPageDoc = XMLConverter.newDOM();
	Element loginPageMessage = loginPageDoc.createElement(GSXML.MESSAGE_ELEM);
	Element loginPageRequest = GSXML.createBasicRequest(loginPageDoc, GSXML.REQUEST_TYPE_PAGE, "", userContext);
	loginPageRequest.setAttribute(GSXML.ACTION_ATT, "p");
	loginPageRequest.setAttribute(GSXML.SUBACTION_ATT, "login");    
	loginPageRequest.setAttribute(GSXML.OUTPUT_ATT, output);
	loginPageRequest.setAttribute(GSXML.BASE_URL, baseURL);
	
	loginPageMessage.appendChild(loginPageRequest);
	
	Element paramList = loginPageDoc.createElement(GSXML.PARAM_ELEM + GSXML.LIST_MODIFIER);
	loginPageRequest.appendChild(paramList);
	
	Element messageParam = loginPageDoc.createElement(GSXML.PARAM_ELEM);
	messageParam.setAttribute(GSXML.NAME_ATT, LOGIN_MESSAGE_PARAM);
	messageParam.setAttribute(GSXML.VALUE_ATT, error_message);
	paramList.appendChild(messageParam);
	
	Element urlParam = loginPageDoc.createElement(GSXML.PARAM_ELEM);
	urlParam.setAttribute(GSXML.NAME_ATT, REDIRECT_URL_PARAM);

	if (redirect_url_string.matches("^[a-z]+://.*$")) {
	    int protocol_boundary_pos = redirect_url_string.indexOf("://");
	    redirect_url_string = redirect_url_string.substring(protocol_boundary_pos+1); // change things like http:// or https:// to //
	}

	urlParam.setAttribute(GSXML.VALUE_ATT, redirect_url_string);
	paramList.appendChild(urlParam);
	
	Node loginPageResponse = this.recept.process(loginPageMessage);
	out.println(XMLConverter.getPrettyString(loginPageResponse));	
    }
    
    private boolean processLoginChanges(HttpServletRequest request, UserContext userContext, PrintWriter out, String baseURL, Map<String, String[]> queryMap, String lang, String output) throws ServletException {

	//logger.info("Start of LibraryServlet::processLoginChanges()");	

	//Check if we need to login or logout
	String username = getFirstParam(GSParams.USERNAME, queryMap);
	String password = getFirstParam(GSParams.PASSWORD, queryMap);
	String logout = getFirstParam(GSParams.LOGOUT, queryMap);
	
	if (logout != null)
	{
	    //logger.info("LibraryServlet::processLoginChanges() logging out (logout cgi param non-null)");
	    request.logout();
	}
	
	if (username != null && password != null)
	{
	    //logger.info("LibrarySerlvet::processLoginChagnes(): username and password not null");

	    //We are changing to another user, so log out first
	    if (request.getAuthType() != null)
	    {
		//logger.info("Logging out (preparing to change to another user) ");
		request.logout();
	    }

	    
	    //This try/catch block catches when the login request fails (e.g. The user enters an incorrect password).
	    try
	    {
		request.login(username, password);
		//logger.info("Global Login successful");
            }
	    catch (Exception ex)
	    {
		// The following is useful for debugging purposes
		//ex.printStackTrace();
		
		try
		{
		    //If the global login fails then try a site-level login
		    String siteName = (String) this.recept.getConfigParams().get(GSConstants.SITE_NAME);
		    String siteUserName = siteName + "-" + username;
		    //logger.info("Global login unsuccessful, trying site-level login with: " + siteUserName);
		    request.login(siteUserName, password);
		    //logger.info("Site-wide login successful");
		}
		catch (Exception exc)
		{
		    //logger.info("Site-wide login unsuccessful => generating login page");		    

		    //The user entered in either the wrong username or the wrong password
		
		    HttpSession session = request.getSession();
		    ServletContext context = session.getServletContext();
		    String redirect_url_string = request.getRequestURI().replaceFirst(context.getContextPath() + "/", "");

		    if ((request.getQueryString() != null) && (request.getQueryString().length() > 0))
		    {
			redirect_url_string += "?" + request.getQueryString().replace("&", "&amp;");
		    }

		    generateLoginPage(redirect_url_string, getTextString("auth.error.un_or_pw_err", lang), userContext, out, baseURL, output);
		    return false;
		}
	    }
	}
	return true;


    }

    private void updateUserContextWithAuthenticatedInfo(HttpServletRequest request, UserContext userContext)
    {
	//logger.info("Start of updateUserContextWithAutenticatedInfo");
	
	//Get the username
	String user_name = request.getUserPrincipal().getName();
	//logger.info("  getUserPrincipal user_name = " + user_name);
	
	userContext.setUsername(user_name);

	//Get the groups for the user
	Document msg_doc = XMLConverter.newDOM();
	Element userInfoMessage = msg_doc.createElement(GSXML.MESSAGE_ELEM);
	Element userInfoRequest = GSXML.createBasicRequest(msg_doc, GSXML.REQUEST_TYPE_PROCESS, Authentication.GET_USER_INFORMATION_SERVICE, userContext);
	userInfoMessage.appendChild(userInfoRequest);

	Element paramList = msg_doc.createElement(GSXML.PARAM_ELEM + GSXML.LIST_MODIFIER);
	userInfoRequest.appendChild(paramList);
	paramList.appendChild(GSXML.createParameter(msg_doc, GSXML.USERNAME_ATT, user_name));

	Element userInfoResponseMessage = (Element) this.recept.process(userInfoMessage);
	Element userInfoResponse = (Element) GSXML.getChildByTagName(userInfoResponseMessage, GSXML.RESPONSE_ELEM);
	Element respParamList = (Element) GSXML.getChildByTagName(userInfoResponse, GSXML.PARAM_ELEM + GSXML.LIST_MODIFIER);

	if (respParamList != null)
	{
	    HashMap<String, Serializable> params = GSXML.extractParams(respParamList, false);
	    String groups = (String) params.get("groups");
	    String editEnabled = (String) params.get("editEnabled");
	    userContext.setGroups(groups.split(","));
	    userContext.setEditEnabled((editEnabled != null) ? editEnabled : "false");
	}
    }

    private void appendUserInformation(Element xml_request, UserContext userContext)
    {
	Element userInformation = xml_request.getOwnerDocument().createElement(GSXML.USER_INFORMATION_ELEM);
	userInformation.setAttribute(GSXML.USERNAME_ATT, userContext.getUsername());


	userInformation.setAttribute(GSXML.GROUPS_ATT, userContext.getGroupsString());
	userInformation.setAttribute(GSXML.EDIT_ENABLED_ATT, userContext.getEditEnabled());
	xml_request.appendChild(userInformation);
    }

    /**
     * request.getRemoteAddr() returns the IP of the client *or* of a proxy.
     * We want the client IP, so we can check if it's truly local or not, since proxies can be
     * local, thus masking a client connecting from an external IP.
     * The following code is from
     * https://www.javaprogramto.com/2020/11/how-to-get-client-ip-address-in-java.html
     */
    public String getClientIpAddress(HttpServletRequest request)
    {
	final String[] VALID_IP_HEADER_CANDIDATES = { 
	    "X-Forwarded-For",
	    "Proxy-Client-IP",
	    "WL-Proxy-Client-IP",
	    "HTTP_X_FORWARDED_FOR",
	    "HTTP_X_FORWARDED",
	    "HTTP_X_CLUSTER_CLIENT_IP",
	    "HTTP_CLIENT_IP",
	    "HTTP_FORWARDED_FOR",
	    "HTTP_FORWARDED",
	    "HTTP_VIA",
	    "REMOTE_ADDR"
	};
	
	
	for (String header : VALID_IP_HEADER_CANDIDATES) {
	    String ipAddress = request.getHeader(header);
	    if (ipAddress != null && ipAddress.length() != 0 && !"unknown".equalsIgnoreCase(ipAddress)) {
		return ipAddress;
	    }
	}
	return request.getRemoteAddr(); // fallback to whatever is the incoming IP (could be proxy)
    }

    
    /**
     * Clients on the same machine as the GS3 server can reconfigure and activate/deactivate,
     * Users in the administrator group also have the right to do such system commands.
     * For collection-level system commands, users with permissions to edit all collections or
     * the specific collection have a right to activate/deactivate a collection.
     * Useful links:
     * https://stackoverflow.com/questions/13563351/how-to-restrict-acces-to-specific-servlet-by-ip-via-container-configuration
     * https://stackoverflow.com/questions/35015514/restricting-access-to-localhost-for-java-servlet-endpoint
     * Reverse of: https://stackoverflow.com/questions/2539461/how-to-access-java-servlet-running-on-my-pc-from-outside
    */
    private boolean configureSecurityCheck(HttpServletRequest request, UserContext userContext,
					   PrintWriter out, String baseURL, String lang,
					   String output, Map<String, String[]> queryMap)
    {
	// if request emanates on same machine as GS server/from local machine, let it go through
	String ipOfClient = getClientIpAddress(request); //request.getRemoteAddr();

	// Load the global.properties file, get the tomcat.server.IPregex and check for any of it
	// matching against the local IP
	String tomcatServerIPregex = GlobalProperties.getProperty("tomcat.server.IPregex", "");	
	if(ipOfClient.equals(request.getLocalAddr()) || ipOfClient.matches(tomcatServerIPregex)) {
	    return true;
	}
	
	// TODO: Want regex for localhost situations and then make it a property in build.props
	// In build.props: comment, you might like to leave the IP of your server machine
	// at the end of the line	
	// Check with another machine: browser, retest cmdline and activate.pl
	// Final phase is testing with apache set up
	
	// if we have sc=colname, user needs to be colname-|all-collection-editor
	// if we have st=collection&sn=colname, same as above
	// otherwise user needs to be administrator

	// We can have MODULE_TYPE=collection And MODULE_NAME=collname,
	// OR SYSTEM_CLUSTER=collname
	// If none of these specified, then assume the user should be administrator
	// If not a collection level operation, the user should be administrator

	String coll = null;
	String module_type = getFirstParam(GSParams.SYSTEM_MODULE_TYPE, queryMap);
	if(module_type != null && module_type.equals("collection")) {
	    // To deactivate demo:library?excerptid=gs_content&a=s&sa=d&sc=lucene-jdbm-demo
	    coll = getFirstParam(GSParams.SYSTEM_MODULE_NAME, queryMap);
	} else {
	    // To reconfigure demo: library?excerptid=gs_content&a=s&sa=c&sc=lucene-jdbm-demo
	    coll = getFirstParam(GSParams.SYSTEM_CLUSTER, queryMap);
	    if (coll != null && coll.equals("")) {
		coll = null;
	    }
	}
	//String collname_editor = (coll == null) ? null : (coll+"-collections-editor");
        String groups = userContext.getGroupsString();
        if (GroupsUtil.isAdministrator(groups)) {
            return true;
        }
        if (coll != null && GroupsUtil.canEditCollection(userContext.getUsername(), groups, coll)) {
          return true;
        }
	errorAndLoginPage(request, userContext, out, baseURL, lang, output);
	return false;	
    }

  // returns true if the user is allowed to access the collection/document
    private boolean runCollectionSecurityCheck(HttpServletRequest request, UserContext userContext, PrintWriter out, String baseURL, String collection, String document, String lang, String output) {

    	// logger.info("LibraryServlet::runCollectionSecurityCheck(): start of check for collection:"+collection);
	
	//Get the security info for this collection
	Document msg_doc = XMLConverter.newDOM();
	Element securityMessage = msg_doc.createElement(GSXML.MESSAGE_ELEM);
	Element securityRequest = GSXML.createBasicRequest(msg_doc, GSXML.REQUEST_TYPE_SECURITY, collection, userContext);
	securityMessage.appendChild(securityRequest);
	if (document != null && !document.equals(""))
	{
	    securityRequest.setAttribute(GSXML.NODE_OID, document);
	}

	Element securityResponse = (Element) GSXML.getChildByTagName(this.recept.process(securityMessage), GSXML.RESPONSE_ELEM);
	if (securityResponse == null)
	{
	    return false;
	}
        if (securityResponse.getAttribute("noSecurity").equals("true")) {
          return true;
        }
	// Groups mentioned in collectionConfig.xml's Security element, se_groups
	ArrayList<String> se_groups = GSXML.getGroupsFromSecurityResponse(securityResponse);

	//If guests are not allowed to access this page then check to see if the user is in a group that is allowed to access the page
	if (!se_groups.contains(""))
	{
	    boolean found = false;
	    for (String se_group : se_groups)
	    {
		if (request.isUserInRole(se_group))
		{
		    found = true;
		    break;
		}
	    }

	    //The current user is not allowed to access the page so produce a login page
	    if (!found)
	    {
		errorAndLoginPage(request, userContext, out, baseURL, lang, output);
		return false;
	    }
	}
	return true;
    }


    private void errorAndLoginPage(HttpServletRequest request, UserContext userContext,
				   PrintWriter out, String baseURL, String lang, String output)
    {	
	String error_message = "";
	if (request.getAuthType() == null)
	    {
		error_message = getTextString("auth.error.login", lang);
	    }
	else
	    {
		error_message = getTextString("auth.error.incorrect_login", lang);
	    }
	
	HttpSession session = request.getSession();
	ServletContext context = session.getServletContext();
	String redirect_url_string = request.getRequestURI().replaceFirst(context.getContextPath() + "/", "");
	
	if (request.getQueryString() != null && request.getQueryString().length() > 0)
	    {
		redirect_url_string += "?" + request.getQueryString().replace("&", "&amp;");
	    }
	
	generateLoginPage(redirect_url_string, error_message, userContext, out, baseURL, output);
    }

    
  private String getTextString(String key, String lang) {
      return Dictionary.getTextString(key, lang, null, null, null, null, null);
  
  }
  //a debugging method
  private void displaySize(Hashtable<String, UserSessionCache> table)
  {
    if (table == null)
      {
	//logger.info("cached table is null");
	return;
      }
    if (table.size() == 0)
      {
	//logger.info("cached table size is zero");
	return;
      }
    int num_cached_coll = 0;
    ArrayList<UserSessionCache> cache_list = new ArrayList<UserSessionCache>(table.values());
    for (int i = 0; i < cache_list.size(); i++)
      {
	num_cached_coll += cache_list.get(i).tableSize();
      }
    //logger.info("Number of sessions : total number of cached collection info = " + table.size() + " : " + num_cached_coll);
  }

  /** merely a debugging method! */
  private String tableToString(Hashtable<String, Hashtable<String, String>> table)
  {
    String str = "";
    Enumeration<String> keys = table.keys();
    while (keys.hasMoreElements())
      {
	String name = keys.nextElement();
	str += name + ", ";
      }
    return str;
  }

  /**
   * this goes through each URL and adds in a session id if needed-- its
   * needed if the browser doesn't accept cookies also escapes things if
   * needed
   */
  protected void encodeURLs(Node dataNode, HttpServletResponse response)
  {
    if (dataNode == null)
      {
	return;
      }

    Element data = null;

    short nodeType = dataNode.getNodeType();
    if (nodeType == Node.DOCUMENT_NODE)
      {
	Document docNode = (Document) dataNode;
	data = docNode.getDocumentElement();
      }
    else if (nodeType == Node.ELEMENT_NODE)
      {
	data = (Element) dataNode;
      }

    // Normally 'dataNode' is an Element or Document, but it is possible for it to be one of the other Node types
    // (e.g., Node.TEXT_NODE, which is generated when using excerptid-text=xxxx)
    // For these simpler cases, there is no URL encoding work to be done, so leaving 'data' as null is sufficient
    
    if (data != null)
      {

	// get all the <a> elements
	NodeList hrefs = data.getElementsByTagName("a");
	// Instead of calculating each iteration...
	int hrefscount = hrefs.getLength();

	for (int i = 0; hrefs != null && i < hrefscount; i++)
	  {
	    Element a = (Element) hrefs.item(i);
	    // ugly hack to get rid of : in the args - interferes with session handling
	    String href = a.getAttribute("href");
	    if (!href.equals(""))
	      {
		if (href.indexOf("?") != -1)
		  {
		    String[] parts = StringUtils.split(href, "\\?", -1);
		    if (parts.length == 1)
		      {
			parts[0] = StringUtils.replace(parts[0], ":", "%3A");
			href = "?" + parts[0];
		      }
		    else
		      {
			parts[1] = StringUtils.replace(parts[1], ":", "%3A");
			href = parts[0] + "?" + parts[1];
		      }

		  }
		a.setAttribute("href", response.encodeURL(href));
	      }
	  }

	// now find any submit bits - get all the <form> elements
	NodeList forms = data.getElementsByTagName("form");
	int formscount = forms.getLength();
	for (int i = 0; forms != null && i < formscount; i++)
	  {
	    Element form = (Element) forms.item(i);
	    form.setAttribute("action", response.encodeURL(form.getAttribute("action")));
	  }
	// are these the only cases where URLs occur??
	// we should only do this for greenstone urls?
      }

  }

  protected String getFirstParam(String name, Map<String, String[]> map)
  {
    String[] val = map.get(name);
    if (val == null || val.length == 0)
      {
	return null;
      }

    return val[0];
  }

  public void doPost(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException
  {

      // **** OIDC
      /*
    String p = normalisePathForOIDC(req);

    if ("/oidc/token".equals(p))        { oidc.handleToken(req, resp); return; }
    if ("1".equals(req.getParameter("__dev_login"))) { handleDevLogin(req, resp); return; }

    resp.setStatus(404);
      */
      
      
    //Check if we need to process a file upload
    if (JakartaServletDiskFileUpload.isMultipartContent(request))
      {
	

	int sizeLimit = System.getProperties().containsKey("servlet.upload.filesize.limit") ? Integer.parseInt(System.getProperty("servlet.upload.filesize.limit")) : 100 * 1024 * 1024;

	File tempDir = new File(GlobalProperties.getGSDL3Home() + File.separator + "tmp");
	if (!tempDir.exists())
	  {
	    tempDir.mkdirs();
	  }

        DiskFileItemFactory fileItemFactory =
          DiskFileItemFactory.builder()
          .setPath(tempDir.toPath())
          .setThreshold(0)
          .get();
        
	JakartaServletDiskFileUpload  uploadHandler =
          new JakartaServletDiskFileUpload(fileItemFactory);
	uploadHandler.setMaxFileSize(sizeLimit);

	HashMap<String, String[]> queryMap = new HashMap<>();
	try
	  {
	    List<DiskFileItem> items = uploadHandler.parseRequest(request);
            for(DiskFileItem current : items) {
	      
		if (current.isFormField())
		  {
		    queryMap.put(current.getFieldName(), new String[] { current.getString(StandardCharsets.UTF_8) });
		  }
		else if (current.getName() != null && !current.getName().isEmpty()) {
		  
		    File file = new File(tempDir, current.getName());
		    current.write(file.toPath());

		    queryMap.put("md___ex.Filesize", new String[] { String.valueOf(file.length()) });
		    queryMap.put("md___ex.Filename", new String[] { current.getName() });
		  }
	      }
	  }
	catch (Exception e)
	  {
	    e.printStackTrace();
	  }

	doGetOrPost(request, response, queryMap);
      }
    else
      {
	doGetOrPost(request, response, request.getParameterMap());
      }
  }



    /*
  private String normalisePathForOIDC(HttpServletRequest req) {
    String servlet_path = req.getServletPath() == null ? "" : req.getServletPath();
    String path_info = req.getPathInfo() == null ? "" : req.getPathInfo();

    String path = (servlet_path + path_info).replace("//", "/"); // avoid any slash double-ups that might have occurred
    
    int i_pos = p.indexOf("/oidc/");    
    if (i_pos >= 0) return p.substring(i_pos);
    
    int w_pos = p.indexOf("/.well-known/");
    if (w_pos >= 0) return path.substring(w_pos);
    
    return path;
  }
    */
    
  private void handleDevLogin(HttpServletRequest req, HttpServletResponse resp) throws IOException {
    String u = req.getParameter("username");
    String pw = req.getParameter("password");
    if (u != null && !u.isEmpty() && pw != null && !pw.isEmpty()) {
      req.getSession(true).setAttribute("gs3_user", u);
      String referer = req.getHeader("Referer");
      resp.setStatus(302);
      resp.setHeader("Location", referer != null ? referer : req.getRequestURL().toString());
    } else {
      resp.setStatus(401);
      resp.getWriter().println("Login failed.");
    }
  }
    
}
