/*
 *    OAIPMH.java
 *    Copyright (C) 2010 New Zealand Digital Library, http://www.nzdl.org
 *
 *    This program is free software; you can redistribute it and/or modify
 *    it under the terms of the GNU General Public License as published by
 *    the Free Software Foundation; either version 2 of the License, or
 *    (at your option) any later version.
 *
 *    This program is distributed in the hope that it will be useful,
 *    but WITHOUT ANY WARRANTY; without even the implied warranty of
 *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *    GNU General Public License for more details.
 *
 *    You should have received a copy of the GNU General Public License
 *    along with this program; if not, write to the Free Software
 *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
 */
package org.greenstone.gsdl3.service;

// Greenstone classes
import org.greenstone.gsdl3.core.GSException;
import org.greenstone.gsdl3.util.GSXML;
import org.greenstone.gsdl3.util.OAIXML;
import org.greenstone.gsdl3.util.OID;
import org.greenstone.gsdl3.util.GSFile;
import org.greenstone.gsdl3.util.XMLConverter;

import org.greenstone.gsdl3.util.SimpleCollectionDatabase;
import org.greenstone.gsdl3.util.DBInfo;
// XML classes
import org.w3c.dom.Document;
import org.w3c.dom.Element;
import org.w3c.dom.Node;
import org.w3c.dom.NodeList;

// General Java classes
import java.io.File;
import java.util.StringTokenizer;
import java.util.Vector;
import java.util.Set;
import java.util.Iterator;
import java.util.ArrayList;
import java.util.Date;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Map.Entry;

import org.apache.log4j.Logger;

/** Implements the oai metadata retrieval service for GS3 collections.
 *  Dig into each collection's database and retrieve the metadata
 *
 */

public class OAIPMH extends ServiceRack {
  
  static Logger logger = Logger.getLogger(org.greenstone.gsdl3.service.OAIPMH.class.getName());
  
  protected SimpleCollectionDatabase coll_db = null;
  protected SimpleCollectionDatabase oaiinf_db = null;

  protected String repository_id = "";
  protected String site_name = "";
  protected String coll_name = "";
 
  // set this up during configure
  protected Element list_sets_response = null;
  
  protected Element meta_formats_definition = null;
  protected HashMap<String, HashSet<String>> format_elements_map = null;
  protected HashMap<String, Element> format_response_map = null;
    
  protected String index_stem = "";
  protected String infodb_type = "";
  
  /** constructor */
  public OAIPMH() {

  }
  
  public void cleanUp() {
    super.cleanUp();//??
	
	if(this.coll_db != null) {		
		this.coll_db.closeDatabase();
		this.coll_db = null;
	}
    if (this.oaiinf_db != null){
    	this.oaiinf_db.closeDatabase();
    }
  }
  
  /** configure this service 
  info is the OAIPMH service rack from collectionConfig.xml, and 
  extra_info is buildConfig.xml */
  public boolean configure(Element info, Element extra_info) {
    if (!super.configure(info, extra_info)){
      logger.info("Configuring ServiceRack.java returns false.");
      return false;
    }
    
    //get the names from ServiceRack.java
    this.site_name = this.router.getSiteName();
    this.coll_name = this.cluster_name;
    
    logger.info("Configuring OAIPMH...");

    this.config_info = info;
    
    // the index stem is either specified in the buildConfig.xml file (extra_info) or uses the collection name
    Element metadata_list = (Element) GSXML.getChildByTagName(extra_info, GSXML.METADATA_ELEM+GSXML.LIST_MODIFIER);

    if (metadata_list != null) {
      
      Element index_stem_elem = (Element) GSXML.getNamedElement(metadata_list, GSXML.METADATA_ELEM, GSXML.NAME_ATT, "indexStem");
      
      if (index_stem_elem != null) {
	this.index_stem = GSXML.getNodeText(index_stem_elem);
      }

      Element infodb_type_elem = (Element) GSXML.getNamedElement(metadata_list, GSXML.METADATA_ELEM, GSXML.NAME_ATT, "infodbType");
      if (infodb_type_elem != null) {
	this.infodb_type = GSXML.getNodeText(infodb_type_elem);
      }

    }

    if (index_stem == null || index_stem.equals("")) {
	this.index_stem = this.cluster_name; // index_stem is the name of the db in indext/text, it is <colname>.<db>
    }
    if (infodb_type == null || infodb_type.equals("")) {
      this.infodb_type = "gdbm"; // the default
    }

	// DB OPENING STUFF MOVED TO configureOAI(), because OAIPMH.configure() is called by the regular MessageRouter when this activates collections for the regular "library" servlet
	// whereas OAIPMH.configureOAI() is only called by OAIMessageRouter when it activates collections for the "oaiserver" servlet (after OAIMessageRouter calls regular configure() first)
	// We don't want the DBs opened twice: once by MessageRouter's call to OAIPMH.configure() and once by OAIMessageRouter calling OAIPMH.configure().
    
    // work out what sets this collection has. Will usually contain the collection itself, optional super collection, and maybe subcolls if appropriate classifiers are present.
    configureSetInfo();
    // the short_service_info is used by the message router to find the method names, 

    Element list_records = this.desc_doc.createElement(GSXML.SERVICE_ELEM);
    list_records.setAttribute(GSXML.NAME_ATT, OAIXML.LIST_RECORDS);
    list_records.setAttribute(GSXML.TYPE_ATT, "oai");
    this.short_service_info.appendChild(list_records);

    Element list_identifiers = this.desc_doc.createElement(GSXML.SERVICE_ELEM);
    list_identifiers.setAttribute(GSXML.NAME_ATT, OAIXML.LIST_IDENTIFIERS);
    list_identifiers.setAttribute(GSXML.TYPE_ATT, "oai");
    this.short_service_info.appendChild(list_identifiers);
    
    Element list_sets = this.desc_doc.createElement(GSXML.SERVICE_ELEM);
    list_sets.setAttribute(GSXML.NAME_ATT, OAIXML.LIST_SETS);
    list_sets.setAttribute(GSXML.TYPE_ATT, "oai");
    this.short_service_info.appendChild(list_sets);
    
    Element list_metadata_formats = this.desc_doc.createElement(GSXML.SERVICE_ELEM);
    list_metadata_formats.setAttribute(GSXML.NAME_ATT, OAIXML.LIST_METADATA_FORMATS);
    list_metadata_formats.setAttribute(GSXML.TYPE_ATT, "oai");
    this.short_service_info.appendChild(list_metadata_formats);

    Element get_record = this.desc_doc.createElement(GSXML.SERVICE_ELEM);
    get_record.setAttribute(GSXML.NAME_ATT, OAIXML.GET_RECORD);
    get_record.setAttribute(GSXML.TYPE_ATT, "oai");
    this.short_service_info.appendChild(get_record);
    
    return true;
  }

  /** oai_config_elem is the content of OAIConfig-xxx.xml found in web/WEB-INF/classes */
  public boolean configureOAI(Element oai_config_elem) {
    this.meta_formats_definition = this.desc_doc.createElement(OAIXML.LIST_METADATA_FORMATS);
    this.format_response_map = new HashMap<String, Element>();
    this.format_elements_map = new HashMap<String, HashSet<String>>();
    
    // get the reposityIdentifier
    Element ri = (Element)GSXML.getChildByTagName(oai_config_elem, OAIXML.REPOSITORY_IDENTIFIER);
    if (ri != null) { 
      this.repository_id = GSXML.getNodeText(ri);
    } else {
      this.repository_id = "";
    }

    // for now, all we want is the metadata prefix description and the mapping list
    Element main_lmf_elem = (Element) GSXML.getChildByTagName(oai_config_elem, OAIXML.LIST_METADATA_FORMATS);
    if (main_lmf_elem == null) {
      logger.error("No listMetadataFormats element found in OAIConfig.xml");
      return false;
    }
    NodeList meta_formats_list = this.config_info.getElementsByTagName(OAIXML.METADATA_FORMAT);
    if (meta_formats_list.getLength() == 0) {
      logger.error("no metadataFormat elements found in OAIPMH serviceRack element");
      return false;
    }
    
    boolean found_meta_format = false;
    for(int i=0; i<meta_formats_list.getLength(); i++) {
      Element mf = (Element) meta_formats_list.item(i);
      String prefix = mf.getAttribute(OAIXML.METADATA_PREFIX);
      if (prefix.equals("")) {
	logger.error("metadataFormat element had no metadataPrefix attribute");
	continue;
      }
      // get the right format from OAIConfig - looks for a <metadataFormat> element
      // where <metadataNamespace> value == prefix
      Element meta_format = findNamedMetadataFormat(main_lmf_elem, prefix);
      if (meta_format == null) {
	logger.error("Couldn't find metadataFormat named "+prefix+" in OAIConfig.xml");
	continue;
      }
      
      // copy the format definition into our stored Element
      // this is the <metadataFormat> element
      Element collection_version_format = (Element) this.desc_doc.importNode(meta_format, true);
      collection_version_format.setAttribute(GSXML.NAME_ATT, prefix); // for convenience
      this.meta_formats_definition.appendChild(collection_version_format);
      // set up the response element for this format
      // this is the <metadataFormat element with only the metadataPrefix,
      // schema and metadataNamespace children
      format_response_map.put(prefix, OAIXML.getMetadataFormatShort(this.desc_doc, collection_version_format));
      // add in collection specific mappings
      addCollectionMappings(collection_version_format, mf);
      // now set up a list of all collection elements for reverse lookup of the mapping
      // This is a set of all GS metadata elements that will be used (from the mappings)
      format_elements_map.put(prefix, getAllCollectionElements(collection_version_format));

    } // end for
	
    // Open the coll db and oai-inf db databases and store handles to them	
    coll_db = new SimpleCollectionDatabase(infodb_type);
    if (!coll_db.databaseOK()) {
      logger.error("Couldn't create the collection database of type "+infodb_type);
      return false;
    }

    oaiinf_db = new SimpleCollectionDatabase(infodb_type);
    if (!oaiinf_db.databaseOK()) {
      logger.error("Couldn't create the oai-inf database of type "+infodb_type);
      return false;
    }

    
    // Open databases for querying
    String coll_db_file = GSFile.collectionDatabaseFile(this.site_home, this.cluster_name, index_stem, infodb_type);
    if (!this.coll_db.openDatabase(coll_db_file, SimpleCollectionDatabase.READ)) {
      logger.error("Could not open collection database!");
      return false;
    }
    // the oaiinf_db is called oai-inf.<infodb_type_extension>
    String oaiinf_db_file = GSFile.OAIInfoDatabaseFile(this.site_home, this.cluster_name, "oai-inf", infodb_type);
    File oaiinfFile = new File(oaiinf_db_file);
    
    if(!oaiinfFile.exists()) {
	logger.warn("oai-inf database for collection + " + this.cluster_name + " does not exist.");
	oaiinf_db = null;
    } else if (!this.oaiinf_db.openDatabase(oaiinf_db_file, SimpleCollectionDatabase.READ)) {
	logger.warn("Could not open oai-inf database for collection + " + this.cluster_name + "!");
	oaiinf_db = null;
    }
	
    return true;
  }

    /**
     * @return the associated OAICollection's OAI_EARLIEST_TIMESTAMP_OID record's 
     * OAI_INF_TIMESTAMP field from the collection's oai-inf.db IN MILLISECONDS
     */
    public long getEarliestTimestamp() {
	long timestamp = -1;
	
	DBInfo oai_info = null;
	if(oaiinf_db != null) {
	    // get internal record containing the earliest timestamp of the collection
	    oai_info = this.oaiinf_db.getInfo(OAIXML.OAI_EARLIEST_TIMESTAMP_OID);
	    if (oai_info == null) {
		logger.warn("Can't get collection " + this.cluster_name + "'s earliest timestamp from oai-inf db. No entry for 'OID' " + OAIXML.OAI_EARLIEST_TIMESTAMP_OID + " in the db.");		
	    } else {
		timestamp = Long.parseLong(oai_info.getInfo(OAIXML.OAI_INF_TIMESTAMP)) * 1000; // stored in seconds, so x1000 to  convert to milliseconds
		//logger.info("@@@ found earliest OAI timestamp for collection " + this.coll_name + ": " + timestamp + " (ms)");
	    }
	}
	return timestamp;
    }
    
  protected Element findNamedMetadataFormat(Element list_meta_formats, String prefix) {
    NodeList formats = list_meta_formats.getElementsByTagName(OAIXML.METADATA_FORMAT);
    for (int i=0; i<formats.getLength(); i++) {
      Element format = (Element)formats.item(i);
      String meta_name = GSXML.getNodeText((Element)GSXML.getChildByTagName(format, OAIXML.METADATA_PREFIX));
      if (prefix.equals(meta_name)) {
	return format;
      }
    }
    return null;
  }
    
    /** goes through the mappings from the collection one, and replaces existing ones in the main one */
  /* both of these elements are <metadataFormat> */
    protected void addCollectionMappings(Element main_meta_format, Element coll_meta_format) {
      boolean simple_format = false;
      String simple_att = main_meta_format.getAttribute("simple");
      if (simple_att.equals("true") || simple_att.equals("")) {
        // if the att is not present, we assume simple
        simple_format = true;
      }
      boolean allow_extra_elements = false;
      String allow_att = main_meta_format.getAttribute("allowExtraElements");
      if (allow_att.equals("true")) {
        allow_extra_elements = true;
        // if its not there, we don't allow extras
      }
      Document doc = main_meta_format.getOwnerDocument();
      
      Element main_metadata_elem = (Element)GSXML.getChildByTagName(main_meta_format, OAIXML.METADATA);
      // get the actual container element, eg oai_dc:dc
      Element main_container = GSXML.getFirstElementChild(main_metadata_elem);

      if (simple_format) {
        NodeList coll_elements = coll_meta_format.getElementsByTagName(OAIXML.ELEMENT);
        if (coll_elements.getLength()==0) {
          // no mappings to include
          return;
        }
        
        for (int i=0; i<coll_elements.getLength(); i++) {
          Element e = (Element)coll_elements.item(i);
          String elem_name = e.getAttribute(GSXML.NAME_ATT);
          Element orig_elem = GSXML.getNamedElement(main_container, OAIXML.ELEMENT, GSXML.NAME_ATT, elem_name);
          if (orig_elem == null) {
            if (allow_extra_elements) {
              main_container.appendChild(doc.importNode(e, true));
            } else {
              logger.error(elem_name+" not found in meta format, and allowExtraElements is not set, or set to false, so not using it");
            }
          } else {
            main_container.replaceChild(doc.importNode(e, true), orig_elem );
          }
        } // for each element
        return;
      } // if simple


      // we have a complex mapping, with nested elements
      //get the top level element of collection (eg rdf:RDF)
      Element coll_container = GSXML.getFirstElementChild(coll_meta_format);
      if (coll_container ==  null) return;
      if (!coll_container.getTagName().equals(main_container.getTagName())) {
        logger.error("Mismatch between metadata node in OAIConfig and collectionConfig, not adding any collection elements" + coll_container.getTagName()+", "+main_container.getTagName());
        return;
      }
      //recurse through, replacing attributeMappings, replacing/adding element mappings
      copyMappings(main_container, coll_container, allow_extra_elements);
      
    } // addCollectionMappings

  protected void copyMappings(Element to_elem, Element from_elem, boolean allow_extra_elements) {
    Document doc = to_elem.getOwnerDocument();
    Node n = from_elem.getFirstChild();
    while(n!= null) {
      if (n.getNodeType() == Node.ELEMENT_NODE) {
        // we are only interested here in Elements
        String node_name = n.getNodeName();
        if (node_name.equals(OAIXML.ATTRIBUTE_MAPPING) || node_name.equals(OAIXML.ELEMENT)) {
          String name_att = ((Element)n).getAttribute(GSXML.NAME_ATT);
          if (!name_att.equals("")) {
            Element orig_elem = GSXML.getNamedElement(to_elem, node_name, GSXML.NAME_ATT, name_att);
            if (orig_elem != null) {
              to_elem.replaceChild(doc.importNode(n, true), orig_elem);
           
            } else if (allow_extra_elements) {
              to_elem.appendChild(doc.importNode(n, true));
            }
          }
        }
        else {
          // its a different element, so lets recurse into it
          Element to_child = GSXML.findEquivalentElement(to_elem, (Element)n);
          if (to_child != null) {
            copyMappings(to_child, (Element)n, allow_extra_elements);
          } else if (allow_extra_elements) {
            // its not already present, add it
            to_elem.appendChild(doc.importNode(n, true));
          }
        }
      }
      n = n.getNextSibling();
    }// while n != null
  }
    /** goes through all the mappings and makes a set of all collection
	metadata names that could become an oai meta element - acts as
	a reverse lookup for the mappings */
    protected HashSet<String> getAllCollectionElements(Element meta_format) {
      HashSet<String> meta_name_set = new HashSet<String>();

      NodeList elements = meta_format.getElementsByTagName(OAIXML.ELEMENT);
      for (int i=0; i<elements.getLength(); i++) {
	Element e = (Element)elements.item(i);
	Element map = (Element)GSXML.getChildByTagName(e, OAIXML.MAPPING);
        NodeList attr_map_list = GSXML.getChildrenByTagName(e, OAIXML.ATTRIBUTE_MAPPING);
        int num_attr_maps = attr_map_list.getLength();
	if (map == null && num_attr_maps == 0) {
	  // there is no mapping, just use the element name
	  meta_name_set.add(e.getAttribute(GSXML.NAME_ATT));
	} else {
          if (map != null) {
            String list_of_names = map.getAttribute(OAIXML.ELEMENTS);
            String[] name_array = list_of_names.split(",");
            for (int j=0; j<name_array.length; j++) {
              meta_name_set.add(name_array[j]);
            }
          }
          if (num_attr_maps>0) {
            for (int a=0; a < num_attr_maps; a++) {
              Element ae = (Element)attr_map_list.item(a);
              String name= ae.getAttribute(OAIXML.ELEMENT);
              if (!name.equals("")) {
                meta_name_set.add(name);
              }
            }
          }
        }
      } // foreach element
      return meta_name_set;
    }
   
  /** returns a specific service description */
  public Element getServiceDescription(Document doc, String service_id, String lang, String subset) {
    
    if (service_id.equals(OAIXML.LIST_RECORDS)) {
      Element list_records = doc.createElement(GSXML.SERVICE_ELEM);
      list_records.setAttribute(GSXML.NAME_ATT, OAIXML.LIST_RECORDS);
      list_records.setAttribute(GSXML.TYPE_ATT, "oai");
      return list_records;
    }
    
    if (service_id.equals(OAIXML.LIST_IDENTIFIERS)) {
      Element list_identifiers = doc.createElement(GSXML.SERVICE_ELEM);
      list_identifiers.setAttribute(GSXML.NAME_ATT, OAIXML.LIST_IDENTIFIERS);
      list_identifiers.setAttribute(GSXML.TYPE_ATT, "oai");
      return list_identifiers;
    }
    if (service_id.equals(OAIXML.LIST_SETS)) {
      Element list_sets = doc.createElement(GSXML.SERVICE_ELEM);
      list_sets.setAttribute(GSXML.NAME_ATT, OAIXML.LIST_SETS);
      list_sets.setAttribute(GSXML.TYPE_ATT, "oai");
      return list_sets;
    }
    if (service_id.equals(OAIXML.LIST_METADATA_FORMATS)) {
      Element list_metadata_formats = doc.createElement(GSXML.SERVICE_ELEM);
      list_metadata_formats.setAttribute(GSXML.NAME_ATT, OAIXML.LIST_METADATA_FORMATS);
      list_metadata_formats.setAttribute(GSXML.TYPE_ATT, "oai");
      return list_metadata_formats;
    }
    
    if (service_id.equals(OAIXML.GET_RECORD)) {
      Element get_record = doc.createElement(GSXML.SERVICE_ELEM);
      get_record.setAttribute(GSXML.NAME_ATT, OAIXML.GET_RECORD);
      get_record.setAttribute(GSXML.TYPE_ATT, "oai");
      return get_record;
    }
    
    return null;
  }

  /** The list sets service returns all the sets that this collection is/is part of/contains. This is gathered by Receptionist from all collections to answer the OAI ListSets request.  */
  protected Element processListSets(Element req) {
    return list_sets_response;
  }
  /** returns the actual record element used in the OAI GetRecord response */
  protected Element processGetRecord(Element req) {
    /** arguments:
        identifier: required
        metadataPrefix: required
     *  Exceptions: badArgument; cannotDisseminateFormat; idDoesNotExist
     */

      String base_url = req.getAttribute(GSXML.BASE_URL); // ends in oaiserver
      String context_url = base_url.substring(0, base_url.lastIndexOf("/")+1); // remove /oaiserver
      //logger.error("base url = "+base_url+", context = "+context_url);
    NodeList params = GSXML.getChildrenByTagName(req, GSXML.PARAM_ELEM);
    HashMap<String, String> param_map = GSXML.getParamMap(params);    
    
    String prefix = param_map.get(OAIXML.METADATA_PREFIX);
    if (prefix == null || prefix.equals("")) {
      //Just a double-check
      logger.error("the value of metadataPrefix att is not present in the request.");
      return OAIXML.createErrorResponse(OAIXML.CANNOT_DISSEMINATE_FORMAT, "");
    }
    
    // check that we support this format
    if (!format_response_map.containsKey(prefix)) {
      logger.error("metadata prefix is not supported for collection "+this.coll_name);
      return OAIXML.createErrorResponse(OAIXML.CANNOT_DISSEMINATE_FORMAT, "");
    }

    Document doc = XMLConverter.newDOM();
    
    String oid = param_map.get(OAIXML.OID); // TODO should this be identifier???
    boolean OID_is_deleted = false;
    long millis = -1;

    DBInfo oai_info = null;
    if(oaiinf_db != null) {
	oai_info = this.oaiinf_db.getInfo(oid);
	if (oai_info == null) {
	    logger.warn("OID: " + oid + " is not present in the collection's oai-inf database.");
	} else  {

	    // indexdb doesn't have info on deleted docs, only oaiinf db does.
	    // So only oaiinfdb has timestamps for deleted docs
	    // For non-deleted doc ids: also obtain timestamp from oaiinf db, 
	    // but if the oaiinf db doesn't exist, resort to oailastmodified fields of indexdb.
	    String timestamp = oai_info.getInfo(OAIXML.OAI_INF_TIMESTAMP); // stored in seconds, like oailastmodified in the collection index db
	    millis = Long.parseLong(timestamp)*1000; // in milliseconds	    
	    
	    String oaiinf_status = oai_info.getInfo(OAIXML.OAI_INF_STATUS);
	    if(oaiinf_status != null && oaiinf_status.equals(OAIXML.OAI_INF_DELETED)) {
		OID_is_deleted = true;
	    }
	}
    }

    //get a DBInfo object of the identifier; if this identifier is not present in the database,
    // null is returned.
    DBInfo info = this.coll_db.getInfo(oid);
    if (info == null) {
	if(!OID_is_deleted) { // we don't expect to find entries for deleted docs in index db.
	    logger.error("OID: " + oid + " is not present in the collection index database.");
	    return OAIXML.createErrorResponse(OAIXML.ID_DOES_NOT_EXIST, "");
	} // if doc deleted, id missing in indexdb is not an error: doc id would exist only in oai-inf db, marked as deleted 'D'
    }
    else if (millis == -1) { // so couldn't get doc lastmod from oaiinf db, get oailastmodified from collection's index db
	millis = getDateStampMillis(info);	
    }
    String oailastmodified = (millis == -1) ? "" : OAIXML.getTime(millis);
    

    Element get_record_response = doc.createElement(GSXML.RESPONSE_ELEM);
    Element get_record = doc.createElement(OAIXML.GET_RECORD);
    get_record_response.appendChild(get_record);
    Element record = doc.createElement(OAIXML.RECORD);
    //compose the header element
    record.appendChild(createHeaderElement(doc, oid, oailastmodified, OID_is_deleted));      
    if(!OID_is_deleted) {
	//compose the metadata element
	record.appendChild(createMetadataResultElement(doc, prefix, context_url, info));
    }
    get_record.appendChild(record);
    return get_record_response;
  }

  /** return a list of records in specified set, containing metadata from specified prefix*/
  protected Element processListRecords(Element req) {
    return processListIdentifiersOrRecords(req, OAIXML.LIST_RECORDS, true);
  }

  /** return a list of identifiers in specified set that contain metadata belonging to specified prefix. */
  protected Element processListIdentifiers(Element req) {
    return processListIdentifiersOrRecords(req, OAIXML.LIST_IDENTIFIERS, false);
  }

  // Get a list of records/identifiers that match the parameters.
  protected Element processListIdentifiersOrRecords(Element req, String response_name, boolean include_metadata) {
    /** arguments:
        metadataPrefix: required
     *  from: optional
     *  until: optional
     *  set: optional
     *  resumptionToken: exclusive and optional (ignored as it has been handled by OAIReceptionist)
     *  Exceptions: badArgument; cannotDisseminateFormat; idDoesNotExist
     */ 

      String base_url = req.getAttribute(GSXML.BASE_URL); // ends in oaiserver
      String context_url = base_url.substring(0, base_url.lastIndexOf("/")+1); // remove /oaiserver
      // logger.error("base url = "+base_url+", context = "+context_url);
      NodeList params = GSXML.getChildrenByTagName(req, GSXML.PARAM_ELEM);
    
    if(params.getLength() == 0) {
      logger.error("must at least have the metadataPrefix parameter, can't be none");
      return OAIXML.createErrorResponse(OAIXML.BAD_ARGUMENT, "");
    }
    
    HashMap<String, String> param_map = GSXML.getParamMap(params);  
    
    String prefix = "";
    Date from_date = null;
    Date until_date = null;
    
    if(param_map.containsKey(OAIXML.METADATA_PREFIX) == false) {    
      //Just a double-check
      logger.error("A param element containing the metadataPrefix is not present.");
      return OAIXML.createErrorResponse(OAIXML.CANNOT_DISSEMINATE_FORMAT, "");
    }
    prefix = param_map.get(OAIXML.METADATA_PREFIX);
    if (prefix == null || prefix.equals("")) {
      //Just a double-check
      logger.error("the value of metadataPrefix att is not present in the request.");
      return OAIXML.createErrorResponse(OAIXML.CANNOT_DISSEMINATE_FORMAT, "");
    }
    
    if(param_map.containsKey(OAIXML.FROM)) {
      String from = param_map.get(OAIXML.FROM);
      from_date = OAIXML.getDate(from);
    }    
    if(param_map.containsKey(OAIXML.UNTIL)) {
      String until = param_map.get(OAIXML.UNTIL);
      until_date = OAIXML.getDate(until);
    }    

    if (!format_response_map.containsKey(prefix)) {
      logger.error(prefix + " metadata prefix is not supported for collection "+this.coll_name);
      return OAIXML.createErrorResponse(OAIXML.CANNOT_DISSEMINATE_FORMAT, "");
    }

    // get list of oids
    ArrayList<String> oid_list = null;
    if(oaiinf_db != null) { // try getting the OIDs from the oaiinf_db
	oid_list = new ArrayList<String>(oaiinf_db.getAllKeys());
    
	if(oid_list == null) { // try getting the OIDs from the oai entries in the index db
	    logger.warn("@@@@@@@@@@@@@ NO OIDs in oai-inf db for " + this.cluster_name);
	    oid_list = getChildrenIds(OAIXML.BROWSELIST);
	}
    }

    if (oid_list == null) {
      logger.error("No matched records found in collection: oai-inf and index db's browselist are empty");
      return OAIXML.createErrorResponse(OAIXML.NO_RECORDS_MATCH, "");
    }
    // all validation is done

    // get the list of elements that are in this metadata prefix
    HashSet<String> set_of_elems = format_elements_map.get(prefix);

    Document doc = XMLConverter.newDOM();
    Element list_items_response = doc.createElement(GSXML.RESPONSE_ELEM);
    Element list_items = doc.createElement(response_name);
    list_items_response.appendChild(list_items);

    for(int i=0; i<oid_list.size(); i++) {
      String oid = oid_list.get(i);

      if(oid.equals(OAIXML.OAI_EARLIEST_TIMESTAMP_OID)) { // internal id not doc id, so skip
	  continue;
      }
      
      boolean OID_is_deleted = false;
      long millis = -1;

      DBInfo oai_info = null;
      if(oaiinf_db != null) {
	  oai_info = this.oaiinf_db.getInfo(oid);
	  if (oai_info == null) {
	      logger.warn("OID: " + oid + " is not present in the collection's oai-inf database.");
	  } else  {
	      
	      // indexdb doesn't have info on deleted docs, only oaiinf db does.
	      // So only oaiinfdb has timestamps for deleted docs
	      // For non-deleted doc ids: also obtain timestamp from oaiinf db, 
	      // but if the oaiinf db doesn't exist, resort to oailastmodified fields of indexdb.
	      String timestamp = oai_info.getInfo(OAIXML.OAI_INF_TIMESTAMP); // stored in seconds like oailastmodified in the collection index db	      
	      millis = Long.parseLong(timestamp)*1000; // in milliseconds
		  
	      String oaiinf_status = oai_info.getInfo(OAIXML.OAI_INF_STATUS);
	      if(oaiinf_status != null && oaiinf_status.equals(OAIXML.OAI_INF_DELETED)) {
		  OID_is_deleted = true;
	      }
	  }
      }
      DBInfo info = this.coll_db.getInfo(oid);
      if (info == null) { // can happen if oid was deleted, in which case only oai_info keeps a record of the oid
	  if(!OID_is_deleted) { // we don't expect to find entries for deleted docs in index db.
	      logger.error("Collection database does not contain information about oid: " +oid);
	  }
      }
      else if (millis == -1) { // so couldn't get doc lastmod from oaiinf db, get oailastmodified from collection's index db
	  
	  millis = getDateStampMillis(info);
      }

      Date this_date = null;
      String oailastmodified = (millis == -1) ? "" : OAIXML.getTime(millis);
      
      if (millis == -1) {
	  if (from_date != null || until_date !=null) {
	      continue; // if this doc doesn't have a date for some reason, and
	      // we are doing a date range, then don't include it.
	  }
      } else {
	  this_date = new Date(millis);
	  if (from_date != null) {
	      if(this_date.before(from_date)) {
		  continue;
	      }
	  }
	  if (until_date != null) {
	      if (this_date.after(until_date)) {
		  continue;
	      }
	  }    
      }
      
      //compose the header element, which we'll be appending no matter what
      Element header = createHeaderElement(doc, oid, oailastmodified, OID_is_deleted);

      if (include_metadata) { // doing ListRecords
	  // compose a record for adding header and metadata
	  Element record = doc.createElement(OAIXML.RECORD);
	  list_items.appendChild(record);
	  //insert the header element
	  record.appendChild(header);
	  //Now check that this id has metadata for the required prefix.
	  if (info != null && documentContainsMetadata(info, set_of_elems)) {
	      // YES, it does have some metadata for this prefix	    
	      //compose the metadata element
	      record.appendChild(createMetadataResultElement(doc, prefix, context_url, info));
	  } // otherwise the oid was 'deleted' and only in the oai-inf db and not in the info (collection index) db
      } else { // doing ListIdentifiers
	  //append the header element
	  list_items.appendChild(header);      
      }      
      
    }//end of for(int i=0; i<oid_list.size(); i++) of doing thru each record
    
    return list_items_response;        
    
  }

  
  // have implemented setDescription as an element, instead of a container containing metadata
  private boolean configureSetInfo() {

    Document doc = XMLConverter.newDOM();
    this.list_sets_response = doc.createElement(GSXML.RESPONSE_ELEM);
    Element list_sets_elem = doc.createElement(OAIXML.LIST_SETS);
    this.list_sets_response.appendChild(list_sets_elem);
    String set_name = this.coll_name;
    String set_description = null;
    Element name_elem = (Element)GSXML.getChildByTagName(this.config_info, OAIXML.SET_NAME);
    if (name_elem!=null) {
      set_name = GSXML.getNodeText(name_elem);
      if (set_name.equals("")) {
	set_name = this.coll_name; // default to coll name if can't find one
      }
    }
    Element description_elem = (Element)GSXML.getChildByTagName(this.config_info, OAIXML.SET_DESCRIPTION);
    if (description_elem!=null) {
      set_description = GSXML.getNodeText(description_elem);
      if (set_description.equals("")) {
	set_description = null;
      }
    }
    Element coll_set = OAIXML.createSet(doc, this.coll_name, set_name, set_description);
    list_sets_elem.appendChild(coll_set);
    
    // are we part of any super sets?
    NodeList super_set_list = GSXML.getChildrenByTagName(this.config_info, OAIXML.OAI_SUPER_SET);
    for (int i=0; i<super_set_list.getLength(); i++) {
      String super_name = ((Element)super_set_list.item(i)).getAttribute(GSXML.NAME_ATT);
      if (super_name != null && !super_name.equals("")) {
	list_sets_elem.appendChild(OAIXML.createSet(doc, super_name, super_name, null));
      }
    }
    return true;
  }

 /** create the metadata element used when processing ListRecords/GetRecord requests, eg
   <metadata>
     <oai_dc:dc>
       actual metadata
     </oai_dc:dc>
   </metadata>
   */
    protected Element createMetadataResultElement(Document doc, String prefix, String context_url, DBInfo info) {
    // the <metadata> element
    Element metadata = doc.createElement(OAIXML.METADATA);
    
    Element meta_format_element = GSXML.getNamedElement(this.meta_formats_definition, OAIXML.METADATA_FORMAT, GSXML.NAME_ATT, prefix);
    //logger.error("meta format element = "+XMLConverter.getPrettyString(meta_format_element));
    Element def_meta_elem = (Element)GSXML.getChildByTagName(meta_format_element, OAIXML.METADATA);
    Element def_meta_main_elem = GSXML.getFirstElementChild(def_meta_elem);
    // just get teh top level <oai_dc:dc> element, with no children
    Element metadata_content_elem = (Element)doc.importNode(def_meta_main_elem, false);
    metadata.appendChild(metadata_content_elem);
    // recurse through the definition, adding nodes to the result as needed
    processElementMappings(metadata_content_elem, def_meta_main_elem, context_url, info);
    return metadata;
  }

  // This recurses through the definition element, and copies over xml nodes/ generates metadata nodes into result element as needed. 
    protected void processElementMappings(Element result, Element definition, String context_url, DBInfo info) {
    Document doc = result.getOwnerDocument();
    Node n=definition.getFirstChild();
    while (n != null) {
      if (n.getNodeType() == Node.ELEMENT_NODE) {
        // we are only interested in Elements
        if (n.getNodeName().equals(OAIXML.ATTRIBUTE_MAPPING)) {
          String att_name = ((Element)n).getAttribute("name");
          String att_source_meta = ((Element)n).getAttribute("element");
          if (!att_name.equals("") && !att_source_meta.equals("")) {
	      Vector<String> values = getMetadataValuesFromDB(att_source_meta, OAIXML.SELECT_SINGLE_VALUE, context_url, info);
                        
            if (!values.isEmpty() ) {
              String att_value = values.get(0);
              if (att_value != null && !att_value.equals("")) {
                result.setAttribute(att_name, att_value);
              }
              else {
                System.err.println("no value found for "+att_source_meta);
              }
            }
          }
        } else if (n.getNodeName().equals(OAIXML.ELEMENT)) {
          // we need to create the new metadata
	    createAndAddNewMetadata(result, (Element)n, context_url, info);
         } else {
          // we just copy the element. however, it may itself contain <element>
          Element new_node = (Element)doc.importNode(n, false);
          processElementMappings(new_node, (Element)n, context_url, info);
          result.appendChild(new_node);
        }
      }
      n = n.getNextSibling();
    }
  }

  // definition is an <element>
  // assume element can contain mapping, but not attributeMapping.
    protected void createAndAddNewMetadata(Element result, Element definition, String context_url, DBInfo info) {
    String new_name = definition.getAttribute("name");
    if (new_name.equals("")) {
      return;
    }
    Element mapping = (Element)GSXML.getChildByTagName(definition, OAIXML.MAPPING);
    if (mapping == null) {
      // no mappings, we just look up the name in DBInfo, and add it if present
      // oai uses : for namespace, gs uses .
      String gs_name = new_name.replace(":",".");
      addMetadataValues(result, new_name, gs_name, OAIXML.SELECT_ALL_VALUES, context_url, info);
    }
    else {
	addMetadataValues(result, new_name, mapping.getAttribute(OAIXML.ELEMENTS), mapping.getAttribute(OAIXML.SELECT), context_url, info);
    } 
  }
    // context url shouldn't include oaiserver, eg localhost:8383/greenstone3
    protected Vector<String> getMetadataValuesFromDB(String meta_name_list, String select_type, String context_url, DBInfo info) {
       
	Vector<String> all_values = new Vector<String>();
    String[] names = meta_name_list.split(",");
    for (int i=0; i<names.length; i++) {
      Vector<String> values;
      // some special words
      String name = names[i];
      if (name.startsWith(OAIXML.GSF_LINK_PREFIX)) {
	values = new Vector<String>();
	String link_url = null;
	if (name.equals(OAIXML.GSF_LINK_PREFIX+OAIXML.LINK_TYPE_DOCUMENT)) {
	  link_url =  context_url + this.library_name+ "/collection/" + this.coll_name + "/document/" + info.getInfo("Identifier");
	} else if (names[i].equals(OAIXML.GSF_LINK_PREFIX+OAIXML.LINK_TYPE_PRINT)) {
	  link_url =  context_url + this.library_name+ "/collection/" + this.coll_name + "/document/" + info.getInfo("Identifier")+"/print";
	} else if (name.equals(OAIXML.GSF_LINK_PREFIX+OAIXML.LINK_TYPE_SOURCE)) {
	  String srcfile = info.getInfo("srclinkFile");
	  if (!srcfile.equals("")) {
            link_url = context_url + this.library_name+ "/collection/" + this.coll_name + "/document/" + info.getInfo("Identifier")+ "/assoc/" + srcfile;
	  }
	} else if (name.equals(OAIXML.GSF_LINK_PREFIX+OAIXML.LINK_TYPE_COVER)) {
          String hascover = info.getInfo("hascover");
          if (hascover.equals("1")) {
            link_url = context_url + this.library_name+ "/collection/" + this.coll_name + "/document/" + info.getInfo("Identifier")+ "/assoc/cover.jpg";
          }
        }
	if (link_url !=null) {
	  values.add(link_url);
	}
      } else {
	values = info.getMultiInfo(name);
      }

      if (values == null || values.size()==0) {
	continue;
      }
      for (int j=0; j<values.size(); j++) {
        all_values.add(values.get(j));
	if (select_type.equals(OAIXML.SELECT_SINGLE_VALUE)) {
	  return all_values; // only want to add one value
	}
      }
      if (select_type.equals(OAIXML.SELECT_FIRST_VALID_META)) {
	return all_values; // we have added all values of this meta elem
      }
      // otherwise, we will keep going through the list and add them all.
    } // for each name in metadata name list
    return all_values;
  }
   
  /** get the name_list metadata values from DB, then add them as elements
      into result */
    protected void addMetadataValues(Element result, String new_meta_name, String name_list, String select_type, String context_url, DBInfo info) {

      Vector<String> values = getMetadataValuesFromDB(name_list, select_type, context_url, info);
    if (values != null && values.size()!=0) {
      for (int j=0; j<values.size(); j++) {
	addMetadataElement(result, new_meta_name, values.get(j));
      }
      
    }
  }
  
  // specific metadata formats might need to do some custom metadata that is not
  //just a standard mapping.
  protected void addCustomMetadata(Element meta_list_elem, String prefix, DBInfo info) {
    

  }
  
  /** create the actual metadata element for the list */
  protected void addMetadataElement(Element meta_list_elem, String name, String value) {
    
    Element meta = GSXML.createTextElement(meta_list_elem.getOwnerDocument(), name, value);
    meta_list_elem.appendChild(meta);
  }


  /** create a header element used when processing requests like ListRecords/GetRecord/ListIdentifiers
   */  
  protected Element createHeaderElement(Document doc, String oid, String oailastmodified, boolean deleted) {

        Element header = doc.createElement(OAIXML.HEADER);
	
	// if deleted, get the date and change oailastmodified to timestamp in oaiinfo
	if(deleted) {
	    header.setAttribute(OAIXML.OAI_INF_STATUS, OAIXML.HEADER_STATUS_ATTR_DELETED); // set the header status to deleted
		// then the timestamp for deletion will be from oai-inf database 
	}
    
        Element identifier = doc.createElement(OAIXML.IDENTIFIER);
	GSXML.setNodeText(identifier, OAIXML.createOAIIdentifier(repository_id,coll_name,oid));
        header.appendChild(identifier);
        Element set_spec = doc.createElement(OAIXML.SET_SPEC);
	GSXML.setNodeText(set_spec, coll_name);
        header.appendChild(set_spec);
        Element datestamp = doc.createElement(OAIXML.DATESTAMP);
        GSXML.setNodeText(datestamp, oailastmodified);
        header.appendChild(datestamp);
        return header;
  }

  /** return the metadata information  */
  protected Element processListMetadataFormats(Element req) {
    // the request sent here must contain an OID. see doListMetadataFormats() in OAIReceptionist
    Element param = GSXML.getNamedElement(req, GSXML.PARAM_ELEM, GSXML.NAME_ATT, OAIXML.OID);
    if (param == null) {
      logger.error("An element containing the OID attribute not is present.");
      return OAIXML.createErrorResponse(OAIXML.ID_DOES_NOT_EXIST, "");
    }
    String oid = param.getAttribute(GSXML.VALUE_ATT);
    if (oid == null || oid.equals("")) {
      logger.error("No OID is present in the request.");
      return OAIXML.createErrorResponse(OAIXML.ID_DOES_NOT_EXIST, "");
    }

    /*
    ArrayList<String> oid_list = null;
    if(oaiinf_db != null) { // try getting the OIDs from the oaiinf_db
	oid_list = new ArrayList<String>(oaiinf_db.getAllKeys());
	
	if(oid_list == null) { // try getting the OIDs from the oai entries in the index db
	    oid_list = getChildrenIds(OAIXML.BROWSELIST);
	}
    }
    */
    // assume meta formats are only for OIDs that have not been deleted
    // so don't need to check oai-inf db, and can just check collection's index db for list of OIDs
    ArrayList<String> oid_list = getChildrenIds(OAIXML.BROWSELIST);
    if (oid_list == null || oid_list.contains(oid) == false) {
      logger.error("OID: " + oid + " is not present in the database.");
      Element e= OAIXML.createErrorResponse(OAIXML.ID_DOES_NOT_EXIST, "");
//      logger.error((new XMLConverter()).getPrettyString (e));
      return e;
    }
    
    DBInfo info = null;    
    info = this.coll_db.getInfo(oid);
    if (info == null) { //just double check
      return OAIXML.createErrorResponse(OAIXML.OAI_SERVICE_UNAVAILABLE, "");
    }
    
    Document doc = XMLConverter.newDOM();
    Element list_metadata_formats_response = doc.createElement(GSXML.RESPONSE_ELEM);
    
    Element list_metadata_formats = doc.createElement(OAIXML.LIST_METADATA_FORMATS);
    list_metadata_formats_response.appendChild(list_metadata_formats);
    boolean has_meta_format = false;
    
    // for each format in format_elements_map
    Iterator<String> it = format_elements_map.keySet().iterator();
    while (it.hasNext()) {
      String format = it.next();
      HashSet<String> set_of_elems = format_elements_map.get(format);
      if (documentContainsMetadata(info, set_of_elems)) {
	// add this format into the response
	has_meta_format = true;
	list_metadata_formats.appendChild(doc.importNode(format_response_map.get(format), true));
      }
    }

    if (has_meta_format == false) {
      logger.error("Specified metadata names are not contained in the database.");
      return OAIXML.createErrorResponse(OAIXML.NO_METADATA_FORMATS, "");
    } else {
      return list_metadata_formats_response;
    }
  }

  protected boolean documentContainsMetadata(DBInfo info, HashSet<String> set_of_elems) {
    if (set_of_elems.size() == 0) {
      return false;
    }
    Iterator<String> i = set_of_elems.iterator();
    while (i.hasNext()) {
      if (!info.getInfo(i.next()).equals("")) {
	return true;
      }
    }
    return false;
  }

  /** returns a list of the child ids in order, null if no children */
  protected ArrayList<String> getChildrenIds(String node_id) {
    DBInfo info = this.coll_db.getInfo(node_id);
    if (info == null) {
      return null;
    }
    
    String contains = info.getInfo("contains");
    if (contains.equals("")) {
      return null;
    }
    ArrayList<String> children = new ArrayList<String>();
    StringTokenizer st = new StringTokenizer(contains, ";");
    while (st.hasMoreTokens()) {
      String child_id = st.nextToken().replaceAll("\"", node_id);
      children.add(child_id);
    }
    return children;    
  }
  /**method to check whether any of the 'metadata_names' is contained in the 'info'.
   * The name may be in the form: <name>,<mapped name>, in which the mapped name is
   * optional. The mapped name is looked up in the DBInfo; if not present, use the first
   * name which is mandatory.
   */
  // IS this used anywhere??
  protected boolean containsMetadata(DBInfo info, String[] metadata_names) {
    if (metadata_names == null) return false;
    logger.info("checking metadata names in db.");
    for(int i=0; i<metadata_names.length; i++) {
      int index = metadata_names[i].indexOf(",");
      String meta_name = (index == -1) ? metadata_names[i] :
                              metadata_names[i].substring(index + 1);
      
      if(info.getInfo(meta_name).equals("") == false) {
        return true;
      }
    }
    return false;
  }

  protected long getDateStampMillis(DBInfo info) {
    // gs.OAIDateStamp is in YYYY-MM-DD
    String time_stamp = info.getInfo(OAIXML.GS_OAI_DATE_STAMP);
    long millis = -1;
    if (!time_stamp.equals("")) {
      millis = OAIXML.getTime(time_stamp);
    }
    if (millis == -1) {
      // oailastmodified is in seconds
      time_stamp = info.getInfo(OAIXML.OAI_LASTMODIFIED);
      if (!time_stamp.equals("")) {
	millis = Long.parseLong(time_stamp)*1000;
      }
    }
    return millis;
    

  }
}