/* * GS2Construct.java * Copyright (C) 2002 New Zealand Digital Library, http://www.nzdl.org * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by * the Free Software Foundation; either version 2 of the License, or * (at your option) any later version. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * along with this program; if not, write to the Free Software * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. */ package org.greenstone.gsdl3.service; import java.io.BufferedWriter; import java.io.File; import java.io.FileWriter; import java.io.Serializable; import java.io.UnsupportedEncodingException; import java.net.URLEncoder; import java.util.Collections; import java.util.Iterator; import java.util.Map.Entry; import java.util.HashMap; import java.util.Map; import java.util.Set; import java.util.regex.Matcher; import java.util.regex.Pattern; import org.apache.log4j.Logger; import org.greenstone.gsdl3.build.GS2PerlConstructor; import org.greenstone.gsdl3.build.GS2PerlListener; import org.greenstone.gsdl3.util.Dictionary; import org.greenstone.gsdl3.util.GroupsUtil; import org.greenstone.gsdl3.util.GSFile; import org.greenstone.gsdl3.util.GSParams; import org.greenstone.gsdl3.util.GSPath; import org.greenstone.gsdl3.util.GSStatus; import org.greenstone.gsdl3.util.GSXML; import org.greenstone.gsdl3.util.OID; import org.greenstone.gsdl3.util.SimpleCollectionDatabase; import org.greenstone.gsdl3.util.UserContext; import org.greenstone.gsdl3.util.XMLConverter; // https://developer.android.com/reference/org/json/JSONObject.html // https://developer.android.com/reference/org/json/JSONArray.html import org.json.JSONArray; import org.json.JSONException; import org.json.JSONObject; import org.w3c.dom.Document; import org.w3c.dom.Element; import org.w3c.dom.Node; import org.w3c.dom.Text; /** * A Services class for building collections provides a wrapper around the old * perl scripts * */ public class GS2Construct extends ServiceRack { static Logger logger = Logger.getLogger(org.greenstone.gsdl3.service.GS2Construct.class.getName()); // default error message private static final String NO_PERMISSIONS_ERROR = "auth.error.no_permissions"; private static final String NO_METADATA_PERMISSIONS_ERROR = "auth.error.no_metadata_permissions"; // services offered private static final String NEW_SERVICE = "NewCollection"; private static final String ADD_DOC_SERVICE = "AddDocument"; private static final String IMPORT_SERVICE = "ImportCollection"; private static final String BUILD_SERVICE = "BuildCollection"; private static final String ACTIVATE_SERVICE = "ActivateCollection"; private static final String BUILD_AND_ACTIVATE_SERVICE = "BuildAndActivateCollection"; private static final String DELETE_SERVICE = "DeleteCollection"; private static final String RELOAD_SERVICE = "ReloadCollection"; private static final String MODIFY_METADATA_SERVICE = "ModifyMetadata"; // set or remove metadata // params used private static final String COL_PARAM = "collection"; private static final String NEW_COL_TITLE_PARAM = "collTitle"; private static final String NEW_COL_ABOUT_PARAM = "collAbout"; private static final String CREATOR_PARAM = "creator"; private static final String NEW_FILE_PARAM = "newfile"; private static final String PROCESS_ID_PARAM = GSParams.PROCESS_ID; private static final String BUILDTYPE_PARAM = "buildType"; private static final String BUILDTYPE_MG = "mg"; private static final String BUILDTYPE_MGPP = "mgpp"; protected static String DATABASE_TYPE = null; protected SimpleCollectionDatabase coll_db = null; // the list of the collections - store between some method calls private String[] collection_list = null; // set of listeners for any construction commands protected Map listeners = null; protected HashMap collectionOperationMap = new HashMap(); public GS2Construct() { this.listeners = Collections.synchronizedMap(new HashMap()); } /** returns a specific service description */ protected Element getServiceDescription(Document doc, String service, String lang, String subset) { Element description = doc.createElement(GSXML.SERVICE_ELEM); description.setAttribute(GSXML.TYPE_ATT, GSXML.SERVICE_TYPE_PROCESS); description.setAttribute(GSXML.NAME_ATT, service); if (subset == null || subset.equals(GSXML.DISPLAY_TEXT_ELEM + GSXML.LIST_MODIFIER)) { description.appendChild(GSXML.createDisplayTextElement(doc, GSXML.DISPLAY_TEXT_NAME, getServiceTextName(service, lang))); description.appendChild(GSXML.createDisplayTextElement(doc, GSXML.DISPLAY_TEXT_DESCRIPTION, getServiceTextDescription(service, lang))); description.appendChild(GSXML.createDisplayTextElement(doc, GSXML.DISPLAY_TEXT_SUBMIT, getServiceTextSubmit(service, lang))); } if (subset == null || subset.equals(GSXML.PARAM_ELEM + GSXML.LIST_MODIFIER)) { Element param_list = doc.createElement(GSXML.PARAM_ELEM + GSXML.LIST_MODIFIER); description.appendChild(param_list); if (service.equals(NEW_SERVICE)) { Element param = GSXML.createParameterDescription(doc, NEW_COL_TITLE_PARAM, getTextString("param." + NEW_COL_TITLE_PARAM, lang), GSXML.PARAM_TYPE_STRING, null, null, null); param_list.appendChild(param); param = GSXML.createParameterDescription(doc, CREATOR_PARAM, getTextString("param." + CREATOR_PARAM, lang), GSXML.PARAM_TYPE_STRING, null, null, null); param_list.appendChild(param); param = GSXML.createParameterDescription(doc, NEW_COL_ABOUT_PARAM, getTextString("param." + NEW_COL_ABOUT_PARAM, lang), GSXML.PARAM_TYPE_TEXT, null, null, null); param_list.appendChild(param); String[] types = { BUILDTYPE_MGPP, BUILDTYPE_MG }; String[] type_texts = { getTextString("param." + BUILDTYPE_PARAM + "." + BUILDTYPE_MGPP, lang), getTextString("param." + BUILDTYPE_PARAM + "." + BUILDTYPE_MG, lang) }; param = GSXML.createParameterDescription(doc, BUILDTYPE_PARAM, getTextString("param." + BUILDTYPE_PARAM, lang), GSXML.PARAM_TYPE_ENUM_SINGLE, BUILDTYPE_MGPP, types, type_texts); param_list.appendChild(param); } else if (service.equals(ACTIVATE_SERVICE) || service.equals(IMPORT_SERVICE) || service.equals(BUILD_SERVICE) || service.equals(RELOAD_SERVICE) || service.equals(DELETE_SERVICE) || service.equals(MODIFY_METADATA_SERVICE)) { Element param = GSXML.createParameterDescription(doc, COL_PARAM, getTextString("param." + COL_PARAM, lang), GSXML.PARAM_TYPE_ENUM_SINGLE, null, this.collection_list, this.collection_list); param_list.appendChild(param); } else { // invalid service name return null; } } return description; } // each service must have a method "process" protected Element processNewCollection(Element request) { if (!userHasCollectionCreationPermissions(request)) { String lang = request.getAttribute(GSXML.LANG_ATT); return errorResponse("processNewCollection", NO_PERMISSIONS_ERROR, lang); } return runCommand(request, GS2PerlConstructor.NEW); } /** TODO:implement this */ protected Element processAddDocument(Element request) { if (!userHasCollectionEditPermissions(request)) { String lang = request.getAttribute(GSXML.LANG_ATT); return errorResponse("processAddDocument", NO_PERMISSIONS_ERROR, lang); } Document result_doc = XMLConverter.newDOM(); // decode the file name, add it to the import directory String name = GSPath.getFirstLink(request.getAttribute(GSXML.TO_ATT)); Element response = result_doc.createElement(GSXML.RESPONSE_ELEM); response.setAttribute(GSXML.FROM_ATT, name); Element status = result_doc.createElement(GSXML.STATUS_ELEM); response.appendChild(status); //String lang = request.getAttribute(GSXML.LANG_ATT); //String request_type = request.getAttribute(GSXML.TYPE_ATT); Text t = result_doc.createTextNode("AddDocument: not implemented yet"); status.appendChild(t); status.setAttribute(GSXML.STATUS_ERROR_CODE_ATT, Integer.toString(GSStatus.ERROR)); return response; } protected Element processBuildAndActivateCollection(Element request) { if (isStatusRequest(request)) { return runStatusRequest(request); } // check permissions if (!userHasCollectionEditPermissions(request)) { String lang = request.getAttribute(GSXML.LANG_ATT); return errorResponse("processBuildAndActivateCollection", NO_PERMISSIONS_ERROR, lang); } waitUntilReady(request); Element buildResponse = processBuildCollection(request); if (buildResponse.getElementsByTagName(GSXML.ERROR_ELEM).getLength() > 0) { signalReady(request); return buildResponse; } Element statusElem = (Element) buildResponse.getElementsByTagName(GSXML.STATUS_ELEM).item(0); String id = statusElem.getAttribute("pid"); GS2PerlListener currentListener = this.listeners.get(id); int statusCode = currentListener.getStatus(); while (!GSStatus.isCompleted(statusCode)) { // wait for the process, and keep checking the status code // there is probably a better way to do this. try { Thread.currentThread().sleep(100); } catch (Exception e) { // ignore } statusCode = currentListener.getStatus(); } Element activateResponse = processActivateCollection(request); signalReady(request); return activateResponse; } protected Element processImportCollection(Element request) { if (isStatusRequest(request)) { return runStatusRequest(request); } if (!userHasCollectionEditPermissions(request)) { String lang = request.getAttribute(GSXML.LANG_ATT); return errorResponse("processImportCollection", NO_PERMISSIONS_ERROR, lang); } Element param_list = (Element) GSXML.getChildByTagName(request, GSXML.PARAM_ELEM + GSXML.LIST_MODIFIER); HashMap params = GSXML.extractParams(param_list, false); if (params == null) { return null; } //If we have been requested to only build certain documents then we need to create a manifest file String documentsParam = (String) params.get("documents"); if (documentsParam != null && !documentsParam.equals("")) { String s = File.separator; String manifestFolderPath = this.site_home + s + "collect" + s + params.get(COL_PARAM) + s + "manifests"; String manifestFilePath = manifestFolderPath + File.separator + "tempManifest.xml"; File manifestFolderFile = new File(manifestFolderPath); if (!manifestFolderFile.exists()) { manifestFolderFile.mkdirs(); } File manifestFile = new File(manifestFilePath); if (!manifestFile.exists()) { try { manifestFile.createNewFile(); } catch (Exception ex) { ex.printStackTrace(); return null; //Probably should return an actual error } } String[] docList = documentsParam.split(","); try { BufferedWriter bw = new BufferedWriter(new FileWriter(manifestFile)); bw.write("\n"); bw.write("\n"); bw.write(" \n"); for (int j = 0; j < docList.length; j++) { bw.write(" " + docList[j] + "\n"); } bw.write(" \n"); bw.write("\n"); bw.close(); } catch (Exception ex) { ex.printStackTrace(); return null; //Probably should return an actual error } } return runCommand(request, GS2PerlConstructor.IMPORT); } protected Element processBuildCollection(Element request) { if (isStatusRequest(request)) { return runStatusRequest(request); } if (!userHasCollectionEditPermissions(request)) { String lang = request.getAttribute(GSXML.LANG_ATT); return errorResponse("processBuildCollection", NO_PERMISSIONS_ERROR, lang); } return runCommand(request, GS2PerlConstructor.BUILD); } protected Element processModifyMetadata(Element request) { // There are now 3 types of operations whereby metadata gets modified: // - document including document-meta editing: user needs document editing powers // - adding user comments: user just needs an account and needs to be logged in // - removing user comments: user needs to be in administrator group // We handle all 3 cases in this service. Element param_list = (Element) GSXML.getChildByTagName(request, GSXML.PARAM_ELEM + GSXML.LIST_MODIFIER); HashMap params = GSXML.extractParams(param_list, false); String metaserver_command = (String) params.get("a"); // e.g. set-archives-metadata or set-metadata-array boolean supportsSettingMultipleMeta = metaserver_command.equals("set-metadata-array") ? true : false; String json_str = (String) params.get("json"); // String or null if no param named "json" String[] docids = null; // For user comments (setting or removing), these are the allowed metadata fields Pattern allowedMetaFieldsPattern = Pattern.compile("^(username|usertimestamp|usercomment)$"); String lang = request.getAttribute(GSXML.LANG_ATT); String collection = (String)params.get(COL_PARAM); if (userHasCollectionEditPermissions(request, collection)) { // means user can modify ANY metadata if (metaserver_command.equals("remove-metadata-array")) { // check if only removing user comments metadata fields docids = getDocIdsWithOptFilter(json_str, allowedMetaFieldsPattern); if(docids == null) { // removing multiple metadata that are Not user comments // can be done by any user with collection edit permissions docids = getDocIdsWithOptFilter(json_str, null); } } // if dealing with an array of meta, then parse out the docids from the json else if(supportsSettingMultipleMeta) { docids = getDocIdsWithOptFilter(json_str, null); } // else set-meta operation on single metadata field of single doc, // and docid will be obtained in runCommand() where it's needed } else { // check if user logged in // shouldn't be able to do any meta modification if not logged in UserContext context = new UserContext(request); if (context.getUsername().equals("")) { return errorResponse("processModifyMetadata", NO_METADATA_PERMISSIONS_ERROR, lang); } else { // User is logged in at least, see whether they can do any restricted set-meta ops // that are open to regular users (those without permissions to edit this collection). // For now, there's only one restricted set-meta operation open to any logged in users // who don't otherwise have editing permissions for the collection: adding user comments. boolean isAddingUserComments = false; if(supportsSettingMultipleMeta) { docids = getDocIdsWithOptFilter(json_str, allowedMetaFieldsPattern); if(docids != null) { isAddingUserComments = true; } } else { String metaname = (String) params.get("metaname"); if(isAllowedToModifyMeta(metaname, allowedMetaFieldsPattern)) { isAddingUserComments = true; } } if(!isAddingUserComments) { // logged in user is attempting to set meta outside restricted set, // In this case, they're attempting to set meta not related to user comments return errorResponse("processModifyMetadata", NO_PERMISSIONS_ERROR, lang); } } } // wait until we can reserve the collection for processing waitUntilReady(request); // process Element response = runCommand(request, GS2PerlConstructor.MODIFY_METADATA_SERVER, docids); if (response.getElementsByTagName(GSXML.ERROR_ELEM).getLength() <= 0) // if no errors, wait for process to finish { Element statusElem = (Element) response.getElementsByTagName(GSXML.STATUS_ELEM).item(0); String id = statusElem.getAttribute("pid"); GS2PerlListener currentListener = this.listeners.get(id); int statusCode = currentListener.getStatus(); while (!GSStatus.isCompleted(statusCode)) { // wait for the process, and keep checking the status code // there is probably a better way to do this. try { Thread.currentThread().sleep(100); } catch (Exception e) { // ignore } statusCode = currentListener.getStatus(); } } Element statusElem = (Element) response.getElementsByTagName(GSXML.STATUS_ELEM).item(0); String statusString = GSXML.getNodeText(statusElem); statusString += " and monitored until done."; // check for errors int initial_response_status_code = Integer.parseInt(statusElem.getAttribute(GSXML.STATUS_ERROR_CODE_ATT)); if (GSStatus.isError(initial_response_status_code)) { logger.info("Got error status code: " + initial_response_status_code); statusString += "But got error status code: " + initial_response_status_code; } else { // check for Construction event errors String id = statusElem.getAttribute("pid"); GS2PerlListener currentListener = this.listeners.get(id); int completed_status_code = currentListener.getStatus(); // update statusElem so we return the status code on completion statusElem.setAttribute(GSXML.STATUS_ERROR_CODE_ATT, Integer.toString(completed_status_code)); if (GSStatus.isError(completed_status_code)) { logger.info("Got construction event error. Error status code: " + completed_status_code); statusString += "But got construction event error, status code: " + completed_status_code; // add the rest of the messages to the statusElem node statusString += "\n" + currentListener.getUpdate(); } } // can finally set statusString GSXML.setNodeText(statusElem, statusString); // release hold on collection signalReady(request); return response; } protected Element processActivateCollection(Element request) { if (isStatusRequest(request)) { return runStatusRequest(request); } String lang = request.getAttribute(GSXML.LANG_ATT); if (!userHasCollectionEditPermissions(request)) { return errorResponse("processActivateCollection", NO_PERMISSIONS_ERROR, lang); } // this activates the collection on disk. but now we need to tell // the MR about it. but we have to wait until the process is finished. Element param_list = (Element) GSXML.getChildByTagName(request, GSXML.PARAM_ELEM + GSXML.LIST_MODIFIER); HashMap params = GSXML.extractParams(param_list, false); String coll_name = (String) params.get(COL_PARAM); UserContext userContext = new UserContext(request); String request_type = request.getAttribute(GSXML.TYPE_ATT); // now we de-activate the collection before running activate.pl, and then re-activate at end // So activate.pl only does the moving, no activation. This way will prevent java from launching // perl, exiting and then leaving dangling file handles (on index/text/col.gdb) in perl. if (!request_type.equals(GSXML.REQUEST_TYPE_STATUS)) { systemRequest("delete", coll_name, null, userContext); // deactivate collection } Element response = runCommand(request, GS2PerlConstructor.ACTIVATE); // if request is for STATUS, then this won't run activate.pl Element status = (Element) GSXML.getChildByTagName(response, GSXML.STATUS_ELEM); // check for finished int status_code = Integer.parseInt(status.getAttribute(GSXML.STATUS_ERROR_CODE_ATT)); if (GSStatus.isCompleted(status_code) && GSStatus.isError(status_code)) { // we shouldn't carry out the next bit, just return the response return response; } String id = status.getAttribute(GSXML.STATUS_PROCESS_ID_ATT); GS2PerlListener listener = this.listeners.get(id); if (listener == null) { logger.error("somethings gone wrong, couldn't find the listener"); status.setAttribute(GSXML.STATUS_ERROR_CODE_ATT, Integer.toString(GSStatus.ERROR)); return response; } while (!GSStatus.isCompleted(status_code)) { // wait for the process, and keep checking the status code // there is probably a better way to do this. try { Thread.currentThread().sleep(100); } catch (Exception e) { // ignore } status_code = listener.getStatus(); } // add the rest of the messages to the status node Text t = status.getOwnerDocument().createTextNode("\n" + listener.getUpdate()); status.appendChild(t); status.setAttribute(GSXML.STATUS_ERROR_CODE_ATT, Integer.toString(listener.getStatus())); if (GSStatus.isError(status_code)) { return response; // without doing the next bit } t = status.getOwnerDocument().createTextNode("\n"); status.appendChild(t); // once have got here, we assume // the first bit proceeded successfully, now reload the collection (sends a collection reactivation request) systemRequest("reload", coll_name, status, userContext); // this will append more messages to the status, and overwrite the error code att return response; } protected Element processDeleteCollection(Element request) { if (!userHasCollectionEditPermissions(request)) { String lang = request.getAttribute(GSXML.LANG_ATT); return errorResponse("processDeleteCollection", NO_PERMISSIONS_ERROR, lang); } Document result_doc = XMLConverter.newDOM(); // the response to send back String name = GSPath.getFirstLink(request.getAttribute(GSXML.TO_ATT)); Element response = result_doc.createElement(GSXML.RESPONSE_ELEM); response.setAttribute(GSXML.FROM_ATT, name); Element status = result_doc.createElement(GSXML.STATUS_ELEM); response.appendChild(status); Text t = null; // the text node for the error/success message String lang = request.getAttribute(GSXML.LANG_ATT); String request_type = request.getAttribute(GSXML.TYPE_ATT); Element param_list = (Element) GSXML.getChildByTagName(request, GSXML.PARAM_ELEM + GSXML.LIST_MODIFIER); HashMap params = GSXML.extractParams(param_list, false); boolean get_status_only = false; if (request_type.equals(GSXML.REQUEST_TYPE_STATUS)) { get_status_only = true; } if (get_status_only) { // at the moment, delete is synchronous. but it may take ages so should do the command in another thread maybe? in which case we will want to ask for status logger.error("had a status request for delete - this shouldn't happen!!"); //t = result_doc.createTextNode(""); //status.appendChild(t); status.setAttribute(GSXML.STATUS_ERROR_CODE_ATT, Integer.toString(GSStatus.ERROR)); return response; } String coll_name = (String) params.get(COL_PARAM); String[] args = { coll_name }; File coll_dir = new File(GSFile.collectionBaseDir(this.site_home, coll_name)); // check that the coll is there in the first place if (!coll_dir.exists()) { t = result_doc.createTextNode(getTextString("delete.exists_error", lang, args)); status.appendChild(t); status.setAttribute(GSXML.STATUS_ERROR_CODE_ATT, Integer.toString(GSStatus.ERROR)); return response; } // try to delete the directory if (!GSFile.deleteFile(coll_dir)) { t = result_doc.createTextNode(getTextString("delete.delete_error", lang, args)); status.setAttribute(GSXML.STATUS_ERROR_CODE_ATT, Integer.toString(GSStatus.ERROR)); status.appendChild(t); return response; } UserContext userContext = new UserContext(request); systemRequest("delete", coll_name, status, userContext); return response; } protected Element processReloadCollection(Element request) { if (!userHasCollectionEditPermissions(request)) { String lang = request.getAttribute(GSXML.LANG_ATT); return errorResponse("processReloadCollection", NO_PERMISSIONS_ERROR, lang); } Document result_doc = XMLConverter.newDOM(); // the response to send back String name = GSPath.getFirstLink(request.getAttribute(GSXML.TO_ATT)); Element response = result_doc.createElement(GSXML.RESPONSE_ELEM); response.setAttribute(GSXML.FROM_ATT, name); Element status = result_doc.createElement(GSXML.STATUS_ELEM); response.appendChild(status); Text t = null; // the text node for the error/success message String lang = request.getAttribute(GSXML.LANG_ATT); String request_type = request.getAttribute(GSXML.TYPE_ATT); Element param_list = (Element) GSXML.getChildByTagName(request, GSXML.PARAM_ELEM + GSXML.LIST_MODIFIER); HashMap params = GSXML.extractParams(param_list, false); boolean get_status_only = false; if (request_type.equals(GSXML.REQUEST_TYPE_STATUS)) { get_status_only = true; } if (get_status_only) { // reload is synchronous - this makes no sense logger.error("had a status request for reload - this shouldn't happen!!"); //t = result_doc.createTextNode(""); //status.appendChild(t); status.setAttribute(GSXML.STATUS_ERROR_CODE_ATT, Integer.toString(GSStatus.ERROR)); return response; } String coll_name = (String) params.get(COL_PARAM); UserContext userContext = new UserContext(request); systemRequest("reload", coll_name, status, userContext); return response; } /** * send a configure request to the message router action name should be * "delete" or "reload" response will be put into the status element */ protected void systemRequest(String operation, String coll_name, Element status, UserContext userContext) { // send the request to the MR Document doc = XMLConverter.newDOM(); Element message = doc.createElement(GSXML.MESSAGE_ELEM); Element request = GSXML.createBasicRequest(doc, GSXML.REQUEST_TYPE_SYSTEM, "", userContext); message.appendChild(request); Element command = doc.createElement(GSXML.SYSTEM_ELEM); request.appendChild(command); command.setAttribute(GSXML.SYSTEM_MODULE_TYPE_ATT, GSXML.COLLECTION_ELEM); command.setAttribute(GSXML.SYSTEM_MODULE_NAME_ATT, coll_name); if (operation.equals("delete")) { command.setAttribute(GSXML.TYPE_ATT, GSXML.SYSTEM_TYPE_DEACTIVATE); } else if (operation.equals("reload")) { command.setAttribute(GSXML.TYPE_ATT, GSXML.SYSTEM_TYPE_ACTIVATE); } else { logger.error("invalid action name passed to systemRequest:" + operation); return; } request.appendChild(command); Node response = this.router.process(message); // at the moment, get no info in response so ignore it Text t; String[] args = { coll_name }; if (status != null) { if (response == null) { t = status.getOwnerDocument().createTextNode(getTextString(operation + ".configure_error", userContext.getLanguage(), args)); status.setAttribute(GSXML.STATUS_ERROR_CODE_ATT, Integer.toString(GSStatus.ERROR)); status.appendChild(t); return; } // if we got here, we have succeeded! t = status.getOwnerDocument().createTextNode(getTextString(operation + ".success", userContext.getLanguage(), args)); status.setAttribute(GSXML.STATUS_ERROR_CODE_ATT, Integer.toString(GSStatus.SUCCESS)); status.appendChild(t); } } /** * configure the service module for now, all services have type=build - need * to think about this */ // in configure we set up any params that should be saved to the session. At this stage, assuming none should be saved. public boolean configure(Element info, Element extra_info) { if (!super.configure(info, extra_info)) { return false; } logger.info("configuring GS2Construct"); // read thru collect folder and set up collectionList this.collection_list = getCollectionList(); Element e = null; // hard code in the services for now // set up short_service_info_ - for now just has name and type e = this.desc_doc.createElement(GSXML.SERVICE_ELEM); e.setAttribute(GSXML.TYPE_ATT, GSXML.SERVICE_TYPE_PROCESS); e.setAttribute(GSXML.NAME_ATT, NEW_SERVICE); this.short_service_info.appendChild(e); e = this.desc_doc.createElement(GSXML.SERVICE_ELEM); e.setAttribute(GSXML.TYPE_ATT, GSXML.SERVICE_TYPE_PROCESS); e.setAttribute(GSXML.NAME_ATT, IMPORT_SERVICE); this.short_service_info.appendChild(e); e = this.desc_doc.createElement(GSXML.SERVICE_ELEM); e.setAttribute(GSXML.TYPE_ATT, GSXML.SERVICE_TYPE_PROCESS); e.setAttribute(GSXML.NAME_ATT, BUILD_SERVICE); this.short_service_info.appendChild(e); e = this.desc_doc.createElement(GSXML.SERVICE_ELEM); e.setAttribute(GSXML.TYPE_ATT, GSXML.SERVICE_TYPE_PROCESS); e.setAttribute(GSXML.NAME_ATT, ACTIVATE_SERVICE); this.short_service_info.appendChild(e); e = this.desc_doc.createElement(GSXML.SERVICE_ELEM); e.setAttribute(GSXML.TYPE_ATT, GSXML.SERVICE_TYPE_PROCESS); e.setAttribute(GSXML.NAME_ATT, BUILD_AND_ACTIVATE_SERVICE); this.short_service_info.appendChild(e); e = this.desc_doc.createElement(GSXML.SERVICE_ELEM); e.setAttribute(GSXML.TYPE_ATT, GSXML.SERVICE_TYPE_PROCESS); e.setAttribute(GSXML.NAME_ATT, DELETE_SERVICE); this.short_service_info.appendChild(e); e = this.desc_doc.createElement(GSXML.SERVICE_ELEM); e.setAttribute(GSXML.TYPE_ATT, GSXML.SERVICE_TYPE_PROCESS); e.setAttribute(GSXML.NAME_ATT, RELOAD_SERVICE); this.short_service_info.appendChild(e); //e = this.desc_doc.createElement(GSXML.SERVICE_ELEM); //e.setAttribute(GSXML.TYPE_ATT, GSXML.SERVICE_TYPE_PROCESS); //e.setAttribute(GSXML.NAME_ATT, ADD_DOC_SERVICE); //this.short_service_info.appendChild(e); e = this.desc_doc.createElement(GSXML.SERVICE_ELEM); e.setAttribute(GSXML.TYPE_ATT, GSXML.SERVICE_TYPE_PROCESS); e.setAttribute(GSXML.NAME_ATT, MODIFY_METADATA_SERVICE); this.short_service_info.appendChild(e); return true; } protected Element runStatusRequest(Element request) { Document result_doc = XMLConverter.newDOM(); // the response to send back String name = GSPath.getFirstLink(request.getAttribute(GSXML.TO_ATT)); Element response = result_doc.createElement(GSXML.RESPONSE_ELEM); response.setAttribute(GSXML.FROM_ATT, name); Element status = result_doc.createElement(GSXML.STATUS_ELEM); response.appendChild(status); String lang = request.getAttribute(GSXML.LANG_ATT); Element param_list = (Element) GSXML.getChildByTagName(request, GSXML.PARAM_ELEM + GSXML.LIST_MODIFIER); HashMap params = GSXML.extractParams(param_list, false); String id = (String) params.get(PROCESS_ID_PARAM); status.setAttribute(GSXML.STATUS_PROCESS_ID_ATT, id); GS2PerlListener listener = this.listeners.get(id); if (listener == null) { Text t = result_doc.createTextNode(getTextString("general.process_id_error", lang)); status.appendChild(t); status.setAttribute(GSXML.STATUS_ERROR_CODE_ATT, Integer.toString(GSStatus.ERROR)); } else { Text t = result_doc.createTextNode(listener.getUpdate()); status.appendChild(t); status.setAttribute(GSXML.STATUS_ERROR_CODE_ATT, Integer.toString(listener.getStatus())); // check that we actually should be removing the listener here if (listener.isFinished()) { // remove this listener - its job is done this.listeners.remove(id); // not working } } return response; } protected Element runCommand(Element request, int type) { return runCommand(request, type, null); } /** returns a response element */ protected Element runCommand(Element request, int type, String[] docids) { if (isStatusRequest(request)) { // leave breadcrumbs in case we should have checked in other places for // status request which we didn't System.err.println("runCommand receieved a status request, which it shouldn't, type="+type); return runStatusRequest(request); } Document result_doc = XMLConverter.newDOM(); // the response to send back String name = GSPath.getFirstLink(request.getAttribute(GSXML.TO_ATT)); Element response = result_doc.createElement(GSXML.RESPONSE_ELEM); response.setAttribute(GSXML.FROM_ATT, name); Element status = result_doc.createElement(GSXML.STATUS_ELEM); response.appendChild(status); String lang = request.getAttribute(GSXML.LANG_ATT); String request_type = request.getAttribute(GSXML.TYPE_ATT); Element param_list = (Element) GSXML.getChildByTagName(request, GSXML.PARAM_ELEM + GSXML.LIST_MODIFIER); //GSXML.elementToLogAsString("### Extracted param_list: ", param_list, true); //GSXML.elementToLogAsUnicodeDebugString("### DEBUG Extracted param_list: ", param_list, true); HashMap params = GSXML.extractParams(param_list, false); // do the actual command String coll_name = null; if (type == GS2PerlConstructor.NEW) { String coll_title = (String) params.get(NEW_COL_TITLE_PARAM); coll_name = createNewCollName(coll_title); } else { coll_name = (String) params.get(COL_PARAM); } // makes a paramList of the relevant params Element other_params = extractOtherParams(params, type); //create the constructor to do the work GS2PerlConstructor constructor = new GS2PerlConstructor("perl_build"); if (!constructor.configure()) { Text t = result_doc.createTextNode(getTextString("general.configure_constructor_error", lang)); status.appendChild(t); status.setAttribute(GSXML.STATUS_ERROR_CODE_ATT, Integer.toString(GSStatus.ERROR)); return response; } constructor.setSiteHome(this.site_home); constructor.setLibraryName(this.library_name); constructor.setCollectionName(coll_name); constructor.setActionType(type); constructor.setProcessParams(other_params); if (type == GS2PerlConstructor.IMPORT) { constructor.setManifestFile(this.site_home + File.separator + "collect" + File.separator + params.get(COL_PARAM) + File.separator + "manifests" + File.separator + "tempManifest.xml"); } else if (type == GS2PerlConstructor.MODIFY_METADATA_SERVER) { StringBuffer querystring = new StringBuffer(); // convert params into a single string again? Set> entries = params.entrySet(); Iterator> i = entries.iterator(); String oid = null; while (i.hasNext()) { Map.Entry entry = i.next(); String paramname = entry.getKey(); paramname = paramname.replace("s1.", ""); // replaces all occurrences if (paramname.equals("collection")) { paramname = "c"; } if (paramname.equals("d")){ oid = (String) entry.getValue(); } String paramvalue = (String) entry.getValue(); // This will be used to set QUERY_STRING in the environment, see build/GS2PerlConstructor.java // Need to also ensure that special characters in param values won't get clobbered on Windows by perl/CGI.pm, // see also https://www.nntp.perl.org/group/perl.perl5.porters/2016/10/msg240120.html // Therefore URL encode CGI param values in the query_string, as at https://stackoverflow.com/questions/10786042/java-url-encoding-of-query-string-parameters // perl/CGI.pm doesn't like us URL encoding the entire QUERY_STRING such as the equal sign between each paramName and paramValue. // So we URL encode each paramValue separately, which is done in GS2Construct.java::runCommand() querystring.append(paramname + "=" + urlEncodeValue(paramname, paramvalue)); if (i.hasNext()) { querystring.append("&"); } } // Mark files for reindexing (e.g. if set-meta or remove-meta was called) // Note that remove-meta doesn't mean the document should be marked for // Deletion: only meta was removed. if (oid != null) { // if we have only one oid markDocumentInFlatDatabase("R", coll_name, OID.getTop(oid)); } else if (docids != null) { // check if we are dealing with many doc ids, as cold in theory happen when set-metadata-array is called for(int index = 0; index < docids.length; index++) { String docid = docids[index]; markDocumentInFlatDatabase("R", coll_name, OID.getTop(docid)); } } else { String msg = getTextString("general.no_valid_docid_error", lang); logger.error("*** " + msg); Text t = result_doc.createTextNode(msg); status.appendChild(t); status.setAttribute(GSXML.STATUS_ERROR_CODE_ATT, Integer.toString(GSStatus.ERROR)); return response; } constructor.setQueryString(querystring.toString()); //logger.info("@@@@ perl querystring: " + querystring.toString()); } GS2PerlListener listener = new GS2PerlListener(); constructor.addListener(listener); constructor.start(); String id = newID(); this.listeners.put(id, listener); status.setAttribute(GSXML.STATUS_PROCESS_ID_ATT, id); status.setAttribute(GSXML.STATUS_ERROR_CODE_ATT, Integer.toString(GSStatus.ACCEPTED)); Text t = result_doc.createTextNode(getTextString("general.process_start", lang)); status.appendChild(t); return response; } //************************ // some helper functions //************************ private String urlEncodeValue(String paramName, String paramVal) { StringBuffer modParamVal = new StringBuffer(); try{ // don't do this, as it will double encode semicolons (and other such regular chars that will get URL encoded). // But then perl won't double-decode to get original semicolon back. // paramVal += URLEncoder.encode(oldParamVal, "UTF-8"); // Instead, just encode chars whose int value is >= 128 for(int i = 0; i < paramVal.length(); i++) { char c = paramVal.charAt(i); if(c >= 128) { modParamVal.append(URLEncoder.encode(""+c, "UTF-8")); // Better way as per https://stackoverflow.com/questions/10786042/java-url-encoding-of-query-string-parameters // but our release kits' Java is too old for importing java.nio.charset.StandardCharsets //paramVal = URLEncoder.encode(""+c, StandardCharsets.UTF_8.name()); } else { modParamVal.append(c); } } //logger.error("@@@@@@@@ old paramVal for paramName " + paramName + "is: |" + paramVal + "|"); paramVal = modParamVal.toString(); //logger.error("@@@@@@@@ new param val for paramName " + paramName + " is: |" + paramVal + "|"); } catch(UnsupportedEncodingException uee) { logger.warn("**** Unable to encode query_string param " + paramName + " in UTF-8, so attempting to continue with its unencoded value."); // don't output param value to log, in case of sensitive data? } return paramVal; } /** parse the collect directory and return a list of collection names */ protected String[] getCollectionList() { File collectDir = new File(GSFile.collectDir(this.site_home)); if (!collectDir.exists()) { logger.error("couldn't find collect dir: " + collectDir.toString()); return null; } logger.info("GS2Construct: reading thru directory " + collectDir.getPath() + " to find collections."); File[] contents = collectDir.listFiles(); int num_colls = 0; for (int i = 0; i < contents.length; i++) { if (contents[i].isDirectory() && !contents[i].getName().startsWith("CVS")) { num_colls++; } } String[] names = new String[num_colls]; for (int i = 0, j = 0; i < contents.length; i++) { if (contents[i].isDirectory()) { String colName = contents[i].getName(); if (!colName.startsWith("CVS")) { names[j] = colName; j++; } } } return names; } /** ids used for process id */ private int current_id = 0; private String newID() { current_id++; return Integer.toString(current_id); } /** creates a new short name from the collection title */ protected String createNewCollName(String coll_title) { String base_name = null; // take the first 6 letters if (coll_title.length() < 6) { base_name = coll_title; } else { base_name = coll_title.substring(0, 6); } File coll_dir = new File(GSFile.collectionBaseDir(this.site_home, base_name)); if (!coll_dir.exists()) { // this name is ok - not used yet return base_name; } // now we have to make a new name until we get a good one // try name1, name2 name3 etc int i = 0; while (coll_dir.exists()) { i++; coll_dir = new File(GSFile.collectionBaseDir(this.site_home, base_name + Integer.toString(i))); } return base_name + Integer.toString(i); } /** * takes the params from the request (in the HashMap) and extracts any that * need to be passed to the constructor and puts them into a paramList * element */ protected Element extractOtherParams(HashMap params, int type) { Document doc = XMLConverter.newDOM(); Element param_list = doc.createElement(GSXML.PARAM_ELEM + GSXML.LIST_MODIFIER); if (type == GS2PerlConstructor.NEW) { Element param = doc.createElement(GSXML.PARAM_ELEM); param.setAttribute(GSXML.NAME_ATT, "creator"); param.setAttribute(GSXML.VALUE_ATT, (String) params.get(CREATOR_PARAM)); param_list.appendChild(param); param = doc.createElement(GSXML.PARAM_ELEM); param.setAttribute(GSXML.NAME_ATT, "about"); param.setAttribute(GSXML.VALUE_ATT, (String) params.get(NEW_COL_ABOUT_PARAM)); param_list.appendChild(param); param = doc.createElement(GSXML.PARAM_ELEM); param.setAttribute(GSXML.NAME_ATT, "title"); param.setAttribute(GSXML.VALUE_ATT, (String) params.get(NEW_COL_TITLE_PARAM)); param_list.appendChild(param); param = doc.createElement(GSXML.PARAM_ELEM); param.setAttribute(GSXML.NAME_ATT, "buildtype"); param.setAttribute(GSXML.VALUE_ATT, (String) params.get(BUILDTYPE_PARAM)); param_list.appendChild(param); return param_list; } // other ones dont have params yet return null; } protected void waitUntilReady(Element request) { Element param_list = (Element) GSXML.getChildByTagName(request, GSXML.PARAM_ELEM + GSXML.LIST_MODIFIER); HashMap params = GSXML.extractParams(param_list, false); String collection = (String) params.get(COL_PARAM); if (checkCollectionIsNotBusy(collection)) { return; } while (!checkCollectionIsNotBusy(collection)) // When the collection ceases to be busy, we place a hold on it { try { Thread.currentThread().sleep(1000); } catch (Exception ex) { ex.printStackTrace(); } } } protected void signalReady(Element request) { Element param_list = (Element) GSXML.getChildByTagName(request, GSXML.PARAM_ELEM + GSXML.LIST_MODIFIER); HashMap params = GSXML.extractParams(param_list, false); String collection = (String) params.get(COL_PARAM); collectionOperationMap.remove(collection); } // If collection is NOT busy, then reserve it protected synchronized boolean checkCollectionIsNotBusy(String collection) { if (collectionOperationMap.get(collection) == null) { collectionOperationMap.put(collection, true); return true; } return false; } protected boolean isAllowedToModifyMeta(String metaname, Pattern allowedMetaFieldsPattern) { if(allowedMetaFieldsPattern == null) { // null when user has edit permissions, so they can set any meta ///logger.info("### User has permissions to set any meta."); return true; } if(metaname == null) { ///logger.info("### Can't check null metaname against pattern"); return false; } Matcher m = allowedMetaFieldsPattern.matcher(metaname); if(!m.matches()) { ///logger.info("### metaname: " + metaname + " doesn't match allowed allowed fields: " + allowedMetaFieldsPattern.toString()); return false; } else { return true; } } protected String[] getDocIdsWithOptFilter(String json_str, Pattern filterFields) // boolean strictOrPermissible { if(json_str == null) { logger.error("### Shouldn't be happening: null json string"); return null; } String[] docids = null; // check that the name of each metadata being set matches the pattern filterFields. // The presence of any other meta means something other than adding user comments is being attempted, // which is invalid try { JSONArray docInfoList = new JSONArray(json_str); docids = new String[docInfoList.length()]; for(int index = 0; index < docInfoList.length(); index++) { JSONObject docInfo = docInfoList.getJSONObject(index); if(docInfo.has("metatable")) { // should exist for metadata arrays docids[index] = docInfo.getString("docid"); // should exist if metatable existed ///logger.info("@@@ Found docid: " + docids[index]); JSONArray metatable = docInfo.getJSONArray("metatable"); for(int i = 0; i < metatable.length(); i++) { JSONObject meta = metatable.getJSONObject(i); String metaname = meta.getString("metaname"); ///logger.info("### metaname: " + metaname); if(!isAllowedToModifyMeta(metaname, filterFields)) { return null; } } } else { // can also have set-metadatat-array without metatable field, like // json=[{"docid":"HASHc5bce2d6d3e5b04e470ec9","metaname":"Title","metavalue":"Tralalala","metamode":"accumulate"},{"docid":"HASHbe483fa4df4e096335d1c8","metaname":"Title","metavalue":"Lala was here","metapos":0, "metamode":"override"}] // though we use the metatable way for setting user comments meta. // The alternative way is described as example usage in perllib/cgiactions/modmetadataaction.pm // and also the way it's already coded up there, so we need to allow it here. docids[index] = docInfo.getString("docid"); String metaname = docInfo.getString("metaname"); if(!isAllowedToModifyMeta(metaname, filterFields)) { return null; } } } } catch(JSONException jsonex) { logger.error("Exception when parsing json string: " + json_str); logger.error(jsonex); } // if we're here, then it means that the JSON only asked for username|usercomment|usertimestamp meta // meaning that the setmeta operation was a valid user comment operation. // In that case, we have a docid for which we need to add a user comment // set-metadata-array can take more docids, but that doesn't happen when creating a user // comment (though it now happens when editing existing user comments). And one comment is // added at a time, but 3 meta fields are set for each comment: username, usercomment and // timestamp, hence the use of set-meta-array. return docids; } protected Element errorResponse(String serviceName, String errorKey, String lang) { String error_message = getTextString(errorKey, lang, Dictionary.core_servlet_dictionary_name); Document result_doc = XMLConverter.newDOM(); Element result = GSXML.createBasicResponse(result_doc, serviceName); GSXML.addError(result, error_message); return result; } protected boolean isStatusRequest(Element request) { String request_type = request.getAttribute(GSXML.TYPE_ATT); if (request_type.equals(GSXML.REQUEST_TYPE_STATUS)) return true; return false; } protected boolean userIsAdministrator(Element request) { UserContext context = new UserContext(request); return GroupsUtil.isAdministrator(context.getGroupsString()); } protected boolean userHasCollectionCreationPermissions(Element request) { UserContext context = new UserContext(request); String groups = context.getGroupsString(); return GroupsUtil.canCreateCollection(groups); } protected boolean userHasCollectionEditPermissions(Element request) { Element param_list = (Element) GSXML.getChildByTagName(request, GSXML.PARAM_ELEM + GSXML.LIST_MODIFIER); HashMap params = GSXML.extractParams(param_list, false); String collection = (String)params.get(COL_PARAM); if (collection == null) return false; return userHasCollectionEditPermissions(request, collection); } protected boolean userHasCollectionEditPermissions(Element request, String collection) { UserContext context = new UserContext(request); return GroupsUtil.canEditCollection(context.getUsername(), context.getGroupsString(), collection); } protected void markDocumentInFlatDatabase(String mark, String collection, String oid) { Document msg_doc = XMLConverter.newDOM(); Element message = msg_doc.createElement(GSXML.MESSAGE_ELEM); UserContext userContext = new UserContext(); Element query_request = GSXML.createBasicRequest(msg_doc, GSXML.REQUEST_TYPE_DESCRIBE , collection, userContext); message.appendChild(query_request); Element result = (Element) this.router.process(message); Element resp_elem = (Element) GSXML.getChildByTagName(result, GSXML.RESPONSE_ELEM); Element coll_elem = (Element) GSXML.getChildByTagName(resp_elem, GSXML.COLLECTION_ELEM); String dbtype = coll_elem.getAttribute(GSXML.DB_TYPE_ATT); SimpleCollectionDatabase coll_db = new SimpleCollectionDatabase(dbtype); if (!coll_db.databaseOK()) { logger.error("Couldn't create the collection database of type " + dbtype); return; } // Open database for reading. It may not exist if collection is pre-built without archives (such as demo collections) String coll_db_file = GSFile.archivesDatabaseFile(this.site_home, collection, dbtype); if (!coll_db.openDatabase(coll_db_file, SimpleCollectionDatabase.READ)) { logger.error("Could not open collection archives database. Database doesn't exist or else somebody's already using it?"); return; } // now we know we have an archives folder String old_value = coll_db.getValue(oid); String new_value = "" + mark; if(old_value == null) { logger.error("### null old_value in flat DB for oid " + oid); } else { new_value = old_value.replace("B", "" + mark); logger.info("### Replacing db entry for oid " + oid + " which has old_value " + old_value); logger.info("### with new value " + new_value); } // Close database for reading coll_db.closeDatabase(); if (!coll_db.openDatabase(coll_db_file, SimpleCollectionDatabase.WRITE)) { logger.error("Could not open collection archives database. Somebody already using this database!"); return; } coll_db.setValue(oid, new_value); coll_db.closeDatabase(); } }