<?xml version="1.0" encoding="UTF-8" ?>

<!--
 Licensed to the Apache Software Foundation (ASF) under one or more
 contributor license agreements.  See the NOTICE file distributed with
 this work for additional information regarding copyright ownership.
 The ASF licenses this file to You under the Apache License, Version 2.0
 (the "License"); you may not use this file except in compliance with
 the License.  You may obtain a copy of the License at

     http://www.apache.org/licenses/LICENSE-2.0

 Unless required by applicable law or agreed to in writing, software
 distributed under the License is distributed on an "AS IS" BASIS,
 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
-->

<!-- This is the basis of a schema for a greenstone collection.
     It will get filled in by solrbuilder.pm and copied to the data folder -->

<!--
    For more information, on how to customize this file, please see
    https://solr.apache.org/guide/solr/latest/indexing-guide/schema-elements.html

PERFORMANCE NOTE: To improve performance one could
  - set stored="false" for all fields possible (esp large fields) when you
    only need to search on the field but don't need to return the original
    value.
  - set indexed="false" if you don't need to search on the field, but only
    return the field as a result of searching on other indexed fields.
  - remove all unneeded copyField statements
  - for best index size and searching performance, set "index" to false
    for all general text fields, use copyField to copy them to the
    catchall "text" field, and use that for searching.
-->

<schema name="gs-default" version="1.7">

     <!-- Valid attributes for fields:
     name: mandatory - the name for the field
     type: mandatory - the name of a field type from the
       fieldTypes section
     indexed: true if this field should be indexed (searchable or sortable)
     stored: true if this field should be retrievable
     docValues: true if this field should have doc values. Doc Values is
       recommended (required, if you are using *Point fields) for faceting,
       grouping, sorting and function queries. Doc Values will make the index
       faster to load, more NRT-friendly and more memory-efficient.
       They are currently only supported by StrField, UUIDField, all
       *PointFields, and depending on the field type, they might require
       the field to be single-valued, be required or have a default value
       (check the documentation of the field type you're interested in for
       more information)
     multiValued: true if this field may contain multiple values per document
     omitNorms: (expert) set to true to omit the norms associated with
       this field (this disables length normalization and index-time
       boosting for the field, and saves some memory).  Only full-text
       fields or fields that need an index-time boost need norms.
       Norms are omitted for primitive (non-analyzed) types by default.
     termVectors: [false] set to true to store the term vector for a
       given field.
       When using MoreLikeThis, fields used for similarity should be
       stored for best performance.
     termPositions: Store position information with the term vector.
       This will increase storage costs.
     termOffsets: Store offset information with the term vector. This
       will increase storage costs.
     required: The field is required.  It will throw an error if the
       value does not exist
     default: a value that should be used if no value is specified
       when adding a document.
    -->

    <!-- field names should consist of alphanumeric or underscore characters only and
      not start with a digit.  This is not currently strictly enforced,
      but other field names will not have first class support from all components
      and back compatibility is not guaranteed.  Names with both leading and
      trailing underscores (e.g. _version_) are reserved.
    -->

    <!-- predefined fields -->

    <!-- field used by solr -->
    <!-- docValues are enabled by default for long type so we don't need to index the version field  -->
    <field name="_version_" type="plong" indexed="false" stored="false"/>

    <!-- all docs have our unique field of docOID -->
    <field name="docOID" type="string" indexed="true" stored="true" required="true"/>

    <!-- if the user has specified 'allfields' then this ZZ field is used to catch all the data from the other fields. If there is no allfields index, then no copyFields will be added to the schema, so this will end up with nothing in it -->
    <field name="ZZ" type="text_en_splitting" indexed="true" stored="false" multiValued="true"/>
    
    <!-- dynamic field for sorting - not tokenised, and single valued. using docValues speeds up sorting and faceting -->
    <dynamicField name="by*" type="string" indexed="true" stored="false" docValues="true"/>
    <!-- dynamic field for facets - not tokenised, and multi-valued valued 
         ( hence strings type) -->
    <dynamicField name="fc*" type="strings" indexed="true" stored="false" docValues="true" />
    <!-- dynamic field for metadata that wasn't predefined. eg if the user specified 'metadata' in the index-->
    <dynamicField name="*_m" type="text_en_splitting" indexed="true" stored="true" multiValued="true"/>
    
    <!-- collection specific fields added by solrbuilder -->
    <!-- ##GREENSTONE-FIELDS## -->
    <!-- end coll specific fields -->
    
    <!-- Field to use to determine and enforce document uniqueness.
         Unless this field is marked with required="false", it will be a required field
    -->
    <uniqueKey>docOID</uniqueKey>

    <!-- copyField commands copy one field to another at the time a document
       is added to the index.  It's used either to index the same field differently,
       or to add multiple fields to the same field for easier/faster searching. -->

    <!-- TODO - could we use this to generate allfields, rather than explicitly
passing in allfields field?? -->
    <!--<copyField source="sourceFieldName" dest="destinationFieldName"/> -->
    
    <!-- field type definitions. The "name" attribute is
       just a label to be used by field definitions.  The "class"
       attribute and any other attributes determine the real
       behavior of the fieldType.
         Class names starting with "solr" refer to java classes in a
       standard package such as org.apache.solr.analysis
    -->

    <!-- sortMissingLast and sortMissingFirst attributes are optional attributes are
         currently supported on types that are sorted internally as strings
         and on numeric types.
       This includes "string", "boolean", "pint", "pfloat", "plong", "pdate", "pdouble".
       - If sortMissingLast="true", then a sort on this field will cause documents
         without the field to come after documents with the field,
         regardless of the requested sort order (asc or desc).
       - If sortMissingFirst="true", then a sort on this field will cause documents
         without the field to come before documents with the field,
         regardless of the requested sort order.
       - If sortMissingLast="false" and sortMissingFirst="false" (the default),
         then default lucene sorting will be used which places docs without the
         field first in an ascending sort and last in a descending sort.
    -->

    <!-- The StrField type is not analyzed, but indexed/stored verbatim. 
     use strings if there can be more than one value per document -->
    <fieldType name="string" class="solr.StrField" sortMissingLast="true" />
    <fieldType name="strings" class="solr.StrField" sortMissingLast="true" multiValued="true" />
    
    <!-- boolean type: "true" or "false" -->
    <fieldType name="boolean" class="solr.BoolField" sortMissingLast="true"/>
    <fieldType name="booleans" class="solr.BoolField" sortMissingLast="true" multiValued="true"/>
    
    <!--
      Numeric field types that index values using KD-trees.
      Point fields don't support FieldCache, so they must have docValues enabled if needed for sorting, faceting, functions, etc.
      This is the default, so it does not need to be set explicitly.
    -->
    <fieldType name="pint" class="solr.IntPointField"/>
    <fieldType name="pfloat" class="solr.FloatPointField"/>
    <fieldType name="plong" class="solr.LongPointField"/>
    <fieldType name="pdouble" class="solr.DoublePointField"/>

    <fieldType name="pints" class="solr.IntPointField" multiValued="true"/>
    <fieldType name="pfloats" class="solr.FloatPointField" multiValued="true"/>
    <fieldType name="plongs" class="solr.LongPointField" multiValued="true"/>
    <fieldType name="pdoubles" class="solr.DoublePointField" multiValued="true"/>
    <fieldType name="random" class="solr.RandomSortField" indexed="true"/>

    <!-- since fields of this type are by default not stored or indexed,
       any data added to them will be ignored outright.  -->
    <fieldType name="ignored" stored="false" indexed="false" multiValued="true" docValues="false" class="solr.StrField" />

        <!-- The format for this date field is of the form 1995-12-31T23:59:59Z, and
         is a more restricted form of the canonical representation of dateTime
         http://www.w3.org/TR/xmlschema-2/#dateTime
         The trailing "Z" designates UTC time and is mandatory.
         Optional fractional seconds are allowed: 1995-12-31T23:59:59.999Z
         All other components are mandatory.

         Expressions can also be used to denote calculations that should be
         performed relative to "NOW" to determine the value, ie...

               NOW/HOUR
                  ... Round to the start of the current hour
               NOW-1DAY
                  ... Exactly 1 day prior to now
               NOW/DAY+6MONTHS+3DAYS
                  ... 6 months and 3 days in the future from the start of
                      the current day

      -->
    <!-- KD-tree versions of date fields -->
    <fieldType name="pdate" class="solr.DatePointField"/>
    <fieldType name="pdates" class="solr.DatePointField" multiValued="true"/>

    <!--Binary data type. The data should be sent/retrieved in as Base64 encoded Strings -->
    <fieldType name="binary" class="solr.BinaryField"/>

        <!-- A general text field that has reasonable, generic
         cross-language defaults: it tokenizes with StandardTokenizer,
         removes stop words from case-insensitive "stopwords.txt"
         (empty by default), and down cases.  At query time only, it
         also applies synonyms.
    -->
    <fieldType name="text_general" class="solr.TextField" positionIncrementGap="100" multiValued="true">
      <analyzer type="index">
        <tokenizer name="standard"/>
        <filter name="stop" ignoreCase="true" words="stopwords.txt" />
        <!-- in this example, we will only use synonyms at query time
        <filter name="synonymGraph" synonyms="index_synonyms.txt" ignoreCase="true" expand="false"/>
        <filter name="flattenGraph"/>
        -->
        <filter name="lowercase"/>
      </analyzer>
      <analyzer type="query">
        <tokenizer name="standard"/>
        <filter name="stop" ignoreCase="true" words="stopwords.txt" />
        <filter name="synonymGraph" synonyms="synonyms.txt" ignoreCase="true" expand="true"/>
        <filter name="lowercase"/>
      </analyzer>
    </fieldType>

    <!-- ##GREENSTONE-FIELDTYPES## -->

    <!-- for now, add in the other two used by gs currently. but make it more dynamic
         , adding in lang variants, and ones specified in collectionConfig -->

    <!-- A specialized field for geospatial search filters and distance sorting. -->
    <fieldType name="location" class="solr.LatLonPointSpatialField"/>

    <!-- a special type for identifier - we don't do tokenisation, but we do do lowercasing -->
    <fieldType name="text_id" class="solr.TextField">
      <analyzer type="index">
	<tokenizer name="keyword"/>	
	<filter name="lowercase"/>
      </analyzer>
      <analyzer type="query">
	<tokenizer name="keyword"/>
	<filter name="lowercase"/>
      </analyzer>
    </fieldType>
    <!-- do we want to use splitting version or not? -->
    <fieldType name="text_en_splitting" class="solr.TextField" positionIncrementGap="100" autoGeneratePhraseQueries="false">
      <analyzer type="index">
        <!-- remove any html tags. this also unescapes ones that are escaped, and removes them-->
        <charFilter class="solr.HTMLStripCharFilterFactory"/>
        <tokenizer name="standard"/>
        <!-- in this example, we will only use synonyms at query time
        <filter name="synonymGraph" synonyms="index_synonyms.txt" ignoreCase="true" expand="false"/>
        -->
        <!-- handles things like camelcase, hyphnes, numbers and mixed tokens-->
        <!-- catenate words and numbers makes eg covid-19 generate covid19 (as well as the standard covid, 19) it can increase index size. If index size becomes an issue, can set these two to 0--> 
        <filter name="wordDelimiterGraph" generateWordParts="1" generateNumberParts="1" catenateWords="1" catenateNumbers="1" catenateAll="0" splitOnCaseChange="1"/>
        <filter name="lowercase"/>
        <filter name="stop"
                ignoreCase="true"
                words="lang/stopwords_en.txt"/>
        <filter name="keywordMarker" protected="protwords.txt"/>
        <!--<filter name="porterStem"/>-->
	<filter name="englishMinimalStem"/>
        <filter name="flattenGraph" />
      </analyzer>
      <analyzer type="query">
        <tokenizer name="standard"/>
        <filter name="synonymGraph" synonyms="synonyms.txt" ignoreCase="true" expand="true"/>
        <filter name="wordDelimiterGraph" generateWordParts="1" generateNumberParts="1" catenateWords="0" catenateNumbers="0" catenateAll="0" splitOnCaseChange="1"/>
        <filter name="lowercase"/>
        <filter name="stop"
                ignoreCase="true"
                words="lang/stopwords_en.txt"
        />
       
        <filter name="keywordMarker" protected="protwords.txt"/>
        <!--<filter name="porterStem"/>-->
	<filter name="englishMinimalStem"/>
      </analyzer>
    </fieldType>
    
  </schema>
