########################################################################### # # lucenebuildproc.pm -- perl wrapper for building index with Lucene # A component of the Greenstone digital library software # from the New Zealand Digital Library Project at the # University of Waikato, New Zealand. # # Copyright (C) 1999 New Zealand Digital Library Project # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; either version 2 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. # ########################################################################### package lucenebuildproc; # This document processor outputs a document # for lucene to process # Use same basic XML structure setup by mgppbuilder/mgppbuildproc use mgppbuildproc; use ghtml; sub BEGIN { @lucenebuildproc::ISA = ('mgppbuildproc'); } sub new { my $class = shift @_; my $self = new mgppbuildproc (@_); return bless $self, $class; } sub is_incremental_capable { my $self = shift (@_); # Unlike MG and MGPP, Lucene supports incremental building return 1; } sub preprocess_text { my $self = shift (@_); my ($text, $strip_html, $para) = @_; my ($outtext) = ""; if ($strip_html) { while ($text =~ /<([^>]*)>/ && $text ne "") { my $tag = $1; $outtext .= $`." "; #add everything before the matched tag $text = $'; #'everything after the matched tag if ($para && $tag =~ /^\s*p\s/i) { $outtext .= $para; } elsif ($tag =~ /^pre$/) { # a pre tag $text =~ /<\/pre>/; # find the closing pre tag my $tmp_text = $`; #everything before the closing pre tag $text = $'; #'everything after the $tmp_text =~ s/[<>]//g; # remove all < and > $outtext.= $tmp_text . " "; } } $outtext .= $text; # add any remaining text } #if strip_html else { $outtext = $text; } #if ($para) { #$text =~ s/(
{'output_handle'};
my $outhandle = $self->{'outhandle'};
my $indexed_doc = 1;
# only output this document if it is one to be indexed
return if ($doc_obj->get_doc_type() ne "indexed_doc");
# see if this document belongs to this subcollection
foreach my $indexexp (@{$self->{'indexexparr'}}) {
$indexed_doc = 0;
my ($field, $exp, $options) = split /\//, $indexexp;
if (defined ($field) && defined ($exp)) {
my ($bool) = $field =~ /^(.)/;
$field =~ s/^.// if $bool eq '!';
if ($field =~ /^filename$/i) {
$field = $doc_obj->get_source_filename();
} else {
$field = $doc_obj->get_metadata_element($doc_obj->get_top_section(), $field);
}
next unless defined $field;
if ($bool eq '!') {
if ($options =~ /^i$/i) {
if ($field !~ /$exp/i) {$indexed_doc = 1; last;}
} else {
if ($field !~ /$exp/) {$indexed_doc = 1; last;}
}
} else {
if ($options =~ /^i$/i) {
if ($field =~ /$exp/i) {$indexed_doc = 1; last;}
} else {
if ($field =~ /$exp/) {$indexed_doc = 1; last;}
}
}
}
}
# if this doc is so far in the sub collection, and we have lang info,
# now we check the languages to see if it matches
if($indexed_doc && defined $self->{'lang_meta'}) {
$indexed_doc = 0;
my $field = $doc_obj->get_metadata_element($doc_obj->get_top_section(), $self->{'lang_meta'});
if (defined $field) {
foreach my $lang (@{$self->{'langarr'}}) {
my ($bool) = $lang =~ /^(.)/;
if ($bool eq '!') {
$lang =~ s/^.//;
if ($field !~ /$lang/) {
$indexed_doc = 1; last;
}
} else {
if ($field =~ /$lang/) {
$indexed_doc = 1; last;
}
}
}
}
}
# this is another document
$self->{'num_docs'} += 1;
# get the parameters for the output
# split on : just in case there is subcoll and lang stuff
my ($fields) = split (/:/, $self->{'index'});
my $doc_level = $mgppbuildproc::level_map{'document'};
my $gs2ns = 'xmlns:gs2="http://www.greenstone.org/gs2"';
my $levels = $self->{'levels'};
my $ldoc_level = $levels->{'document'};
my $lsec_level = $levels->{'section'};
my $lpar_level = $levels->{'paragraph'};
my $docid="";
if ($ldoc_level) {
if ($self->{'gdbm_level'} eq 'document') {
my $doc_sec_num = $self->{'num_docs'};
$docid = "gs2:id=\"$doc_sec_num\"";
} else {
# default is section level
my $doc_sec_num = $self->{'num_sections'}+1;
$docid = "gs2:id=\"$doc_sec_num\"";
}
}
my $documenttag = "<$doc_level $gs2ns file=\"$file\" $docid >\n";
my $documentendtag = "\n$doc_level>\n";
my ($sectiontag) = "";
if ($lsec_level) {
$sectiontag = $mgppbuildproc::level_map{'section'};
}
my ($parastarttag) = "";
my ($paraendtag) = "";
if ($self->{'levels'}->{'paragraph'}) {
if ($self->{'strip_html'}) {
$parastarttag = "<".$mgppbuildproc::level_map{'paragraph'}.">";
$paraendtag = "".$mgppbuildproc::level_map{'paragraph'}.">";
} else {
print $outhandle "Paragraph level can not be used with no_strip_html!. Not indexing Paragraphs.\n";
}
}
my $doc_section = 0; # just for this document
my $text = "";
$text .= $documenttag;
# get the text for this document
my $section = $doc_obj->get_top_section();
while (defined $section) {
# update a few statistics
$doc_section++;
$self->{'num_sections'} += 1;
if ($sectiontag ne "") {
my $secid = "gs2:id=\"".$self->{'num_sections'}."\"";
$text .= "\n<$sectiontag $secid >\n";
}
# if we are doing subcollections, then some docs shouldn't be indexed.
# but we need to put the section tag placeholders in there so the
# sections match up with gdbm db
if ($indexed_doc) {
#if ($self->{'indexing_text'}) {
# $text .= "$parastarttag"; # only add para tags for indexing
# note that we assume that metadata will not be asked for for the compressed text, so we add para tags without checking for indexing_text
# }
$self->{'num_bytes'} += $doc_obj->get_text_length ($section);
foreach my $field (split (/,/, $fields)) {
# only deal with this field if it doesn't start with top or
# this is the first section
my $real_field = $field;
if (!($real_field =~ s/^top//) || ($doc_section == 1)) {
my $new_text = "";
my $tmp_text = "";
if ($real_field eq "text") {
if ($self->{'indexing_text'}) { #tag the text with