/************************************************************************** * * invf.h -- Data structures for inverted files * Copyright (C) 1999 Rodger McNab * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by * the Free Software Foundation; either version 2 of the License, or * (at your option) any later version. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * along with this program; if not, write to the Free Software * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. * * **************************************************************************/ #ifndef H_INVF #define H_INVF #include "mglong.h" #include #include "UCArray.h" // NOTE: This does not include the magic number // header info for .invf.dict file struct invf_dict_header { mg_u_long lookback; mg_u_long word_dict_start; mg_u_long word_dict_size; mg_u_long tag_dict_start; mg_u_long tag_dict_size; mg_u_long num_docs; mg_u_long num_frags; mg_u_long num_words; mg_u_long total_bytes; mg_u_long index_string_bytes; mg_u_long num_levels; invf_dict_header (); virtual ~invf_dict_header (); virtual void Clear(); virtual bool Read (FILE *f); virtual bool Write (FILE *f) const; }; struct dict_el { UCArray el; // word or tag mg_u_long frag_occur; mg_u_long freq; virtual void Clear (); dict_el () { Clear (); } virtual ~dict_el () { } // Read assumes that the last word is in el bool Read (FILE *f); bool Write (FILE *f, const UCArray *lastEl) const; }; struct word_dict_el : public dict_el { mg_u_long *levelFreqs; void Clear (); word_dict_el () { levelFreqs = NULL; Clear (); } ~word_dict_el (); void SetNumLevels (mg_u_long numLevels); // SetNumLevels should be called before either // reading or writing using Read and Write // Read assumes that the last word is in el bool Read (FILE *f, mg_u_long numLevels); bool Write (FILE *f, const UCArray *lastEl, mg_u_long numLevels) const; }; // wblk = word block // tblk = tag block // this version of the blocked dictionary uses a fixed number // of entries per block, not a fixed block size // info for .invf.dict.blocked file // blocked dict has a heap of blocks, some for words, some for tags // and an index into each set of blocks. The index has pointers to // the first entry in each block. Can do a binary search on the index // to find out which block an elemnet is in struct block_dict_header : public invf_dict_header { // note: word_dict_start and tag_dict_start are undefined // for blocked dictionaries mg_u_long entries_per_wblk; // word blocks mg_u_long num_wblks; mg_u_long max_wblk_size; mg_u_long wblk_start; mg_u_long wblk_idx_start; mg_u_long entries_per_tblk; // tag blocks mg_u_long num_tblks; mg_u_long max_tblk_size; mg_u_long tblk_start; mg_u_long tblk_idx_start; block_dict_header (); void Clear (); bool Read (FILE *f); bool Write (FILE *f) const; }; struct block_dict_el { UCArray el; // word or tag mg_u_long frag_occur; // # entries in invf file - if have a // word level index, this is the same as freq, otherwise, its the number // of fragments containing this word mg_u_long freq; // # of times this word occurs mg_u_long invf_ptr; // pointer into inverted file virtual void Clear (); block_dict_el () { Clear (); } virtual ~block_dict_el () { } // Read assumes that the last word is in el // set lastEl = NULL when no lookback is wanted (eg // for the start of a block bool Read (FILE *f); bool Write (FILE *f, const UCArray *lastEl) const; }; struct word_block_dict_el : public block_dict_el { mg_u_long *levelFreqs; // freq of the word at each level void Clear (); word_block_dict_el () { levelFreqs = NULL; Clear (); } ~word_block_dict_el (); void SetNumLevels (mg_u_long numLevels); // SetNumLevels should be called before either // reading or writing using Read and Write // Read assumes that the last word is in el bool Read (FILE *f, mg_u_long numLevels); bool Write (FILE *f, const UCArray *lastEl, mg_u_long numLevels) const; }; typedef vector word_block_dict_el_array; struct block_idx_info { UCArray el; mg_u_long block_ptr; block_idx_info (); void Clear (); bool Read (FILE *f); bool Write (FILE *f) const; }; // used for an index into the word and tag blocks typedef vector block_idx; bool ReadBlockIdx (FILE *f, block_idx &blockIdx); bool WriteBlockIdx (FILE *f, const block_idx &blockIdx); struct stem_idx_header { mg_u_long lookback; mg_u_long dict_size; mg_u_long entries_per_block; mg_u_long num_blocks; mg_u_long max_block_size; mg_u_long blocks_start; mg_u_long block_idx_start; mg_u_long stemmer_num; mg_u_long stem_method; stem_idx_header (); void Clear (); bool Read (FILE *f); bool Write (FILE *f) const; }; struct stem_block_dict_el { UCArray el; // word or tag vector equivWords; stem_block_dict_el (); void Clear (); // Read assumes that the last word is in el // set lastEl = NULL when no lookback is wanted (eg // for the start of a block bool Read (FILE *f); bool Write (FILE *f, const UCArray *lastEl) const; }; #define SKIP_MODE_NO_SKIPS 0 // invf file - has a list of frags for each word, but the word is not // stored in the invf file - the dictionaries store the words, along // with num entries, and a pointer into invf file struct invf_file_header { mg_u_long no_of_words; mg_u_long no_of_tags; mg_u_long skip_mode; mg_u_long word_level_index; // 1 if word level index mg_u_long params[16]; invf_file_header (); void Clear (); bool Read (FILE *f); bool Write (FILE *f) const; }; // the search functions returns true if a block that could // satisfy the request is found. these functions assume that // the block index is sorted by DictCompare (or DictLTUCArray) bool SearchElNum (const block_idx &bIdx, mg_u_long entriesPerBlock, mg_u_long elNum, mg_u_long &blockIdxNum, mg_u_long &blockStartElNum); bool SearchEl (const block_idx &bIdx, mg_u_long entriesPerBlock, const UCArray &el, mg_u_long &blockIdxNum, mg_u_long &blockStartElNum); // The next six functions use SearchElNum and SearchEl // for a particular type of dictionary (Block, WordBlock, or StemBlock) // and then look up the entry bool SearchBlockDictElNum (FILE *dictFile, const block_idx &bIdx, mg_u_long entriesPerBlock, mg_u_long dictSize, mg_u_long elNum, block_dict_el &dictEl); bool SearchBlockDictEl (FILE *dictFile, const block_idx &bIdx, mg_u_long entriesPerBlock, mg_u_long dictSize, const UCArray &el, block_dict_el &dictEl, mg_u_long &elNum); // assumes the numLevels has been set for dictEl bool SearchWordBlockDictElNum (FILE *dictFile, const block_idx &bIdx, mg_u_long entriesPerBlock, mg_u_long dictSize, mg_u_long numLevels, mg_u_long elNum, word_block_dict_el &dictEl); bool SearchWordBlockDictEl (FILE *dictFile, const block_idx &bIdx, mg_u_long entriesPerBlock, mg_u_long dictSize, mg_u_long numLevels, const UCArray &el, word_block_dict_el &dictEl, mg_u_long &elNum); bool SearchStemBlockDictElNum (FILE *dictFile, const block_idx &bIdx, mg_u_long entriesPerBlock, mg_u_long dictSize, mg_u_long elNum, stem_block_dict_el &dictEl); bool SearchStemBlockDictEl (FILE *dictFile, const block_idx &bIdx, mg_u_long entriesPerBlock, mg_u_long dictSize, const UCArray &el, stem_block_dict_el &dictEl, mg_u_long &elNum); //----------------------------------------------------- // New functions for partial matching // returns a list of word numbers from the main block index whose // words start with el. bool PartialMatchSearchWordBlockDictEl (FILE *dictFile, const block_idx &bIdx, mg_u_long entriesPerBlock, mg_u_long dictSize, mg_u_long numLevels, const UCArray &el, word_block_dict_el &dictEl, vector &elNumList, bool casefold); //---------------------------------------------------------- // new functions for full text browse bool NearestSearchWordBlockDictEl (FILE *dictFile, const block_idx &bIdx, mg_u_long entriesPerBlock, mg_u_long dictSize, mg_u_long numLevels, const UCArray &el, word_block_dict_el &dictEl, mg_u_long &elNum); // returns a list of word_block_dict_el, with no levelfreqs bool SearchWordBlockDictElNumRange (FILE *dictFile, const block_idx &bIdx, mg_u_long entriesPerBlock, mg_u_long dictSize, mg_u_long numLevels, mg_u_long elNum, mg_u_long numWords, word_block_dict_el_array &terms); // just returns a list of terms bool SearchWordBlockDictElNumRange (FILE *dictFile, const block_idx &bIdx, mg_u_long entriesPerBlock, mg_u_long dictSize, mg_u_long numLevels, mg_u_long elNum, mg_u_long numWords, UCArrayVector &terms); #endif