/************************************************************************** * * ivf.pass2.c -- Memory efficient pass 2 inversion * Copyright (C) 1994 Neil Sharman * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by * the Free Software Foundation; either version 2 of the License, or * (at your option) any later version. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * along with this program; if not, write to the Free Software * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. * * $Id$ * **************************************************************************/ /* $Log$ Revision 1.3 2004/06/10 05:07:43 kjdon have to declare vars before calling functions!! Revision 1.2 2004/06/10 03:02:05 kjdon fixed the bug that was causing it not to be able to create a second index using jni - basically had to reset all the static variables at the start of each pass. the tricky thing to find was the static variables in occur_to_lexical in ivf.pass2 Revision 1.1 2003/02/20 21:18:23 mdewsnip Addition of MG package for search and retrieval Revision 1.2 2001/09/21 12:46:42 kjm18 updated mg to be in line with mg_1.3f. Now uses long long for some variables to enable indexing of very large collections. * Revision 1.2 1997/08/02 05:01:57 wew * changed literal values of 32 for the bit size of magic numbers of * files to sizeof (mg_u_long) * 8, increased the gap at the start * of the invf during processing to 200 bytes Revision 1.1 1999/08/10 21:17:54 sjboddie renamed mg-1.3d directory mg Revision 1.3 1998/12/17 09:12:51 rjmcnab Altered mg to process utf-8 encoded Unicode. The main changes are in the parsing of the input, the casefolding, and the stemming. Revision 1.2 1998/11/25 07:55:43 rjmcnab Modified mg to that you can specify the stemmer you want to use via a command line option. You specify it to mg_passes during the build process. The number of the stemmer that you used is stored within the inverted dictionary header and the stemmed dictionary header so the correct stemmer is used in later stages of building and querying. Revision 1.1 1998/11/17 09:34:45 rjmcnab *** empty log message *** * Revision 1.3 1994/10/20 03:56:49 tes * I have rewritten the boolean query optimiser and abstracted out the * components of the boolean query. * * Revision 1.2 1994/09/20 04:41:35 tes * For version 1.1 * */ /* * Modified: * - long long disk pointers and bit counts for inverted file * (1999-08-03 Tim Bell ) * Code provided by Owen de Kretser */ static char *RCSID = "$Id$"; #include "local_strings.h" #include "sysfuncs.h" #include "memlib.h" #include "messages.h" #include "stemmer.h" #include "perf_hash.h" #include "bitio_m.h" #include "bitio_m_mems.h" #include "bitio_gen.h" #include "bitio_random.h" #include "bitio_stdio.h" #include "netorder.h" /* [RPAP - Jan 97: Endian Ordering] */ #include "mg_files.h" #include "invf.h" #include "locallib.h" #include "mg.h" #include "build.h" #include "words.h" #include "hash.h" #include "longlong.h" #ifdef USE_LONG_LONG #define BIO_Random_Seek_X BIO_Random_Seek_LL #define BIO_Random_Tell_X BIO_Random_Tell_LL #else #define BIO_Random_Seek_X BIO_Random_Seek #define BIO_Random_Tell_X BIO_Random_Tell #endif /* [RPAP - Feb 97: WIN32 Port] */ #ifdef __WIN32__ #include #endif #ifndef RND_BUF_SIZE #define RND_BUF_SIZE 8*1024 /*#define RND_BUF_SIZE 128 */ #endif #define print_fsize(file)\ do\ {\ struct stat file_state;\ fstat(fileno(invf_out), &file_state);\ Message("len(invf) = %d", file_state.st_size);\ }while(0) typedef struct word_rec { mg_u_long ptr; mg_u_long last; } word_rec; typedef struct invf_state_rec { mg_ullong Disk_Ptr; mg_ullong Disk_Last; mg_u_long Disk_B; } invf_state_rec; typedef struct chunk { mg_u_long start_doc; mg_u_long params_pos; mg_u_long disk_pos; mg_u_long N; } chunk; static FILE *dict; /* Stemmed dictionary file */ static FILE *hash; /* Stemmed dictionary hash file */ static FILE *invf; /* Inverted file */ static FILE *invf_in; /* Inverted file */ static FILE *invf_out; /* Inverted file */ static FILE *invf_idx; /* Inverted index file */ static FILE *count; /* Count file */ static FILE *count_trans; /* Count translation file */ static FILE *invf_state; /* Inverted file State */ static FILE *chunk_state; /* Chunk state */ static FILE *chunks; /* Chunk state */ static FILE *invf_para = NULL; /* Paragraph counts file */ static FILE *weights = NULL; /* Weights file */ static stdio_bitio_state sbs; static random_bitio_state crbs; static chunk *chunk_data = NULL; static random_bitio_state rbs, rbsp; static int docs_left = 0, next_docs_left = 0; static mg_u_long N; static word_rec *WordRecs; static u_char *lg_bs; static float *idf = NULL; static char *MemoryBuffer = NULL; static mg_u_long MemBufSize; static mg_u_long BufToUse; static struct invf_dict_header idh; static perf_hash_data *phd; static mg_u_long *word_list = NULL; static mg_u_long wl_size = 0; static mg_u_long dict_size; static mg_u_long no_of_ptrs = 0; static mg_u_long chunks_read = 0; static mg_u_long Disk_pos = 0; static mg_u_long callnum = 0; static mg_u_long wordnum = 0; static mg_u_long totalIbytes = 0; static mg_u_long totalDbytes = 0; static mg_u_long totalHbytes = 0; static mg_u_long MemInUse = 0; static mg_u_long MaxMemInUse = 0; static mg_u_long max_buffer_len; void ChangeMemInUse (int mem) { MemInUse += mem; if (MemInUse > MaxMemInUse) MaxMemInUse = MemInUse; } void ResetStaticI2Vars() { docs_left = 0; next_docs_left = 0; N = 0; MemBufSize=0; BufToUse=0; memset(&idh, 0, sizeof(idh)); wl_size = 0; dict_size = 0; no_of_ptrs = 0; chunks_read = 0; Disk_pos = 0; callnum = 0; wordnum = 0; totalIbytes = 0; totalDbytes = 0; totalHbytes = 0; MemInUse = 0; MaxMemInUse = 0; max_buffer_len = 0; } static int open_files (char *file_name) { char FName[200]; if (!(dict = open_file (file_name, INVF_DICT_SUFFIX, "rb", MAGIC_STEM_BUILD, MG_CONTINUE))) /* [RPAP - Feb 97: WIN32 Port] */ return (COMPERROR); if (!(hash = open_file (file_name, INVF_DICT_HASH_SUFFIX, "rb", MAGIC_HASH, MG_CONTINUE))) /* [RPAP - Feb 97: WIN32 Port] */ return (COMPERROR); if (!(count = open_file (file_name, INVF_CHUNK_SUFFIX, "rb", MAGIC_CHUNK, MG_CONTINUE))) /* [RPAP - Feb 97: WIN32 Port] */ return (COMPERROR); fread (&max_buffer_len, sizeof (max_buffer_len), 1, count); NTOHUL(max_buffer_len); /* [RPAP - Jan 97: Endian Ordering] */ BIO_Stdio_Decode_Start (count, &sbs); next_docs_left = BIO_Stdio_Gamma_Decode (&sbs, NULL) - 1; if (!(count_trans = open_file (file_name, INVF_CHUNK_TRANS_SUFFIX, "rb", MAGIC_CHUNK_TRANS, MG_CONTINUE))) /* [RPAP - Feb 97: WIN32 Port] */ return (COMPERROR); if (!(invf = create_file (file_name, INVF_SUFFIX, "w+b", MAGIC_INVF, MG_CONTINUE))) /* [RPAP - Feb 97: WIN32 Port] */ return (COMPERROR); fflush (invf); if (!(invf_in = open_file (file_name, INVF_SUFFIX, "rb", MAGIC_INVF, MG_CONTINUE))) /* [RPAP - Feb 97: WIN32 Port] */ return (COMPERROR); if (!(invf_out = create_file (file_name, INVF_SUFFIX, "wb", MAGIC_INVF, MG_CONTINUE))) /* [RPAP - Feb 97: WIN32 Port] */ return (COMPERROR); BIO_Random_Start (invf, RND_BUF_SIZE, &rbs); BIO_Random_Start (invf, RND_BUF_SIZE, &rbsp); ChangeMemInUse (RND_BUF_SIZE * 2); if (!(invf_idx = create_file (file_name, INVF_IDX_SUFFIX, "wb", MAGIC_INVI, MG_CONTINUE))) /* [RPAP - Feb 97: WIN32 Port] */ return (COMPERROR); if (InvfLevel == 3) if (!(invf_para = create_file (file_name, INVF_PARAGRAPH_SUFFIX, "wb", MAGIC_PARAGRAPH, MG_CONTINUE))) /* [RPAP - Feb 97: WIN32 Port] */ return (COMPERROR); sprintf (FName, FILE_NAME_FORMAT ".%d", get_basepath (), file_name, ".invf.state", (mg_s_long) getpid ()); /* [RPAP - Feb 97: WIN32 Port] */ if (!(invf_state = fopen (FName, "w+b"))) /* [RPAP - Feb 97: WIN32 Port] */ { Message ("Unable to create \"%s\"", FName); return (COMPERROR); } unlink (FName); sprintf (FName, FILE_NAME_FORMAT ".%d", get_basepath (), file_name, ".chunk.state", (mg_s_long) getpid ()); /* [RPAP - Feb 97: WIN32 Port] */ if (!(chunk_state = fopen (FName, "w+b"))) /* [RPAP - Feb 97: WIN32 Port] */ { Message ("Unable to create \"%s\"", FName); return (COMPERROR); } unlink (FName); BIO_Random_Start (chunk_state, RND_BUF_SIZE, &crbs); ChangeMemInUse (RND_BUF_SIZE); sprintf (FName, FILE_NAME_FORMAT ".%d", get_basepath (), file_name, ".chunks", (mg_s_long) getpid ()); /* [RPAP - Feb 97: WIN32 Port] */ if (!(chunks = fopen (FName, "w+b"))) /* [RPAP - Feb 97: WIN32 Port] */ { Message ("Unable to create \"%s\"", FName); return (COMPERROR); } unlink (FName); return (COMPALLOK); } #define ISR_CACHE 1024 #define ISR_ENTRY_SIZE (sizeof(mg_ullong)*2 + sizeof(mg_u_long)) invf_state_rec * in_cache (int pos) { static char isr_data[ISR_CACHE * ISR_ENTRY_SIZE]; static invf_state_rec isr; static int isr_base = 0, isr_num = -1, isr_pos = -1; if (isr_pos >= 0) bcopy ((char *) &isr, &isr_data[isr_pos * ISR_ENTRY_SIZE], ISR_ENTRY_SIZE); if (pos < isr_base || pos >= isr_base + isr_num) { if (isr_num >= 0) { fseek (invf_state, isr_base * ISR_ENTRY_SIZE, 0); fwrite (isr_data, 1, ISR_ENTRY_SIZE * isr_num, invf_state); } isr_base = pos; fseek (invf_state, isr_base * ISR_ENTRY_SIZE, 0); fread (isr_data, 1, ISR_ENTRY_SIZE * ISR_CACHE, invf_state); isr_num = ISR_CACHE; } isr_pos = pos - isr_base; bcopy (&isr_data[isr_pos * ISR_ENTRY_SIZE], (char *) &isr, ISR_ENTRY_SIZE); return &isr; } mg_u_long occur_to_lexical (mg_s_long occ, int clear_state) { static mg_s_long pos = -1; static random_bitio_state rbs; static int val = 0; if (clear_state) { pos = -1; val = 0; return 0; } if (pos == -1) { BIO_Random_Start (count_trans, RND_BUF_SIZE, &rbs); pos = 0x7fffffff; } if (occ < pos) { if (occ == -1) { BIO_Random_Done (&rbs); return 0; } BIO_Random_Seek_X (sizeof (mg_u_long) * 8, &rbs); pos = 0; } while (pos <= occ) { val = BIO_Random_Binary_Decode (dict_size + 1, &rbs, NULL) - 1; pos++; } return (val); } void add_chunk_state (mg_u_long pos, mg_u_long start_doc, mg_u_long N) { chunk_data[chunks_read].params_pos = pos; chunk_data[chunks_read].start_doc = start_doc; chunk_data[chunks_read].N = N; chunks_read++; } int init_ivf_2 (char *file_name) { u_char prev[MAXSTEMLEN + 1]; int i; mg_ullong totalIbits; mg_ullong lasttotalIbits; double logN = 0.0; ResetStaticI2Vars(); /* clear the global statics */ occur_to_lexical(0, 1); /* clear the statics in here*/ if (open_files (file_name) == COMPERROR) return COMPERROR; /* Read in the stemmed dictionary file header */ fread ((char *) &idh, sizeof (idh), 1, dict); /* [RPAP - Jan 97: Endian Ordering] */ NTOHUL(idh.lookback); NTOHUL(idh.dict_size); NTOHUL(idh.total_bytes); NTOHUL(idh.index_string_bytes); NTOHD(idh.input_bytes); /* [RJM 07/97: 4G limit] */ NTOHUL(idh.num_of_docs); NTOHUL(idh.static_num_of_docs); NTOHUL(idh.num_of_words); NTOHUL(idh.stemmer_num); NTOHUL(idh.stem_method); dict_size = idh.dict_size; N = idh.num_of_docs; if (!(phd = read_perf_hash_data (hash))) { Message ("Unable to read in hash data"); return COMPERROR; } totalHbytes = sizeof (perf_hash_data) + sizeof (u_char) * 256 + sizeof (int) * (phd->MAX_N + 1) + sizeof (int *) * 3 * phd->MAX_CH + sizeof (mg_s_long) * phd->MAX_CH * phd->MAX_L; if (!(WordRecs = Xmalloc (sizeof (word_rec) * idh.dict_size))) { Message ("No memory for word entries"); return COMPERROR; } totalDbytes += sizeof (word_rec) * idh.dict_size; /* separate storage for the log(b) values, one byte each */ if (!(lg_bs = Xmalloc (sizeof (u_char) * idh.dict_size))) { Message ("No memory for lg b's"); return COMPERROR; } totalDbytes += sizeof (u_char) * idh.dict_size; if (MakeWeights) { /* separate storage for the idf values, one single each */ if (!(idf = Xmalloc (sizeof (float) * idh.dict_size))) { Message ("No memory for idf's"); return COMPERROR; } totalDbytes += sizeof (float) * idh.dict_size; if (!(weights = create_file (file_name, WEIGHTS_SUFFIX, "wb", MAGIC_WGHT, MG_CONTINUE))) { /* [RPAP - Feb 97: WIN32 Port] */ Message ("Couldn't open weights file for writing"); return (COMPERROR); } } else { unlink (make_name (file_name, WEIGHTS_SUFFIX, NULL)); } chunk_data = Xmalloc (sizeof (chunk) * (ChunkLimit + 2)); totalDbytes += sizeof (chunk) * (ChunkLimit + 2); totalIbits = sizeof (mg_u_long) * 8; /* The magic number */ totalIbits += 8 * 200; /* A 200 byte gap */ if (MakeWeights) { wl_size = 1024; if (!(word_list = Xmalloc (sizeof (*word_list) * wl_size))) { Message ("No memory for word_list"); return COMPERROR; } logN = log ((double) N); } for (i = 0; i < idh.dict_size; i++) { invf_state_rec *isr; register mg_u_long copy, suff, p; mg_u_long fcnt, wcnt; lasttotalIbits = totalIbits; copy = fgetc (dict); suff = fgetc (dict); *prev = copy + suff; fread (prev + copy + 1, sizeof (u_char), suff, dict); fread ((char *) &fcnt, sizeof (fcnt), 1, dict); fread ((char *) &wcnt, sizeof (wcnt), 1, dict); /* [RPAP - Jan 97: Endian Ordering] */ NTOHUL(fcnt); NTOHUL(wcnt); WordRecs[i].last = 0; WordRecs[i].ptr = 0; p = fcnt; if (MakeWeights) idf[i] = logN - log ((double) fcnt); isr = in_cache (i); isr->Disk_Last = 0; isr->Disk_Ptr = totalIbits; isr->Disk_B = BIO_Bblock_Init (N, p); totalIbits += BIO_Bblock_Bound_b (N, p, isr->Disk_B); if (InvfLevel >= 2) totalIbits += BIO_Gamma_Bound (wcnt, fcnt); #ifdef USE_LONG_LONG totalIbits = (totalIbits + 7ull) & 0xfffffffffffffff8ull; #else totalIbits = (totalIbits + 7ul) & 0xfffffff8ul; #endif if (totalIbits < lasttotalIbits) { fprintf(stderr, "ERROR: The totalIbits counter (%lu byte unsigned integer) has overflowed.\n", sizeof (mg_ullong)); if (sizeof (mg_ullong) < 8) { fprintf(stderr, " Try compiling with GCC to enable use of 8 bytes for this counter.\n"); } fprintf(stderr, " Build aborted.\n"); exit(1); } } /* now convert to bytes, and actually get the space */ #ifdef USE_LONG_LONG totalIbytes = (totalIbits + 7ull) >> 3ull; #else totalIbytes = (totalIbits + 7ul) >> 3ul; #endif return (COMPALLOK); } static void LoadCounts (void) { mg_u_long numwords, i, last_total; static mg_u_long local_N = 0; mg_u_long totalIbits, crbs_pos; word_rec *wr; mg_u_long *counts; if (MemoryBuffer == NULL) { MemBufSize = sizeof (mg_u_long) * dict_size; if (max_buffer_len > MemBufSize) MemBufSize = max_buffer_len; if (!(MemoryBuffer = Xmalloc (MemBufSize))) FatalError (1, "Unable to allocate memory for buffer"); ChangeMemInUse (MemBufSize); } counts = (mg_u_long *) MemoryBuffer; /* bzero ((char *) counts, sizeof (mg_u_long) * dict_size); */ bzero ((char *) counts, MemBufSize); docs_left = next_docs_left; if (!docs_left) FatalError (1, "The number of docs in the current chunk is 0"); BufToUse = BIO_Stdio_Gamma_Decode (&sbs, NULL) - 1; numwords = BIO_Stdio_Gamma_Decode (&sbs, NULL) - 1; local_N = docs_left; for (wr = WordRecs, i = 0; i < dict_size; i++, wr++) wr->ptr = 0; bzero ((char *) lg_bs, dict_size); for (i = 0; i < numwords; i++) { mg_u_long word_num, wcnt, fcnt, p; word_num = occur_to_lexical (i,0); wr = &WordRecs[word_num]; wcnt = BIO_Stdio_Gamma_Decode (&sbs, NULL) - 1; if (wcnt >= 2) fcnt = BIO_Stdio_Gamma_Decode (&sbs, NULL); else fcnt = wcnt; p = fcnt; if (wcnt) { register mg_u_long length; counts[word_num] = p; length = BIO_Bblock_Bound (local_N, p); if (InvfLevel >= 2) length += wcnt; wr->ptr = length; lg_bs[word_num] = floorlog_2 (BIO_Bblock_Init_W (local_N, p)); } } crbs_pos = BIO_Random_Tell (&crbs); totalIbits = 0; last_total = 0; for (wr = WordRecs, i = 0; i < dict_size; i++, wr++) { register mg_u_long length; length = wr->ptr; wr->last = callnum; BIO_Random_Gamma_Encode (counts[i] + 1, &crbs, NULL); if (counts[i]) { if (i) BIO_Random_Delta_Encode (totalIbits - last_total + 1, &crbs, NULL); else BIO_Random_Delta_Encode (1, &crbs, NULL); last_total = totalIbits; } wr->ptr = totalIbits; totalIbits += length; } add_chunk_state (crbs_pos, callnum, local_N); if ((totalIbits + 7ul) >> 3ul > BufToUse) FatalError (1, "Pointers exceed buffer size"); next_docs_left = BIO_Stdio_Gamma_Decode (&sbs, NULL) - 1; } static void DumpChunk (void) { chunk_data[chunks_read - 1].disk_pos = Disk_pos << 3; fseek (chunks, Disk_pos, 0); fwrite (MemoryBuffer, sizeof (char), BufToUse, chunks); Disk_pos += BufToUse; } static void DiskMerge (void) { random_bitio_state *rbsi; random_bitio_state *chks = NULL; mg_u_long *chunk_ptrs; int i; BIO_Random_Flush (&crbs); chunk_ptrs = Xmalloc (chunks_read * sizeof (mg_u_long)); ChangeMemInUse (chunks_read * sizeof (mg_u_long)); bzero ((char *) chunk_ptrs, chunks_read * sizeof (mg_u_long)); rbsi = Xmalloc (chunks_read * sizeof (random_bitio_state)); ChangeMemInUse (chunks_read * sizeof (random_bitio_state)); for (i = 0; i < chunks_read; i++) { rbsi[i] = crbs; rbsi[i].Buf = Xmalloc (rbsi[i].len); ChangeMemInUse (rbsi[i].len); bcopy ((char *) (crbs.Buf), (char *) (rbsi[i].Buf), rbsi[i].len); BIO_Random_Seek (chunk_data[i].params_pos, &rbsi[i]); } if (chunks_read > 1) { int j; chks = Xmalloc ((chunks_read - 1) * sizeof (random_bitio_state)); ChangeMemInUse ((chunks_read - 1) * sizeof (random_bitio_state)); BIO_Random_Start (chunks, RND_BUF_SIZE, &chks[0]); ChangeMemInUse (RND_BUF_SIZE); for (j = 1; j < chunks_read - 1; j++) { chks[j] = chks[0]; chks[j].Buf = Xmalloc (chks[0].len); ChangeMemInUse (chks[0].len); bcopy ((char *) (chks[0].Buf), (char *) (chks[j].Buf), chks[0].len); } } for (i = 0; i < dict_size; i++) { int j; invf_state_rec *isr = in_cache (i); register int B; BIO_Random_Seek_X (isr->Disk_Ptr, &rbs); /* Position in invf file */ B = isr->Disk_B; for (j = 0; j < chunks_read; j++) { int p; p = BIO_Random_Gamma_Decode (&rbsi[j], NULL) - 1; if (p) { int ptr, b; chunk_ptrs[j] += BIO_Random_Delta_Decode (&rbsi[j], NULL) - 1; ptr = chunk_ptrs[j]; b = 1 << floorlog_2 (BIO_Bblock_Init_W (chunk_data[j].N, p)); if (j == chunks_read - 1) { int k, CurrDoc; DECODE_START ((u_char *) MemoryBuffer, ptr) CurrDoc = isr->Disk_Last; for (k = 0; k < p; k++) { register mg_u_long x, tf; BBLOCK_DECODE (x, b); if (k == 0) x = x + chunk_data[j].start_doc - isr->Disk_Last; CurrDoc += x; BIO_Random_Bblock_Encode (x, B, &rbs, NULL); if (InvfLevel >= 2) { UNARY_DECODE (tf); BIO_Random_Gamma_Encode (tf, &rbs, NULL); } } DECODE_DONE isr->Disk_Last = CurrDoc; } else { int k, CurrDoc; random_bitio_state *Chks = chks + j; BIO_Random_Seek (chunk_data[j].disk_pos + ptr, Chks); CurrDoc = isr->Disk_Last; for (k = 0; k < p; k++) { register mg_u_long x, tf; x = BIO_Random_Bblock_Decode (b, Chks, NULL); if (k == 0) x = x + chunk_data[j].start_doc - isr->Disk_Last; CurrDoc += x; BIO_Random_Bblock_Encode (x, B, &rbs, NULL); if (InvfLevel >= 2) { tf = BIO_Random_Unary_Decode (Chks, NULL); BIO_Random_Gamma_Encode (tf, &rbs, NULL); } } isr->Disk_Last = CurrDoc; } } } isr->Disk_Ptr = BIO_Random_Tell_X (&rbs); } if (chunks_read > 1) { int j; for (j = 0; j < chunks_read - 1; j++) { Xfree (chks[j].Buf); ChangeMemInUse (-chks[j].len); } Xfree (chks); ChangeMemInUse (-(chunks_read - 1) * sizeof (random_bitio_state)); } for (i = 0; i < chunks_read; i++) { Xfree (rbsi[i].Buf); ChangeMemInUse (-rbsi[i].len); } Xfree (rbsi); ChangeMemInUse (-chunks_read * sizeof (random_bitio_state)); /* chunks_read = 0; */ Xfree (chunk_ptrs); ChangeMemInUse (-chunks_read * sizeof (mg_u_long)); chunks_read = 0; Disk_pos = 0; BIO_Random_Seek (0, &crbs); } static void MergeIn (void) { static int disk_chunks = 0; static int header = 0; if (!header) { fprintf (stderr, "ivf.pass2 : "); header = 1; } if (disk_chunks == ChunkLimit || next_docs_left == 0) { fprintf (stderr, "M"); DiskMerge (); disk_chunks = 0; } else { fprintf (stderr, "-"); DumpChunk (); disk_chunks++; } if (next_docs_left == 0) fprintf (stderr, "\n"); } static int wl_comp (const void *a, const void *b) { return *((int *) a) - *((int *) b); } static int process_doc (u_char * s_in, int l_in) { int res; u_char *end = s_in + l_in - 1; mg_u_long tocode; mg_u_long wl_pos = 0; if (!docs_left) LoadCounts (); callnum++; if (!inaword (s_in, end)) if (SkipSGML) PARSE_NON_STEM_WORD_OR_SGML_TAG (s_in, end); else PARSE_NON_STEM_WORD (s_in, end); while (s_in <= end) { u_char Word[MAXSTEMLEN + 1]; PARSE_STEM_WORD (Word, s_in, end); stemmer (idh.stem_method, idh.stemmer_num, Word); if (SkipSGML) PARSE_NON_STEM_WORD_OR_SGML_TAG (s_in, end); else PARSE_NON_STEM_WORD (s_in, end); if (*Word == 0) continue; res = perf_hash (phd, Word); { word_rec *arr = &WordRecs[res]; int b = 1 << lg_bs[res]; wordnum++; tocode = callnum; ENCODE_START ((u_char *) MemoryBuffer, arr->ptr) if (tocode > arr->last) { register int x; x = tocode - arr->last - 1; BBLOCK_ENCODE (x + 1, b); if (InvfLevel >= 2) ENCODE_BIT (1); no_of_ptrs++; arr->last = tocode; } else if (InvfLevel >= 2) { __pos--; ENCODE_BIT (0); ENCODE_BIT (1); } arr->ptr = __pos; ENCODE_DONE } if (MakeWeights) { if (wl_pos >= wl_size) { wl_size += (wl_size >> 1); word_list = Xrealloc (word_list, sizeof (*word_list) * wl_size); } word_list[wl_pos++] = res; } } if (MakeWeights) { float doc_weight = 0.0; if (wl_pos) { mg_u_long *wl = word_list; mg_u_long i, count, val; qsort (wl, wl_pos, sizeof (*wl), wl_comp); count = 1; val = *wl++; for (i = 1; i <= wl_pos; i++, wl++) if (i == wl_pos || val != *wl) { double weight = count * idf[val]; doc_weight += weight * weight; count = 1; val = *wl; } else count++; } HTONF(doc_weight); /* [RPAP - Jan 97: Endian Ordering] */ fwrite ((char *) &doc_weight, sizeof (doc_weight), 1, weights); } docs_left--; if (!docs_left) MergeIn (); return COMPALLOK; } int process_ivf_2 (u_char * s_in, int l_in) { if (InvfLevel <= 2) return process_doc (s_in, l_in); else { int count = 0; int pos = 0; u_char *start = s_in; while (pos < l_in) { if (s_in[pos] == TERMPARAGRAPH) { int len = pos + s_in + 1 - start; if (process_doc (start, len) != COMPALLOK) return (COMPERROR); start = s_in + pos + 1; count++; } pos++; } if (start < s_in + pos) { if (process_doc (start, pos + s_in - start) != COMPALLOK) return (COMPERROR); count++; } HTONSI(count); /* [RPAP - Jan 97: Endian Ordering] */ fwrite ((char *) &count, sizeof (count), 1, invf_para); } return COMPALLOK; } static void stats (mg_u_long len) { #ifndef SILENT fseek (count, 0, 2); fseek (count_trans, 0, 2); fseek (invf_state, 0, 2); fseek (invf, 0, 0); fseek (invf, 0, 2); fseek (chunks, 0, 2); fseek (chunk_state, 0, 2); Message ("File sizes\n"); Message (" Chunk desc : %10u bytes\n", ftell (count)); Message (" Chunk trans : %10u bytes\n", ftell (count_trans)); Message (" Chunks : %10u bytes\n", ftell (chunks)); Message (" Chunk state : %10u bytes\n", ftell (chunk_state)); Message (" Invf state : %10u bytes\n", ftell (invf_state)); Message (" Peak invf : %10u bytes\n", len); Message (" Final invf : %10u bytes\n", ftell (invf)); Message ("Peak disk usage : %10.2f %%\n", (double) (ftell (count) + ftell (count_trans) + ftell (invf_state) + ftell (chunks) + ftell (chunk_state) + len) / ftell (invf) * 100.0); #endif } /* ARGSUSED */ int done_ivf_2 (char *FileName) { mg_s_long i; mg_ullong totalIbits; mg_u_long invf_len; mg_u_long bytes_output; struct invf_file_header ifh; if (weights) fclose (weights); if (invf_para) fclose (invf_para); free_perf_hash (phd); phd = NULL; Xfree (MemoryBuffer); MemoryBuffer = NULL; ChangeMemInUse (-MemBufSize); BIO_Random_Done (&rbs); BIO_Random_Done (&rbsp); fflush (invf); fseek (invf, 0, 2); invf_len = ftell (invf); fseek (invf_out, sizeof (mg_s_long), 0); /* [RPAP - Jan 97: Endian Ordering] */ HTONUL2(dict_size, ifh.no_of_words); HTONUL2(no_of_ptrs, ifh.no_of_ptrs); ifh.skip_mode = 0; bzero ((char *) ifh.params, sizeof (ifh.params)); HTONUL2(InvfLevel, ifh.InvfLevel); fwrite ((char *) &ifh, sizeof (ifh), 1, invf_out); bytes_output = ftell (invf_out); totalIbits = sizeof (mg_u_long) * 8; /* The magic number */ totalIbits += 8 * 200; /* A 200 byte gap */ /* find the right place in the file to start reading p values */ fseek (dict, sizeof (mg_u_long) + sizeof (struct invf_dict_header), 0); for (i = 0; i < dict_size; i++) { invf_state_rec *isr; mg_u_long fcnt, wcnt, s, e; register mg_u_long p; u_char dummy1, dummy2[MAXSTEMLEN + 1]; /* output location to the invf_idx */ HTONUL(bytes_output); /* [RPAP - Jan 97: Endian Ordering] */ fwrite ((char *) &bytes_output, sizeof (bytes_output), 1, invf_idx); NTOHUL(bytes_output); /* [RPAP - Jan 97: Endian Ordering] */ /* read an entry for a word, just to get p value */ dummy1 = fgetc (dict); dummy1 = fgetc (dict); fread (dummy2, sizeof (u_char), dummy1, dict); fread ((char *) &fcnt, sizeof (fcnt), 1, dict); fread ((char *) &wcnt, sizeof (wcnt), 1, dict); /* [RPAP - Jan 97: Endian Ordering] */ NTOHUL(fcnt); NTOHUL(wcnt); p = fcnt; isr = in_cache (i); e = (isr->Disk_Ptr + 7ul) >> 3ul; s = totalIbits >> 3; fseek (invf_in, s, 0); while (s < e) { u_char c = getc (invf_in); if (s == e - 1) { u_char ands[8] = {0xff, 0x80, 0xc0, 0xe0, 0xf0, 0xf8, 0xfc, 0xfe}; c &= ands[isr->Disk_Ptr & 7ul]; } putc (c, invf_out); bytes_output++; s++; } totalIbits += BIO_Bblock_Bound_b (N, p, isr->Disk_B); if (InvfLevel >= 2) totalIbits += BIO_Gamma_Bound (wcnt, fcnt); #ifdef USE_LONG_LONG totalIbits = (totalIbits + 7ull) & 0xfffffffffffffff8ull; #else totalIbits = (totalIbits + 7ul) & 0xfffffff8ul; #endif } fclose (invf_in); /* [RPAP - Feb 97: WIN32 Port] */ #ifdef __WIN32__ if (!(_chsize (_fileno (invf_out), bytes_output))) Message ("Could not truncate invf."); #else ftruncate (fileno (invf_out), bytes_output); #endif fclose (invf_out); HTONUL(bytes_output); /* [RPAP - Jan 97: Endian Ordering] */ fwrite ((char *) &bytes_output, sizeof (bytes_output), 1, invf_idx); NTOHUL(bytes_output); /* [RPAP - Jan 97: Endian Ordering] */ fclose (invf_idx); #ifndef SILENT { char *temp_str = msg_prefix; mg_u_long total; msg_prefix = "ivf.pass2"; stats (invf_len); Message ("Pass two data structures : %6.3f Mbyte\n", (double) totalDbytes / 1024 / 1024); total = totalDbytes; Message ("Pass two hash structure(s) : %6.3f Mbyte\n", (double) totalHbytes / 1024 / 1024); total += totalHbytes; Message ("Peak extra memory in use : %6.3f Mbyte\n", (double) MaxMemInUse / 1024 / 1024); total += MaxMemInUse; Message ("Peak total memory in use : %6.3f Mbyte\n", (double) total / 1024 / 1024); msg_prefix = temp_str; } #endif Xfree(chunk_data); chunk_data = NULL; Xfree (WordRecs); WordRecs = NULL; Xfree (lg_bs); lg_bs = NULL; Xfree (idf); idf = NULL; Xfree (word_list); word_list = NULL; /* Free the memory allocated for the BIO_Random */ occur_to_lexical (-1,1); BIO_Random_Done (&crbs); fclose (invf); fclose (dict); fclose (hash); fclose (count); fclose (count_trans); fclose (chunk_state); fclose (chunks); fclose (invf_state); return (COMPALLOK); }