/************************************************************************** * * mg_stem_idx.c -- Memory efficient stem index builder * Copyright (C) 1997 Ross Peeters * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by * the Free Software Foundation; either version 2 of the License, or * (at your option) any later version. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * along with this program; if not, write to the Free Software * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. * **************************************************************************/ #include "sysfuncs.h" #include "memlib.h" #include "messages.h" #include "filestats.h" #include "netorder.h" /* [RPAP - Jan 97: Endian Ordering] */ #include "mg_files.h" #include "invf.h" #include "mg.h" #include "locallib.h" #include "backend.h" /* For struct stemmed_dict */ #include "words.h" #include "stemmer.h" #include "hash.h" #include "local_strings.h" typedef struct PosEntry { unsigned int num_cases; unsigned int blk; unsigned short blk_index; unsigned short offset; } PosEntry; typedef struct PosList { unsigned int list_size; unsigned int num_entries; PosEntry PE[1]; } PosList; typedef struct idx_hash_rec { u_char *word; PosList *PL; } idx_hash_rec; #define POOL_SIZE 1024*1024 #define HASH_POOL_SIZE 8192 #define INITIAL_HASH_SIZE 7927 static mg_u_long MaxMemInUse = 0; static mg_u_long MemInUse = 0; static idx_hash_rec **IdxHashTable; static mg_u_long IdxHashSize; static mg_u_long IdxHashUsed; static u_char *IdxPool; static int IdxPoolLeft; static idx_hash_rec *idx_hr_pool; static int idx_hr_PoolLeft; static idx_hash_rec **idx_first_occr; static int idx_max_first_occr; int block_size = 1024 * 4; int force = 0; static mg_s_long lookback = 4; static void ChangeMem (int Change) { MemInUse += Change; if (MemInUse > MaxMemInUse) MaxMemInUse = MemInUse; } /* ========================================================================= * Function: MakePosList * Description: * Input: * Output: * ========================================================================= */ PosList * MakePosList (int n) { PosList *pl; int list_size = (n == 0 ? 1 : n); /* always allocate at least one node */ pl = Xmalloc (sizeof (PosList) + (list_size - 1) * sizeof (PosEntry)); if (!pl) FatalError (1, "Unable to allocate term list"); ChangeMem (sizeof (PosList) + (list_size - 1) * sizeof (PosEntry)); pl->num_entries = n; pl->list_size = list_size; return pl; } /* ========================================================================= * Function: ResizePosList * Description: * Input: * Output: * ========================================================================= */ #define GROWTH_FACTOR 2 #define MIN_SIZE 2 static void ResizePosList (PosList ** pos_list) { PosList *pl = *pos_list; ChangeMem (-(sizeof (PosList) + (pl->list_size - 1) * sizeof (PosEntry))); if (pl->num_entries > pl->list_size) { if (pl->list_size) pl->list_size *= GROWTH_FACTOR; else pl->list_size = MIN_SIZE; } pl = Xrealloc (pl, sizeof (PosList) + (pl->list_size - 1) * sizeof (PosEntry)); if (!pl) FatalError (1, "Unable to resize pos list"); ChangeMem (sizeof (PosList) + (pl->list_size - 1) * sizeof (PosEntry)); *pos_list = pl; } /* ========================================================================= * Function: FreePosList * Description: * Input: * Output: * ========================================================================= */ void FreePosList (PosList ** the_pl) { PosList *pl = *the_pl; ChangeMem (-(sizeof(PosList) + sizeof (PosEntry) * (pl->list_size - 1))); Xfree (pl); *the_pl = NULL; } /* ========================================================================= * Function: ResetPosList * Description: * Input: * Output: * ========================================================================= */ void ResetPosList (PosList ** pl) { if (*pl) FreePosList (pl); *pl = MakePosList (0); } /* ========================================================================= * Function: AddPosEntry * Description: * Input: * Output: * ========================================================================= */ int AddPosEntry (PosList ** pos_list, PosEntry * pe) { PosList *pl = *pos_list; pl->num_entries++; ResizePosList (pos_list); pl = *pos_list; /* copy the structure contents */ bcopy ((char *) pe, (char *) &(pl->PE[pl->num_entries - 1]), sizeof (PosEntry)); return pl->num_entries - 1; } /* Modified from stem_search.c */ stemmed_dict * ReadStemDictBlk (File * stem_file) { mg_u_long i; stemmed_dict *sd; u_char *buffer; if (!(sd = Xmalloc (sizeof (stemmed_dict)))) FatalError (1, "Could not allocate memory for stemmed dict"); sd->stem_file = stem_file; sd->MemForStemDict = 0; Fread (&sd->sdh, sizeof (sd->sdh), 1, stem_file); /* [RPAP - Jan 97: Endian Ordering] */ NTOHUL(sd->sdh.lookback); NTOHUL(sd->sdh.block_size); NTOHUL(sd->sdh.num_blocks); NTOHUL(sd->sdh.blocks_start); NTOHUL(sd->sdh.index_chars); NTOHUL(sd->sdh.num_of_docs); NTOHUL(sd->sdh.static_num_of_docs); NTOHUL(sd->sdh.num_of_words); NTOHUL(sd->sdh.stemmer_num); NTOHUL(sd->sdh.stem_method); NTOHUL(sd->sdh.indexed); if (!(buffer = Xmalloc (sd->sdh.index_chars))) { Xfree (sd); FatalError (1, "Could not allocate memory for stemmed dict"); return (NULL); }; sd->MemForStemDict += sd->sdh.index_chars; if (!(sd->index = Xmalloc (sd->sdh.num_blocks * sizeof (*sd->index)))) { Xfree (sd); Xfree (buffer); FatalError (1, "Could not allocate memory for stemmed dict"); return (NULL); }; sd->MemForStemDict += sd->sdh.num_blocks * sizeof (*sd->index); if (!(sd->pos = Xmalloc (sd->sdh.num_blocks * sizeof (*sd->pos)))) { Xfree (sd); Xfree (buffer); Xfree (sd->index); FatalError (1, "Could not allocate memory for stemmed dict"); return (NULL); }; sd->MemForStemDict += sd->sdh.num_blocks * sizeof (*sd->pos); if (!(sd->buffer = Xmalloc (sd->sdh.block_size * sizeof (*sd->buffer)))) { Xfree (sd); Xfree (buffer); Xfree (sd->index); Xfree (sd->buffer); FatalError (1, "Could not allocate memory for stemmed dict"); return (NULL); }; sd->MemForStemDict += sd->sdh.block_size * sizeof (*sd->buffer); sd->active = -1; for (i = 0; i < sd->sdh.num_blocks; i++) { register u_char len; sd->index[i] = buffer; len = Getc (stem_file); *buffer++ = len; Fread (buffer, sizeof (u_char), len, stem_file); buffer += len; Fread (&sd->pos[i], sizeof (*sd->pos), 1, stem_file); NTOHUL(sd->pos[i]); /* [RPAP - Jan 97: Endian Ordering] */ } return sd; } void mg_init_process () { /* Allocate memory for idx hash table */ if (!(IdxPool = Xmalloc (POOL_SIZE))) FatalError (1, "Unable to allocate memory for idx pool"); IdxPoolLeft = POOL_SIZE; ChangeMem (POOL_SIZE); if (!(idx_hr_pool = Xmalloc (HASH_POOL_SIZE * sizeof (idx_hash_rec)))) FatalError (1, "Unable to allocate memory for idx pool"); idx_hr_PoolLeft = HASH_POOL_SIZE; ChangeMem (HASH_POOL_SIZE * sizeof (idx_hash_rec)); IdxHashSize = INITIAL_HASH_SIZE; IdxHashUsed = 0; if (!(IdxHashTable = Xmalloc (sizeof (idx_hash_rec *) * IdxHashSize))) FatalError (1, "Unable to allocate memory for idx table"); ChangeMem (sizeof (idx_hash_rec *) * IdxHashSize); bzero ((char *) IdxHashTable, sizeof (idx_hash_rec *) * IdxHashSize); idx_max_first_occr = 8192; if (!(idx_first_occr = Xmalloc (sizeof (idx_hash_rec *) * idx_max_first_occr))) FatalError (1, "Unable to allocate memory for idx_first_occur"); ChangeMem (sizeof (idx_hash_rec *) * idx_max_first_occr); } void PackIdxHashTable (void) { int s, d; for (s = d = 0; s < IdxHashSize; s++) if (IdxHashTable[s]) IdxHashTable[d++] = IdxHashTable[s]; ChangeMem (-sizeof (idx_hash_rec *) * IdxHashSize); ChangeMem (sizeof (idx_hash_rec *) * IdxHashUsed); if (!(IdxHashTable = Xrealloc (IdxHashTable, sizeof (idx_hash_rec *) * IdxHashUsed))) FatalError (1, "Out of memory"); IdxHashSize = IdxHashUsed; } void process_stem_dict (stemmed_dict * sd, int stem_method) { int block; short blk_index = -1; int wordnum = -1; PosEntry *prevPE = NULL; idx_hash_rec *prevIdx = NULL; u_char word[MAXSTEMLEN + 1]; u_char prev[MAXSTEMLEN + 1]; /* For each block in stem dict... */ for (block = 0; block < sd->sdh.num_blocks; block++) { register unsigned int res; int num_indexes; mg_u_long *first_word, *last_invf_len; unsigned short *num_words; u_char *base; unsigned short *index; /* Read block into buffer */ Fseek (sd->stem_file, sd->pos[block] + sd->sdh.blocks_start, 0); Fread (sd->buffer, sd->sdh.block_size, sizeof (u_char), sd->stem_file); sd->active = sd->pos[block]; /* Move through block header */ first_word = (mg_u_long *) (sd->buffer); NTOHUL(*first_word); /* [RPAP - Jan 97: Endian Ordering] */ last_invf_len = (mg_u_long *) (first_word + 1); NTOHUL(*last_invf_len); /* [RPAP - Jan 97: Endian Ordering] */ num_words = (unsigned short *) (last_invf_len + 1); NTOHUS(*num_words); /* [RPAP - Jan 97: Endian Ordering] */ index = num_words + 1; num_indexes = ((*num_words - 1) / sd->sdh.lookback) + 1; { /* [RPAP - Jan 97: Endian Ordering] */ int i; for (i = 0; i < num_indexes; i++) NTOHUS(index[i]); } base = (u_char *) (index + num_indexes); blk_index = -1; /* For each word in block... */ for (res = 0; res < *num_words; res++) { unsigned copy, suff; register idx_hash_rec *idx_ent = 0; /* Update blk_index */ if (!(res % sd->sdh.lookback)) blk_index++; copy = *base++; suff = *base++; bcopy ((char *) base, (char *) (prev + copy + 1), suff); base += suff; *prev = copy + suff; /* Skip irrelevant word info */ base += sizeof (mg_u_long); base += sizeof (mg_u_long); base += sizeof (mg_u_long); /* Stem word */ bcopy ((char *) prev, (char *) word, *prev + 1); stemmer (stem_method, sd->sdh.stemmer_num, word); /* Check if word follows straight on from previous word */ if (prevIdx && !compare (word, prevIdx->word)) prevPE->num_cases++; else { /* Search the idx hash table for word */ register mg_u_long hashval, step; register int hsize = IdxHashSize; HASH (hashval, step, word, hsize); for (;;) { register u_char *s1; register u_char *s2; register int len; idx_ent = IdxHashTable[hashval]; if (!idx_ent) { /* Create a next entry in the hash table */ int len = *word + 1; if (!idx_hr_PoolLeft) { if (!(idx_hr_pool = Xmalloc (HASH_POOL_SIZE * sizeof (idx_hash_rec)))) FatalError (1, "Unable to allocate memory for pool"); idx_hr_PoolLeft = HASH_POOL_SIZE; ChangeMem (HASH_POOL_SIZE * sizeof (idx_hash_rec)); } idx_ent = idx_hr_pool++; idx_hr_PoolLeft--; if (len > IdxPoolLeft) { if (!(IdxPool = Xmalloc (POOL_SIZE))) FatalError (1, "Unable to allocate memory for pool"); IdxPoolLeft = POOL_SIZE; ChangeMem (POOL_SIZE); } wordnum++; idx_ent->word = IdxPool; idx_ent->PL = MakePosList (0); { PosEntry PE; PE.num_cases = 1; PE.blk = block; PE.blk_index = (unsigned short) blk_index; PE.offset = res % sd->sdh.lookback; AddPosEntry (&(idx_ent->PL), &PE); } prevIdx = idx_ent; prevPE = &(idx_ent->PL->PE[idx_ent->PL->num_entries - 1]); bcopy ((char *) word, (char *) IdxPool, len); IdxPool += len; IdxPoolLeft -= len; if (IdxHashUsed == idx_max_first_occr - 1) { ChangeMem (-sizeof (idx_hash_rec *) * idx_max_first_occr); idx_max_first_occr *= 2; if (!(idx_first_occr = Xrealloc (idx_first_occr, sizeof (idx_hash_rec *) * idx_max_first_occr))) FatalError (1, "Unable to allocate memory for idx_first_occr"); ChangeMem (sizeof (idx_hash_rec *) * idx_max_first_occr); } idx_first_occr[IdxHashUsed] = idx_ent; IdxHashUsed++; IdxHashTable[hashval] = idx_ent; break; } /* if (!idx_ent) */ /* Compare the words */ s1 = word; s2 = idx_ent->word; len = *s1 + 1; for (; len; len--) if (*s1++ != *s2++) break; if (len) { /* Entry is not the right one - move to next hash index */ hashval = (hashval + step); if (hashval >= hsize) hashval -= hsize; } else { /* Entry is correct - added PosEntry to word */ PosEntry PE; PE.num_cases = 1; PE.blk = block; PE.blk_index = (unsigned short) blk_index; PE.offset = res % sd->sdh.lookback; AddPosEntry (&(idx_ent->PL), &PE); prevIdx = idx_ent; prevPE = &(idx_ent->PL->PE[idx_ent->PL->num_entries - 1]); break; } } /* for (;;) */ } /* else (the word does not follow on from previous one) */ if (IdxHashUsed >= IdxHashSize >> 1) { idx_hash_rec **ht; mg_u_long size; mg_u_long i; size = prime (IdxHashSize * 2); if (!(ht = Xmalloc (sizeof (idx_hash_rec *) * size))) FatalError (1, "Unable to allocate memory for idx table"); bzero ((char *) ht, sizeof (idx_hash_rec *) * size); ChangeMem (sizeof (idx_hash_rec *) * size); for (i = 0; i < IdxHashSize; i++) if (IdxHashTable[i]) { register u_char *wptr; idx_hash_rec *ent; register mg_u_long hashval, step; wptr = IdxHashTable[i]->word; HASH (hashval, step, wptr, size); ent = ht[hashval]; while (ent) { hashval += step; if (hashval >= size) hashval -= size; ent = ht[hashval]; } ht[hashval] = IdxHashTable[i]; } Xfree (IdxHashTable); ChangeMem (-sizeof (idx_hash_rec *) * IdxHashSize); IdxHashTable = ht; IdxHashSize = size; } } /* end for each word */ } /* end for each block */ } static int idx_comp (const void *A, const void *B) { u_char *s1 = (*((idx_hash_rec **) A))->word; u_char *s2 = (*((idx_hash_rec **) B))->word; return (casecompare (s1, s2)); } void save_idx (char * filename, int stem_method) { char *FName; mg_u_long i, j, pos, First_word, num; struct stem_idx_header sih; u_char *buffer, *last_word = NULL; unsigned short *pointers; int buf_in_use; unsigned short ptrs_in_use, word_num; FILE *idbi = NULL, *tmp = NULL; FName = make_name (filename, ".tmp", NULL); if (!(tmp = fopen (FName, "w+b"))) FatalError (1, "Unable to open \"%s\".\n", FName); /* Delete the file now */ unlink (FName); /* Create appropriate stem index file */ switch (stem_method) { case (1): { idbi = create_file (filename, INVF_DICT_BLOCKED_1_SUFFIX, "wb", MAGIC_STEM_1, MG_ABORT); break; } case (2): { idbi = create_file (filename, INVF_DICT_BLOCKED_2_SUFFIX, "wb", MAGIC_STEM_2, MG_ABORT); break; } case (3): { idbi = create_file (filename, INVF_DICT_BLOCKED_3_SUFFIX, "wb", MAGIC_STEM_3, MG_ABORT); break; } } if (!idbi) FatalError (1, "Could NOT create .invf.blocked.%d file", stem_method); PackIdxHashTable(); qsort (IdxHashTable, IdxHashUsed, sizeof (idx_hash_rec *), idx_comp); sih.lookback = lookback; sih.block_size = block_size; sih.num_blocks = 0; sih.blocks_start = 0; sih.index_chars = 0; sih.num_of_words = IdxHashUsed; fwrite ((char *) &sih, sizeof (sih), 1, idbi); if (!(buffer = Xmalloc (block_size + 512))) FatalError (1, "Unable to allocate memory for \"buffer\"\n"); if (!(pointers = Xmalloc (block_size + 512))) FatalError (1, "Unable to allocate memory for \"buffer\"\n"); buf_in_use = 0; pos = 0; word_num = 0; ptrs_in_use = 0; First_word = 0; /* For each word in the hashtable... */ for (i = 0; i < IdxHashUsed; i++) { register mg_u_long extra, copy, suff; register struct idx_hash_rec *ent = IdxHashTable[i]; /* build a new word on top of prev */ if (last_word != NULL) copy = prefixlen (last_word, ent->word); else copy = 0; suff = *(ent->word) - copy; last_word = ent->word; if (word_num % sih.lookback == 0) /* Will need copy chars to add + a pointer in index */ extra = copy + sizeof (*pointers); else extra = 0; if ((ptrs_in_use + 1) * sizeof (*pointers) + sizeof (ptrs_in_use) + extra + buf_in_use + sizeof (First_word) + suff + 1 + sizeof (ent->PL->num_entries) + ent->PL->num_entries * sizeof (PosEntry) > block_size) { /* Dump buffer to tmp file */ int chunk; HTONUL(First_word); /* [RPAP - Jan 97: Endian Ordering] */ HTONUS(word_num); /* [RPAP - Jan 97: Endian Ordering] */ fwrite (&First_word, sizeof (First_word), 1, tmp); fwrite (&word_num, sizeof (word_num), 1, tmp); fwrite (pointers, sizeof (*pointers), ptrs_in_use, tmp); fwrite (buffer, sizeof (u_char), buf_in_use, tmp); bzero ((char *) buffer, block_size); chunk = buf_in_use + ptrs_in_use * sizeof (*pointers) + sizeof (ptrs_in_use) + sizeof (First_word); if (force && chunk < block_size) { fwrite (buffer, sizeof (u_char), block_size - chunk, tmp); chunk = block_size; } pos += chunk; buf_in_use = 0; word_num = 0; ptrs_in_use = 0; sih.num_blocks++; /* Check that entry will fit into new block */ if (sizeof (*pointers) + sizeof (ptrs_in_use) + extra + sizeof (First_word) + suff + 1 + sizeof (ent->PL->num_entries) + ent->PL->num_entries * sizeof (PosEntry) > block_size) FatalError (1, "Block size to small"); } /* if size > block_size */ if (word_num % sih.lookback == 0) { HTONUS2(buf_in_use, pointers[ptrs_in_use++]); /* [RPAP - Jan 97: Endian Ordering] */ suff += copy; copy = 0; } /* Output Word information */ buffer[buf_in_use++] = copy; buffer[buf_in_use++] = suff; bcopy ((char *) (ent->word + copy + 1), (char *) (buffer + buf_in_use), suff); buf_in_use += suff; HTONUI(ent->PL->num_entries); /* [RPAP - Jan 97: Endian Ordering] */ bcopy ((char *) &(ent->PL->num_entries), (char *) (buffer + buf_in_use), sizeof (ent->PL->num_entries)); NTOHUI(ent->PL->num_entries); /* [RPAP - Jan 97: Endian Ordering] */ buf_in_use += sizeof (ent->PL->num_entries); for (j = 0; j < ent->PL->num_entries; j++) { register PosEntry *pe = &(ent->PL->PE[j]); HTONUI(pe->num_cases); /* [RPAP - Jan 97: Endian Ordering] */ bcopy ((char *) &(pe->num_cases), (char *) (buffer + buf_in_use), sizeof (pe->num_cases)); buf_in_use += sizeof (pe->num_cases); HTONUI(pe->blk); /* [RPAP - Jan 97: Endian Ordering] */ bcopy ((char *) &(pe->blk), (char *) (buffer + buf_in_use), sizeof (pe->blk)); buf_in_use += sizeof (pe->blk); HTONUS(pe->blk_index); /* [RPAP - Jan 97: Endian Ordering] */ bcopy ((char *) &(pe->blk_index), (char *) (buffer + buf_in_use), sizeof (pe->blk_index)); buf_in_use += sizeof (pe->blk_index); HTONUS(pe->offset); /* [RPAP - Jan 97: Endian Ordering] */ bcopy ((char *) &(pe->offset), (char *) (buffer + buf_in_use), sizeof (pe->offset)); buf_in_use += sizeof (pe->offset); } if (buf_in_use + ptrs_in_use * sizeof (*pointers) + sizeof (ptrs_in_use) > block_size) FatalError (1, "Fatal Internal Error # 64209258\n"); if (word_num == 0) /* the first word in each block - word_num set to 0 when buffer written to tmp file */ { /* Write word to main index */ fwrite (ent->word, sizeof (u_char), *(ent->word) + 1, idbi); HTONUL(pos); /* [RPAP - Jan 97: Endian Ordering] */ fwrite (&pos, sizeof (pos), 1, idbi); NTOHUL(pos); /* [RPAP - Jan 97: Endian Ordering] */ sih.index_chars += *(ent->word) + 1; First_word = i; } word_num++; } /* end for each word */ /* for each hash entry */ if (buf_in_use) { /* Write last buffer to tmp file */ int chunk; /* [RPAP - Jan 97: Endian Ordering] */ HTONUL(First_word); HTONUS(word_num); fwrite (&First_word, sizeof (First_word), 1, tmp); fwrite (&word_num, sizeof (word_num), 1, tmp); fwrite (pointers, sizeof (*pointers), ptrs_in_use, tmp); fwrite (buffer, sizeof (u_char), buf_in_use, tmp); bzero ((char *) buffer, block_size); chunk = buf_in_use + ptrs_in_use * sizeof (*pointers) + sizeof (ptrs_in_use) + sizeof (First_word); if (force && chunk < block_size) { fwrite (buffer, sizeof (u_char), block_size - chunk, tmp); chunk = block_size; } sih.num_blocks++; } rewind (tmp); sih.blocks_start = sih.index_chars + sizeof (mg_u_long) + sizeof (sih) + sih.num_blocks * sizeof (pos); if (force) { int amount; amount = sih.blocks_start % block_size; if (amount != 0) { bzero ((char *) buffer, block_size); fwrite (buffer, sizeof (u_char), block_size - amount, idbi); sih.blocks_start += block_size - amount; } } while ((num = fread (buffer, sizeof (u_char), block_size, tmp)) != 0) fwrite (buffer, sizeof (u_char), num, idbi); fclose (tmp); /* skip over the magic number */ fseek (idbi, sizeof (mg_u_long), 0); /* [RPAP - Jan 97: Endian Ordering] */ HTONUL(sih.lookback); HTONUL(sih.block_size); HTONUL(sih.num_blocks); HTONUL(sih.blocks_start); HTONUL(sih.index_chars); HTONUL(sih.num_of_words); fwrite (&sih, sizeof (sih), 1, idbi); fclose (idbi); #ifndef SILENT Message ("Stem %d:\n", stem_method); Message (" Block size : %10d\n", block_size); Message (" Num_blocks : %10d\n", NTOHUL(sih.num_blocks)); /* [RPAP - Jan 97: Endian Ordering] */ Message (" Max mem used : %10.1f Mb\n", (double) MaxMemInUse / 1024.0 / 1024.0); Message (" Num_of_words : %10d\n", NTOHUL(sih.num_of_words)); /* [RPAP - Jan 97: Endian Ordering] */ #endif } void UpdateStemDict (char * filename, int stem_method) { FILE *idb; struct stem_dict_header sdh; if (!(idb = open_file (filename, INVF_DICT_BLOCKED_SUFFIX, "r+b", MAGIC_STEM, MG_CONTINUE))) FatalError (1, "Could not update stemmed dict"); fread ((char *) &sdh, sizeof (sdh), 1, idb); NTOHUL(sdh.indexed); /* [RPAP - Jan 97: Endian Ordering] */ sdh.indexed |= 1 << (stem_method - 1); HTONUL(sdh.indexed); fseek (idb, sizeof (mg_u_long), 0); fwrite ((char *) &sdh, sizeof (sdh), 1, idb); fclose (idb); } /* Main */ int main (int argc, char **argv) { File *idb; /* File to .invf.dict.blocked */ char *filename = ""; stemmed_dict *sd; /* Stemmed dictionary */ int ch; char path[512]; int stem_method = 0; msg_prefix = argv[0]; opterr = 0; while ((ch = getopt (argc, argv, "f:d:b:hFs:")) != -1) switch (ch) { case 'f': /* input file */ filename = optarg; break; case 'd': set_basepath (optarg); break; case 'b': block_size = atoi (optarg); break; case 'F': force = 1; break; case 's': stem_method = atoi (optarg); break; case 'h': case '?': fprintf (stderr, "usage: %s [-d directory] " "[-b num] [-F] [-h] -s 1|2|3 -f name\n", argv[0]); exit (1); } if (stem_method < 1 || stem_method > 3) FatalError (1, "Stem method must be 1, 2 or 3"); /* Open required stem dict file */ sprintf (path, FILE_NAME_FORMAT, get_basepath (), filename, INVF_DICT_BLOCKED_SUFFIX); if (!(idb = Fopen (path, "rb", MAGIC_STEM))) FatalError (1, "Unable to open \"%s\"", path); /* Read in idb header and index to blocks */ if (!(sd = ReadStemDictBlk (idb))) FatalError (1, "Could not read stemmed dictionary"); /* Process stemmed dictionary */ mg_init_process (); process_stem_dict (sd, stem_method); save_idx (filename, stem_method); /* Close stemmed dict */ Fclose (idb); /* Update stemmed dict */ UpdateStemDict (filename, stem_method); return 0; }