Rename third_party/hunspell_new back to third_party/hunspell.

third_party/hunspell_new/src/hunspell/hashmgr.cxx

-#include "license.hunspell"
-#include "license.myspell"
-
-#include <stdlib.h> 
-#include <string.h>
-#include <stdio.h> 
-#include <ctype.h>
-
-#include "hashmgr.hxx"
-#include "csutil.hxx"
-#include "atypes.hxx"
-
-// build a hash table from a munched word list
-
-#ifdef HUNSPELL_CHROME_CLIENT
-HashMgr::HashMgr(hunspell::BDictReader* reader)
-{
-  bdict_reader = reader;
-#else
-HashMgr::HashMgr(const char * tpath, const char * apath, const char * key)
-{
-#endif
-  tablesize = 0;
-  tableptr = NULL;
-  flag_mode = FLAG_CHAR;
-  complexprefixes = 0;
-  utf8 = 0;
-  langnum = 0;
-  lang = NULL;
-  enc = NULL;
-  csconv = 0;
-  ignorechars = NULL;
-  ignorechars_utf16 = NULL;
-  ignorechars_utf16_len = 0;
-  numaliasf = 0;
-  aliasf = NULL;
-  numaliasm = 0;
-  aliasm = NULL;
-  forbiddenword = FORBIDDENWORD; // forbidden word signing flag
-#ifdef HUNSPELL_CHROME_CLIENT
-  // No tables to load, just the AF lines.
-  load_config(NULL, NULL);
-  int ec = LoadAFLines();
-#else
-  load_config(apath, key);
-  int ec = load_tables(tpath, key);
-#endif
-  if (ec) {
-    /* error condition - what should we do here */
-    HUNSPELL_WARNING(stderr, "Hash Manager Error : %d\n",ec);
-    if (tableptr) {
-      free(tableptr);
-      tableptr = NULL;
-    }
-    tablesize = 0;
-  }
-}
-
-
-HashMgr::~HashMgr()
-{
-  if (tableptr) {
-    // now pass through hash table freeing up everything
-    // go through column by column of the table
-    for (int i=0; i < tablesize; i++) {
-      struct hentry * pt = tableptr[i];
-      struct hentry * nt = NULL;
-      while(pt) {
-        nt = pt->next;
-        if (pt->astr && (!aliasf || TESTAFF(pt->astr, ONLYUPCASEFLAG, pt->alen))) free(pt->astr);
-        free(pt);
-        pt = nt;
-      }
-    }
-    free(tableptr);
-  }
-  tablesize = 0;
-
-  if (aliasf) {
-    for (int j = 0; j < (numaliasf); j++) free(aliasf[j]);
-    free(aliasf);
-    aliasf = NULL;
-    if (aliasflen) {
-      free(aliasflen);
-      aliasflen = NULL;
-    }
-  }
-  if (aliasm) {
-    for (int j = 0; j < (numaliasm); j++) free(aliasm[j]);
-    free(aliasm);
-    aliasm = NULL;
-  }  
-
-#ifndef OPENOFFICEORG
-#ifndef MOZILLA_CLIENT
-  if (utf8) free_utf_tbl();
-#endif
-#endif
-
-  if (enc) free(enc);
-  if (lang) free(lang);
-  
-  if (ignorechars) free(ignorechars);
-  if (ignorechars_utf16) free(ignorechars_utf16);
-
-#ifdef HUNSPELL_CHROME_CLIENT
-  EmptyHentryCache();
-  for (std::vector<std::string*>::iterator it = pointer_to_strings_.begin();
-       it != pointer_to_strings_.end(); ++it) {
-    delete *it;
-  }
-#endif
-#ifdef MOZILLA_CLIENT
-    delete [] csconv;
-#endif
-}
-
-#ifdef HUNSPELL_CHROME_CLIENT
-void HashMgr::EmptyHentryCache() {
-  // We need to delete each cache entry, and each additional one in the linked
-  // list of homonyms.
-  for (HEntryCache::iterator i = hentry_cache.begin();
-       i != hentry_cache.end(); ++i) {
-    hentry* cur = i->second;
-    while (cur) {
-      hentry* next = cur->next_homonym;
-      DeleteHashEntry(cur);
-      cur = next;
-    }
-  }
-  hentry_cache.clear();
-}
-#endif
-
-// lookup a root word in the hashtable
-
-struct hentry * HashMgr::lookup(const char *word) const
-{
-#ifdef HUNSPELL_CHROME_CLIENT
-  int affix_ids[hunspell::BDict::MAX_AFFIXES_PER_WORD];
-  int affix_count = bdict_reader->FindWord(word, affix_ids);
-  if (affix_count == 0) { // look for custom added word
-    std::map<base::StringPiece, int>::const_iterator iter = 
-      custom_word_to_affix_id_map_.find(word);
-    if (iter != custom_word_to_affix_id_map_.end()) {
-      affix_count = 1;
-      affix_ids[0] = iter->second;
-    }
-  }
-
-  static const int kMaxWordLen = 128;
-  static char word_buf[kMaxWordLen];
-  // To take account of null-termination, we use upto 127.
-  strncpy(word_buf, word, kMaxWordLen - 1);
-
-  return AffixIDsToHentry(word_buf, affix_ids, affix_count);
-#else
-    struct hentry * dp;
-    if (tableptr) {
-       dp = tableptr[hash(word)];
-       if (!dp) return NULL;
-       for (  ;  dp != NULL;  dp = dp->next) {
-          if (strcmp(word, dp->word) == 0) return dp;
-       }
-    }
-    return NULL;
-#endif
-}
-
-// add a word to the hash table (private)
-int HashMgr::add_word(const char * word, int wbl, int wcl, unsigned short * aff,
-    int al, const char * desc, bool onlyupcase)
-{
-#ifndef HUNSPELL_CHROME_CLIENT
-    bool upcasehomonym = false;
-    int descl = desc ? (aliasm ? sizeof(short) : strlen(desc) + 1) : 0;
-    // variable-length hash record with word and optional fields
-    struct hentry* hp = 
-        (struct hentry *) malloc (sizeof(struct hentry) + wbl + descl);
-    if (!hp) return 1;
-    char * hpw = hp->word;
-    strcpy(hpw, word);
-    if (ignorechars != NULL) {
-      if (utf8) {
-        remove_ignored_chars_utf(hpw, ignorechars_utf16, ignorechars_utf16_len);
-      } else {
-        remove_ignored_chars(hpw, ignorechars);
-      }
-    }
-    if (complexprefixes) {
-        if (utf8) reverseword_utf(hpw); else reverseword(hpw);
-    }
-
-    int i = hash(hpw);
-
-    hp->blen = (unsigned char) wbl;
-    hp->clen = (unsigned char) wcl;
-    hp->alen = (short) al;
-    hp->astr = aff;
-    hp->next = NULL;      
-    hp->next_homonym = NULL;
-
-    // store the description string or its pointer
-    if (desc) {
-        hp->var = H_OPT;
-        if (aliasm) {
-            hp->var += H_OPT_ALIASM;
-            store_pointer(hpw + wbl + 1, get_aliasm(atoi(desc)));
-        } else {
-            strcpy(hpw + wbl + 1, desc);
-            if (complexprefixes) {
-                if (utf8) reverseword_utf(HENTRY_DATA(hp));
-                else reverseword(HENTRY_DATA(hp));
-            }
-        }
-        if (strstr(HENTRY_DATA(hp), MORPH_PHON)) hp->var += H_OPT_PHON;
-    } else hp->var = 0;
-
-       struct hentry * dp = tableptr[i];
-       if (!dp) {
-         tableptr[i] = hp;
-         return 0;
-       }
-       while (dp->next != NULL) {
-         if ((!dp->next_homonym) && (strcmp(hp->word, dp->word) == 0)) {
-            // remove hidden onlyupcase homonym
-            if (!onlyupcase) {
-                if ((dp->astr) && TESTAFF(dp->astr, ONLYUPCASEFLAG, dp->alen)) {
-                    free(dp->astr);
-                    dp->astr = hp->astr;
-                    dp->alen = hp->alen;
-                    free(hp);
-                    return 0;
-                } else {
-                    dp->next_homonym = hp;
-                }
-            } else {
-                upcasehomonym = true;
-            }
-         }
-         dp=dp->next;
-       }
-       if (strcmp(hp->word, dp->word) == 0) {
-            // remove hidden onlyupcase homonym
-            if (!onlyupcase) {
-                if ((dp->astr) && TESTAFF(dp->astr, ONLYUPCASEFLAG, dp->alen)) {
-                    free(dp->astr);
-                    dp->astr = hp->astr;
-                    dp->alen = hp->alen;
-                    free(hp);
-                    return 0;
-                } else {
-                    dp->next_homonym = hp;
-                }
-            } else {
-                upcasehomonym = true;
-            }
-       }
-       if (!upcasehomonym) {
-            dp->next = hp;
-       } else {
-            // remove hidden onlyupcase homonym
-            if (hp->astr) free(hp->astr);
-            free(hp);
-       }
-#else
-    std::map<base::StringPiece, int>::iterator iter =
-        custom_word_to_affix_id_map_.find(word);
-    if(iter == custom_word_to_affix_id_map_.end()) {  // word needs to be added
-      std::string* new_string_word = new std::string(word);
-      pointer_to_strings_.push_back(new_string_word);
-      base::StringPiece sp(*(new_string_word));
-      custom_word_to_affix_id_map_[sp] = 0; // no affixes for custom words
-      return 1;
-    }
-#endif
-    return 0;
-}     
-
-int HashMgr::add_hidden_capitalized_word(char * word, int wbl, int wcl,
-    unsigned short * flags, int al, char * dp, int captype)
-{
-    // add inner capitalized forms to handle the following allcap forms:
-    // Mixed caps: OpenOffice.org -> OPENOFFICE.ORG
-    // Allcaps with suffixes: CIA's -> CIA'S    
-    if (((captype == HUHCAP) || (captype == HUHINITCAP) ||
-      ((captype == ALLCAP) && (flags != NULL))) &&
-      !((flags != NULL) && TESTAFF(flags, forbiddenword, al))) {
-          unsigned short * flags2 = (unsigned short *) malloc (sizeof(unsigned short) * (al+1));
-          if (!flags2) return 1;
-          if (al) memcpy(flags2, flags, al * sizeof(unsigned short));
-          flags2[al] = ONLYUPCASEFLAG;
-          if (utf8) {
-              char st[BUFSIZE];
-              w_char w[BUFSIZE];
-              int wlen = u8_u16(w, BUFSIZE, word);
-              mkallsmall_utf(w, wlen, langnum);
-              mkallcap_utf(w, 1, langnum);
-              u16_u8(st, BUFSIZE, w, wlen);
-              return add_word(st,wbl,wcl,flags2,al+1,dp, true);
-           } else {
-               mkallsmall(word, csconv);
-               mkinitcap(word, csconv);
-               return add_word(word,wbl,wcl,flags2,al+1,dp, true);
-           }
-    }
-    return 0;
-}
-
-// detect captype and modify word length for UTF-8 encoding
-int HashMgr::get_clen_and_captype(const char * word, int wbl, int * captype) {
-    int len;
-    if (utf8) {
-      w_char dest_utf[BUFSIZE];
-      len = u8_u16(dest_utf, BUFSIZE, word);
-      *captype = get_captype_utf8(dest_utf, len, langnum);
-    } else {
-      len = wbl;
-      *captype = get_captype((char *) word, len, csconv);
-    }
-    return len;
-}
-
-// remove word (personal dictionary function for standalone applications)
-int HashMgr::remove(const char * word)
-{
-#ifdef HUNSPELL_CHROME_CLIENT
-    std::map<base::StringPiece, int>::iterator iter =
-        custom_word_to_affix_id_map_.find(word);
-    if (iter != custom_word_to_affix_id_map_.end())
-        custom_word_to_affix_id_map_.erase(iter);
-#else
-    struct hentry * dp = lookup(word);
-    while (dp) {
-        if (dp->alen == 0 || !TESTAFF(dp->astr, forbiddenword, dp->alen)) {
-            unsigned short * flags =
-                (unsigned short *) malloc(sizeof(short) * (dp->alen + 1));
-            if (!flags) return 1;
-            for (int i = 0; i < dp->alen; i++) flags[i] = dp->astr[i];
-            flags[dp->alen] = forbiddenword;
-            dp->astr = flags;
-            dp->alen++;
-            flag_qsort(flags, 0, dp->alen);
-        }
-        dp = dp->next_homonym;
-    }
-#endif
-    return 0;
-}
-
-/* remove forbidden flag to add a personal word to the hash */
-int HashMgr::remove_forbidden_flag(const char * word) {
-    struct hentry * dp = lookup(word);
-    if (!dp) return 1;
-    while (dp) {
-         if (dp->astr && TESTAFF(dp->astr, forbiddenword, dp->alen)) {
-            if (dp->alen == 1) dp->alen = 0; // XXX forbidden words of personal dic.
-            else {
-                unsigned short * flags2 =
-                    (unsigned short *) malloc(sizeof(short) * (dp->alen - 1));
-                if (!flags2) return 1;
-                int i, j = 0;
-                for (i = 0; i < dp->alen; i++) {
-                    if (dp->astr[i] != forbiddenword) flags2[j++] = dp->astr[i];
-                }
-                dp->alen--;
-                dp->astr = flags2; // XXX allowed forbidden words
-            }
-         }
-         dp = dp->next_homonym;
-       }
-   return 0;
-}
-
-// add a custom dic. word to the hash table (public)
-int HashMgr::add(const char * word)
-{
-    unsigned short * flags = NULL;
-    int al = 0;
-    if (remove_forbidden_flag(word)) {
-        int captype;
-        int wbl = strlen(word);
-        int wcl = get_clen_and_captype(word, wbl, &captype);
-        add_word(word, wbl, wcl, flags, al, NULL, false);
-        return add_hidden_capitalized_word((char *) word, wbl, wcl, flags, al, NULL, captype);
-    }
-    return 0;
-}
-
-int HashMgr::add_with_affix(const char * word, const char * example)
-{
-    // detect captype and modify word length for UTF-8 encoding
-    struct hentry * dp = lookup(example);
-    remove_forbidden_flag(word);
-    if (dp && dp->astr) {
-        int captype;
-        int wbl = strlen(word);
-        int wcl = get_clen_and_captype(word, wbl, &captype);
-        if (aliasf) {
-            add_word(word, wbl, wcl, dp->astr, dp->alen, NULL, false);  
-        } else {
-            unsigned short * flags = (unsigned short *) malloc (dp->alen * sizeof(short));
-            if (flags) {
-                memcpy((void *) flags, (void *) dp->astr, dp->alen * sizeof(short));
-                add_word(word, wbl, wcl, flags, dp->alen, NULL, false);
-            } else return 1;
-        }
-        return add_hidden_capitalized_word((char *) word, wbl, wcl, dp->astr, dp->alen, NULL, captype);
-    }
-    return 1;
-}
-
-// walk the hash table entry by entry - null at end
-// initialize: col=-1; hp = NULL; hp = walk_hashtable(&col, hp);
-struct hentry * HashMgr::walk_hashtable(int &col, struct hentry * hp) const
-{  
-#ifdef HUNSPELL_CHROME_CLIENT
-  // Return NULL if dictionary is not valid.
-  if (!bdict_reader->IsValid())
-    return NULL;
-
-  // This function is only ever called by one place and not nested. We can
-  // therefore keep static state between calls and use |col| as a "reset" flag
-  // to avoid changing the API. It is set to -1 for the first call.
-  // Allocate the iterator on the heap to prevent an exit time destructor.
-  static hunspell::WordIterator& word_iterator =
-      *new hunspell::WordIterator(bdict_reader->GetAllWordIterator());
-  if (col < 0) {
-    col = 1;
-    word_iterator = bdict_reader->GetAllWordIterator();
-  }
-
-  int affix_ids[hunspell::BDict::MAX_AFFIXES_PER_WORD];
-  static const int kMaxWordLen = 128;
-  static char word[kMaxWordLen];
-  int affix_count = word_iterator.Advance(word, kMaxWordLen, affix_ids);
-  if (affix_count == 0)
-    return NULL;
-  short word_len = static_cast<short>(strlen(word));
-
-  // Since hunspell 1.2.8, an hentry struct becomes a variable-length struct,
-  // i.e. a struct which uses its array 'word[1]' as a variable-length array.
-  // As noted above, this function is not nested. So, we just use a static
-  // struct which consists of an hentry and a char[kMaxWordLen], and initialize
-  // the static struct and return it for now.
-  // No need to create linked lists for the extra affixes.
-  static struct {
-    hentry entry;
-    char word[kMaxWordLen];
-  } hash_entry;
-
-  return InitHashEntry(&hash_entry.entry, sizeof(hash_entry),
-                       &word[0], word_len, affix_ids[0]);
-#else
-  if (hp && hp->next != NULL) return hp->next;
-  for (col++; col < tablesize; col++) {
-    if (tableptr[col]) return tableptr[col];
-  }
-  // null at end and reset to start
-  col = -1;
-  return NULL;
-#endif
-}
-
-// load a munched word list and build a hash table on the fly
-int HashMgr::load_tables(const char * tpath, const char * key)
-{
-#ifndef HUNSPELL_CHROME_CLIENT
-  int al;
-  char * ap;
-  char * dp;
-  char * dp2;
-  unsigned short * flags;
-  char * ts;
-
-  // open dictionary file
-  FileMgr * dict = new FileMgr(tpath, key);
-  if (dict == NULL) return 1;
-
-  // first read the first line of file to get hash table size */
-  if ((ts = dict->getline()) == NULL) {
-    HUNSPELL_WARNING(stderr, "error: empty dic file %s\n", tpath);
-    delete dict;
-    return 2;
-  }
-  mychomp(ts);
-
-  /* remove byte order mark */
-  if (strncmp(ts,"\xEF\xBB\xBF",3) == 0) {
-    memmove(ts, ts+3, strlen(ts+3)+1);
-    // warning: dic file begins with byte order mark: possible incompatibility with old Hunspell versions
-  }
-
-  tablesize = atoi(ts);
-  if (tablesize == 0) {
-    HUNSPELL_WARNING(stderr, "error: line 1: missing or bad word count in the dic file\n");
-    delete dict;
-    return 4;
-  }
-  tablesize = tablesize + 5 + USERWORD;
-  if ((tablesize %2) == 0) tablesize++;
-
-  // allocate the hash table
-  tableptr = (struct hentry **) malloc(tablesize * sizeof(struct hentry *));
-  if (! tableptr) {
-    delete dict;
-    return 3;
-  }
-  for (int i=0; i<tablesize; i++) tableptr[i] = NULL;
-
-  // loop through all words on much list and add to hash
-  // table and create word and affix strings
-
-  while ((ts = dict->getline()) != NULL) {
-    mychomp(ts);
-    // split each line into word and morphological description
-    dp = ts;
-    while ((dp = strchr(dp, ':')) != NULL) {
-        if ((dp > ts + 3) && (*(dp - 3) == ' ' || *(dp - 3) == '\t')) {
-            for (dp -= 4; dp >= ts && (*dp == ' ' || *dp == '\t'); dp--);
-            if (dp < ts) { // missing word
-                dp = NULL;
-            } else {
-                *(dp + 1) = '\0';
-                dp = dp + 2;
-            }
-            break;
-        }
-        dp++;
-    }
-
-    // tabulator is the old morphological field separator
-    dp2 = strchr(ts, '\t');
-    if (dp2 && (!dp || dp2 < dp)) {
-        *dp2 = '\0';
-        dp = dp2 + 1;
-    }
-
-    // split each line into word and affix char strings
-    // "\/" signs slash in words (not affix separator)
-    // "/" at beginning of the line is word character (not affix separator)
-    ap = strchr(ts,'/');
-    while (ap) {
-        if (ap == ts) {
-            ap++;
-            continue;
-        } else if (*(ap - 1) != '\\') break;
-        // replace "\/" with "/"
-        for (char * sp = ap - 1; *sp; *sp = *(sp + 1), sp++);
-        ap = strchr(ap,'/');
-    }
-
-    if (ap) {
-      *ap = '\0';
-      if (aliasf) {
-        int index = atoi(ap + 1);
-        al = get_aliasf(index, &flags, dict);
-        if (!al) {
-            HUNSPELL_WARNING(stderr, "error: line %d: bad flag vector alias\n", dict->getlinenum());
-            *ap = '\0';
-        }
-      } else {
-        al = decode_flags(&flags, ap + 1, dict);
-        if (al == -1) {
-            HUNSPELL_WARNING(stderr, "Can't allocate memory.\n");
-            delete dict;
-            return 6;
-        }
-        flag_qsort(flags, 0, al);
-      }
-    } else {
-      al = 0;
-      ap = NULL;
-      flags = NULL;
-    }
-
-    int captype;
-    int wbl = strlen(ts);
-    int wcl = get_clen_and_captype(ts, wbl, &captype);
-    // add the word and its index plus its capitalized form optionally
-    if (add_word(ts,wbl,wcl,flags,al,dp, false) ||
-        add_hidden_capitalized_word(ts, wbl, wcl, flags, al, dp, captype)) {
-        delete dict;
-        return 5;
-    }
-  }
-
-  delete dict;
-#endif
-  return 0;
-}
-
-// the hash function is a simple load and rotate
-// algorithm borrowed
-
-int HashMgr::hash(const char * word) const
-{
-#ifdef HUNSPELL_CHROME_CLIENT
-    return 0;
-#else
-    long  hv = 0;
-    for (int i=0; i < 4  &&  *word != 0; i++)
-        hv = (hv << 8) | (*word++);
-    while (*word != 0) {
-      ROTATE(hv,ROTATE_LEN);
-      hv ^= (*word++);
-    }
-    return (unsigned long) hv % tablesize;
-#endif
-}
-
-int HashMgr::decode_flags(unsigned short ** result, char * flags, FileMgr * af) {
-    int len;
-    if (*flags == '\0') {
-        *result = NULL;
-        return 0;
-    }
-    switch (flag_mode) {
-      case FLAG_LONG: { // two-character flags (1x2yZz -> 1x 2y Zz)
-        len = strlen(flags);
-        if (len%2 == 1) HUNSPELL_WARNING(stderr, "error: line %d: bad flagvector\n", af->getlinenum());
-        len /= 2;
-        *result = (unsigned short *) malloc(len * sizeof(short));
-        if (!*result) return -1;
-        for (int i = 0; i < len; i++) {
-            (*result)[i] = (((unsigned short) flags[i * 2]) << 8) + (unsigned short) flags[i * 2 + 1]; 
-        }
-        break;
-      }
-      case FLAG_NUM: { // decimal numbers separated by comma (4521,23,233 -> 4521 23 233)
-        int i;
-        len = 1;
-        char * src = flags; 
-        unsigned short * dest;
-        char * p;
-        for (p = flags; *p; p++) {
-          if (*p == ',') len++;
-        }
-        *result = (unsigned short *) malloc(len * sizeof(short));
-        if (!*result) return -1;
-        dest = *result;
-        for (p = flags; *p; p++) {
-          if (*p == ',') {
-            i = atoi(src);
-            if (i >= DEFAULTFLAGS) HUNSPELL_WARNING(stderr, "error: line %d: flag id %d is too large (max: %d)\n",
-              af->getlinenum(), i, DEFAULTFLAGS - 1);
-            *dest = (unsigned short) i;
-            if (*dest == 0) HUNSPELL_WARNING(stderr, "error: line %d: 0 is wrong flag id\n", af->getlinenum());
-            src = p + 1;
-            dest++;
-          }
-        }
-        i = atoi(src);
-        if (i >= DEFAULTFLAGS) HUNSPELL_WARNING(stderr, "error: line %d: flag id %d is too large (max: %d)\n",
-          af->getlinenum(), i, DEFAULTFLAGS - 1);
-        *dest = (unsigned short) i;
-        if (*dest == 0) HUNSPELL_WARNING(stderr, "error: line %d: 0 is wrong flag id\n", af->getlinenum());
-        break;
-      }    
-      case FLAG_UNI: { // UTF-8 characters
-        w_char w[BUFSIZE/2];
-        len = u8_u16(w, BUFSIZE/2, flags);
-        *result = (unsigned short *) malloc(len * sizeof(short));
-        if (!*result) return -1;
-        memcpy(*result, w, len * sizeof(short));
-        break;
-      }
-      default: { // Ispell's one-character flags (erfg -> e r f g)
-        unsigned short * dest;
-        len = strlen(flags);
-        *result = (unsigned short *) malloc(len * sizeof(short));
-        if (!*result) return -1;
-        dest = *result;
-        for (unsigned char * p = (unsigned char *) flags; *p; p++) {
-          *dest = (unsigned short) *p;
-          dest++;
-        }
-      }
-    }
-    return len;
-}
-
-unsigned short HashMgr::decode_flag(const char * f) {
-    unsigned short s = 0;
-    int i;
-    switch (flag_mode) {
-      case FLAG_LONG:
-        s = ((unsigned short) f[0] << 8) + (unsigned short) f[1];
-        break;
-      case FLAG_NUM:
-        i = atoi(f);
-        if (i >= DEFAULTFLAGS) HUNSPELL_WARNING(stderr, "error: flag id %d is too large (max: %d)\n", i, DEFAULTFLAGS - 1);
-        s = (unsigned short) i;
-        break;
-      case FLAG_UNI:
-        u8_u16((w_char *) &s, 1, f);
-        break;
-      default:
-        s = (unsigned short) *((unsigned char *)f);
-    }
-    if (s == 0) HUNSPELL_WARNING(stderr, "error: 0 is wrong flag id\n");
-    return s;
-}
-
-char * HashMgr::encode_flag(unsigned short f) {
-    unsigned char ch[10];
-    if (f==0) return mystrdup("(NULL)");
-    if (flag_mode == FLAG_LONG) {
-        ch[0] = (unsigned char) (f >> 8);
-        ch[1] = (unsigned char) (f - ((f >> 8) << 8));
-        ch[2] = '\0';
-    } else if (flag_mode == FLAG_NUM) {
-        sprintf((char *) ch, "%d", f);
-    } else if (flag_mode == FLAG_UNI) {
-        u16_u8((char *) &ch, 10, (w_char *) &f, 1);
-    } else {
-        ch[0] = (unsigned char) (f);
-        ch[1] = '\0';
-    }
-    return mystrdup((char *) ch);
-}
-
-// read in aff file and set flag mode
-int  HashMgr::load_config(const char * affpath, const char * key)
-{
-  char * line; // io buffers
-  int firstline = 1;
- 
-  // open the affix file
-#ifdef HUNSPELL_CHROME_CLIENT
-  hunspell::LineIterator iterator = bdict_reader->GetOtherLineIterator();
-  FileMgr * afflst = new FileMgr(&iterator);
-#else
-  FileMgr * afflst = new FileMgr(affpath, key);
-#endif
-  if (!afflst) {
-    HUNSPELL_WARNING(stderr, "Error - could not open affix description file %s\n",affpath);
-    return 1;
-  }
-
-    // read in each line ignoring any that do not
-    // start with a known line type indicator
-
-    while ((line = afflst->getline()) != NULL) {
-        mychomp(line);
-
-       /* remove byte order mark */
-       if (firstline) {
-         firstline = 0;
-         if (strncmp(line,"\xEF\xBB\xBF",3) == 0) memmove(line, line+3, strlen(line+3)+1);
-       }
-
-        /* parse in the try string */
-        if ((strncmp(line,"FLAG",4) == 0) && isspace(line[4])) {
-            if (flag_mode != FLAG_CHAR) {
-                HUNSPELL_WARNING(stderr, "error: line %d: multiple definitions of the FLAG affix file parameter\n", afflst->getlinenum());
-            }
-            if (strstr(line, "long")) flag_mode = FLAG_LONG;
-            if (strstr(line, "num")) flag_mode = FLAG_NUM;
-            if (strstr(line, "UTF-8")) flag_mode = FLAG_UNI;
-            if (flag_mode == FLAG_CHAR) {
-                HUNSPELL_WARNING(stderr, "error: line %d: FLAG needs `num', `long' or `UTF-8' parameter\n", afflst->getlinenum());
-            }
-        }
-        if (strncmp(line,"FORBIDDENWORD",13) == 0) {
-          char * st = NULL;
-          if (parse_string(line, &st, afflst->getlinenum())) {
-             delete afflst;
-             return 1;
-          }
-          forbiddenword = decode_flag(st);
-          free(st);
-        }
-        if (strncmp(line, "SET", 3) == 0) {
-          if (parse_string(line, &enc, afflst->getlinenum())) {
-             delete afflst;
-             return 1;
-          }         
-          if (strcmp(enc, "UTF-8") == 0) {
-            utf8 = 1;
-#ifndef OPENOFFICEORG
-#ifndef MOZILLA_CLIENT
-            initialize_utf_tbl();
-#endif
-#endif
-          } else csconv = get_current_cs(enc);
-        }
-        if (strncmp(line, "LANG", 4) == 0) {
-          if (parse_string(line, &lang, afflst->getlinenum())) {
-             delete afflst;
-             return 1;
-          }         
-          langnum = get_lang_num(lang);
-        }
-
-       /* parse in the ignored characters (for example, Arabic optional diacritics characters */
-       if (strncmp(line,"IGNORE",6) == 0) {
-          if (parse_array(line, &ignorechars, &ignorechars_utf16,
-                 &ignorechars_utf16_len, utf8, afflst->getlinenum())) {
-             delete afflst;
-             return 1;
-          }
-       }
-
-       if ((strncmp(line,"AF",2) == 0) && isspace(line[2])) {
-          if (parse_aliasf(line, afflst)) {
-             delete afflst;
-             return 1;
-          }
-       }
-
-       if ((strncmp(line,"AM",2) == 0) && isspace(line[2])) {
-          if (parse_aliasm(line, afflst)) {
-             delete afflst;
-             return 1;
-          }
-       }
-
-       if (strncmp(line,"COMPLEXPREFIXES",15) == 0) complexprefixes = 1;
-       if (((strncmp(line,"SFX",3) == 0) || (strncmp(line,"PFX",3) == 0)) && isspace(line[3])) break;
-    }
-    if (csconv == NULL) csconv = get_current_cs(SPELL_ENCODING);
-    delete afflst;
-    return 0;
-}
-
-/* parse in the ALIAS table */
-int  HashMgr::parse_aliasf(char * line, FileMgr * af)
-{
-   if (numaliasf != 0) {
-      HUNSPELL_WARNING(stderr, "error: line %d: multiple table definitions\n", af->getlinenum());
-      return 1;
-   }
-   char * tp = line;
-   char * piece;
-   int i = 0;
-   int np = 0;
-   piece = mystrsep(&tp, 0);
-   while (piece) {
-       if (*piece != '\0') {
-          switch(i) {
-             case 0: { np++; break; }
-             case 1: { 
-                       numaliasf = atoi(piece);
-                       if (numaliasf < 1) {
-                          numaliasf = 0;
-                          aliasf = NULL;
-                          aliasflen = NULL;
-                          HUNSPELL_WARNING(stderr, "error: line %d: bad entry number\n", af->getlinenum());
-                          return 1;
-                       }
-                       aliasf = (unsigned short **) malloc(numaliasf * sizeof(unsigned short *));
-                       aliasflen = (unsigned short *) malloc(numaliasf * sizeof(short));
-                       if (!aliasf || !aliasflen) {
-                          numaliasf = 0;
-                          if (aliasf) free(aliasf);
-                          if (aliasflen) free(aliasflen);
-                          aliasf = NULL;
-                          aliasflen = NULL;
-                          return 1;
-                       }
-                       np++;
-                       break;
-                     }
-             default: break;
-          }
-          i++;
-       }
-       piece = mystrsep(&tp, 0);
-   }
-   if (np != 2) {
-      numaliasf = 0;
-      free(aliasf);
-      free(aliasflen);
-      aliasf = NULL;
-      aliasflen = NULL;
-      HUNSPELL_WARNING(stderr, "error: line %d: missing data\n", af->getlinenum());
-      return 1;
-   } 
- 
-   /* now parse the numaliasf lines to read in the remainder of the table */
-   char * nl;
-   for (int j=0; j < numaliasf; j++) {
-        if ((nl = af->getline()) == NULL) return 1;
-        mychomp(nl);
-        tp = nl;
-        i = 0;
-        aliasf[j] = NULL;
-        aliasflen[j] = 0;
-        piece = mystrsep(&tp, 0);
-        while (piece) {
-           if (*piece != '\0') {
-               switch(i) {
-                  case 0: {
-                             if (strncmp(piece,"AF",2) != 0) {
-                                 numaliasf = 0;
-                                 free(aliasf);
-                                 free(aliasflen);
-                                 aliasf = NULL;
-                                 aliasflen = NULL;
-                                 HUNSPELL_WARNING(stderr, "error: line %d: table is corrupt\n", af->getlinenum());
-                                 return 1;
-                             }
-                             break;
-                          }
-                  case 1: {
-                            aliasflen[j] = (unsigned short) decode_flags(&(aliasf[j]), piece, af);
-                            flag_qsort(aliasf[j], 0, aliasflen[j]);
-                            break; 
-                          }
-                  default: break;
-               }
-               i++;
-           }
-           piece = mystrsep(&tp, 0);
-        }
-        if (!aliasf[j]) {
-             free(aliasf);
-             free(aliasflen);
-             aliasf = NULL;
-             aliasflen = NULL;
-             numaliasf = 0;
-             HUNSPELL_WARNING(stderr, "error: line %d: table is corrupt\n", af->getlinenum());
-             return 1;
-        }
-   }
-   return 0;
-}
-
-#ifdef HUNSPELL_CHROME_CLIENT
-int HashMgr::LoadAFLines()
-{
-  utf8 = 1;  // We always use UTF-8.
-
-  // Read in all the AF lines which tell us the rules for each affix group ID.
-  hunspell::LineIterator iterator = bdict_reader->GetAfLineIterator();
-  FileMgr afflst(&iterator);
-  while (char* line = afflst.getline()) {
-    int rv = parse_aliasf(line, &afflst);
-    if (rv)
-      return rv;
-  }
-
-  return 0;
-}
-
-hentry* HashMgr::InitHashEntry(hentry* entry,
-                               size_t item_size,
-                               const char* word,
-                               int word_length,
-                               int affix_index) const {
-  // Return if the given buffer doesn't have enough space for a hentry struct
-  // or the given word is too long.
-  // Our BDICT cannot handle words longer than (128 - 1) bytes. So, it is
-  // better to return an error if the given word is too long and prevent
-  // an unexpected result caused by a long word.
-  const int kMaxWordLen = 128;
-  if (item_size < sizeof(hentry) + word_length + 1 ||
-      word_length >= kMaxWordLen)
-    return NULL;
-
-  // Initialize a hentry struct with the given parameters, and
-  // append the given string at the end of this hentry struct.
-  memset(entry, 0, item_size);
-  FileMgr af(NULL);
-  entry->alen = static_cast<short>(
-      const_cast<HashMgr*>(this)->get_aliasf(affix_index, &entry->astr, &af));
-  entry->blen = static_cast<unsigned char>(word_length);
-  memcpy(&entry->word, word, word_length);
-
-  return entry;
-}
-
-hentry* HashMgr::CreateHashEntry(const char* word,
-                                 int word_length,
-                                 int affix_index) const {
-  // Return if the given word is too long.
-  // (See the comment in HashMgr::InitHashEntry().)
-  const int kMaxWordLen = 128;
-  if (word_length >= kMaxWordLen)
-    return NULL;
-
-  const size_t kEntrySize = sizeof(hentry) + word_length + 1;
-  struct hentry* entry = reinterpret_cast<hentry*>(malloc(kEntrySize));
-  if (entry)
-    InitHashEntry(entry, kEntrySize, word, word_length, affix_index);
-
-  return entry;
-}
-
-void HashMgr::DeleteHashEntry(hentry* entry) const {
-  free(entry);
-}
-
-hentry* HashMgr::AffixIDsToHentry(char* word,
-                                  int* affix_ids, 
-                                  int affix_count) const
-{
-  if (affix_count == 0)
-    return NULL;
-
-  HEntryCache& cache = const_cast<HashMgr*>(this)->hentry_cache;
-  std::string std_word(word);
-  HEntryCache::iterator found = cache.find(std_word);
-  if (found != cache.end()) {
-    // We must return an existing hentry for the same word if we've previously
-    // handed one out. Hunspell will compare pointers in some cases to see if
-    // two words it has found are the same.
-    return found->second;
-  }
-
-  short word_len = static_cast<short>(strlen(word));
-
-  // We can get a number of prefixes per word. There will normally be only one,
-  // but if not, there will be a linked list of "hentry"s for the "homonym"s 
-  // for the word.
-  struct hentry* first_he = NULL;
-  struct hentry* prev_he = NULL;  // For making linked list.
-  for (int i = 0; i < affix_count; i++) {
-    struct hentry* he = CreateHashEntry(word, word_len, affix_ids[i]);
-    if (!he)
-      break;
-    if (i == 0)
-      first_he = he;
-    if (prev_he)
-      prev_he->next_homonym = he;
-    prev_he = he;
-  }
-
-  cache[std_word] = first_he;  // Save this word in the cache for later.
-  return first_he;
-}
-
-hentry* HashMgr::GetHentryFromHEntryCache(char* word) {
-  HEntryCache& cache = const_cast<HashMgr*>(this)->hentry_cache;
-  std::string std_word(word);
-  HEntryCache::iterator found = cache.find(std_word);
-  if (found != cache.end())
-    return found->second;
-  else
-    return NULL;
-}
-#endif
-
-int HashMgr::is_aliasf() {
-    return (aliasf != NULL);
-}
-
-int HashMgr::get_aliasf(int index, unsigned short ** fvec, FileMgr * af) {
-    if ((index > 0) && (index <= numaliasf)) {
-        *fvec = aliasf[index - 1];
-        return aliasflen[index - 1];
-    }
-    HUNSPELL_WARNING(stderr, "error: line %d: bad flag alias index: %d\n", af->getlinenum(), index);
-    *fvec = NULL;
-    return 0;
-}
-
-/* parse morph alias definitions */
-int  HashMgr::parse_aliasm(char * line, FileMgr * af)
-{
-   if (numaliasm != 0) {
-      HUNSPELL_WARNING(stderr, "error: line %d: multiple table definitions\n", af->getlinenum());
-      return 1;
-   }
-   char * tp = line;
-   char * piece;
-   int i = 0;
-   int np = 0;
-   piece = mystrsep(&tp, 0);
-   while (piece) {
-       if (*piece != '\0') {
-          switch(i) {
-             case 0: { np++; break; }
-             case 1: { 
-                       numaliasm = atoi(piece);
-                       if (numaliasm < 1) {
-                          HUNSPELL_WARNING(stderr, "error: line %d: bad entry number\n", af->getlinenum());
-                          return 1;
-                       }
-                       aliasm = (char **) malloc(numaliasm * sizeof(char *));
-                       if (!aliasm) {
-                          numaliasm = 0;
-                          return 1;
-                       }
-                       np++;
-                       break;
-                     }
-             default: break;
-          }
-          i++;
-       }
-       piece = mystrsep(&tp, 0);
-   }
-   if (np != 2) {
-      numaliasm = 0;
-      free(aliasm);
-      aliasm = NULL;
-      HUNSPELL_WARNING(stderr, "error: line %d: missing data\n", af->getlinenum());
-      return 1;
-   } 
-
-   /* now parse the numaliasm lines to read in the remainder of the table */
-   char * nl = line;
-   for (int j=0; j < numaliasm; j++) {
-        if ((nl = af->getline()) == NULL) return 1;
-        mychomp(nl);
-        tp = nl;
-        i = 0;
-        aliasm[j] = NULL;
-        piece = mystrsep(&tp, ' ');
-        while (piece) {
-           if (*piece != '\0') {
-               switch(i) {
-                  case 0: {
-                             if (strncmp(piece,"AM",2) != 0) {
-                                 HUNSPELL_WARNING(stderr, "error: line %d: table is corrupt\n", af->getlinenum());
-                                 numaliasm = 0;
-                                 free(aliasm);
-                                 aliasm = NULL;
-                                 return 1;
-                             }
-                             break;
-                          }
-                  case 1: {
-                            // add the remaining of the line
-                            if (*tp) {
-                                *(tp - 1) = ' ';
-                                tp = tp + strlen(tp);
-                            }
-                            if (complexprefixes) {
-                                if (utf8) reverseword_utf(piece);
-                                    else reverseword(piece);
-                            }
-                            aliasm[j] = mystrdup(piece);
-                            if (!aliasm[j]) {
-                                 numaliasm = 0;
-                                 free(aliasm);
-                                 aliasm = NULL;
-                                 return 1;
-                            }
-                            break; }
-                  default: break;
-               }
-               i++;
-           }
-           piece = mystrsep(&tp, ' ');
-        }
-        if (!aliasm[j]) {
-             numaliasm = 0;
-             free(aliasm);
-             aliasm = NULL;
-             HUNSPELL_WARNING(stderr, "error: line %d: table is corrupt\n", af->getlinenum());
-             return 1;
-        }
-   }
-   return 0;
-}
-
-int HashMgr::is_aliasm() {
-    return (aliasm != NULL);
-}
-
-char * HashMgr::get_aliasm(int index) {
-    if ((index > 0) && (index <= numaliasm)) return aliasm[index - 1];
-    HUNSPELL_WARNING(stderr, "error: bad morph. alias index: %d\n", index);
-    return NULL;
-}