Add spellcheck and word suggestion to the prediction service

third_party/prediction/suggest/policyimpl/dictionary/header/header_policy.cpp

+/*
+ * Copyright (C) 2013, The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "third_party/prediction/suggest/policyimpl/dictionary/header/header_policy.h"
+
+#include <algorithm>
+
+namespace latinime {
+
+// Note that these are corresponding definitions in Java side in
+// DictionaryHeader.
+const char* const HeaderPolicy::MULTIPLE_WORDS_DEMOTION_RATE_KEY =
+    "MULTIPLE_WORDS_DEMOTION_RATE";
+const char* const HeaderPolicy::REQUIRES_GERMAN_UMLAUT_PROCESSING_KEY =
+    "REQUIRES_GERMAN_UMLAUT_PROCESSING";
+// TODO: Change attribute string to "IS_DECAYING_DICT".
+const char* const HeaderPolicy::IS_DECAYING_DICT_KEY = "USES_FORGETTING_CURVE";
+const char* const HeaderPolicy::DATE_KEY = "date";
+const char* const HeaderPolicy::LAST_DECAYED_TIME_KEY = "LAST_DECAYED_TIME";
+const char* const HeaderPolicy::UNIGRAM_COUNT_KEY = "UNIGRAM_COUNT";
+const char* const HeaderPolicy::BIGRAM_COUNT_KEY = "BIGRAM_COUNT";
+const char* const HeaderPolicy::EXTENDED_REGION_SIZE_KEY =
+    "EXTENDED_REGION_SIZE";
+// Historical info is information that is needed to support decaying such as
+// timestamp, level and
+// count.
+const char* const HeaderPolicy::HAS_HISTORICAL_INFO_KEY = "HAS_HISTORICAL_INFO";
+const char* const HeaderPolicy::LOCALE_KEY =
+    "locale";  // match Java declaration
+const char* const HeaderPolicy::FORGETTING_CURVE_OCCURRENCES_TO_LEVEL_UP_KEY =
+    "FORGETTING_CURVE_OCCURRENCES_TO_LEVEL_UP";
+const char* const
+    HeaderPolicy::FORGETTING_CURVE_PROBABILITY_VALUES_TABLE_ID_KEY =
+        "FORGETTING_CURVE_PROBABILITY_VALUES_TABLE_ID";
+const char* const
+    HeaderPolicy::FORGETTING_CURVE_DURATION_TO_LEVEL_DOWN_IN_SECONDS_KEY =
+        "FORGETTING_CURVE_DURATION_TO_LEVEL_DOWN_IN_SECONDS";
+
+const char* const HeaderPolicy::MAX_UNIGRAM_COUNT_KEY = "MAX_UNIGRAM_COUNT";
+const char* const HeaderPolicy::MAX_BIGRAM_COUNT_KEY = "MAX_BIGRAM_COUNT";
+
+const int HeaderPolicy::DEFAULT_MULTIPLE_WORDS_DEMOTION_RATE = 100;
+const float HeaderPolicy::MULTIPLE_WORD_COST_MULTIPLIER_SCALE = 100.0f;
+const int HeaderPolicy::DEFAULT_FORGETTING_CURVE_OCCURRENCES_TO_LEVEL_UP = 2;
+const int HeaderPolicy::DEFAULT_FORGETTING_CURVE_PROBABILITY_VALUES_TABLE_ID =
+    3;
+// 30 days
+const int
+    HeaderPolicy::DEFAULT_FORGETTING_CURVE_DURATION_TO_LEVEL_DOWN_IN_SECONDS =
+        30 * 24 * 60 * 60;
+
+const int HeaderPolicy::DEFAULT_MAX_UNIGRAM_COUNT = 10000;
+const int HeaderPolicy::DEFAULT_MAX_BIGRAM_COUNT = 10000;
+
+// Used for logging. Question mark is used to indicate that the key is not
+// found.
+void HeaderPolicy::readHeaderValueOrQuestionMark(const char* const key,
+                                                 int* outValue,
+                                                 int outValueSize) const {
+  if (outValueSize <= 0)
+    return;
+  if (outValueSize == 1) {
+    outValue[0] = '\0';
+    return;
+  }
+  std::vector<int> keyCodePointVector;
+  HeaderReadWriteUtils::insertCharactersIntoVector(key, &keyCodePointVector);
+  DictionaryHeaderStructurePolicy::AttributeMap::const_iterator it =
+      mAttributeMap.find(keyCodePointVector);
+  if (it == mAttributeMap.end()) {
+    // The key was not found.
+    outValue[0] = '?';
+    outValue[1] = '\0';
+    return;
+  }
+  const int terminalIndex =
+      std::min(static_cast<int>(it->second.size()), outValueSize - 1);
+  for (int i = 0; i < terminalIndex; ++i) {
+    outValue[i] = it->second[i];
+  }
+  outValue[terminalIndex] = '\0';
+}
+
+const std::vector<int> HeaderPolicy::readLocale() const {
+  return HeaderReadWriteUtils::readCodePointVectorAttributeValue(&mAttributeMap,
+                                                                 LOCALE_KEY);
+}
+
+float HeaderPolicy::readMultipleWordCostMultiplier() const {
+  const int demotionRate = HeaderReadWriteUtils::readIntAttributeValue(
+      &mAttributeMap, MULTIPLE_WORDS_DEMOTION_RATE_KEY,
+      DEFAULT_MULTIPLE_WORDS_DEMOTION_RATE);
+  if (demotionRate <= 0) {
+    return static_cast<float>(MAX_VALUE_FOR_WEIGHTING);
+  }
+  return MULTIPLE_WORD_COST_MULTIPLIER_SCALE / static_cast<float>(demotionRate);
+}
+
+bool HeaderPolicy::readRequiresGermanUmlautProcessing() const {
+  return HeaderReadWriteUtils::readBoolAttributeValue(
+      &mAttributeMap, REQUIRES_GERMAN_UMLAUT_PROCESSING_KEY, false);
+}
+
+bool HeaderPolicy::fillInAndWriteHeaderToBuffer(
+    const bool updatesLastDecayedTime,
+    const int unigramCount,
+    const int bigramCount,
+    const int extendedRegionSize,
+    BufferWithExtendableBuffer* const outBuffer) const {
+  int writingPos = 0;
+  DictionaryHeaderStructurePolicy::AttributeMap attributeMapToWrite(
+      mAttributeMap);
+  fillInHeader(updatesLastDecayedTime, unigramCount, bigramCount,
+               extendedRegionSize, &attributeMapToWrite);
+  if (!HeaderReadWriteUtils::writeDictionaryVersion(
+          outBuffer, mDictFormatVersion, &writingPos)) {
+    return false;
+  }
+  if (!HeaderReadWriteUtils::writeDictionaryFlags(outBuffer, mDictionaryFlags,
+                                                  &writingPos)) {
+    return false;
+  }
+  // Temporarily writes a dummy header size.
+  int headerSizeFieldPos = writingPos;
+  if (!HeaderReadWriteUtils::writeDictionaryHeaderSize(outBuffer, 0 /* size */,
+                                                       &writingPos)) {
+    return false;
+  }
+  if (!HeaderReadWriteUtils::writeHeaderAttributes(
+          outBuffer, &attributeMapToWrite, &writingPos)) {
+    return false;
+  }
+  // Writes the actual header size.
+  if (!HeaderReadWriteUtils::writeDictionaryHeaderSize(outBuffer, writingPos,
+                                                       &headerSizeFieldPos)) {
+    return false;
+  }
+  return true;
+}
+
+void HeaderPolicy::fillInHeader(
+    const bool updatesLastDecayedTime,
+    const int unigramCount,
+    const int bigramCount,
+    const int extendedRegionSize,
+    DictionaryHeaderStructurePolicy::AttributeMap* outAttributeMap) const {
+  HeaderReadWriteUtils::setIntAttribute(outAttributeMap, UNIGRAM_COUNT_KEY,
+                                        unigramCount);
+  HeaderReadWriteUtils::setIntAttribute(outAttributeMap, BIGRAM_COUNT_KEY,
+                                        bigramCount);
+  HeaderReadWriteUtils::setIntAttribute(
+      outAttributeMap, EXTENDED_REGION_SIZE_KEY, extendedRegionSize);
+  // Set the current time as the generation time.
+  HeaderReadWriteUtils::setIntAttribute(outAttributeMap, DATE_KEY,
+                                        TimeKeeper::peekCurrentTime());
+  HeaderReadWriteUtils::setCodePointVectorAttribute(outAttributeMap, LOCALE_KEY,
+                                                    mLocale);
+  if (updatesLastDecayedTime) {
+    // Set current time as the last updated time.
+    HeaderReadWriteUtils::setIntAttribute(
+        outAttributeMap, LAST_DECAYED_TIME_KEY, TimeKeeper::peekCurrentTime());
+  }
+}
+
+/* static */ DictionaryHeaderStructurePolicy::AttributeMap
+HeaderPolicy::createAttributeMapAndReadAllAttributes(
+    const uint8_t* const dictBuf) {
+  DictionaryHeaderStructurePolicy::AttributeMap attributeMap;
+  HeaderReadWriteUtils::fetchAllHeaderAttributes(dictBuf, &attributeMap);
+  return attributeMap;
+}
+
+}  // namespace latinime