Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(317)

Unified Diff: base/i18n/icu_encoding_detection.cc

Issue 1647803004: Move base to DEPS (Closed) Base URL: git@github.com:domokit/mojo.git@master
Patch Set: Created 4 years, 11 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View side-by-side diff with in-line comments
Download patch
« no previous file with comments | « base/i18n/icu_encoding_detection.h ('k') | base/i18n/icu_string_conversions.h » ('j') | no next file with comments »
Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
Index: base/i18n/icu_encoding_detection.cc
diff --git a/base/i18n/icu_encoding_detection.cc b/base/i18n/icu_encoding_detection.cc
deleted file mode 100644
index ccd5cde2098dbce7574d2cbeb80f6f4e263fb83f..0000000000000000000000000000000000000000
--- a/base/i18n/icu_encoding_detection.cc
+++ /dev/null
@@ -1,104 +0,0 @@
-// Copyright (c) 2011 The Chromium Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file.
-
-#include "base/i18n/icu_encoding_detection.h"
-
-#include <set>
-
-#include "base/strings/string_util.h"
-#include "third_party/icu/source/i18n/unicode/ucsdet.h"
-
-namespace base {
-
-bool DetectEncoding(const std::string& text, std::string* encoding) {
- if (IsStringASCII(text)) {
- *encoding = std::string();
- return true;
- }
-
- UErrorCode status = U_ZERO_ERROR;
- UCharsetDetector* detector = ucsdet_open(&status);
- ucsdet_setText(detector, text.data(), static_cast<int32_t>(text.length()),
- &status);
- const UCharsetMatch* match = ucsdet_detect(detector, &status);
- if (match == NULL)
- return false;
- const char* detected_encoding = ucsdet_getName(match, &status);
- ucsdet_close(detector);
-
- if (U_FAILURE(status))
- return false;
-
- *encoding = detected_encoding;
- return true;
-}
-
-bool DetectAllEncodings(const std::string& text,
- std::vector<std::string>* encodings) {
- UErrorCode status = U_ZERO_ERROR;
- UCharsetDetector* detector = ucsdet_open(&status);
- ucsdet_setText(detector, text.data(), static_cast<int32_t>(text.length()),
- &status);
- int matches_count = 0;
- const UCharsetMatch** matches = ucsdet_detectAll(detector,
- &matches_count,
- &status);
- if (U_FAILURE(status)) {
- ucsdet_close(detector);
- return false;
- }
-
- // ICU has some heuristics for encoding detection, such that the more likely
- // encodings should be returned first. However, it doesn't always return
- // all encodings that properly decode |text|, so we'll append more encodings
- // later. To make that efficient, keep track of encodings sniffed in this
- // first phase.
- std::set<std::string> sniffed_encodings;
-
- encodings->clear();
- for (int i = 0; i < matches_count; i++) {
- UErrorCode get_name_status = U_ZERO_ERROR;
- const char* encoding_name = ucsdet_getName(matches[i], &get_name_status);
-
- // If we failed to get the encoding's name, ignore the error.
- if (U_FAILURE(get_name_status))
- continue;
-
- int32_t confidence = ucsdet_getConfidence(matches[i], &get_name_status);
-
- // We also treat this error as non-fatal.
- if (U_FAILURE(get_name_status))
- continue;
-
- // A confidence level >= 10 means that the encoding is expected to properly
- // decode the text. Drop all encodings with lower confidence level.
- if (confidence < 10)
- continue;
-
- encodings->push_back(encoding_name);
- sniffed_encodings.insert(encoding_name);
- }
-
- // Append all encodings not included earlier, in arbitrary order.
- // TODO(jshin): This shouldn't be necessary, possible ICU bug.
- // See also http://crbug.com/65917.
- UEnumeration* detectable_encodings = ucsdet_getAllDetectableCharsets(detector,
- &status);
- int detectable_count = uenum_count(detectable_encodings, &status);
- for (int i = 0; i < detectable_count; i++) {
- int name_length;
- const char* name_raw = uenum_next(detectable_encodings,
- &name_length,
- &status);
- std::string name(name_raw, name_length);
- if (sniffed_encodings.find(name) == sniffed_encodings.end())
- encodings->push_back(name);
- }
- uenum_close(detectable_encodings);
-
- ucsdet_close(detector);
- return !encodings->empty();
-}
-
-} // namespace base
« no previous file with comments | « base/i18n/icu_encoding_detection.h ('k') | base/i18n/icu_string_conversions.h » ('j') | no next file with comments »

Powered by Google App Engine
This is Rietveld 408576698