Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(236)

Side by Side Diff: components/visitedlink/browser/visitedlink_master.cc

Issue 1417943002: Load the table of visited links from database file asynchronously. (Closed) Base URL: https://chromium.googlesource.com/chromium/src.git@master
Patch Set: Created 5 years, 2 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch
OLDNEW
1 // Copyright (c) 2012 The Chromium Authors. All rights reserved. 1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be 2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file. 3 // found in the LICENSE file.
4 4
5 #include "components/visitedlink/browser/visitedlink_master.h" 5 #include "components/visitedlink/browser/visitedlink_master.h"
6 6
7 #if defined(OS_WIN) 7 #if defined(OS_WIN)
8 #include <windows.h> 8 #include <windows.h>
9 #include <io.h> 9 #include <io.h>
10 #include <shlobj.h> 10 #include <shlobj.h>
(...skipping 98 matching lines...) Expand 10 before | Expand all | Expand 10 after
109 // of FILE* value. Double pointer to FILE is used because file may still not 109 // of FILE* value. Double pointer to FILE is used because file may still not
110 // be opened by the time of scheduling the task for execution. 110 // be opened by the time of scheduling the task for execution.
111 void AsyncClose(FILE** file) { 111 void AsyncClose(FILE** file) {
112 if (*file) 112 if (*file)
113 base::IgnoreResult(fclose(*file)); 113 base::IgnoreResult(fclose(*file));
114 free(file); 114 free(file);
115 } 115 }
116 116
117 } // namespace 117 } // namespace
118 118
119 struct VisitedLinkMaster::LoadFromFileResult
120 : public base::RefCountedThreadSafe<LoadFromFileResult> {
121 LoadFromFileResult(base::ScopedFILE file,
122 scoped_ptr<base::SharedMemory> shared_memory,
123 Fingerprint* hash_table,
124 int32 num_entries,
125 int32 used_count,
126 uint8 salt[LINK_SALT_LENGTH]);
127
128 base::ScopedFILE file;
129 scoped_ptr<base::SharedMemory> shared_memory;
130 Fingerprint* hash_table;
131 int32 num_entries;
132 int32 used_count;
133 uint8 salt[LINK_SALT_LENGTH];
134
135 private:
136 friend class base::RefCountedThreadSafe<LoadFromFileResult>;
137 virtual ~LoadFromFileResult();
138
139 DISALLOW_COPY_AND_ASSIGN(LoadFromFileResult);
140 };
141
142 VisitedLinkMaster::LoadFromFileResult::LoadFromFileResult(
143 base::ScopedFILE file,
144 scoped_ptr<base::SharedMemory> shared_memory,
145 Fingerprint* hash_table,
146 int32 num_entries,
147 int32 used_count,
148 uint8 salt[LINK_SALT_LENGTH])
149 : file(file.Pass()),
150 shared_memory(shared_memory.Pass()),
151 hash_table(hash_table),
152 num_entries(num_entries),
153 used_count(used_count) {
154 memcpy(this->salt, salt, LINK_SALT_LENGTH);
155 }
156
157 VisitedLinkMaster::LoadFromFileResult::~LoadFromFileResult() {
158 }
159
119 // TableBuilder --------------------------------------------------------------- 160 // TableBuilder ---------------------------------------------------------------
120 161
121 // How rebuilding from history works 162 // How rebuilding from history works
122 // --------------------------------- 163 // ---------------------------------
123 // 164 //
124 // We mark that we're rebuilding from history by setting the table_builder_ 165 // We mark that we're rebuilding from history by setting the table_builder_
125 // member in VisitedLinkMaster to the TableBuilder we create. This builder 166 // member in VisitedLinkMaster to the TableBuilder we create. This builder
126 // will be called on the history thread by the history system for every URL 167 // will be called on the history thread by the history system for every URL
127 // in the database. 168 // in the database.
128 // 169 //
(...skipping 45 matching lines...) Expand 10 before | Expand all | Expand 10 after
174 }; 215 };
175 216
176 // VisitedLinkMaster ---------------------------------------------------------- 217 // VisitedLinkMaster ----------------------------------------------------------
177 218
178 VisitedLinkMaster::VisitedLinkMaster(content::BrowserContext* browser_context, 219 VisitedLinkMaster::VisitedLinkMaster(content::BrowserContext* browser_context,
179 VisitedLinkDelegate* delegate, 220 VisitedLinkDelegate* delegate,
180 bool persist_to_disk) 221 bool persist_to_disk)
181 : browser_context_(browser_context), 222 : browser_context_(browser_context),
182 delegate_(delegate), 223 delegate_(delegate),
183 listener_(new VisitedLinkEventListener(this, browser_context)), 224 listener_(new VisitedLinkEventListener(this, browser_context)),
184 persist_to_disk_(persist_to_disk) { 225 persist_to_disk_(persist_to_disk),
226 table_is_loading_from_file_(false),
227 weak_ptr_factory_(this) {
185 InitMembers(); 228 InitMembers();
186 } 229 }
187 230
188 VisitedLinkMaster::VisitedLinkMaster(Listener* listener, 231 VisitedLinkMaster::VisitedLinkMaster(Listener* listener,
189 VisitedLinkDelegate* delegate, 232 VisitedLinkDelegate* delegate,
190 bool persist_to_disk, 233 bool persist_to_disk,
191 bool suppress_rebuild, 234 bool suppress_rebuild,
192 const base::FilePath& filename, 235 const base::FilePath& filename,
193 int32 default_table_size) 236 int32 default_table_size)
194 : browser_context_(NULL), 237 : browser_context_(NULL),
195 delegate_(delegate), 238 delegate_(delegate),
196 persist_to_disk_(persist_to_disk) { 239 persist_to_disk_(persist_to_disk),
240 table_is_loading_from_file_(false),
241 weak_ptr_factory_(this) {
197 listener_.reset(listener); 242 listener_.reset(listener);
198 DCHECK(listener_.get()); 243 DCHECK(listener_.get());
199 InitMembers(); 244 InitMembers();
200 245
201 database_name_override_ = filename; 246 database_name_override_ = filename;
202 table_size_override_ = default_table_size; 247 table_size_override_ = default_table_size;
203 suppress_rebuild_ = suppress_rebuild; 248 suppress_rebuild_ = suppress_rebuild;
204 } 249 }
205 250
206 VisitedLinkMaster::~VisitedLinkMaster() { 251 VisitedLinkMaster::~VisitedLinkMaster() {
207 if (table_builder_.get()) { 252 if (table_builder_.get()) {
208 // Prevent the table builder from calling us back now that we're being 253 // Prevent the table builder from calling us back now that we're being
209 // destroyed. Note that we DON'T delete the object, since the history 254 // destroyed. Note that we DON'T delete the object, since the history
210 // system is still writing into it. When that is complete, the table 255 // system is still writing into it. When that is complete, the table
211 // builder will destroy itself when it finds we are gone. 256 // builder will destroy itself when it finds we are gone.
212 table_builder_->DisownMaster(); 257 table_builder_->DisownMaster();
213 } 258 }
214 FreeURLTable(); 259 FreeURLTable();
215 // FreeURLTable() will schedule closing of the file and deletion of |file_|. 260 // FreeURLTable() will schedule closing of the file and deletion of |file_|.
216 // So nothing should be done here. 261 // So nothing should be done here.
262
263 if (table_is_loading_from_file_ &&
264 (!added_since_rebuild_or_load_.empty() ||
265 !deleted_since_rebuild_or_load_.empty())) {
266 // Delete the database file if exist because we don't have enough time to
267 // load the table from the database file and now we have inconsistent
268 // state. On the next start table will be rebuilt.
269 base::FilePath filename;
270 GetDatabaseFileName(&filename);
271 PostIOTask(FROM_HERE,
272 base::Bind(IgnoreResult(&base::DeleteFile), filename, false));
273 }
217 } 274 }
218 275
219 void VisitedLinkMaster::InitMembers() { 276 void VisitedLinkMaster::InitMembers() {
220 file_ = NULL; 277 file_ = NULL;
221 shared_memory_ = NULL; 278 shared_memory_ = NULL;
222 shared_memory_serial_ = 0; 279 shared_memory_serial_ = 0;
223 used_items_ = 0; 280 used_items_ = 0;
224 table_size_override_ = 0; 281 table_size_override_ = 0;
225 suppress_rebuild_ = false; 282 suppress_rebuild_ = false;
226 sequence_token_ = BrowserThread::GetBlockingPool()->GetSequenceToken(); 283 sequence_token_ = BrowserThread::GetBlockingPool()->GetSequenceToken();
227
228 #ifndef NDEBUG
229 posted_asynchronous_operation_ = false;
230 #endif
231 } 284 }
232 285
233 bool VisitedLinkMaster::Init() { 286 bool VisitedLinkMaster::Init() {
234 // We probably shouldn't be loading this from the UI thread,
235 // but it does need to happen early on in startup.
236 // http://code.google.com/p/chromium/issues/detail?id=24163
237 base::ThreadRestrictions::ScopedAllowIO allow_io;
238
239 if (persist_to_disk_) { 287 if (persist_to_disk_) {
240 if (InitFromFile()) 288 if (InitFromFile())
241 return true; 289 return true;
242 } 290 }
243 return InitFromScratch(suppress_rebuild_); 291 return InitFromScratch(suppress_rebuild_);
244 } 292 }
245 293
246 VisitedLinkMaster::Hash VisitedLinkMaster::TryToAddURL(const GURL& url) { 294 VisitedLinkMaster::Hash VisitedLinkMaster::TryToAddURL(const GURL& url) {
247 // Extra check that we are not incognito. This should not happen. 295 // Extra check that we are not incognito. This should not happen.
248 // TODO(boliu): Move this check to HistoryService when IsOffTheRecord is 296 // TODO(boliu): Move this check to HistoryService when IsOffTheRecord is
249 // removed from BrowserContext. 297 // removed from BrowserContext.
250 if (browser_context_ && browser_context_->IsOffTheRecord()) { 298 if (browser_context_ && browser_context_->IsOffTheRecord()) {
251 NOTREACHED(); 299 NOTREACHED();
252 return null_hash_; 300 return null_hash_;
253 } 301 }
254 302
255 if (!url.is_valid()) 303 if (!url.is_valid())
256 return null_hash_; // Don't add invalid URLs. 304 return null_hash_; // Don't add invalid URLs.
257 305
258 Fingerprint fingerprint = ComputeURLFingerprint(url.spec().data(), 306 Fingerprint fingerprint = ComputeURLFingerprint(url.spec().data(),
259 url.spec().size(), 307 url.spec().size(),
260 salt_); 308 salt_);
261 if (table_builder_.get()) { 309 if (table_builder_.get() || table_is_loading_from_file_) {
262 // If we have a pending delete for this fingerprint, cancel it. 310 // If we have a pending delete for this fingerprint, cancel it.
263 std::set<Fingerprint>::iterator found = 311 std::set<Fingerprint>::iterator found =
264 deleted_since_rebuild_.find(fingerprint); 312 deleted_since_rebuild_or_load_.find(fingerprint);
265 if (found != deleted_since_rebuild_.end()) 313 if (found != deleted_since_rebuild_or_load_.end())
266 deleted_since_rebuild_.erase(found); 314 deleted_since_rebuild_or_load_.erase(found);
267 315
268 // A rebuild is in progress, save this addition in the temporary list so 316 // A rebuild or load is in progress, save this addition in the temporary
269 // it can be added once rebuild is complete. 317 // list so it can be added once rebuild is complete.
270 added_since_rebuild_.insert(fingerprint); 318 added_since_rebuild_or_load_.insert(fingerprint);
271 } 319 }
272 320
273 // If the table is "full", we don't add URLs and just drop them on the floor. 321 // If the table is "full", we don't add URLs and just drop them on the floor.
274 // This can happen if we get thousands of new URLs and something causes 322 // This can happen if we get thousands of new URLs and something causes
275 // the table resizing to fail. This check prevents a hang in that case. Note 323 // the table resizing to fail. This check prevents a hang in that case. Note
276 // that this is *not* the resize limit, this is just a sanity check. 324 // that this is *not* the resize limit, this is just a sanity check.
277 if (used_items_ / 8 > table_length_ / 10) 325 if (used_items_ / 8 > table_length_ / 10)
278 return null_hash_; // Table is more than 80% full. 326 return null_hash_; // Table is more than 80% full.
279 327
280 return AddFingerprint(fingerprint, true); 328 return AddFingerprint(fingerprint, true);
281 } 329 }
282 330
283 void VisitedLinkMaster::PostIOTask(const tracked_objects::Location& from_here, 331 void VisitedLinkMaster::PostIOTask(const tracked_objects::Location& from_here,
284 const base::Closure& task) { 332 const base::Closure& task) {
285 DCHECK(persist_to_disk_); 333 DCHECK(persist_to_disk_);
286 BrowserThread::GetBlockingPool()->PostSequencedWorkerTask(sequence_token_, 334 BrowserThread::GetBlockingPool()->PostSequencedWorkerTask(sequence_token_,
287 from_here, task); 335 from_here, task);
288 } 336 }
289 337
290 void VisitedLinkMaster::AddURL(const GURL& url) { 338 void VisitedLinkMaster::AddURL(const GURL& url) {
291 Hash index = TryToAddURL(url); 339 Hash index = TryToAddURL(url);
292 if (!table_builder_.get() && index != null_hash_) { 340 if (!table_builder_.get() &&
341 !table_is_loading_from_file_ &&
342 index != null_hash_) {
293 // Not rebuilding, so we want to keep the file on disk up-to-date. 343 // Not rebuilding, so we want to keep the file on disk up-to-date.
294 if (persist_to_disk_) { 344 if (persist_to_disk_) {
295 WriteUsedItemCountToFile(); 345 WriteUsedItemCountToFile();
296 WriteHashRangeToFile(index, index); 346 WriteHashRangeToFile(index, index);
297 } 347 }
298 ResizeTableIfNecessary(); 348 ResizeTableIfNecessary();
299 } 349 }
300 } 350 }
301 351
302 void VisitedLinkMaster::AddURLs(const std::vector<GURL>& url) { 352 void VisitedLinkMaster::AddURLs(const std::vector<GURL>& url) {
303 for (std::vector<GURL>::const_iterator i = url.begin(); 353 for (std::vector<GURL>::const_iterator i = url.begin();
304 i != url.end(); ++i) { 354 i != url.end(); ++i) {
305 Hash index = TryToAddURL(*i); 355 Hash index = TryToAddURL(*i);
306 if (!table_builder_.get() && index != null_hash_) 356 if (!table_builder_.get() &&
357 !table_is_loading_from_file_ &&
358 index != null_hash_)
307 ResizeTableIfNecessary(); 359 ResizeTableIfNecessary();
308 } 360 }
309 361
310 // Keeps the file on disk up-to-date. 362 // Keeps the file on disk up-to-date.
311 if (!table_builder_.get() && persist_to_disk_) 363 if (!table_builder_.get() &&
364 !table_is_loading_from_file_ &&
365 persist_to_disk_)
312 WriteFullTable(); 366 WriteFullTable();
313 } 367 }
314 368
315 void VisitedLinkMaster::DeleteAllURLs() { 369 void VisitedLinkMaster::DeleteAllURLs() {
316 // Any pending modifications are invalid. 370 // Any pending modifications are invalid.
317 added_since_rebuild_.clear(); 371 added_since_rebuild_or_load_.clear();
318 deleted_since_rebuild_.clear(); 372 deleted_since_rebuild_or_load_.clear();
373 table_is_loading_from_file_ = false;
319 374
320 // Clear the hash table. 375 // Clear the hash table.
321 used_items_ = 0; 376 used_items_ = 0;
322 memset(hash_table_, 0, this->table_length_ * sizeof(Fingerprint)); 377 memset(hash_table_, 0, this->table_length_ * sizeof(Fingerprint));
323 378
324 // Resize it if it is now too empty. Resize may write the new table out for 379 // Resize it if it is now too empty. Resize may write the new table out for
325 // us, otherwise, schedule writing the new table to disk ourselves. 380 // us, otherwise, schedule writing the new table to disk ourselves.
326 if (!ResizeTableIfNecessary() && persist_to_disk_) 381 if (!ResizeTableIfNecessary() && persist_to_disk_)
327 WriteFullTable(); 382 WriteFullTable();
328 383
329 listener_->Reset(); 384 listener_->Reset();
330 } 385 }
331 386
332 VisitedLinkDelegate* VisitedLinkMaster::GetDelegate() { 387 VisitedLinkDelegate* VisitedLinkMaster::GetDelegate() {
333 return delegate_; 388 return delegate_;
334 } 389 }
335 390
336 void VisitedLinkMaster::DeleteURLs(URLIterator* urls) { 391 void VisitedLinkMaster::DeleteURLs(URLIterator* urls) {
337 if (!urls->HasNextURL()) 392 if (!urls->HasNextURL())
338 return; 393 return;
339 394
340 listener_->Reset(); 395 listener_->Reset();
341 396
342 if (table_builder_.get()) { 397 if (table_builder_.get() || table_is_loading_from_file_) {
343 // A rebuild is in progress, save this deletion in the temporary list so 398 // A rebuild is in progress, save this deletion in the temporary list so
344 // it can be added once rebuild is complete. 399 // it can be added once rebuild is complete.
345 while (urls->HasNextURL()) { 400 while (urls->HasNextURL()) {
346 const GURL& url(urls->NextURL()); 401 const GURL& url(urls->NextURL());
347 if (!url.is_valid()) 402 if (!url.is_valid())
348 continue; 403 continue;
349 404
350 Fingerprint fingerprint = 405 Fingerprint fingerprint =
351 ComputeURLFingerprint(url.spec().data(), url.spec().size(), salt_); 406 ComputeURLFingerprint(url.spec().data(), url.spec().size(), salt_);
352 deleted_since_rebuild_.insert(fingerprint); 407 deleted_since_rebuild_or_load_.insert(fingerprint);
353 408
354 // If the URL was just added and now we're deleting it, it may be in the 409 // If the URL was just added and now we're deleting it, it may be in the
355 // list of things added since the last rebuild. Delete it from that list. 410 // list of things added since the last rebuild or load. Delete it from
411 // that list.
356 std::set<Fingerprint>::iterator found = 412 std::set<Fingerprint>::iterator found =
357 added_since_rebuild_.find(fingerprint); 413 added_since_rebuild_or_load_.find(fingerprint);
358 if (found != added_since_rebuild_.end()) 414 if (found != added_since_rebuild_or_load_.end())
359 added_since_rebuild_.erase(found); 415 added_since_rebuild_or_load_.erase(found);
360 416
361 // Delete the URLs from the in-memory table, but don't bother writing 417 // Delete the URLs from the in-memory table, but don't bother writing
362 // to disk since it will be replaced soon. 418 // to disk since it will be replaced soon.
363 DeleteFingerprint(fingerprint, false); 419 DeleteFingerprint(fingerprint, false);
364 } 420 }
365 return; 421 return;
366 } 422 }
367 423
368 // Compute the deleted URLs' fingerprints and delete them 424 // Compute the deleted URLs' fingerprints and delete them
369 std::set<Fingerprint> deleted_fingerprints; 425 std::set<Fingerprint> deleted_fingerprints;
(...skipping 157 matching lines...) Expand 10 before | Expand all | Expand 10 after
527 583
528 // Write the hash data. 584 // Write the hash data.
529 WriteToFile(file_, kFileHeaderSize, 585 WriteToFile(file_, kFileHeaderSize,
530 hash_table_, table_length_ * sizeof(Fingerprint)); 586 hash_table_, table_length_ * sizeof(Fingerprint));
531 587
532 // The hash table may have shrunk, so make sure this is the end. 588 // The hash table may have shrunk, so make sure this is the end.
533 PostIOTask(FROM_HERE, base::Bind(&AsyncTruncate, file_)); 589 PostIOTask(FROM_HERE, base::Bind(&AsyncTruncate, file_));
534 } 590 }
535 591
536 bool VisitedLinkMaster::InitFromFile() { 592 bool VisitedLinkMaster::InitFromFile() {
537 DCHECK(file_ == NULL); 593 DCHECK_CURRENTLY_ON(BrowserThread::UI);
594
595 DCHECK(file_ == nullptr);
538 DCHECK(persist_to_disk_); 596 DCHECK(persist_to_disk_);
539 597
540 base::FilePath filename; 598 base::FilePath filename;
541 GetDatabaseFileName(&filename); 599 if (!GetDatabaseFileName(&filename))
600 return false;
601
602 // Create the temporary table.
603 // The salt must be generated before the table so that it can be copied to
604 // the shared memory.
605 GenerateSalt(salt_);
606 if (!CreateURLTable(DefaultTableSize()))
607 return false;
608
609 #ifndef NDEBUG
610 DebugValidate();
611 #endif
612
613 table_is_loading_from_file_ = true;
614
615 TableLoadCompleteCallback callback = base::Bind(
616 &VisitedLinkMaster::OnTableLoadComplete, weak_ptr_factory_.GetWeakPtr());
617
618 PostIOTask(FROM_HERE,
619 base::Bind(&VisitedLinkMaster::LoadFromFile, filename, callback));
620
621 return true;
622 }
623
624 // static
625 void VisitedLinkMaster::LoadFromFile(
626 const base::FilePath& filename,
627 const TableLoadCompleteCallback& callback) {
628 scoped_refptr<LoadFromFileResult> load_from_file_result;
629 bool success = LoadApartFromFile(filename, &load_from_file_result);
630
631 BrowserThread::PostTask(BrowserThread::UI, FROM_HERE,
632 base::Bind(callback, success, load_from_file_result));
633 }
634
635 // static
636 bool VisitedLinkMaster::LoadApartFromFile(
637 const base::FilePath& filename,
638 scoped_refptr<LoadFromFileResult>* load_from_file_result) {
639 DCHECK(load_from_file_result);
640
542 base::ScopedFILE file_closer(base::OpenFile(filename, "rb+")); 641 base::ScopedFILE file_closer(base::OpenFile(filename, "rb+"));
543 if (!file_closer.get()) 642 if (!file_closer.get())
544 return false; 643 return false;
545 644
546 int32 num_entries, used_count; 645 int32 num_entries, used_count;
547 if (!ReadFileHeader(file_closer.get(), &num_entries, &used_count, salt_)) 646 uint8 salt[LINK_SALT_LENGTH];
647 if (!ReadFileHeader(file_closer.get(), &num_entries, &used_count, salt))
548 return false; // Header isn't valid. 648 return false; // Header isn't valid.
549 649
550 // Allocate and read the table. 650 // Allocate and read the table.
551 if (!CreateURLTable(num_entries, false)) 651 scoped_ptr<base::SharedMemory> shared_memory;
652 VisitedLinkCommon::Fingerprint* hash_table;
653 if (!CreateApartURLTable(num_entries, salt, &shared_memory, &hash_table))
552 return false; 654 return false;
553 if (!ReadFromFile(file_closer.get(), kFileHeaderSize, 655
554 hash_table_, num_entries * sizeof(Fingerprint))) { 656 if (!ReadFromFile(file_closer.get(), kFileHeaderSize, hash_table,
555 FreeURLTable(); 657 num_entries * sizeof(Fingerprint))) {
556 return false; 658 return false;
557 } 659 }
558 used_items_ = used_count; 660
661 *load_from_file_result = new LoadFromFileResult(file_closer.Pass(),
662 shared_memory.Pass(),
663 hash_table,
664 num_entries,
665 used_count,
666 salt);
667 return true;
668 }
669
670 void VisitedLinkMaster::OnTableLoadComplete(
671 bool success,
672 scoped_refptr<LoadFromFileResult> load_from_file_result) {
673 DCHECK_CURRENTLY_ON(BrowserThread::UI);
674 DCHECK(persist_to_disk_);
675 DCHECK(!table_builder_.get());
676
677 // When the apart table was loading from the database file the current table
678 // was cleared.
brettw 2015/11/02 05:09:23 "was" -> "could have been" (As-is, I think the com
679 if (!table_is_loading_from_file_)
680 return;
681
682 table_is_loading_from_file_ = false;
683
684 if (!success) {
685 // If the table isn't loaded the table will be rebuilt.
686 if (!suppress_rebuild_) {
687 RebuildTableFromDelegate();
688 } else {
689 // When we disallow rebuilds (normally just unit tests), just use the
690 // current empty table.
691 WriteFullTable();
692 }
693 return;
694 }
695
696 DCHECK(load_from_file_result.get());
697
698 // Delete the previous table.
699 DCHECK(shared_memory_);
700 delete shared_memory_;
701 shared_memory_ = nullptr;
702
703 // Assign the open file.
704 DCHECK(!file_);
705 DCHECK(load_from_file_result->file.get());
706 file_ = static_cast<FILE**>(malloc(sizeof(*file_)));
707 *file_ = load_from_file_result->file.release();
708
709 // Assign the loaded table.
710 DCHECK(load_from_file_result->shared_memory.get());
711 DCHECK(load_from_file_result->hash_table);
712 memcpy(salt_, load_from_file_result->salt, LINK_SALT_LENGTH);
713 shared_memory_ = load_from_file_result->shared_memory.release();
714 hash_table_ = load_from_file_result->hash_table;
715 table_length_ = load_from_file_result->num_entries;
716 used_items_ = load_from_file_result->used_count;
559 717
560 #ifndef NDEBUG 718 #ifndef NDEBUG
561 DebugValidate(); 719 DebugValidate();
562 #endif 720 #endif
563 721
564 file_ = static_cast<FILE**>(malloc(sizeof(*file_))); 722 if (!added_since_rebuild_or_load_.empty() ||
565 *file_ = file_closer.release(); 723 !deleted_since_rebuild_or_load_.empty()) {
566 return true; 724 int new_size =
725 NewTableSizeForCount(used_items_ + added_since_rebuild_or_load_.size());
726 if (new_size > table_length_)
727 ResizeTable(new_size);
728
729 // Also add anything that was added while we were asynchronously
730 // loading the table.
731 for (const auto& fingerprint : added_since_rebuild_or_load_)
brettw 2015/11/02 05:09:23 From here to the end of the condition is the same
732 AddFingerprint(fingerprint, false);
733 added_since_rebuild_or_load_.clear();
734
735 // Now handle deletions.
736 DeleteFingerprintsFromCurrentTable(deleted_since_rebuild_or_load_);
737 deleted_since_rebuild_or_load_.clear();
738
739 if (persist_to_disk_)
740 WriteFullTable();
741 }
742
743 // Send an update notification to all child processes.
brettw 2015/11/02 05:09:23 This will duplicate the message if the table is re
744 listener_->NewTable(shared_memory_);
brettw 2015/11/02 05:09:23 I *think* here we also need a Reset() after the Ne
567 } 745 }
568 746
569 bool VisitedLinkMaster::InitFromScratch(bool suppress_rebuild) { 747 bool VisitedLinkMaster::InitFromScratch(bool suppress_rebuild) {
570 int32 table_size = kDefaultTableSize;
571 if (table_size_override_)
572 table_size = table_size_override_;
573
574 // The salt must be generated before the table so that it can be copied to 748 // The salt must be generated before the table so that it can be copied to
575 // the shared memory. 749 // the shared memory.
576 GenerateSalt(salt_); 750 GenerateSalt(salt_);
577 if (!CreateURLTable(table_size, true)) 751 if (!CreateURLTable(DefaultTableSize()))
578 return false; 752 return false;
579 753
580 #ifndef NDEBUG 754 #ifndef NDEBUG
581 DebugValidate(); 755 DebugValidate();
582 #endif 756 #endif
583 757
584 if (suppress_rebuild && persist_to_disk_) { 758 if (suppress_rebuild && persist_to_disk_) {
585 // When we disallow rebuilds (normally just unit tests), just use the 759 // When we disallow rebuilds (normally just unit tests), just use the
586 // current empty table. 760 // current empty table.
587 WriteFullTable(); 761 WriteFullTable();
588 return true; 762 return true;
589 } 763 }
590 764
591 // This will build the table from history. On the first run, history will 765 // This will build the table from history. On the first run, history will
592 // be empty, so this will be correct. This will also write the new table 766 // be empty, so this will be correct. This will also write the new table
593 // to disk. We don't want to save explicitly here, since the rebuild may 767 // to disk. We don't want to save explicitly here, since the rebuild may
594 // not complete, leaving us with an empty but valid visited link database. 768 // not complete, leaving us with an empty but valid visited link database.
595 // In the future, we won't know we need to try rebuilding again. 769 // In the future, we won't know we need to try rebuilding again.
596 return RebuildTableFromDelegate(); 770 return RebuildTableFromDelegate();
597 } 771 }
598 772
773 // static
599 bool VisitedLinkMaster::ReadFileHeader(FILE* file, 774 bool VisitedLinkMaster::ReadFileHeader(FILE* file,
600 int32* num_entries, 775 int32* num_entries,
601 int32* used_count, 776 int32* used_count,
602 uint8 salt[LINK_SALT_LENGTH]) { 777 uint8 salt[LINK_SALT_LENGTH]) {
603 DCHECK(persist_to_disk_);
604
605 // Get file size. 778 // Get file size.
606 // Note that there is no need to seek back to the original location in the 779 // Note that there is no need to seek back to the original location in the
607 // file since ReadFromFile() [which is the next call accessing the file] 780 // file since ReadFromFile() [which is the next call accessing the file]
608 // seeks before reading. 781 // seeks before reading.
609 if (fseek(file, 0, SEEK_END) == -1) 782 if (fseek(file, 0, SEEK_END) == -1)
610 return false; 783 return false;
611 size_t file_size = ftell(file); 784 size_t file_size = ftell(file);
612 785
613 if (file_size <= kFileHeaderSize) 786 if (file_size <= kFileHeaderSize)
614 return false; 787 return false;
(...skipping 43 matching lines...) Expand 10 before | Expand all | Expand 10 after
658 if (!browser_context_ || browser_context_->GetPath().empty()) 831 if (!browser_context_ || browser_context_->GetPath().empty())
659 return false; 832 return false;
660 833
661 base::FilePath profile_dir = browser_context_->GetPath(); 834 base::FilePath profile_dir = browser_context_->GetPath();
662 *filename = profile_dir.Append(FILE_PATH_LITERAL("Visited Links")); 835 *filename = profile_dir.Append(FILE_PATH_LITERAL("Visited Links"));
663 return true; 836 return true;
664 } 837 }
665 838
666 // Initializes the shared memory structure. The salt should already be filled 839 // Initializes the shared memory structure. The salt should already be filled
667 // in so that it can be written to the shared memory 840 // in so that it can be written to the shared memory
668 bool VisitedLinkMaster::CreateURLTable(int32 num_entries, bool init_to_empty) { 841 bool VisitedLinkMaster::CreateURLTable(int32 num_entries) {
842 scoped_ptr<base::SharedMemory> shared_memory;
843 VisitedLinkCommon::Fingerprint* hash_table;
844 if (CreateApartURLTable(num_entries, salt_, &shared_memory, &hash_table)) {
845 shared_memory_ = shared_memory.release();
846 hash_table_ = hash_table;
847 table_length_ = num_entries;
848 used_items_ = 0;
849 return true;
850 }
851
852 return false;
853 }
854
855 // static
856 bool VisitedLinkMaster::CreateApartURLTable(
857 int32 num_entries,
858 const uint8 salt[LINK_SALT_LENGTH],
859 scoped_ptr<base::SharedMemory>* shared_memory,
860 VisitedLinkCommon::Fingerprint** hash_table) {
861 DCHECK(salt);
862 DCHECK(shared_memory);
863 DCHECK(hash_table);
864
669 // The table is the size of the table followed by the entries. 865 // The table is the size of the table followed by the entries.
670 uint32 alloc_size = num_entries * sizeof(Fingerprint) + sizeof(SharedHeader); 866 uint32 alloc_size = num_entries * sizeof(Fingerprint) + sizeof(SharedHeader);
671 867
672 // Create the shared memory object. 868 // Create the shared memory object.
673 shared_memory_ = new base::SharedMemory(); 869 scoped_ptr<base::SharedMemory> sh_mem(new base::SharedMemory());
674 if (!shared_memory_) 870 if (!sh_mem)
675 return false; 871 return false;
676 872
677 base::SharedMemoryCreateOptions options; 873 base::SharedMemoryCreateOptions options;
678 options.size = alloc_size; 874 options.size = alloc_size;
679 options.share_read_only = true; 875 options.share_read_only = true;
680 876
681 if (!shared_memory_->Create(options) || !shared_memory_->Map(alloc_size)) { 877 if (!sh_mem->Create(options) || !sh_mem->Map(alloc_size))
682 delete shared_memory_;
683 shared_memory_ = NULL;
684 return false; 878 return false;
685 }
686 879
687 if (init_to_empty) { 880 memset(sh_mem->memory(), 0, alloc_size);
688 memset(shared_memory_->memory(), 0, alloc_size);
689 used_items_ = 0;
690 }
691 table_length_ = num_entries;
692 881
693 // Save the header for other processes to read. 882 // Save the header for other processes to read.
694 SharedHeader* header = static_cast<SharedHeader*>(shared_memory_->memory()); 883 SharedHeader* header = static_cast<SharedHeader*>(sh_mem->memory());
695 header->length = table_length_; 884 header->length = num_entries;
696 memcpy(header->salt, salt_, LINK_SALT_LENGTH); 885 memcpy(header->salt, salt, LINK_SALT_LENGTH);
697 886
698 // Our table pointer is just the data immediately following the size. 887 // Our table pointer is just the data immediately following the size.
699 hash_table_ = reinterpret_cast<Fingerprint*>( 888 *hash_table = reinterpret_cast<Fingerprint*>(
700 static_cast<char*>(shared_memory_->memory()) + sizeof(SharedHeader)); 889 static_cast<char*>(sh_mem->memory()) + sizeof(SharedHeader));
890
891 *shared_memory = sh_mem.Pass();
701 892
702 return true; 893 return true;
703 } 894 }
704 895
705 bool VisitedLinkMaster::BeginReplaceURLTable(int32 num_entries) { 896 bool VisitedLinkMaster::BeginReplaceURLTable(int32 num_entries) {
706 base::SharedMemory *old_shared_memory = shared_memory_; 897 base::SharedMemory *old_shared_memory = shared_memory_;
707 Fingerprint* old_hash_table = hash_table_; 898 Fingerprint* old_hash_table = hash_table_;
708 int32 old_table_length = table_length_; 899 int32 old_table_length = table_length_;
709 if (!CreateURLTable(num_entries, true)) { 900 if (!CreateURLTable(num_entries)) {
710 // Try to put back the old state. 901 // Try to put back the old state.
711 shared_memory_ = old_shared_memory; 902 shared_memory_ = old_shared_memory;
712 hash_table_ = old_hash_table; 903 hash_table_ = old_hash_table;
713 table_length_ = old_table_length; 904 table_length_ = old_table_length;
714 return false; 905 return false;
715 } 906 }
716 907
717 #ifndef NDEBUG 908 #ifndef NDEBUG
718 DebugValidate(); 909 DebugValidate();
719 #endif 910 #endif
(...skipping 69 matching lines...) Expand 10 before | Expand all | Expand 10 after
789 980
790 #ifndef NDEBUG 981 #ifndef NDEBUG
791 DebugValidate(); 982 DebugValidate();
792 #endif 983 #endif
793 984
794 // The new table needs to be written to disk. 985 // The new table needs to be written to disk.
795 if (persist_to_disk_) 986 if (persist_to_disk_)
796 WriteFullTable(); 987 WriteFullTable();
797 } 988 }
798 989
990 uint32 VisitedLinkMaster::DefaultTableSize() const {
991 if (table_size_override_)
992 return table_size_override_;
993
994 return kDefaultTableSize;
995 }
996
799 uint32 VisitedLinkMaster::NewTableSizeForCount(int32 item_count) const { 997 uint32 VisitedLinkMaster::NewTableSizeForCount(int32 item_count) const {
800 // These table sizes are selected to be the maximum prime number less than 998 // These table sizes are selected to be the maximum prime number less than
801 // a "convenient" multiple of 1K. 999 // a "convenient" multiple of 1K.
802 static const int table_sizes[] = { 1000 static const int table_sizes[] = {
803 16381, // 16K = 16384 <- don't shrink below this table size 1001 16381, // 16K = 16384 <- don't shrink below this table size
804 // (should be == default_table_size) 1002 // (should be == default_table_size)
805 32767, // 32K = 32768 1003 32767, // 32K = 32768
806 65521, // 64K = 65536 1004 65521, // 64K = 65536
807 130051, // 128K = 131072 1005 130051, // 128K = 131072
808 262127, // 256K = 262144 1006 262127, // 256K = 262144
(...skipping 34 matching lines...) Expand 10 before | Expand all | Expand 10 after
843 bool success, 1041 bool success,
844 const std::vector<Fingerprint>& fingerprints) { 1042 const std::vector<Fingerprint>& fingerprints) {
845 if (success) { 1043 if (success) {
846 // Replace the old table with a new blank one. 1044 // Replace the old table with a new blank one.
847 shared_memory_serial_++; 1045 shared_memory_serial_++;
848 1046
849 // We are responsible for freeing it AFTER it has been replaced if 1047 // We are responsible for freeing it AFTER it has been replaced if
850 // replacement succeeds. 1048 // replacement succeeds.
851 base::SharedMemory* old_shared_memory = shared_memory_; 1049 base::SharedMemory* old_shared_memory = shared_memory_;
852 1050
853 int new_table_size = NewTableSizeForCount( 1051 int new_table_size =
854 static_cast<int>(fingerprints.size() + added_since_rebuild_.size())); 1052 NewTableSizeForCount(static_cast<int>(fingerprints.size() +
1053 added_since_rebuild_or_load_.size()));
855 if (BeginReplaceURLTable(new_table_size)) { 1054 if (BeginReplaceURLTable(new_table_size)) {
856 // Free the old table. 1055 // Free the old table.
857 delete old_shared_memory; 1056 delete old_shared_memory;
858 1057
859 // Add the stored fingerprints to the hash table. 1058 // Add the stored fingerprints to the hash table.
860 for (size_t i = 0; i < fingerprints.size(); i++) 1059 for (size_t i = 0; i < fingerprints.size(); i++)
861 AddFingerprint(fingerprints[i], false); 1060 AddFingerprint(fingerprints[i], false);
862 1061
863 // Also add anything that was added while we were asynchronously 1062 // Also add anything that was added while we were asynchronously
864 // generating the new table. 1063 // generating the new table.
865 for (std::set<Fingerprint>::iterator i = added_since_rebuild_.begin(); 1064 for (std::set<Fingerprint>::iterator i =
866 i != added_since_rebuild_.end(); ++i) 1065 added_since_rebuild_or_load_.begin();
1066 i != added_since_rebuild_or_load_.end(); ++i)
867 AddFingerprint(*i, false); 1067 AddFingerprint(*i, false);
868 added_since_rebuild_.clear(); 1068 added_since_rebuild_or_load_.clear();
869 1069
870 // Now handle deletions. 1070 // Now handle deletions.
871 DeleteFingerprintsFromCurrentTable(deleted_since_rebuild_); 1071 DeleteFingerprintsFromCurrentTable(deleted_since_rebuild_or_load_);
872 deleted_since_rebuild_.clear(); 1072 deleted_since_rebuild_or_load_.clear();
873 1073
874 // Send an update notification to all child processes. 1074 // Send an update notification to all child processes.
875 listener_->NewTable(shared_memory_); 1075 listener_->NewTable(shared_memory_);
876 1076
877 if (persist_to_disk_) 1077 if (persist_to_disk_)
878 WriteFullTable(); 1078 WriteFullTable();
879 } 1079 }
880 } 1080 }
881 table_builder_ = NULL; // Will release our reference to the builder. 1081 table_builder_ = NULL; // Will release our reference to the builder.
882 1082
883 // Notify the unit test that the rebuild is complete (will be NULL in prod.) 1083 // Notify the unit test that the rebuild is complete (will be NULL in prod.)
884 if (!rebuild_complete_task_.is_null()) { 1084 if (!rebuild_complete_task_.is_null()) {
885 rebuild_complete_task_.Run(); 1085 rebuild_complete_task_.Run();
886 rebuild_complete_task_.Reset(); 1086 rebuild_complete_task_.Reset();
887 } 1087 }
888 } 1088 }
889 1089
890 void VisitedLinkMaster::WriteToFile(FILE** file, 1090 void VisitedLinkMaster::WriteToFile(FILE** file,
891 off_t offset, 1091 off_t offset,
892 void* data, 1092 void* data,
893 int32 data_size) { 1093 int32 data_size) {
894 DCHECK(persist_to_disk_); 1094 DCHECK(persist_to_disk_);
895 #ifndef NDEBUG 1095 DCHECK(!table_is_loading_from_file_);
896 posted_asynchronous_operation_ = true;
897 #endif
898 PostIOTask(FROM_HERE, 1096 PostIOTask(FROM_HERE,
899 base::Bind(&AsyncWrite, file, offset, 1097 base::Bind(&AsyncWrite, file, offset,
900 std::string(static_cast<const char*>(data), data_size))); 1098 std::string(static_cast<const char*>(data), data_size)));
901 } 1099 }
902 1100
903 void VisitedLinkMaster::WriteUsedItemCountToFile() { 1101 void VisitedLinkMaster::WriteUsedItemCountToFile() {
904 DCHECK(persist_to_disk_); 1102 DCHECK(persist_to_disk_);
905 if (!file_) 1103 if (!file_)
906 return; // See comment on the file_ variable for why this might happen. 1104 return; // See comment on the file_ variable for why this might happen.
907 WriteToFile(file_, kFileHeaderUsedOffset, &used_items_, sizeof(used_items_)); 1105 WriteToFile(file_, kFileHeaderUsedOffset, &used_items_, sizeof(used_items_));
(...skipping 14 matching lines...) Expand all
922 WriteToFile(file_, kFileHeaderSize, hash_table_, 1120 WriteToFile(file_, kFileHeaderSize, hash_table_,
923 (last_hash + 1) * sizeof(Fingerprint)); 1121 (last_hash + 1) * sizeof(Fingerprint));
924 } else { 1122 } else {
925 // Normal case, just write the range. 1123 // Normal case, just write the range.
926 WriteToFile(file_, first_hash * sizeof(Fingerprint) + kFileHeaderSize, 1124 WriteToFile(file_, first_hash * sizeof(Fingerprint) + kFileHeaderSize,
927 &hash_table_[first_hash], 1125 &hash_table_[first_hash],
928 (last_hash - first_hash + 1) * sizeof(Fingerprint)); 1126 (last_hash - first_hash + 1) * sizeof(Fingerprint));
929 } 1127 }
930 } 1128 }
931 1129
1130 // static
932 bool VisitedLinkMaster::ReadFromFile(FILE* file, 1131 bool VisitedLinkMaster::ReadFromFile(FILE* file,
933 off_t offset, 1132 off_t offset,
934 void* data, 1133 void* data,
935 size_t data_size) { 1134 size_t data_size) {
936 DCHECK(persist_to_disk_);
937 #ifndef NDEBUG
938 // Since this function is synchronous, we require that no asynchronous
939 // operations could possibly be pending.
940 DCHECK(!posted_asynchronous_operation_);
941 #endif
942
943 if (fseek(file, offset, SEEK_SET) != 0) 1135 if (fseek(file, offset, SEEK_SET) != 0)
944 return false; 1136 return false;
945 1137
946 size_t num_read = fread(data, 1, data_size, file); 1138 size_t num_read = fread(data, 1, data_size, file);
947 return num_read == data_size; 1139 return num_read == data_size;
948 } 1140 }
949 1141
950 // VisitedLinkTableBuilder ---------------------------------------------------- 1142 // VisitedLinkTableBuilder ----------------------------------------------------
951 1143
952 VisitedLinkMaster::TableBuilder::TableBuilder( 1144 VisitedLinkMaster::TableBuilder::TableBuilder(
(...skipping 28 matching lines...) Expand all
981 BrowserThread::UI, FROM_HERE, 1173 BrowserThread::UI, FROM_HERE,
982 base::Bind(&TableBuilder::OnCompleteMainThread, this)); 1174 base::Bind(&TableBuilder::OnCompleteMainThread, this));
983 } 1175 }
984 1176
985 void VisitedLinkMaster::TableBuilder::OnCompleteMainThread() { 1177 void VisitedLinkMaster::TableBuilder::OnCompleteMainThread() {
986 if (master_) 1178 if (master_)
987 master_->OnTableRebuildComplete(success_, fingerprints_); 1179 master_->OnTableRebuildComplete(success_, fingerprints_);
988 } 1180 }
989 1181
990 } // namespace visitedlink 1182 } // namespace visitedlink
OLDNEW
« no previous file with comments | « components/visitedlink/browser/visitedlink_master.h ('k') | components/visitedlink/test/visitedlink_unittest.cc » ('j') | no next file with comments »

Powered by Google App Engine
This is Rietveld 408576698