| OLD | NEW |
| 1 // Copyright (c) 2010 The Chromium Authors. All rights reserved. | 1 // Copyright (c) 2010 The Chromium Authors. All rights reserved. |
| 2 // Use of this source code is governed by a BSD-style license that can be | 2 // Use of this source code is governed by a BSD-style license that can be |
| 3 // found in the LICENSE file. | 3 // found in the LICENSE file. |
| 4 | 4 |
| 5 #include "chrome/browser/safe_browsing/safe_browsing_store_file.h" | 5 #include "chrome/browser/safe_browsing/safe_browsing_store_file.h" |
| 6 | 6 |
| 7 #include "base/callback.h" | 7 #include "base/callback.h" |
| 8 | 8 |
| 9 namespace { | 9 namespace { |
| 10 | 10 |
| 11 // NOTE(shess): kFileMagic should not be a byte-wise palindrome, so | 11 // NOTE(shess): kFileMagic should not be a byte-wise palindrome, so |
| 12 // that byte-order changes force corruption. | 12 // that byte-order changes force corruption. |
| 13 const int32 kFileMagic = 0x600D71FE; | 13 const int32 kFileMagic = 0x600D71FE; |
| 14 const int32 kFileVersion = 7; // SQLite storage was 6... | 14 const int32 kFileVersion = 7; // SQLite storage was 6... |
| 15 const size_t kFileHeaderSize = 8 * sizeof(int32); | |
| 16 | 15 |
| 17 bool ReadInt32(FILE* fp, int32* value) { | 16 // Header at the front of the main database file. |
| 18 DCHECK(value); | 17 struct FileHeader { |
| 19 const size_t ret = fread(value, sizeof(*value), 1, fp); | 18 int32 magic, version; |
| 20 return ret == 1; | 19 int32 add_chunk_count, sub_chunk_count; |
| 21 } | 20 int32 add_prefix_count, sub_prefix_count; |
| 21 int32 add_hash_count, sub_hash_count; |
| 22 }; |
| 22 | 23 |
| 23 bool WriteInt32(FILE* fp, int32 value) { | 24 // Header for each chunk in the chunk-accumulation file. |
| 24 const size_t ret = fwrite(&value, sizeof(value), 1, fp); | 25 struct ChunkHeader { |
| 25 return ret == 1; | 26 int32 add_prefix_count, sub_prefix_count; |
| 26 } | 27 int32 add_hash_count, sub_hash_count; |
| 28 }; |
| 27 | 29 |
| 28 bool ReadHash(FILE* fp, SBFullHash* value) { | 30 // Rewind the file. Using fseek(2) because rewind(3) errors are |
| 29 DCHECK(value); | 31 // weird. |
| 30 const size_t ret = fread(&value->full_hash, sizeof(value->full_hash), | 32 bool FileRewind(FILE* fp) { |
| 31 1, fp); | 33 int rv = fseek(fp, 0, SEEK_SET); |
| 32 return ret == 1; | |
| 33 } | |
| 34 | |
| 35 bool WriteHash(FILE* fp, SBFullHash value) { | |
| 36 const size_t ret = fwrite(&value.full_hash, sizeof(value.full_hash), | |
| 37 1, fp); | |
| 38 return ret == 1; | |
| 39 } | |
| 40 | |
| 41 bool FileSeek(FILE* fp, size_t offset) { | |
| 42 int rv = fseek(fp, offset, SEEK_SET); | |
| 43 DCHECK_EQ(rv, 0); | 34 DCHECK_EQ(rv, 0); |
| 44 return rv == 0; | 35 return rv == 0; |
| 45 } | 36 } |
| 46 | 37 |
| 38 // Read an array of |nmemb| items from |fp| into |ptr|. Return true |
| 39 // on success. |
| 40 template <class T> |
| 41 bool ReadArray(T* ptr, size_t nmemb, FILE* fp) { |
| 42 const size_t ret = fread(ptr, sizeof(T), nmemb, fp); |
| 43 if (ret != nmemb) |
| 44 return false; |
| 45 return true; |
| 46 } |
| 47 |
| 48 // Write an array of |nmemb| items from |ptr| to |fp|. Return true on |
| 49 // success. |
| 50 template <class T> |
| 51 bool WriteArray(const T* ptr, size_t nmemb, FILE* fp) { |
| 52 const size_t ret = fwrite(ptr, sizeof(T), nmemb, fp); |
| 53 if (ret != nmemb) |
| 54 return false; |
| 55 return true; |
| 56 } |
| 57 |
| 58 // Expand |values| to fit |count| new items, and read those items from |
| 59 // |fp|. Returns true on success. |
| 60 template <class T> |
| 61 bool ReadToVector(std::vector<T>* values, size_t count, FILE* fp) { |
| 62 // Pointers into an empty vector may not be valid. |
| 63 if (!count) |
| 64 return true; |
| 65 |
| 66 // Grab the size for purposes of finding where to read to. The |
| 67 // resize could invalidate any iterator captured here. |
| 68 const size_t original_size = values->size(); |
| 69 values->resize(original_size + count); |
| 70 |
| 71 // Sayeth Herb Sutter: Vectors are guaranteed to be contiguous. So |
| 72 // get a pointer to where to read the data to. |
| 73 T* ptr = &((*values)[original_size]); |
| 74 if (!ReadArray(ptr, count, fp)) { |
| 75 values->resize(original_size); |
| 76 return false; |
| 77 } |
| 78 |
| 79 return true; |
| 80 } |
| 81 |
| 82 // Write all of |values| to |fp|. Returns true on succsess. |
| 83 template <class T> |
| 84 bool WriteVector(const std::vector<T>& values, FILE* fp) { |
| 85 // Pointers into empty vectors may not be valid. |
| 86 if (values.empty()) |
| 87 return true; |
| 88 |
| 89 // Sayeth Herb Sutter: Vectors are guaranteed to be contiguous. So |
| 90 // get a pointer to where to write from. |
| 91 const T* ptr = &(values[0]); |
| 92 return WriteArray(ptr, values.size(), fp); |
| 93 } |
| 94 |
| 95 // Remove deleted items (|chunk_id| in |del_set|) from the vector |
| 96 // starting at |offset| running to |end()|. |
| 97 template <class T> |
| 98 void RemoveDeleted(std::vector<T>* vec, size_t offset, |
| 99 const base::hash_set<int32>& del_set) { |
| 100 DCHECK(vec); |
| 101 |
| 102 // Scan through the items read, dropping the items in |del_set|. |
| 103 typename std::vector<T>::iterator add_iter = vec->begin() + offset; |
| 104 for (typename std::vector<T>::iterator iter = add_iter; |
| 105 iter != vec->end(); ++iter) { |
| 106 if (del_set.count(iter->chunk_id) == 0) { |
| 107 *add_iter = *iter; |
| 108 ++add_iter; |
| 109 } |
| 110 } |
| 111 vec->erase(add_iter, vec->end()); |
| 112 } |
| 113 |
| 114 // Combine |ReadToVector()| and |RemoveDeleted()|. Returns true on |
| 115 // success. |
| 116 template <class T> |
| 117 bool ReadToVectorAndDelete(std::vector<T>* values, size_t count, FILE* fp, |
| 118 const base::hash_set<int32>& del_set) { |
| 119 const size_t original_size = values->size(); |
| 120 if (!ReadToVector(values, count, fp)) |
| 121 return false; |
| 122 |
| 123 RemoveDeleted(values, original_size, del_set); |
| 124 return true; |
| 125 } |
| 126 |
| 127 // Read an array of |count| integers and add them to |values|. |
| 128 // Returns true on success. |
| 129 bool ReadToChunkSet(std::set<int32>* values, size_t count, FILE* fp) { |
| 130 if (!count) |
| 131 return true; |
| 132 |
| 133 std::vector<int32> flat_values; |
| 134 if (!ReadToVector(&flat_values, count, fp)) |
| 135 return false; |
| 136 |
| 137 values->insert(flat_values.begin(), flat_values.end()); |
| 138 return true; |
| 139 } |
| 140 |
| 141 // Write the contents of |values| as an array of integers. Returns |
| 142 // true on success. |
| 143 bool WriteChunkSet(const std::set<int32>& values, FILE* fp) { |
| 144 if (values.empty()) |
| 145 return true; |
| 146 |
| 147 const std::vector<int32> flat_values(values.begin(), values.end()); |
| 148 return WriteVector(flat_values, fp); |
| 149 } |
| 150 |
| 47 // Delete the chunks in |deleted| from |chunks|. | 151 // Delete the chunks in |deleted| from |chunks|. |
| 48 void DeleteChunksFromSet(const base::hash_set<int32>& deleted, | 152 void DeleteChunksFromSet(const base::hash_set<int32>& deleted, |
| 49 std::set<int32>* chunks) { | 153 std::set<int32>* chunks) { |
| 50 for (std::set<int32>::iterator iter = chunks->begin(); | 154 for (std::set<int32>::iterator iter = chunks->begin(); |
| 51 iter != chunks->end();) { | 155 iter != chunks->end();) { |
| 52 std::set<int32>::iterator prev = iter++; | 156 std::set<int32>::iterator prev = iter++; |
| 53 if (deleted.count(*prev) > 0) | 157 if (deleted.count(*prev) > 0) |
| 54 chunks->erase(prev); | 158 chunks->erase(prev); |
| 55 } | 159 } |
| 56 } | 160 } |
| (...skipping 48 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 105 | 209 |
| 106 bool SafeBrowsingStoreFile::Close() { | 210 bool SafeBrowsingStoreFile::Close() { |
| 107 ClearUpdateBuffers(); | 211 ClearUpdateBuffers(); |
| 108 | 212 |
| 109 // Make sure the files are closed. | 213 // Make sure the files are closed. |
| 110 file_.reset(); | 214 file_.reset(); |
| 111 new_file_.reset(); | 215 new_file_.reset(); |
| 112 return true; | 216 return true; |
| 113 } | 217 } |
| 114 | 218 |
| 115 bool SafeBrowsingStoreFile::ReadChunksToSet(FILE* fp, std::set<int32>* chunks, | |
| 116 int count) { | |
| 117 DCHECK(fp); | |
| 118 | |
| 119 for (int i = 0; i < count; ++i) { | |
| 120 int32 chunk_id; | |
| 121 if (!ReadInt32(fp, &chunk_id)) | |
| 122 return false; | |
| 123 chunks->insert(chunk_id); | |
| 124 } | |
| 125 return true; | |
| 126 } | |
| 127 | |
| 128 bool SafeBrowsingStoreFile::WriteChunksFromSet(const std::set<int32>& chunks) { | |
| 129 DCHECK(new_file_.get()); | |
| 130 | |
| 131 for (std::set<int32>::const_iterator iter = chunks.begin(); | |
| 132 iter != chunks.end(); ++iter) { | |
| 133 if (!WriteInt32(new_file_.get(), *iter)) | |
| 134 return false; | |
| 135 } | |
| 136 return true; | |
| 137 } | |
| 138 | |
| 139 bool SafeBrowsingStoreFile::ReadAddPrefixes( | |
| 140 FILE* fp, std::vector<SBAddPrefix>* add_prefixes, int count) { | |
| 141 DCHECK(fp && add_prefixes); | |
| 142 | |
| 143 add_prefixes->reserve(add_prefixes->size() + count); | |
| 144 | |
| 145 for (int32 i = 0; i < count; ++i) { | |
| 146 int32 chunk_id; | |
| 147 SBPrefix prefix; | |
| 148 DCHECK_EQ(sizeof(int32), sizeof(prefix)); | |
| 149 | |
| 150 if (!ReadInt32(fp, &chunk_id) || !ReadInt32(fp, &prefix)) | |
| 151 return false; | |
| 152 | |
| 153 if (add_del_cache_.count(chunk_id) > 0) | |
| 154 continue; | |
| 155 | |
| 156 add_prefixes->push_back(SBAddPrefix(chunk_id, prefix)); | |
| 157 } | |
| 158 | |
| 159 return true; | |
| 160 } | |
| 161 | |
| 162 bool SafeBrowsingStoreFile::WriteAddPrefixes( | |
| 163 const std::vector<SBAddPrefix>& add_prefixes) { | |
| 164 DCHECK(new_file_.get()); | |
| 165 | |
| 166 for (std::vector<SBAddPrefix>::const_iterator iter = add_prefixes.begin(); | |
| 167 iter != add_prefixes.end(); ++iter) { | |
| 168 DCHECK_EQ(sizeof(int32), sizeof(iter->prefix)); | |
| 169 if (!WriteInt32(new_file_.get(), iter->chunk_id) || | |
| 170 !WriteInt32(new_file_.get(), iter->prefix)) | |
| 171 return false; | |
| 172 } | |
| 173 return true; | |
| 174 } | |
| 175 | |
| 176 bool SafeBrowsingStoreFile::ReadSubPrefixes( | |
| 177 FILE* fp, std::vector<SBSubPrefix>* sub_prefixes, int count) { | |
| 178 DCHECK(fp && sub_prefixes); | |
| 179 | |
| 180 sub_prefixes->reserve(sub_prefixes->size() + count); | |
| 181 | |
| 182 for (int32 i = 0; i < count; ++i) { | |
| 183 int32 chunk_id, add_chunk_id; | |
| 184 SBPrefix add_prefix; | |
| 185 DCHECK_EQ(sizeof(int32), sizeof(add_prefix)); | |
| 186 | |
| 187 if (!ReadInt32(fp, &chunk_id) || | |
| 188 !ReadInt32(fp, &add_chunk_id) || !ReadInt32(fp, &add_prefix)) | |
| 189 return false; | |
| 190 | |
| 191 if (sub_del_cache_.count(chunk_id) > 0) | |
| 192 continue; | |
| 193 | |
| 194 sub_prefixes->push_back(SBSubPrefix(chunk_id, add_chunk_id, add_prefix)); | |
| 195 } | |
| 196 | |
| 197 return true; | |
| 198 } | |
| 199 | |
| 200 bool SafeBrowsingStoreFile::WriteSubPrefixes( | |
| 201 std::vector<SBSubPrefix>& sub_prefixes) { | |
| 202 DCHECK(new_file_.get()); | |
| 203 | |
| 204 for (std::vector<SBSubPrefix>::const_iterator iter = sub_prefixes.begin(); | |
| 205 iter != sub_prefixes.end(); ++iter) { | |
| 206 if (!WriteInt32(new_file_.get(), iter->chunk_id) || | |
| 207 !WriteInt32(new_file_.get(), iter->add_chunk_id) || | |
| 208 !WriteInt32(new_file_.get(), iter->add_prefix)) | |
| 209 return false; | |
| 210 } | |
| 211 return true; | |
| 212 } | |
| 213 | |
| 214 bool SafeBrowsingStoreFile::ReadAddHashes( | |
| 215 FILE* fp, std::vector<SBAddFullHash>* add_hashes, int count) { | |
| 216 DCHECK(fp && add_hashes); | |
| 217 | |
| 218 add_hashes->reserve(add_hashes->size() + count); | |
| 219 | |
| 220 for (int i = 0; i < count; ++i) { | |
| 221 int32 chunk_id; | |
| 222 int32 received; | |
| 223 SBFullHash full_hash; | |
| 224 | |
| 225 if (!ReadInt32(fp, &chunk_id) || | |
| 226 !ReadInt32(fp, &received) || | |
| 227 !ReadHash(fp, &full_hash)) | |
| 228 return false; | |
| 229 | |
| 230 if (add_del_cache_.count(chunk_id) > 0) | |
| 231 continue; | |
| 232 | |
| 233 add_hashes->push_back(SBAddFullHash(chunk_id, received, full_hash)); | |
| 234 } | |
| 235 | |
| 236 return true; | |
| 237 } | |
| 238 | |
| 239 bool SafeBrowsingStoreFile::WriteAddHashes( | |
| 240 const std::vector<SBAddFullHash>& add_hashes) { | |
| 241 DCHECK(new_file_.get()); | |
| 242 | |
| 243 for (std::vector<SBAddFullHash>::const_iterator iter = add_hashes.begin(); | |
| 244 iter != add_hashes.end(); ++iter) { | |
| 245 if (!WriteInt32(new_file_.get(), iter->chunk_id) || | |
| 246 !WriteInt32(new_file_.get(), iter->received) || | |
| 247 !WriteHash(new_file_.get(), iter->full_hash)) | |
| 248 return false; | |
| 249 } | |
| 250 return true; | |
| 251 } | |
| 252 | |
| 253 bool SafeBrowsingStoreFile::ReadSubHashes( | |
| 254 FILE* fp, std::vector<SBSubFullHash>* sub_hashes, int count) { | |
| 255 DCHECK(fp); | |
| 256 | |
| 257 sub_hashes->reserve(sub_hashes->size() + count); | |
| 258 | |
| 259 for (int i = 0; i < count; ++i) { | |
| 260 int32 chunk_id; | |
| 261 int32 add_chunk_id; | |
| 262 SBFullHash add_full_hash; | |
| 263 | |
| 264 if (!ReadInt32(fp, &chunk_id) || | |
| 265 !ReadInt32(fp, &add_chunk_id) || | |
| 266 !ReadHash(fp, &add_full_hash)) | |
| 267 return false; | |
| 268 | |
| 269 if (sub_del_cache_.count(chunk_id) > 0) | |
| 270 continue; | |
| 271 | |
| 272 sub_hashes->push_back(SBSubFullHash(chunk_id, add_chunk_id, add_full_hash)); | |
| 273 } | |
| 274 | |
| 275 return true; | |
| 276 } | |
| 277 | |
| 278 bool SafeBrowsingStoreFile::WriteSubHashes( | |
| 279 std::vector<SBSubFullHash>& sub_hashes) { | |
| 280 DCHECK(new_file_.get()); | |
| 281 | |
| 282 for (std::vector<SBSubFullHash>::const_iterator iter = sub_hashes.begin(); | |
| 283 iter != sub_hashes.end(); ++iter) { | |
| 284 if (!WriteInt32(new_file_.get(), iter->chunk_id) || | |
| 285 !WriteInt32(new_file_.get(), iter->add_chunk_id) || | |
| 286 !WriteHash(new_file_.get(), iter->full_hash)) | |
| 287 return false; | |
| 288 } | |
| 289 return true; | |
| 290 } | |
| 291 | |
| 292 bool SafeBrowsingStoreFile::BeginUpdate() { | 219 bool SafeBrowsingStoreFile::BeginUpdate() { |
| 293 DCHECK(!file_.get() && !new_file_.get()); | 220 DCHECK(!file_.get() && !new_file_.get()); |
| 294 | 221 |
| 295 // Structures should all be clear unless something bad happened. | 222 // Structures should all be clear unless something bad happened. |
| 296 DCHECK(add_chunks_cache_.empty()); | 223 DCHECK(add_chunks_cache_.empty()); |
| 297 DCHECK(sub_chunks_cache_.empty()); | 224 DCHECK(sub_chunks_cache_.empty()); |
| 298 DCHECK(add_del_cache_.empty()); | 225 DCHECK(add_del_cache_.empty()); |
| 299 DCHECK(sub_del_cache_.empty()); | 226 DCHECK(sub_del_cache_.empty()); |
| 300 DCHECK(add_prefixes_.empty()); | 227 DCHECK(add_prefixes_.empty()); |
| 301 DCHECK(sub_prefixes_.empty()); | 228 DCHECK(sub_prefixes_.empty()); |
| (...skipping 11 matching lines...) Expand all Loading... |
| 313 if (empty_) { | 240 if (empty_) { |
| 314 // If the file exists but cannot be opened, try to delete it (not | 241 // If the file exists but cannot be opened, try to delete it (not |
| 315 // deleting directly, the bloom filter needs to be deleted, too). | 242 // deleting directly, the bloom filter needs to be deleted, too). |
| 316 if (file_util::PathExists(filename_)) | 243 if (file_util::PathExists(filename_)) |
| 317 return OnCorruptDatabase(); | 244 return OnCorruptDatabase(); |
| 318 | 245 |
| 319 new_file_.swap(new_file); | 246 new_file_.swap(new_file); |
| 320 return true; | 247 return true; |
| 321 } | 248 } |
| 322 | 249 |
| 323 int32 magic, version; | 250 FileHeader header; |
| 324 if (!ReadInt32(file.get(), &magic) || !ReadInt32(file.get(), &version)) | 251 if (!ReadArray(&header, 1, file.get())) |
| 325 return OnCorruptDatabase(); | 252 return OnCorruptDatabase(); |
| 326 | 253 |
| 327 if (magic != kFileMagic || version != kFileVersion) | 254 if (header.magic != kFileMagic || header.version != kFileVersion) |
| 328 return OnCorruptDatabase(); | 255 return OnCorruptDatabase(); |
| 329 | 256 |
| 330 int32 add_chunk_count, sub_chunk_count; | 257 if (!ReadToChunkSet(&add_chunks_cache_, header.add_chunk_count, file.get()) || |
| 331 if (!ReadInt32(file.get(), &add_chunk_count) || | 258 !ReadToChunkSet(&sub_chunks_cache_, header.sub_chunk_count, file.get())) |
| 332 !ReadInt32(file.get(), &sub_chunk_count)) | |
| 333 return OnCorruptDatabase(); | |
| 334 | |
| 335 if (!FileSeek(file.get(), kFileHeaderSize)) | |
| 336 return OnCorruptDatabase(); | |
| 337 | |
| 338 if (!ReadChunksToSet(file.get(), &add_chunks_cache_, add_chunk_count) || | |
| 339 !ReadChunksToSet(file.get(), &sub_chunks_cache_, sub_chunk_count)) | |
| 340 return OnCorruptDatabase(); | 259 return OnCorruptDatabase(); |
| 341 | 260 |
| 342 file_.swap(file); | 261 file_.swap(file); |
| 343 new_file_.swap(new_file); | 262 new_file_.swap(new_file); |
| 344 return true; | 263 return true; |
| 345 } | 264 } |
| 346 | 265 |
| 347 bool SafeBrowsingStoreFile::FinishChunk() { | 266 bool SafeBrowsingStoreFile::FinishChunk() { |
| 348 if (!add_prefixes_.size() && !sub_prefixes_.size() && | 267 if (!add_prefixes_.size() && !sub_prefixes_.size() && |
| 349 !add_hashes_.size() && !sub_hashes_.size()) | 268 !add_hashes_.size() && !sub_hashes_.size()) |
| 350 return true; | 269 return true; |
| 351 | 270 |
| 352 if (!WriteInt32(new_file_.get(), add_prefixes_.size()) || | 271 ChunkHeader header; |
| 353 !WriteInt32(new_file_.get(), sub_prefixes_.size()) || | 272 header.add_prefix_count = add_prefixes_.size(); |
| 354 !WriteInt32(new_file_.get(), add_hashes_.size()) || | 273 header.sub_prefix_count = sub_prefixes_.size(); |
| 355 !WriteInt32(new_file_.get(), sub_hashes_.size())) | 274 header.add_hash_count = add_hashes_.size(); |
| 275 header.sub_hash_count = sub_hashes_.size(); |
| 276 if (!WriteArray(&header, 1, new_file_.get())) |
| 356 return false; | 277 return false; |
| 357 | 278 |
| 358 if (!WriteAddPrefixes(add_prefixes_) || | 279 if (!WriteVector(add_prefixes_, new_file_.get()) || |
| 359 !WriteSubPrefixes(sub_prefixes_) || | 280 !WriteVector(sub_prefixes_, new_file_.get()) || |
| 360 !WriteAddHashes(add_hashes_) || | 281 !WriteVector(add_hashes_, new_file_.get()) || |
| 361 !WriteSubHashes(sub_hashes_)) | 282 !WriteVector(sub_hashes_, new_file_.get())) |
| 362 return false; | 283 return false; |
| 363 | 284 |
| 364 ++chunks_written_; | 285 ++chunks_written_; |
| 365 | 286 |
| 366 // Clear everything to save memory. | 287 // Clear everything to save memory. |
| 367 return ClearChunkBuffers(); | 288 return ClearChunkBuffers(); |
| 368 } | 289 } |
| 369 | 290 |
| 370 bool SafeBrowsingStoreFile::DoUpdate( | 291 bool SafeBrowsingStoreFile::DoUpdate( |
| 371 const std::vector<SBAddFullHash>& pending_adds, | 292 const std::vector<SBAddFullHash>& pending_adds, |
| 372 std::vector<SBAddPrefix>* add_prefixes_result, | 293 std::vector<SBAddPrefix>* add_prefixes_result, |
| 373 std::vector<SBAddFullHash>* add_full_hashes_result) { | 294 std::vector<SBAddFullHash>* add_full_hashes_result) { |
| 374 DCHECK(file_.get() || empty_); | 295 DCHECK(file_.get() || empty_); |
| 375 DCHECK(new_file_.get()); | 296 DCHECK(new_file_.get()); |
| 376 | 297 |
| 377 std::vector<SBAddPrefix> add_prefixes; | 298 std::vector<SBAddPrefix> add_prefixes; |
| 378 std::vector<SBSubPrefix> sub_prefixes; | 299 std::vector<SBSubPrefix> sub_prefixes; |
| 379 std::vector<SBAddFullHash> add_full_hashes; | 300 std::vector<SBAddFullHash> add_full_hashes; |
| 380 std::vector<SBSubFullHash> sub_full_hashes; | 301 std::vector<SBSubFullHash> sub_full_hashes; |
| 381 | 302 |
| 382 // Read |file_| into the vectors. | 303 // Read |file_| into the vectors. |
| 383 if (!empty_) { | 304 if (!empty_) { |
| 384 DCHECK(file_.get()); | 305 DCHECK(file_.get()); |
| 385 | 306 |
| 386 int32 magic, version; | 307 if (!FileRewind(file_.get())) |
| 387 int32 add_chunk_count, sub_chunk_count; | |
| 388 int32 add_prefix_count, sub_prefix_count; | |
| 389 int32 add_hash_count, sub_hash_count; | |
| 390 | |
| 391 if (!FileSeek(file_.get(), 0)) | |
| 392 return OnCorruptDatabase(); | 308 return OnCorruptDatabase(); |
| 393 | 309 |
| 394 if (!ReadInt32(file_.get(), &magic) || | 310 // Read the file header and make sure it looks right. |
| 395 !ReadInt32(file_.get(), &version) || | 311 FileHeader header; |
| 396 !ReadInt32(file_.get(), &add_chunk_count) || | 312 if (!ReadArray(&header, 1, file_.get())) |
| 397 !ReadInt32(file_.get(), &sub_chunk_count) || | |
| 398 !ReadInt32(file_.get(), &add_prefix_count) || | |
| 399 !ReadInt32(file_.get(), &sub_prefix_count) || | |
| 400 !ReadInt32(file_.get(), &add_hash_count) || | |
| 401 !ReadInt32(file_.get(), &sub_hash_count)) | |
| 402 return OnCorruptDatabase(); | 313 return OnCorruptDatabase(); |
| 403 | 314 |
| 404 if (magic != kFileMagic || version != kFileVersion) | 315 if (header.magic != kFileMagic || header.version != kFileVersion) |
| 405 return OnCorruptDatabase(); | 316 return OnCorruptDatabase(); |
| 406 | 317 |
| 407 const size_t prefixes_offset = kFileHeaderSize + | 318 // Re-read the chunks-seen data to get to the later data in the |
| 408 (add_chunk_count + sub_chunk_count) * sizeof(int32); | 319 // file. No new elements should be added to the sets. |
| 409 if (!FileSeek(file_.get(), prefixes_offset)) | 320 // NOTE(shess): Reading rather than fseek() because calculating |
| 321 // checksums (future CL) will need to scan all data. The code |
| 322 // could just remember state from |BeginUpdate()|, but that call |
| 323 // may be far removed from this call in time, so this seems like a |
| 324 // reasonable trade-off. |
| 325 if (!ReadToChunkSet(&add_chunks_cache_, header.add_chunk_count, |
| 326 file_.get()) || |
| 327 !ReadToChunkSet(&sub_chunks_cache_, header.sub_chunk_count, |
| 328 file_.get())) |
| 410 return OnCorruptDatabase(); | 329 return OnCorruptDatabase(); |
| 411 | 330 |
| 412 if (!ReadAddPrefixes(file_.get(), &add_prefixes, add_prefix_count) || | 331 if (!ReadToVectorAndDelete(&add_prefixes, header.add_prefix_count, |
| 413 !ReadSubPrefixes(file_.get(), &sub_prefixes, sub_prefix_count) || | 332 file_.get(), add_del_cache_) || |
| 414 !ReadAddHashes(file_.get(), &add_full_hashes, add_hash_count) || | 333 !ReadToVectorAndDelete(&sub_prefixes, header.sub_prefix_count, |
| 415 !ReadSubHashes(file_.get(), &sub_full_hashes, sub_hash_count)) | 334 file_.get(), sub_del_cache_) || |
| 335 !ReadToVectorAndDelete(&add_full_hashes, header.add_hash_count, |
| 336 file_.get(), add_del_cache_) || |
| 337 !ReadToVectorAndDelete(&sub_full_hashes, header.sub_hash_count, |
| 338 file_.get(), sub_del_cache_)) |
| 416 return OnCorruptDatabase(); | 339 return OnCorruptDatabase(); |
| 417 | 340 |
| 418 // Close the file so we can later rename over it. | 341 // Close the file so we can later rename over it. |
| 419 file_.reset(); | 342 file_.reset(); |
| 420 } | 343 } |
| 421 DCHECK(!file_.get()); | 344 DCHECK(!file_.get()); |
| 422 | 345 |
| 423 // Rewind the temporary storage. | 346 // Rewind the temporary storage. |
| 424 if (!FileSeek(new_file_.get(), 0)) | 347 if (!FileRewind(new_file_.get())) |
| 425 return false; | 348 return false; |
| 426 | 349 |
| 427 // Append the accumulated chunks onto the vectors from file_. | 350 // Append the accumulated chunks onto the vectors from file_. |
| 428 for (int i = 0; i < chunks_written_; ++i) { | 351 for (int i = 0; i < chunks_written_; ++i) { |
| 429 int32 add_prefix_count, sub_prefix_count; | 352 ChunkHeader header; |
| 430 int32 add_hash_count, sub_hash_count; | |
| 431 | 353 |
| 432 if (!ReadInt32(new_file_.get(), &add_prefix_count) || | 354 if (!ReadArray(&header, 1, new_file_.get())) |
| 433 !ReadInt32(new_file_.get(), &sub_prefix_count) || | |
| 434 !ReadInt32(new_file_.get(), &add_hash_count) || | |
| 435 !ReadInt32(new_file_.get(), &sub_hash_count)) | |
| 436 return false; | 355 return false; |
| 437 | 356 |
| 438 // TODO(shess): If the vectors were kept sorted, then this code | 357 // TODO(shess): If the vectors were kept sorted, then this code |
| 439 // could use std::inplace_merge() to merge everything together in | 358 // could use std::inplace_merge() to merge everything together in |
| 440 // sorted order. That might still be slower than just sorting at | 359 // sorted order. That might still be slower than just sorting at |
| 441 // the end if there were a large number of chunks. In that case | 360 // the end if there were a large number of chunks. In that case |
| 442 // some sort of recursive binary merge might be in order (merge | 361 // some sort of recursive binary merge might be in order (merge |
| 443 // chunks pairwise, merge those chunks pairwise, and so on, then | 362 // chunks pairwise, merge those chunks pairwise, and so on, then |
| 444 // merge the result with the main list). | 363 // merge the result with the main list). |
| 445 if (!ReadAddPrefixes(new_file_.get(), &add_prefixes, add_prefix_count) || | 364 if (!ReadToVectorAndDelete(&add_prefixes, header.add_prefix_count, |
| 446 !ReadSubPrefixes(new_file_.get(), &sub_prefixes, sub_prefix_count) || | 365 new_file_.get(), add_del_cache_) || |
| 447 !ReadAddHashes(new_file_.get(), &add_full_hashes, add_hash_count) || | 366 !ReadToVectorAndDelete(&sub_prefixes, header.sub_prefix_count, |
| 448 !ReadSubHashes(new_file_.get(), &sub_full_hashes, sub_hash_count)) | 367 new_file_.get(), sub_del_cache_) || |
| 368 !ReadToVectorAndDelete(&add_full_hashes, header.add_hash_count, |
| 369 new_file_.get(), add_del_cache_) || |
| 370 !ReadToVectorAndDelete(&sub_full_hashes, header.sub_hash_count, |
| 371 new_file_.get(), sub_del_cache_)) |
| 449 return false; | 372 return false; |
| 450 } | 373 } |
| 451 | 374 |
| 452 // Add the pending adds which haven't since been deleted. | 375 // Append items from |pending_adds| which haven't been deleted. |
| 453 for (std::vector<SBAddFullHash>::const_iterator iter = pending_adds.begin(); | 376 for (std::vector<SBAddFullHash>::const_iterator iter = pending_adds.begin(); |
| 454 iter != pending_adds.end(); ++iter) { | 377 iter != pending_adds.end(); ++iter) { |
| 455 if (add_del_cache_.count(iter->chunk_id) == 0) | 378 if (add_del_cache_.count(iter->chunk_id) == 0) |
| 456 add_full_hashes.push_back(*iter); | 379 add_full_hashes.push_back(*iter); |
| 457 } | 380 } |
| 458 | 381 |
| 459 // Knock the subs from the adds. | 382 // Knock the subs from the adds. |
| 460 SBProcessSubs(&add_prefixes, &sub_prefixes, | 383 SBProcessSubs(&add_prefixes, &sub_prefixes, |
| 461 &add_full_hashes, &sub_full_hashes); | 384 &add_full_hashes, &sub_full_hashes); |
| 462 | 385 |
| 463 // We no longer need to track deleted chunks. | 386 // We no longer need to track deleted chunks. |
| 464 DeleteChunksFromSet(add_del_cache_, &add_chunks_cache_); | 387 DeleteChunksFromSet(add_del_cache_, &add_chunks_cache_); |
| 465 DeleteChunksFromSet(sub_del_cache_, &sub_chunks_cache_); | 388 DeleteChunksFromSet(sub_del_cache_, &sub_chunks_cache_); |
| 466 | 389 |
| 467 // Write the new data to new_file_. | 390 // Write the new data to new_file_. |
| 468 // TODO(shess): If we receive a lot of subs relative to adds, | 391 // TODO(shess): If we receive a lot of subs relative to adds, |
| 469 // overwriting the temporary chunk data in new_file_ with the | 392 // overwriting the temporary chunk data in new_file_ with the |
| 470 // permanent data could leave additional data at the end. Won't | 393 // permanent data could leave additional data at the end. Won't |
| 471 // cause any problems, but does waste space. There is no truncate() | 394 // cause any problems, but does waste space. There is no truncate() |
| 472 // for stdio. Could use ftruncate() or re-open the file. Or maybe | 395 // for stdio. Could use ftruncate() or re-open the file. Or maybe |
| 473 // ignore it, since we'll likely rewrite soon enough. | 396 // ignore it, since we'll likely rewrite the file soon enough. |
| 474 if (!FileSeek(new_file_.get(), 0)) | 397 if (!FileRewind(new_file_.get())) |
| 475 return false; | 398 return false; |
| 476 | 399 |
| 477 if (!WriteInt32(new_file_.get(), kFileMagic) || | 400 FileHeader header; |
| 478 !WriteInt32(new_file_.get(), kFileVersion) || | 401 header.magic = kFileMagic; |
| 479 !WriteInt32(new_file_.get(), add_chunks_cache_.size()) || | 402 header.version = kFileVersion; |
| 480 !WriteInt32(new_file_.get(), sub_chunks_cache_.size()) || | 403 header.add_chunk_count = add_chunks_cache_.size(); |
| 481 !WriteInt32(new_file_.get(), add_prefixes.size()) || | 404 header.sub_chunk_count = sub_chunks_cache_.size(); |
| 482 !WriteInt32(new_file_.get(), sub_prefixes.size()) || | 405 header.add_prefix_count = add_prefixes.size(); |
| 483 !WriteInt32(new_file_.get(), add_full_hashes.size()) || | 406 header.sub_prefix_count = sub_prefixes.size(); |
| 484 !WriteInt32(new_file_.get(), sub_full_hashes.size())) | 407 header.add_hash_count = add_full_hashes.size(); |
| 408 header.sub_hash_count = sub_full_hashes.size(); |
| 409 if (!WriteArray(&header, 1, new_file_.get())) |
| 485 return false; | 410 return false; |
| 486 | 411 |
| 487 if (!WriteChunksFromSet(add_chunks_cache_) || | 412 // Write all the chunk data. |
| 488 !WriteChunksFromSet(sub_chunks_cache_) || | 413 if (!WriteChunkSet(add_chunks_cache_, new_file_.get()) || |
| 489 !WriteAddPrefixes(add_prefixes) || | 414 !WriteChunkSet(sub_chunks_cache_, new_file_.get()) || |
| 490 !WriteSubPrefixes(sub_prefixes) || | 415 !WriteVector(add_prefixes, new_file_.get()) || |
| 491 !WriteAddHashes(add_full_hashes) || | 416 !WriteVector(sub_prefixes, new_file_.get()) || |
| 492 !WriteSubHashes(sub_full_hashes)) | 417 !WriteVector(add_full_hashes, new_file_.get()) || |
| 418 !WriteVector(sub_full_hashes, new_file_.get())) |
| 493 return false; | 419 return false; |
| 494 | 420 |
| 495 // Close the file handle and swizzle the file into place. | 421 // Close the file handle and swizzle the file into place. |
| 496 new_file_.reset(); | 422 new_file_.reset(); |
| 497 if (!file_util::Delete(filename_, false) && | 423 if (!file_util::Delete(filename_, false) && |
| 498 file_util::PathExists(filename_)) | 424 file_util::PathExists(filename_)) |
| 499 return false; | 425 return false; |
| 500 | 426 |
| 501 const FilePath new_filename = TemporaryFileForFilename(filename_); | 427 const FilePath new_filename = TemporaryFileForFilename(filename_); |
| 502 if (!file_util::Move(new_filename, filename_)) { | 428 if (!file_util::Move(new_filename, filename_)) |
| 503 return false; | 429 return false; |
| 504 } | |
| 505 | 430 |
| 506 // Pass the resulting data off to the caller. | 431 // Pass the resulting data off to the caller. |
| 507 add_prefixes_result->swap(add_prefixes); | 432 add_prefixes_result->swap(add_prefixes); |
| 508 add_full_hashes_result->swap(add_full_hashes); | 433 add_full_hashes_result->swap(add_full_hashes); |
| 509 | 434 |
| 510 return true; | 435 return true; |
| 511 } | 436 } |
| 512 | 437 |
| 513 bool SafeBrowsingStoreFile::FinishUpdate( | 438 bool SafeBrowsingStoreFile::FinishUpdate( |
| 514 const std::vector<SBAddFullHash>& pending_adds, | 439 const std::vector<SBAddFullHash>& pending_adds, |
| 515 std::vector<SBAddPrefix>* add_prefixes_result, | 440 std::vector<SBAddPrefix>* add_prefixes_result, |
| 516 std::vector<SBAddFullHash>* add_full_hashes_result) { | 441 std::vector<SBAddFullHash>* add_full_hashes_result) { |
| 517 bool ret = DoUpdate(pending_adds, | 442 bool ret = DoUpdate(pending_adds, |
| 518 add_prefixes_result, add_full_hashes_result); | 443 add_prefixes_result, add_full_hashes_result); |
| 519 | 444 |
| 520 if (!ret) { | 445 if (!ret) { |
| 521 CancelUpdate(); | 446 CancelUpdate(); |
| 522 return false; | 447 return false; |
| 523 } | 448 } |
| 524 | 449 |
| 525 DCHECK(!new_file_.get()); | 450 DCHECK(!new_file_.get()); |
| 526 DCHECK(!file_.get()); | 451 DCHECK(!file_.get()); |
| 527 | 452 |
| 528 return Close(); | 453 return Close(); |
| 529 } | 454 } |
| 530 | 455 |
| 531 bool SafeBrowsingStoreFile::CancelUpdate() { | 456 bool SafeBrowsingStoreFile::CancelUpdate() { |
| 532 return Close(); | 457 return Close(); |
| 533 } | 458 } |
| OLD | NEW |