OLD | NEW |
1 // Copyright (c) 2010 The Chromium Authors. All rights reserved. | 1 // Copyright (c) 2010 The Chromium Authors. All rights reserved. |
2 // Use of this source code is governed by a BSD-style license that can be | 2 // Use of this source code is governed by a BSD-style license that can be |
3 // found in the LICENSE file. | 3 // found in the LICENSE file. |
4 | 4 |
5 #include "chrome/browser/safe_browsing/safe_browsing_store_file.h" | 5 #include "chrome/browser/safe_browsing/safe_browsing_store_file.h" |
6 | 6 |
7 #include "base/callback.h" | 7 #include "base/callback.h" |
8 | 8 |
9 namespace { | 9 namespace { |
10 | 10 |
11 // NOTE(shess): kFileMagic should not be a byte-wise palindrome, so | 11 // NOTE(shess): kFileMagic should not be a byte-wise palindrome, so |
12 // that byte-order changes force corruption. | 12 // that byte-order changes force corruption. |
13 const int32 kFileMagic = 0x600D71FE; | 13 const int32 kFileMagic = 0x600D71FE; |
14 const int32 kFileVersion = 7; // SQLite storage was 6... | 14 const int32 kFileVersion = 7; // SQLite storage was 6... |
15 const size_t kFileHeaderSize = 8 * sizeof(int32); | |
16 | 15 |
17 bool ReadInt32(FILE* fp, int32* value) { | 16 // Header at the front of the main database file. |
18 DCHECK(value); | 17 struct FileHeader { |
19 const size_t ret = fread(value, sizeof(*value), 1, fp); | 18 int32 magic, version; |
20 return ret == 1; | 19 int32 add_chunk_count, sub_chunk_count; |
21 } | 20 int32 add_prefix_count, sub_prefix_count; |
| 21 int32 add_hash_count, sub_hash_count; |
| 22 }; |
22 | 23 |
23 bool WriteInt32(FILE* fp, int32 value) { | 24 // Header for each chunk in the chunk-accumulation file. |
24 const size_t ret = fwrite(&value, sizeof(value), 1, fp); | 25 struct ChunkHeader { |
25 return ret == 1; | 26 int32 add_prefix_count, sub_prefix_count; |
26 } | 27 int32 add_hash_count, sub_hash_count; |
| 28 }; |
27 | 29 |
28 bool ReadHash(FILE* fp, SBFullHash* value) { | 30 // Rewind the file. Using fseek(2) because rewind(3) errors are |
29 DCHECK(value); | 31 // weird. |
30 const size_t ret = fread(&value->full_hash, sizeof(value->full_hash), | 32 bool FileRewind(FILE* fp) { |
31 1, fp); | 33 int rv = fseek(fp, 0, SEEK_SET); |
32 return ret == 1; | |
33 } | |
34 | |
35 bool WriteHash(FILE* fp, SBFullHash value) { | |
36 const size_t ret = fwrite(&value.full_hash, sizeof(value.full_hash), | |
37 1, fp); | |
38 return ret == 1; | |
39 } | |
40 | |
41 bool FileSeek(FILE* fp, size_t offset) { | |
42 int rv = fseek(fp, offset, SEEK_SET); | |
43 DCHECK_EQ(rv, 0); | 34 DCHECK_EQ(rv, 0); |
44 return rv == 0; | 35 return rv == 0; |
45 } | 36 } |
46 | 37 |
| 38 // Read an array of |nmemb| items from |fp| into |ptr|. Return true |
| 39 // on success. |
| 40 template <class T> |
| 41 bool ReadArray(T* ptr, size_t nmemb, FILE* fp) { |
| 42 const size_t ret = fread(ptr, sizeof(T), nmemb, fp); |
| 43 if (ret != nmemb) |
| 44 return false; |
| 45 return true; |
| 46 } |
| 47 |
| 48 // Write an array of |nmemb| items from |ptr| to |fp|. Return true on |
| 49 // success. |
| 50 template <class T> |
| 51 bool WriteArray(const T* ptr, size_t nmemb, FILE* fp) { |
| 52 const size_t ret = fwrite(ptr, sizeof(T), nmemb, fp); |
| 53 if (ret != nmemb) |
| 54 return false; |
| 55 return true; |
| 56 } |
| 57 |
| 58 // Expand |values| to fit |count| new items, and read those items from |
| 59 // |fp|. Returns true on success. |
| 60 template <class T> |
| 61 bool ReadToVector(std::vector<T>* values, size_t count, FILE* fp) { |
| 62 // Pointers into an empty vector may not be valid. |
| 63 if (!count) |
| 64 return true; |
| 65 |
| 66 // Grab the size for purposes of finding where to read to. The |
| 67 // resize could invalidate any iterator captured here. |
| 68 const size_t original_size = values->size(); |
| 69 values->resize(original_size + count); |
| 70 |
| 71 // Sayeth Herb Sutter: Vectors are guaranteed to be contiguous. So |
| 72 // get a pointer to where to read the data to. |
| 73 T* ptr = &((*values)[original_size]); |
| 74 if (!ReadArray(ptr, count, fp)) { |
| 75 values->resize(original_size); |
| 76 return false; |
| 77 } |
| 78 |
| 79 return true; |
| 80 } |
| 81 |
| 82 // Write all of |values| to |fp|. Returns true on succsess. |
| 83 template <class T> |
| 84 bool WriteVector(const std::vector<T>& values, FILE* fp) { |
| 85 // Pointers into empty vectors may not be valid. |
| 86 if (values.empty()) |
| 87 return true; |
| 88 |
| 89 // Sayeth Herb Sutter: Vectors are guaranteed to be contiguous. So |
| 90 // get a pointer to where to write from. |
| 91 const T* ptr = &(values[0]); |
| 92 return WriteArray(ptr, values.size(), fp); |
| 93 } |
| 94 |
| 95 // Remove deleted items (|chunk_id| in |del_set|) from the vector |
| 96 // starting at |offset| running to |end()|. |
| 97 template <class T> |
| 98 void RemoveDeleted(std::vector<T>* vec, size_t offset, |
| 99 const base::hash_set<int32>& del_set) { |
| 100 DCHECK(vec); |
| 101 |
| 102 // Scan through the items read, dropping the items in |del_set|. |
| 103 typename std::vector<T>::iterator add_iter = vec->begin() + offset; |
| 104 for (typename std::vector<T>::iterator iter = add_iter; |
| 105 iter != vec->end(); ++iter) { |
| 106 if (del_set.count(iter->chunk_id) == 0) { |
| 107 *add_iter = *iter; |
| 108 ++add_iter; |
| 109 } |
| 110 } |
| 111 vec->erase(add_iter, vec->end()); |
| 112 } |
| 113 |
| 114 // Combine |ReadToVector()| and |RemoveDeleted()|. Returns true on |
| 115 // success. |
| 116 template <class T> |
| 117 bool ReadToVectorAndDelete(std::vector<T>* values, size_t count, FILE* fp, |
| 118 const base::hash_set<int32>& del_set) { |
| 119 const size_t original_size = values->size(); |
| 120 if (!ReadToVector(values, count, fp)) |
| 121 return false; |
| 122 |
| 123 RemoveDeleted(values, original_size, del_set); |
| 124 return true; |
| 125 } |
| 126 |
| 127 // Read an array of |count| integers and add them to |values|. |
| 128 // Returns true on success. |
| 129 bool ReadToChunkSet(std::set<int32>* values, size_t count, FILE* fp) { |
| 130 if (!count) |
| 131 return true; |
| 132 |
| 133 std::vector<int32> flat_values; |
| 134 if (!ReadToVector(&flat_values, count, fp)) |
| 135 return false; |
| 136 |
| 137 values->insert(flat_values.begin(), flat_values.end()); |
| 138 return true; |
| 139 } |
| 140 |
| 141 // Write the contents of |values| as an array of integers. Returns |
| 142 // true on success. |
| 143 bool WriteChunkSet(const std::set<int32>& values, FILE* fp) { |
| 144 if (values.empty()) |
| 145 return true; |
| 146 |
| 147 const std::vector<int32> flat_values(values.begin(), values.end()); |
| 148 return WriteVector(flat_values, fp); |
| 149 } |
| 150 |
47 // Delete the chunks in |deleted| from |chunks|. | 151 // Delete the chunks in |deleted| from |chunks|. |
48 void DeleteChunksFromSet(const base::hash_set<int32>& deleted, | 152 void DeleteChunksFromSet(const base::hash_set<int32>& deleted, |
49 std::set<int32>* chunks) { | 153 std::set<int32>* chunks) { |
50 for (std::set<int32>::iterator iter = chunks->begin(); | 154 for (std::set<int32>::iterator iter = chunks->begin(); |
51 iter != chunks->end();) { | 155 iter != chunks->end();) { |
52 std::set<int32>::iterator prev = iter++; | 156 std::set<int32>::iterator prev = iter++; |
53 if (deleted.count(*prev) > 0) | 157 if (deleted.count(*prev) > 0) |
54 chunks->erase(prev); | 158 chunks->erase(prev); |
55 } | 159 } |
56 } | 160 } |
(...skipping 48 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
105 | 209 |
106 bool SafeBrowsingStoreFile::Close() { | 210 bool SafeBrowsingStoreFile::Close() { |
107 ClearUpdateBuffers(); | 211 ClearUpdateBuffers(); |
108 | 212 |
109 // Make sure the files are closed. | 213 // Make sure the files are closed. |
110 file_.reset(); | 214 file_.reset(); |
111 new_file_.reset(); | 215 new_file_.reset(); |
112 return true; | 216 return true; |
113 } | 217 } |
114 | 218 |
115 bool SafeBrowsingStoreFile::ReadChunksToSet(FILE* fp, std::set<int32>* chunks, | |
116 int count) { | |
117 DCHECK(fp); | |
118 | |
119 for (int i = 0; i < count; ++i) { | |
120 int32 chunk_id; | |
121 if (!ReadInt32(fp, &chunk_id)) | |
122 return false; | |
123 chunks->insert(chunk_id); | |
124 } | |
125 return true; | |
126 } | |
127 | |
128 bool SafeBrowsingStoreFile::WriteChunksFromSet(const std::set<int32>& chunks) { | |
129 DCHECK(new_file_.get()); | |
130 | |
131 for (std::set<int32>::const_iterator iter = chunks.begin(); | |
132 iter != chunks.end(); ++iter) { | |
133 if (!WriteInt32(new_file_.get(), *iter)) | |
134 return false; | |
135 } | |
136 return true; | |
137 } | |
138 | |
139 bool SafeBrowsingStoreFile::ReadAddPrefixes( | |
140 FILE* fp, std::vector<SBAddPrefix>* add_prefixes, int count) { | |
141 DCHECK(fp && add_prefixes); | |
142 | |
143 add_prefixes->reserve(add_prefixes->size() + count); | |
144 | |
145 for (int32 i = 0; i < count; ++i) { | |
146 int32 chunk_id; | |
147 SBPrefix prefix; | |
148 DCHECK_EQ(sizeof(int32), sizeof(prefix)); | |
149 | |
150 if (!ReadInt32(fp, &chunk_id) || !ReadInt32(fp, &prefix)) | |
151 return false; | |
152 | |
153 if (add_del_cache_.count(chunk_id) > 0) | |
154 continue; | |
155 | |
156 add_prefixes->push_back(SBAddPrefix(chunk_id, prefix)); | |
157 } | |
158 | |
159 return true; | |
160 } | |
161 | |
162 bool SafeBrowsingStoreFile::WriteAddPrefixes( | |
163 const std::vector<SBAddPrefix>& add_prefixes) { | |
164 DCHECK(new_file_.get()); | |
165 | |
166 for (std::vector<SBAddPrefix>::const_iterator iter = add_prefixes.begin(); | |
167 iter != add_prefixes.end(); ++iter) { | |
168 DCHECK_EQ(sizeof(int32), sizeof(iter->prefix)); | |
169 if (!WriteInt32(new_file_.get(), iter->chunk_id) || | |
170 !WriteInt32(new_file_.get(), iter->prefix)) | |
171 return false; | |
172 } | |
173 return true; | |
174 } | |
175 | |
176 bool SafeBrowsingStoreFile::ReadSubPrefixes( | |
177 FILE* fp, std::vector<SBSubPrefix>* sub_prefixes, int count) { | |
178 DCHECK(fp && sub_prefixes); | |
179 | |
180 sub_prefixes->reserve(sub_prefixes->size() + count); | |
181 | |
182 for (int32 i = 0; i < count; ++i) { | |
183 int32 chunk_id, add_chunk_id; | |
184 SBPrefix add_prefix; | |
185 DCHECK_EQ(sizeof(int32), sizeof(add_prefix)); | |
186 | |
187 if (!ReadInt32(fp, &chunk_id) || | |
188 !ReadInt32(fp, &add_chunk_id) || !ReadInt32(fp, &add_prefix)) | |
189 return false; | |
190 | |
191 if (sub_del_cache_.count(chunk_id) > 0) | |
192 continue; | |
193 | |
194 sub_prefixes->push_back(SBSubPrefix(chunk_id, add_chunk_id, add_prefix)); | |
195 } | |
196 | |
197 return true; | |
198 } | |
199 | |
200 bool SafeBrowsingStoreFile::WriteSubPrefixes( | |
201 std::vector<SBSubPrefix>& sub_prefixes) { | |
202 DCHECK(new_file_.get()); | |
203 | |
204 for (std::vector<SBSubPrefix>::const_iterator iter = sub_prefixes.begin(); | |
205 iter != sub_prefixes.end(); ++iter) { | |
206 if (!WriteInt32(new_file_.get(), iter->chunk_id) || | |
207 !WriteInt32(new_file_.get(), iter->add_chunk_id) || | |
208 !WriteInt32(new_file_.get(), iter->add_prefix)) | |
209 return false; | |
210 } | |
211 return true; | |
212 } | |
213 | |
214 bool SafeBrowsingStoreFile::ReadAddHashes( | |
215 FILE* fp, std::vector<SBAddFullHash>* add_hashes, int count) { | |
216 DCHECK(fp && add_hashes); | |
217 | |
218 add_hashes->reserve(add_hashes->size() + count); | |
219 | |
220 for (int i = 0; i < count; ++i) { | |
221 int32 chunk_id; | |
222 int32 received; | |
223 SBFullHash full_hash; | |
224 | |
225 if (!ReadInt32(fp, &chunk_id) || | |
226 !ReadInt32(fp, &received) || | |
227 !ReadHash(fp, &full_hash)) | |
228 return false; | |
229 | |
230 if (add_del_cache_.count(chunk_id) > 0) | |
231 continue; | |
232 | |
233 add_hashes->push_back(SBAddFullHash(chunk_id, received, full_hash)); | |
234 } | |
235 | |
236 return true; | |
237 } | |
238 | |
239 bool SafeBrowsingStoreFile::WriteAddHashes( | |
240 const std::vector<SBAddFullHash>& add_hashes) { | |
241 DCHECK(new_file_.get()); | |
242 | |
243 for (std::vector<SBAddFullHash>::const_iterator iter = add_hashes.begin(); | |
244 iter != add_hashes.end(); ++iter) { | |
245 if (!WriteInt32(new_file_.get(), iter->chunk_id) || | |
246 !WriteInt32(new_file_.get(), iter->received) || | |
247 !WriteHash(new_file_.get(), iter->full_hash)) | |
248 return false; | |
249 } | |
250 return true; | |
251 } | |
252 | |
253 bool SafeBrowsingStoreFile::ReadSubHashes( | |
254 FILE* fp, std::vector<SBSubFullHash>* sub_hashes, int count) { | |
255 DCHECK(fp); | |
256 | |
257 sub_hashes->reserve(sub_hashes->size() + count); | |
258 | |
259 for (int i = 0; i < count; ++i) { | |
260 int32 chunk_id; | |
261 int32 add_chunk_id; | |
262 SBFullHash add_full_hash; | |
263 | |
264 if (!ReadInt32(fp, &chunk_id) || | |
265 !ReadInt32(fp, &add_chunk_id) || | |
266 !ReadHash(fp, &add_full_hash)) | |
267 return false; | |
268 | |
269 if (sub_del_cache_.count(chunk_id) > 0) | |
270 continue; | |
271 | |
272 sub_hashes->push_back(SBSubFullHash(chunk_id, add_chunk_id, add_full_hash)); | |
273 } | |
274 | |
275 return true; | |
276 } | |
277 | |
278 bool SafeBrowsingStoreFile::WriteSubHashes( | |
279 std::vector<SBSubFullHash>& sub_hashes) { | |
280 DCHECK(new_file_.get()); | |
281 | |
282 for (std::vector<SBSubFullHash>::const_iterator iter = sub_hashes.begin(); | |
283 iter != sub_hashes.end(); ++iter) { | |
284 if (!WriteInt32(new_file_.get(), iter->chunk_id) || | |
285 !WriteInt32(new_file_.get(), iter->add_chunk_id) || | |
286 !WriteHash(new_file_.get(), iter->full_hash)) | |
287 return false; | |
288 } | |
289 return true; | |
290 } | |
291 | |
292 bool SafeBrowsingStoreFile::BeginUpdate() { | 219 bool SafeBrowsingStoreFile::BeginUpdate() { |
293 DCHECK(!file_.get() && !new_file_.get()); | 220 DCHECK(!file_.get() && !new_file_.get()); |
294 | 221 |
295 // Structures should all be clear unless something bad happened. | 222 // Structures should all be clear unless something bad happened. |
296 DCHECK(add_chunks_cache_.empty()); | 223 DCHECK(add_chunks_cache_.empty()); |
297 DCHECK(sub_chunks_cache_.empty()); | 224 DCHECK(sub_chunks_cache_.empty()); |
298 DCHECK(add_del_cache_.empty()); | 225 DCHECK(add_del_cache_.empty()); |
299 DCHECK(sub_del_cache_.empty()); | 226 DCHECK(sub_del_cache_.empty()); |
300 DCHECK(add_prefixes_.empty()); | 227 DCHECK(add_prefixes_.empty()); |
301 DCHECK(sub_prefixes_.empty()); | 228 DCHECK(sub_prefixes_.empty()); |
(...skipping 11 matching lines...) Expand all Loading... |
313 if (empty_) { | 240 if (empty_) { |
314 // If the file exists but cannot be opened, try to delete it (not | 241 // If the file exists but cannot be opened, try to delete it (not |
315 // deleting directly, the bloom filter needs to be deleted, too). | 242 // deleting directly, the bloom filter needs to be deleted, too). |
316 if (file_util::PathExists(filename_)) | 243 if (file_util::PathExists(filename_)) |
317 return OnCorruptDatabase(); | 244 return OnCorruptDatabase(); |
318 | 245 |
319 new_file_.swap(new_file); | 246 new_file_.swap(new_file); |
320 return true; | 247 return true; |
321 } | 248 } |
322 | 249 |
323 int32 magic, version; | 250 FileHeader header; |
324 if (!ReadInt32(file.get(), &magic) || !ReadInt32(file.get(), &version)) | 251 if (!ReadArray(&header, 1, file.get())) |
325 return OnCorruptDatabase(); | 252 return OnCorruptDatabase(); |
326 | 253 |
327 if (magic != kFileMagic || version != kFileVersion) | 254 if (header.magic != kFileMagic || header.version != kFileVersion) |
328 return OnCorruptDatabase(); | 255 return OnCorruptDatabase(); |
329 | 256 |
330 int32 add_chunk_count, sub_chunk_count; | 257 if (!ReadToChunkSet(&add_chunks_cache_, header.add_chunk_count, file.get()) || |
331 if (!ReadInt32(file.get(), &add_chunk_count) || | 258 !ReadToChunkSet(&sub_chunks_cache_, header.sub_chunk_count, file.get())) |
332 !ReadInt32(file.get(), &sub_chunk_count)) | |
333 return OnCorruptDatabase(); | |
334 | |
335 if (!FileSeek(file.get(), kFileHeaderSize)) | |
336 return OnCorruptDatabase(); | |
337 | |
338 if (!ReadChunksToSet(file.get(), &add_chunks_cache_, add_chunk_count) || | |
339 !ReadChunksToSet(file.get(), &sub_chunks_cache_, sub_chunk_count)) | |
340 return OnCorruptDatabase(); | 259 return OnCorruptDatabase(); |
341 | 260 |
342 file_.swap(file); | 261 file_.swap(file); |
343 new_file_.swap(new_file); | 262 new_file_.swap(new_file); |
344 return true; | 263 return true; |
345 } | 264 } |
346 | 265 |
347 bool SafeBrowsingStoreFile::FinishChunk() { | 266 bool SafeBrowsingStoreFile::FinishChunk() { |
348 if (!add_prefixes_.size() && !sub_prefixes_.size() && | 267 if (!add_prefixes_.size() && !sub_prefixes_.size() && |
349 !add_hashes_.size() && !sub_hashes_.size()) | 268 !add_hashes_.size() && !sub_hashes_.size()) |
350 return true; | 269 return true; |
351 | 270 |
352 if (!WriteInt32(new_file_.get(), add_prefixes_.size()) || | 271 ChunkHeader header; |
353 !WriteInt32(new_file_.get(), sub_prefixes_.size()) || | 272 header.add_prefix_count = add_prefixes_.size(); |
354 !WriteInt32(new_file_.get(), add_hashes_.size()) || | 273 header.sub_prefix_count = sub_prefixes_.size(); |
355 !WriteInt32(new_file_.get(), sub_hashes_.size())) | 274 header.add_hash_count = add_hashes_.size(); |
| 275 header.sub_hash_count = sub_hashes_.size(); |
| 276 if (!WriteArray(&header, 1, new_file_.get())) |
356 return false; | 277 return false; |
357 | 278 |
358 if (!WriteAddPrefixes(add_prefixes_) || | 279 if (!WriteVector(add_prefixes_, new_file_.get()) || |
359 !WriteSubPrefixes(sub_prefixes_) || | 280 !WriteVector(sub_prefixes_, new_file_.get()) || |
360 !WriteAddHashes(add_hashes_) || | 281 !WriteVector(add_hashes_, new_file_.get()) || |
361 !WriteSubHashes(sub_hashes_)) | 282 !WriteVector(sub_hashes_, new_file_.get())) |
362 return false; | 283 return false; |
363 | 284 |
364 ++chunks_written_; | 285 ++chunks_written_; |
365 | 286 |
366 // Clear everything to save memory. | 287 // Clear everything to save memory. |
367 return ClearChunkBuffers(); | 288 return ClearChunkBuffers(); |
368 } | 289 } |
369 | 290 |
370 bool SafeBrowsingStoreFile::DoUpdate( | 291 bool SafeBrowsingStoreFile::DoUpdate( |
371 const std::vector<SBAddFullHash>& pending_adds, | 292 const std::vector<SBAddFullHash>& pending_adds, |
372 std::vector<SBAddPrefix>* add_prefixes_result, | 293 std::vector<SBAddPrefix>* add_prefixes_result, |
373 std::vector<SBAddFullHash>* add_full_hashes_result) { | 294 std::vector<SBAddFullHash>* add_full_hashes_result) { |
374 DCHECK(file_.get() || empty_); | 295 DCHECK(file_.get() || empty_); |
375 DCHECK(new_file_.get()); | 296 DCHECK(new_file_.get()); |
376 | 297 |
377 std::vector<SBAddPrefix> add_prefixes; | 298 std::vector<SBAddPrefix> add_prefixes; |
378 std::vector<SBSubPrefix> sub_prefixes; | 299 std::vector<SBSubPrefix> sub_prefixes; |
379 std::vector<SBAddFullHash> add_full_hashes; | 300 std::vector<SBAddFullHash> add_full_hashes; |
380 std::vector<SBSubFullHash> sub_full_hashes; | 301 std::vector<SBSubFullHash> sub_full_hashes; |
381 | 302 |
382 // Read |file_| into the vectors. | 303 // Read |file_| into the vectors. |
383 if (!empty_) { | 304 if (!empty_) { |
384 DCHECK(file_.get()); | 305 DCHECK(file_.get()); |
385 | 306 |
386 int32 magic, version; | 307 if (!FileRewind(file_.get())) |
387 int32 add_chunk_count, sub_chunk_count; | |
388 int32 add_prefix_count, sub_prefix_count; | |
389 int32 add_hash_count, sub_hash_count; | |
390 | |
391 if (!FileSeek(file_.get(), 0)) | |
392 return OnCorruptDatabase(); | 308 return OnCorruptDatabase(); |
393 | 309 |
394 if (!ReadInt32(file_.get(), &magic) || | 310 // Read the file header and make sure it looks right. |
395 !ReadInt32(file_.get(), &version) || | 311 FileHeader header; |
396 !ReadInt32(file_.get(), &add_chunk_count) || | 312 if (!ReadArray(&header, 1, file_.get())) |
397 !ReadInt32(file_.get(), &sub_chunk_count) || | |
398 !ReadInt32(file_.get(), &add_prefix_count) || | |
399 !ReadInt32(file_.get(), &sub_prefix_count) || | |
400 !ReadInt32(file_.get(), &add_hash_count) || | |
401 !ReadInt32(file_.get(), &sub_hash_count)) | |
402 return OnCorruptDatabase(); | 313 return OnCorruptDatabase(); |
403 | 314 |
404 if (magic != kFileMagic || version != kFileVersion) | 315 if (header.magic != kFileMagic || header.version != kFileVersion) |
405 return OnCorruptDatabase(); | 316 return OnCorruptDatabase(); |
406 | 317 |
407 const size_t prefixes_offset = kFileHeaderSize + | 318 // Re-read the chunks-seen data to get to the later data in the |
408 (add_chunk_count + sub_chunk_count) * sizeof(int32); | 319 // file. No new elements should be added to the sets. |
409 if (!FileSeek(file_.get(), prefixes_offset)) | 320 // NOTE(shess): Reading rather than fseek() because calculating |
| 321 // checksums (future CL) will need to scan all data. The code |
| 322 // could just remember state from |BeginUpdate()|, but that call |
| 323 // may be far removed from this call in time, so this seems like a |
| 324 // reasonable trade-off. |
| 325 if (!ReadToChunkSet(&add_chunks_cache_, header.add_chunk_count, |
| 326 file_.get()) || |
| 327 !ReadToChunkSet(&sub_chunks_cache_, header.sub_chunk_count, |
| 328 file_.get())) |
410 return OnCorruptDatabase(); | 329 return OnCorruptDatabase(); |
411 | 330 |
412 if (!ReadAddPrefixes(file_.get(), &add_prefixes, add_prefix_count) || | 331 if (!ReadToVectorAndDelete(&add_prefixes, header.add_prefix_count, |
413 !ReadSubPrefixes(file_.get(), &sub_prefixes, sub_prefix_count) || | 332 file_.get(), add_del_cache_) || |
414 !ReadAddHashes(file_.get(), &add_full_hashes, add_hash_count) || | 333 !ReadToVectorAndDelete(&sub_prefixes, header.sub_prefix_count, |
415 !ReadSubHashes(file_.get(), &sub_full_hashes, sub_hash_count)) | 334 file_.get(), sub_del_cache_) || |
| 335 !ReadToVectorAndDelete(&add_full_hashes, header.add_hash_count, |
| 336 file_.get(), add_del_cache_) || |
| 337 !ReadToVectorAndDelete(&sub_full_hashes, header.sub_hash_count, |
| 338 file_.get(), sub_del_cache_)) |
416 return OnCorruptDatabase(); | 339 return OnCorruptDatabase(); |
417 | 340 |
418 // Close the file so we can later rename over it. | 341 // Close the file so we can later rename over it. |
419 file_.reset(); | 342 file_.reset(); |
420 } | 343 } |
421 DCHECK(!file_.get()); | 344 DCHECK(!file_.get()); |
422 | 345 |
423 // Rewind the temporary storage. | 346 // Rewind the temporary storage. |
424 if (!FileSeek(new_file_.get(), 0)) | 347 if (!FileRewind(new_file_.get())) |
425 return false; | 348 return false; |
426 | 349 |
427 // Append the accumulated chunks onto the vectors from file_. | 350 // Append the accumulated chunks onto the vectors from file_. |
428 for (int i = 0; i < chunks_written_; ++i) { | 351 for (int i = 0; i < chunks_written_; ++i) { |
429 int32 add_prefix_count, sub_prefix_count; | 352 ChunkHeader header; |
430 int32 add_hash_count, sub_hash_count; | |
431 | 353 |
432 if (!ReadInt32(new_file_.get(), &add_prefix_count) || | 354 if (!ReadArray(&header, 1, new_file_.get())) |
433 !ReadInt32(new_file_.get(), &sub_prefix_count) || | |
434 !ReadInt32(new_file_.get(), &add_hash_count) || | |
435 !ReadInt32(new_file_.get(), &sub_hash_count)) | |
436 return false; | 355 return false; |
437 | 356 |
438 // TODO(shess): If the vectors were kept sorted, then this code | 357 // TODO(shess): If the vectors were kept sorted, then this code |
439 // could use std::inplace_merge() to merge everything together in | 358 // could use std::inplace_merge() to merge everything together in |
440 // sorted order. That might still be slower than just sorting at | 359 // sorted order. That might still be slower than just sorting at |
441 // the end if there were a large number of chunks. In that case | 360 // the end if there were a large number of chunks. In that case |
442 // some sort of recursive binary merge might be in order (merge | 361 // some sort of recursive binary merge might be in order (merge |
443 // chunks pairwise, merge those chunks pairwise, and so on, then | 362 // chunks pairwise, merge those chunks pairwise, and so on, then |
444 // merge the result with the main list). | 363 // merge the result with the main list). |
445 if (!ReadAddPrefixes(new_file_.get(), &add_prefixes, add_prefix_count) || | 364 if (!ReadToVectorAndDelete(&add_prefixes, header.add_prefix_count, |
446 !ReadSubPrefixes(new_file_.get(), &sub_prefixes, sub_prefix_count) || | 365 new_file_.get(), add_del_cache_) || |
447 !ReadAddHashes(new_file_.get(), &add_full_hashes, add_hash_count) || | 366 !ReadToVectorAndDelete(&sub_prefixes, header.sub_prefix_count, |
448 !ReadSubHashes(new_file_.get(), &sub_full_hashes, sub_hash_count)) | 367 new_file_.get(), sub_del_cache_) || |
| 368 !ReadToVectorAndDelete(&add_full_hashes, header.add_hash_count, |
| 369 new_file_.get(), add_del_cache_) || |
| 370 !ReadToVectorAndDelete(&sub_full_hashes, header.sub_hash_count, |
| 371 new_file_.get(), sub_del_cache_)) |
449 return false; | 372 return false; |
450 } | 373 } |
451 | 374 |
452 // Add the pending adds which haven't since been deleted. | 375 // Append items from |pending_adds| which haven't been deleted. |
453 for (std::vector<SBAddFullHash>::const_iterator iter = pending_adds.begin(); | 376 for (std::vector<SBAddFullHash>::const_iterator iter = pending_adds.begin(); |
454 iter != pending_adds.end(); ++iter) { | 377 iter != pending_adds.end(); ++iter) { |
455 if (add_del_cache_.count(iter->chunk_id) == 0) | 378 if (add_del_cache_.count(iter->chunk_id) == 0) |
456 add_full_hashes.push_back(*iter); | 379 add_full_hashes.push_back(*iter); |
457 } | 380 } |
458 | 381 |
459 // Knock the subs from the adds. | 382 // Knock the subs from the adds. |
460 SBProcessSubs(&add_prefixes, &sub_prefixes, | 383 SBProcessSubs(&add_prefixes, &sub_prefixes, |
461 &add_full_hashes, &sub_full_hashes); | 384 &add_full_hashes, &sub_full_hashes); |
462 | 385 |
463 // We no longer need to track deleted chunks. | 386 // We no longer need to track deleted chunks. |
464 DeleteChunksFromSet(add_del_cache_, &add_chunks_cache_); | 387 DeleteChunksFromSet(add_del_cache_, &add_chunks_cache_); |
465 DeleteChunksFromSet(sub_del_cache_, &sub_chunks_cache_); | 388 DeleteChunksFromSet(sub_del_cache_, &sub_chunks_cache_); |
466 | 389 |
467 // Write the new data to new_file_. | 390 // Write the new data to new_file_. |
468 // TODO(shess): If we receive a lot of subs relative to adds, | 391 // TODO(shess): If we receive a lot of subs relative to adds, |
469 // overwriting the temporary chunk data in new_file_ with the | 392 // overwriting the temporary chunk data in new_file_ with the |
470 // permanent data could leave additional data at the end. Won't | 393 // permanent data could leave additional data at the end. Won't |
471 // cause any problems, but does waste space. There is no truncate() | 394 // cause any problems, but does waste space. There is no truncate() |
472 // for stdio. Could use ftruncate() or re-open the file. Or maybe | 395 // for stdio. Could use ftruncate() or re-open the file. Or maybe |
473 // ignore it, since we'll likely rewrite soon enough. | 396 // ignore it, since we'll likely rewrite the file soon enough. |
474 if (!FileSeek(new_file_.get(), 0)) | 397 if (!FileRewind(new_file_.get())) |
475 return false; | 398 return false; |
476 | 399 |
477 if (!WriteInt32(new_file_.get(), kFileMagic) || | 400 FileHeader header; |
478 !WriteInt32(new_file_.get(), kFileVersion) || | 401 header.magic = kFileMagic; |
479 !WriteInt32(new_file_.get(), add_chunks_cache_.size()) || | 402 header.version = kFileVersion; |
480 !WriteInt32(new_file_.get(), sub_chunks_cache_.size()) || | 403 header.add_chunk_count = add_chunks_cache_.size(); |
481 !WriteInt32(new_file_.get(), add_prefixes.size()) || | 404 header.sub_chunk_count = sub_chunks_cache_.size(); |
482 !WriteInt32(new_file_.get(), sub_prefixes.size()) || | 405 header.add_prefix_count = add_prefixes.size(); |
483 !WriteInt32(new_file_.get(), add_full_hashes.size()) || | 406 header.sub_prefix_count = sub_prefixes.size(); |
484 !WriteInt32(new_file_.get(), sub_full_hashes.size())) | 407 header.add_hash_count = add_full_hashes.size(); |
| 408 header.sub_hash_count = sub_full_hashes.size(); |
| 409 if (!WriteArray(&header, 1, new_file_.get())) |
485 return false; | 410 return false; |
486 | 411 |
487 if (!WriteChunksFromSet(add_chunks_cache_) || | 412 // Write all the chunk data. |
488 !WriteChunksFromSet(sub_chunks_cache_) || | 413 if (!WriteChunkSet(add_chunks_cache_, new_file_.get()) || |
489 !WriteAddPrefixes(add_prefixes) || | 414 !WriteChunkSet(sub_chunks_cache_, new_file_.get()) || |
490 !WriteSubPrefixes(sub_prefixes) || | 415 !WriteVector(add_prefixes, new_file_.get()) || |
491 !WriteAddHashes(add_full_hashes) || | 416 !WriteVector(sub_prefixes, new_file_.get()) || |
492 !WriteSubHashes(sub_full_hashes)) | 417 !WriteVector(add_full_hashes, new_file_.get()) || |
| 418 !WriteVector(sub_full_hashes, new_file_.get())) |
493 return false; | 419 return false; |
494 | 420 |
495 // Close the file handle and swizzle the file into place. | 421 // Close the file handle and swizzle the file into place. |
496 new_file_.reset(); | 422 new_file_.reset(); |
497 if (!file_util::Delete(filename_, false) && | 423 if (!file_util::Delete(filename_, false) && |
498 file_util::PathExists(filename_)) | 424 file_util::PathExists(filename_)) |
499 return false; | 425 return false; |
500 | 426 |
501 const FilePath new_filename = TemporaryFileForFilename(filename_); | 427 const FilePath new_filename = TemporaryFileForFilename(filename_); |
502 if (!file_util::Move(new_filename, filename_)) { | 428 if (!file_util::Move(new_filename, filename_)) |
503 return false; | 429 return false; |
504 } | |
505 | 430 |
506 // Pass the resulting data off to the caller. | 431 // Pass the resulting data off to the caller. |
507 add_prefixes_result->swap(add_prefixes); | 432 add_prefixes_result->swap(add_prefixes); |
508 add_full_hashes_result->swap(add_full_hashes); | 433 add_full_hashes_result->swap(add_full_hashes); |
509 | 434 |
510 return true; | 435 return true; |
511 } | 436 } |
512 | 437 |
513 bool SafeBrowsingStoreFile::FinishUpdate( | 438 bool SafeBrowsingStoreFile::FinishUpdate( |
514 const std::vector<SBAddFullHash>& pending_adds, | 439 const std::vector<SBAddFullHash>& pending_adds, |
515 std::vector<SBAddPrefix>* add_prefixes_result, | 440 std::vector<SBAddPrefix>* add_prefixes_result, |
516 std::vector<SBAddFullHash>* add_full_hashes_result) { | 441 std::vector<SBAddFullHash>* add_full_hashes_result) { |
517 bool ret = DoUpdate(pending_adds, | 442 bool ret = DoUpdate(pending_adds, |
518 add_prefixes_result, add_full_hashes_result); | 443 add_prefixes_result, add_full_hashes_result); |
519 | 444 |
520 if (!ret) { | 445 if (!ret) { |
521 CancelUpdate(); | 446 CancelUpdate(); |
522 return false; | 447 return false; |
523 } | 448 } |
524 | 449 |
525 DCHECK(!new_file_.get()); | 450 DCHECK(!new_file_.get()); |
526 DCHECK(!file_.get()); | 451 DCHECK(!file_.get()); |
527 | 452 |
528 return Close(); | 453 return Close(); |
529 } | 454 } |
530 | 455 |
531 bool SafeBrowsingStoreFile::CancelUpdate() { | 456 bool SafeBrowsingStoreFile::CancelUpdate() { |
532 return Close(); | 457 return Close(); |
533 } | 458 } |
OLD | NEW |