| OLD | NEW |
| 1 // | 1 // |
| 2 // Copyright 2014 Google Inc. All rights reserved. | 2 // Copyright 2014 Google Inc. All rights reserved. |
| 3 // | 3 // |
| 4 // Use of this source code is governed by a BSD-style | 4 // Use of this source code is governed by a BSD-style |
| 5 // license that can be found in the LICENSE file or at | 5 // license that can be found in the LICENSE file or at |
| 6 // https://developers.google.com/open-source/licenses/bsd | 6 // https://developers.google.com/open-source/licenses/bsd |
| 7 // | 7 // |
| 8 | 8 |
| 9 part of charted.charts; | 9 part of charted.charts; |
| 10 | 10 |
| 11 ///Function callback to filter items in the input | 11 ///Function callback to filter items in the input |
| 12 typedef bool AggregationFilterFunc(var item); | 12 typedef bool AggregationFilterFunc(var item); |
| 13 | 13 |
| 14 typedef int CompareFunc(dynamic a, dynamic b); |
| 15 |
| 14 typedef dynamic FieldAccessor(dynamic item, dynamic key); | 16 typedef dynamic FieldAccessor(dynamic item, dynamic key); |
| 15 | 17 |
| 16 /// Given list of items, dimensions and facts, compute | 18 /// Given list of items, dimensions and facts, compute |
| 17 /// aggregates (COUNT, SUM, MIN, MAX) for facts at each dimension level. | 19 /// aggregates (COUNT, SUM, MIN, MAX) for facts at each dimension level. |
| 18 class AggregationModel { | 20 class AggregationModel { |
| 19 // Number of aggregations we collect on each fact | 21 // Number of aggregations we collect on each fact |
| 20 int _aggregationTypesCount = 0; | 22 int _aggregationTypesCount = 0; |
| 21 | 23 |
| 22 // Currently supported list of aggregations. | 24 // Currently supported list of aggregations. |
| 23 static final List<String> supportedAggregationTypes = [ | 25 static final List<String> supportedAggregationTypes = [ |
| (...skipping 58 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 82 // Cache of entities created for the facts on this aggregation view. | 84 // Cache of entities created for the facts on this aggregation view. |
| 83 Map<String, AggregationItem> _entityCache; | 85 Map<String, AggregationItem> _entityCache; |
| 84 | 86 |
| 85 // List of field names that aggregation items will have. | 87 // List of field names that aggregation items will have. |
| 86 List<String> _itemFieldNamesCache; | 88 List<String> _itemFieldNamesCache; |
| 87 | 89 |
| 88 // Walk through the map, by splitting key at '.' | 90 // Walk through the map, by splitting key at '.' |
| 89 final bool walkThroughMap; | 91 final bool walkThroughMap; |
| 90 | 92 |
| 91 // Map of fieldName to comparator function. | 93 // Map of fieldName to comparator function. |
| 92 final Map<String, Function> comparators; | 94 final Map<String, CompareFunc> comparators; |
| 93 | 95 |
| 94 // Timing operations | 96 // Timing operations |
| 95 static final Logger _logger = new Logger('aggregations'); | 97 static final Logger _logger = new Logger('aggregations'); |
| 96 Stopwatch _timeItWatch; | 98 Stopwatch _timeItWatch; |
| 97 String _timeItName; | 99 String _timeItName; |
| 98 | 100 |
| 99 FieldAccessor dimensionAccessor; | 101 FieldAccessor dimensionAccessor; |
| 100 FieldAccessor factsAccessor; | 102 FieldAccessor factsAccessor; |
| 101 | 103 |
| 102 /// Create a new [AggregationModel] from a [collection] of items, | 104 /// Create a new [AggregationModel] from a [collection] of items, |
| 103 /// list of [dimensions] on which the items are grouped and a list of [facts] | 105 /// list of [dimensions] on which the items are grouped and a list of [facts] |
| 104 /// on which aggregations are computed. | 106 /// on which aggregations are computed. |
| 105 AggregationModel(List collection, List dimensions, List facts, | 107 AggregationModel(Iterable collection, List dimensions, List facts, |
| 106 {List<String> aggregationTypes, | 108 {List<String> aggregationTypes, |
| 107 this.walkThroughMap: false, | 109 this.walkThroughMap: false, |
| 108 this.comparators, | 110 this.comparators: const <String, CompareFunc>{}, |
| 109 this.dimensionAccessor, | 111 this.dimensionAccessor, |
| 110 this.factsAccessor}) { | 112 this.factsAccessor}) { |
| 111 _init(collection, dimensions, facts, aggregationTypes); | 113 _init(collection, dimensions, facts, aggregationTypes); |
| 112 } | 114 } |
| 113 | 115 |
| 114 void _timeItStart(String name) { | 116 void _timeItStart(String name) { |
| 115 _timeItName = name; | 117 _timeItName = name; |
| 116 _timeItWatch = new Stopwatch(); | 118 _timeItWatch = new Stopwatch(); |
| 117 _timeItWatch.start(); | 119 _timeItWatch.start(); |
| 118 } | 120 } |
| 119 | 121 |
| 120 void _timeItEnd() { | 122 void _timeItEnd() { |
| 121 _timeItWatch.stop(); | 123 _timeItWatch.stop(); |
| 122 _logger.info('[aggregations/$_timeItName] ' | 124 _logger.info('[aggregations/$_timeItName] ' |
| 123 '${_timeItWatch.elapsed.inMilliseconds}ms/${_rows.length}r'); | 125 '${_timeItWatch.elapsed.inMilliseconds}ms/${_rows.length}r'); |
| 124 } | 126 } |
| 125 | 127 |
| 126 List get factFields => _factFields; | 128 List get factFields => _factFields; |
| 127 List get dimensionFields => _dimFields; | 129 List get dimensionFields => _dimFields; |
| 128 | 130 |
| 129 /// Initialize the view | 131 /// Initialize the view |
| 130 void _init(List collection, List dimensions, List facts, | 132 void _init(Iterable collection, List dimensions, List facts, |
| 131 List<String> aggregationTypes) { | 133 List<String> aggregationTypes) { |
| 132 if (collection == null) { | 134 if (collection == null) { |
| 133 throw new ArgumentError('Data cannot be empty or null'); | 135 throw new ArgumentError('Data cannot be empty or null'); |
| 134 } | 136 } |
| 135 | 137 |
| 136 if (facts == null || facts.isEmpty) { | 138 if (facts == null || facts.isEmpty) { |
| 137 throw new ArgumentError('Facts cannot be empty or null'); | 139 throw new ArgumentError('Facts cannot be empty or null'); |
| 138 } | 140 } |
| 139 | 141 |
| 140 if (dimensions == null) { | 142 if (dimensions == null) { |
| (...skipping 17 matching lines...) Expand all Loading... |
| 158 } | 160 } |
| 159 } else { | 161 } else { |
| 160 aggregationTypes = ['sum']; | 162 aggregationTypes = ['sum']; |
| 161 } | 163 } |
| 162 | 164 |
| 163 // Always adding 'count' for correct computation of average and count. | 165 // Always adding 'count' for correct computation of average and count. |
| 164 if (!aggregationTypes.contains('valid')) { | 166 if (!aggregationTypes.contains('valid')) { |
| 165 aggregationTypes.add('valid'); | 167 aggregationTypes.add('valid'); |
| 166 } | 168 } |
| 167 | 169 |
| 168 _rows = collection; | 170 _rows = new List.from(collection, growable: false); |
| 169 _dimFields = new List.from(dimensions, growable: false); | 171 _dimFields = new List.from(dimensions, growable: false); |
| 170 _factFields = new List.from(facts, growable: false); | 172 _factFields = new List.from(facts, growable: false); |
| 171 _entityCache = new Map<String, AggregationItem>(); | 173 _entityCache = new Map<String, AggregationItem>(); |
| 172 | 174 |
| 173 _createBuffers(); | 175 _createBuffers(); |
| 174 | 176 |
| 175 _aggregationTypesCount = aggregationTypes.length; | 177 _aggregationTypesCount = aggregationTypes.length; |
| 176 for (int i = 0; i < _aggregationTypesCount; i++) { | 178 for (int i = 0; i < _aggregationTypesCount; i++) { |
| 177 switch (aggregationTypes[i]) { | 179 switch (aggregationTypes[i]) { |
| 178 case 'sum': | 180 case 'sum': |
| (...skipping 84 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 263 entity.update(); | 265 entity.update(); |
| 264 } else { | 266 } else { |
| 265 _entityCache.remove(key); | 267 _entityCache.remove(key); |
| 266 entity.clear(); | 268 entity.clear(); |
| 267 } | 269 } |
| 268 } | 270 } |
| 269 }); | 271 }); |
| 270 } | 272 } |
| 271 | 273 |
| 272 final Map<String, List> _parsedKeys = {}; | 274 final Map<String, List> _parsedKeys = {}; |
| 275 |
| 273 /// Get value from a map-like object | 276 /// Get value from a map-like object |
| 274 dynamic _fetch(var item, String key) { | 277 dynamic _fetch(var item, String key) { |
| 275 if (walkThroughMap && key.contains('.')) { | 278 if (walkThroughMap && key.contains('.')) { |
| 276 return walk(item, key, _parsedKeys); | 279 return walk(item, key, _parsedKeys); |
| 277 } else { | 280 } else { |
| 278 return item[key]; | 281 return item[key]; |
| 279 } | 282 } |
| 280 } | 283 } |
| 281 | 284 |
| 282 /// Preprocess Data | 285 /// Preprocess Data |
| (...skipping 37 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 320 _dimToIntMap[di][dimensionVal] = dimensionValCount[di]; | 323 _dimToIntMap[di][dimensionVal] = dimensionValCount[di]; |
| 321 dimensionValEnum = dimensionValCount[di]++; | 324 dimensionValEnum = dimensionValCount[di]++; |
| 322 } | 325 } |
| 323 _dimEnumCache[dimDataOffset + di] = dimensionValEnum; | 326 _dimEnumCache[dimDataOffset + di] = dimensionValEnum; |
| 324 } | 327 } |
| 325 } | 328 } |
| 326 | 329 |
| 327 // Sort all dimensions internally | 330 // Sort all dimensions internally |
| 328 // The resulting arrays would be used to sort the entire data | 331 // The resulting arrays would be used to sort the entire data |
| 329 | 332 |
| 330 List oldSortOrders = _dimSortOrders; | 333 List<List<int>> oldSortOrders = _dimSortOrders; |
| 331 _dimSortOrders = new List.generate(dimensionsCount, (i) { | 334 _dimSortOrders = new List.generate(dimensionsCount, (i) { |
| 332 if (groupBy && i < _dimPrefixLength) { | 335 if (groupBy && i < _dimPrefixLength) { |
| 333 return oldSortOrders[i]; | 336 return oldSortOrders[i]; |
| 334 } | 337 } |
| 335 | 338 |
| 336 List dimensionVals = new List.from(_dimToIntMap[i].keys); | 339 List dimensionVals = new List.from(_dimToIntMap[i].keys); |
| 337 List retval = new List(_dimToIntMap[i].length); | 340 List<int> retval = new List<int>(_dimToIntMap[i].length); |
| 338 | 341 |
| 339 // When a comparator is not specified, our implementation of the | 342 // When a comparator is not specified, our implementation of the |
| 340 // comparator tries to gracefully handle null values. | 343 // comparator tries to gracefully handle null values. |
| 341 dimensionVals.sort( | 344 if (comparators.containsKey(_dimFields[i])) { |
| 342 comparators != null && comparators.containsKey(_dimFields[i]) | 345 dimensionVals.sort(comparators[_dimFields[i]]); |
| 343 ? comparators[_dimFields[i]] | 346 } else { |
| 344 : _defaultDimComparator); | 347 dimensionVals.sort(_defaultDimComparator); |
| 348 } |
| 345 | 349 |
| 346 for (int si = 0; si < retval.length; ++si) { | 350 for (int si = 0; si < retval.length; ++si) { |
| 347 retval[_dimToIntMap[i][dimensionVals[si]]] = si; | 351 retval[_dimToIntMap[i][dimensionVals[si]]] = si; |
| 348 } | 352 } |
| 349 return retval; | 353 return retval; |
| 350 }, growable: false); | 354 }, growable: false); |
| 351 | 355 |
| 352 // Create a list of sorted indices - only if we are not having a full | 356 // Create a list of sorted indices - only if we are not having a full |
| 353 // overlap of dimensionFields. | 357 // overlap of dimensionFields. |
| 354 if (!groupBy || _dimPrefixLength != _dimFields.length) { | 358 if (!groupBy || _dimPrefixLength != _dimFields.length) { |
| 355 _sorted = new List<int>.generate(_rows.length, (i) => i, growable: false); | 359 _sorted = new List<int>.generate(_rows.length, (i) => i, growable: false); |
| 356 _sorted.sort(_comparator); | 360 _sorted.sort(_comparator); |
| 357 } | 361 } |
| 358 | 362 |
| 359 // Pre-compute frequently used values | 363 // Pre-compute frequently used values |
| 360 _offsetSortedIndex = factsCount * _aggregationTypesCount; | 364 _offsetSortedIndex = factsCount * _aggregationTypesCount; |
| 361 _offsetFilteredCount = factsCount * _aggregationTypesCount + 1; | 365 _offsetFilteredCount = factsCount * _aggregationTypesCount + 1; |
| 362 | 366 |
| 363 _timeItEnd(); | 367 _timeItEnd(); |
| 364 } | 368 } |
| 365 | 369 |
| 366 // Ensures that null dimension values don't cause an issue with sorting | 370 // Ensures that null dimension values don't cause an issue with sorting |
| 367 _defaultDimComparator(Comparable left, Comparable right) => | 371 int _defaultDimComparator(Comparable left, Comparable right) => |
| 368 (left == null && right == null) | 372 (left == null && right == null) |
| 369 ? 0 | 373 ? 0 |
| 370 : (left == null) ? -1 : (right == null) ? 1 : left.compareTo(right); | 374 : (left == null) ? -1 : (right == null) ? 1 : left.compareTo(right); |
| 371 | 375 |
| 372 /// Given item indices in rows, compare them based | 376 /// Given item indices in rows, compare them based |
| 373 /// on the sort orders created while pre-processing data. | 377 /// on the sort orders created while pre-processing data. |
| 374 _comparator(int one, int two) { | 378 int _comparator(int one, int two) { |
| 375 if (one == two) { | 379 if (one == two) { |
| 376 return 0; | 380 return 0; |
| 377 } | 381 } |
| 378 | 382 |
| 379 int offsetOne = _dimFields.length * one; | 383 int offsetOne = _dimFields.length * one; |
| 380 int offsetTwo = _dimFields.length * two; | 384 int offsetTwo = _dimFields.length * two; |
| 381 | 385 |
| 382 for (int i = 0; i < _dimFields.length; ++i) { | 386 for (int i = 0; i < _dimFields.length; ++i) { |
| 383 int diff = _dimSortOrders[i][_dimEnumCache[offsetOne + i]] - | 387 int diff = _dimSortOrders[i][_dimEnumCache[offsetOne + i]] - |
| 384 _dimSortOrders[i][_dimEnumCache[offsetTwo + i]]; | 388 _dimSortOrders[i][_dimEnumCache[offsetTwo + i]]; |
| (...skipping 157 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 542 throw new UnimplementedError(); | 546 throw new UnimplementedError(); |
| 543 } | 547 } |
| 544 | 548 |
| 545 /// Return an [AggregationItem] that represents facts for dimension | 549 /// Return an [AggregationItem] that represents facts for dimension |
| 546 /// represented by [dimension] Only one instance of an entity is created | 550 /// represented by [dimension] Only one instance of an entity is created |
| 547 /// per dimension (even if this function is called multiple times) | 551 /// per dimension (even if this function is called multiple times) |
| 548 /// | 552 /// |
| 549 /// Callers of this method can observe the returned entity for updates to | 553 /// Callers of this method can observe the returned entity for updates to |
| 550 /// aggregations caused by changes to filter or done through add, remove | 554 /// aggregations caused by changes to filter or done through add, remove |
| 551 /// or modify of items in the collection. | 555 /// or modify of items in the collection. |
| 552 AggregationItem facts(List dimension) { | 556 AggregationItem facts(List<String> dimension) { |
| 553 List<int> enumeratedList = new List<int>(); | 557 List<int> enumeratedList = new List<int>(); |
| 554 for (int i = 0; i < dimension.length; ++i) { | 558 for (int i = 0; i < dimension.length; ++i) { |
| 555 enumeratedList.add(_dimToIntMap[i][dimension[i]]); | 559 enumeratedList.add(_dimToIntMap[i][dimension[i]]); |
| 556 } | 560 } |
| 557 | 561 |
| 558 String key = enumeratedList.join(':'); | 562 String key = enumeratedList.join(':'); |
| 559 AggregationItem item = _entityCache[key]; | 563 AggregationItem item = _entityCache[key]; |
| 560 | 564 |
| 561 if (item == null && _dimToAggrMap.containsKey(key)) { | 565 if (item == null && _dimToAggrMap.containsKey(key)) { |
| 562 item = new _AggregationItemImpl(this, dimension, key); | 566 item = new _AggregationItemImpl(this, dimension, key); |
| 563 _entityCache[key] = item; | 567 _entityCache[key] = item; |
| 564 } | 568 } |
| 565 | 569 |
| 566 return item; | 570 return item; |
| 567 } | 571 } |
| 568 | 572 |
| 569 /// Return a list of values that are present for a dimension field. | 573 /// Return a list of values that are present for a dimension field. |
| 570 List valuesForDimension(dynamic dimensionFieldName) { | 574 List valuesForDimension(dynamic dimensionFieldName) { |
| 571 int di = _dimFields.indexOf(dimensionFieldName); | 575 int di = _dimFields.indexOf(dimensionFieldName); |
| 572 if (di < 0) { | 576 if (di < 0) { |
| 573 return null; | 577 return null; |
| 574 } | 578 } |
| 575 List values = new List.from(_dimToIntMap[di].keys); | 579 List values = new List.from(_dimToIntMap[di].keys); |
| 576 values.sort( | 580 if (comparators.containsKey(dimensionFieldName)) { |
| 577 comparators != null && comparators.containsKey(dimensionFieldName) | 581 values.sort(comparators[dimensionFieldName]); |
| 578 ? comparators[dimensionFieldName] | 582 } else { |
| 579 : _defaultDimComparator); | 583 values.sort(_defaultDimComparator); |
| 584 } |
| 580 return values; | 585 return values; |
| 581 } | 586 } |
| 582 } | 587 } |
| 583 | 588 |
| 584 /// Parse a path for nested map-like objects. | 589 /// Parse a path for nested map-like objects. |
| 585 /// Caches the parsed key in the passed map. | 590 /// Caches the parsed key in the passed map. |
| 586 /// | 591 /// |
| 587 /// Takes map keys of the format: | 592 /// Takes map keys of the format: |
| 588 /// "list(key=val;val=m).another(state=good).state" | 593 /// "list(key=val;val=m).another(state=good).state" |
| 589 /// and outputs: | 594 /// and outputs: |
| (...skipping 91 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 681 }); | 686 }); |
| 682 if (match) { | 687 if (match) { |
| 683 return current[i]; | 688 return current[i]; |
| 684 } | 689 } |
| 685 } | 690 } |
| 686 } else { | 691 } else { |
| 687 return current[part]; | 692 return current[part]; |
| 688 } | 693 } |
| 689 }); | 694 }); |
| 690 } | 695 } |
| OLD | NEW |