[Devtools] Updated format to store product registry data

third_party/WebKit/Source/devtools/scripts/convert-3pas-product-registry.js

+// Copyright 2017 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
 const fs = require('fs');
 
 /*
 How to use:
 1) Get dump of data as CSV format and name it 3pas.csv same directory as this script.
 2) Header fields in the CSV will be used as keys when destructing into JSON objects [ie: top row data should not have spaces or special chars]
 3) The two important column names are: 'name_legal_product' and 'domain'.
 4) There may not be a header named 'prefix'.
 5) 'name_legal_product' Will have it's data cleaned up a bit, so be prepared for it to change.
 6) This script tries to de-duplicate any data, so be prepared for many entries to go away if it finds a shorter one.
 7) This script will output a javascript file in the product_registry's data format.
 */
 
 /*
  * Configurable variables. You may need to tweak these to be compatible with
  * the server-side, but the defaults work in most cases.
  */
 const hexcase = 0;  /* hex output format. 0 - lowercase; 1 - uppercase        */
 const b64pad = '='; /* base-64 pad character. "=" for strict RFC compliance   */
 const chrsz = 8;    /* bits per input character. 8 - ASCII; 16 - Unicode      */
 
+const typeClassifications = new Map([
+  ['cdn_provider', 'CDN'], ['cdn_commercial_owner', 'CDN'], ['cdn_creative_agency', 'CDN'], ['ad_blocking', 'Ad'],
+  ['ad_exchange', 'Ad'], ['ad_server_ad_network', 'Ad'], ['ad_server_advertiser', 'Ad'], ['demand_side_platform', 'Ad'],
+  ['vast_provider', 'Ad'], ['data_management_platform', 'Tracking'], ['research_analytics', 'Tracking'],
+  ['research_verification', 'Tracking'], ['research_brand_lift', 'Tracking']
+]);
+
 var data = fs.readFileSync('3pas.csv', 'utf8');
 var headerLine = data.split('\n', 1)[0];
 data = data.substr(headerLine.length);
 var headerLineOrigLength = headerLine.length;
 
 var columnNames = Array.from(csvUnmarshaller(headerLine)).map(v => v[0]);
 var lineObjs = [];
 
 var marshaller = csvUnmarshaller(data, 2);
 var lineObj = {};
@@ skipping 75 matching lines @@
   if (!prefixMap) {
     prefixMap = new Map();
     mapOfSubdomains.set(lineObj.domain, prefixMap);
   }
   if (prefixMap.has(prefix))
     console.log('Problem with: ', domain, lineObj.domain);
   prefixMap.set(prefix, lineObj);
 }
 
 var outputProducts = [];
+var outputTypes = [];
 var outputObj = new Map();
 for (var [baseDomain, subdomains] of map) {
   for (var prefixes of subdomains.values()) {
     SKIP_ENTRY: for (var lineObj of prefixes.values()) {
       var prefix = lineObj.prefix;
       var wildLineObj = prefixes.get('*');
       if (wildLineObj && prefix !== '*') {
         if (wildLineObj.name_legal_product === lineObj.name_legal_product) {
           // Skip entry, since wild card is there and already in table.
           continue SKIP_ENTRY;
@@ skipping 20 matching lines @@
               continue SKIP_ENTRY;
           }
         }
         previousDomainPart = domainParts.shift();
       }
       var outputPart = outputObj.get(fullSubdomain);
       if (!outputPart) {
         outputPart = {hash: hex_sha1(fullSubdomain).substr(0, 16), prefixes: {}};
         outputObj.set(fullSubdomain, outputPart);
       }
-      outputPart.prefixes[lineObj.prefix] = registerOutputProduct(lineObj.name_legal_product);
+      outputPart.prefixes[lineObj.prefix] = registerOutputProduct(lineObj.name_legal_product, lineObj.type_vendor);
     }
   }
 }
 
 console.log(
     '// Copyright 2017 The Chromium Authors. All rights reserved.\n' +
     '// Use of this source code is governed by a BSD-style license that can be\n' +
     '// found in the LICENSE file.\n' +
     '// clang-format off\n' +
     '/* eslint-disable */\n' +
     'ProductRegistry.register([');
+if (outputTypes.length) {
+  var data = JSON.stringify(outputTypes).replace(/","/g, '",\n  "');
+  console.log('  ' + data.substring(1, data.length - 1));
+}
+console.log('],');
+console.log('[');
 var data = JSON.stringify(outputProducts).replace(/","/g, '",\n  "');
 console.log('  ' + data.substring(1, data.length - 1));
 console.log('],');
 console.log('[');
 var outputObjArray = Array.from(outputObj.values());
 for (var i = 0; i < outputObjArray.length; i++) {
   var obj = outputObjArray[i];
   var lineEnding = (i === outputObjArray.length - 1) ? '' : ',';
   var comments = [];
-  for (var prefix in obj.prefixes)
-    comments.push('[' + outputProducts[obj.prefixes[prefix]] + ']');
+  for (var prefix in obj.prefixes) {
+    var typeName = outputTypes[obj.prefixes[prefix].type];
+    if (!typeName)
+      typeName = '';
+    else
+      typeName = ':' + typeName;
+    comments.push('[' + outputProducts[obj.prefixes[prefix].product] + typeName + ']');
+  }
   console.log('  ' + JSON.stringify(obj) + lineEnding + ' // ' + comments.join(' '));
 }
 console.log(']);');
 
 
 // items.forEach(lineObj => console.log(lineObj.name_legal_product.padStart(50), lineObj.domain.padStart(30)));
 // console.log("With *: ", items.filter(v => v.domain.indexOf('*') !== -1).length);
 // console.log("Total: ", items.length);
 
 
 
 // Linear but meh.
-function registerOutputProduct(name) {
+function registerOutputProduct(name, type) {
   var index = outputProducts.indexOf(name);
+  var typeIndex = registerOutputType(type);
+  var outObj = {product: index};
   if (index === -1) {
     outputProducts.push(name);
-    return outputProducts.length - 1;
+    outObj.product = outputProducts.length - 1;
+  }
+  if (typeIndex !== -1)
+    outObj.type = typeIndex;
+  return outObj;
+}
+
+function registerOutputType(type) {
+  var name = typeClassifications.get(type);
+  if (!name)
+    return -1;
+  var index = outputTypes.indexOf(name);
+  if (index === -1) {
+    outputTypes.push(name);
+    return outputTypes.length - 1;
   }
   return index;
 }
 
 function* csvUnmarshaller(data, lineOffset) {
   var origLen = data.length;
   var colLength = 0;
   var lineNo = lineOffset || 1;
   while (data.length) {
     var colData;
@@ skipping 213 matching lines @@
         ((binarray[i + 2 >> 2] >> 8 * (3 - (i + 2) % 4)) & 0xFF);
     for (var j = 0; j < 4; j++) {
       if (i * 8 + j * 6 > binarray.length * 32)
         str += b64pad;
       else
         str += tab.charAt((triplet >> 6 * (3 - j)) & 0x3F);
     }
   }
   return str;
 }