Parsing

Visualization of automated processing.

Objects are ingested and parsed (or not), then assessed for properties in several categories.

Uses point-along-path interpolation; paths generated in Sketch.

Inspired by Stephanie Yee and Tony Chu's A Visual Introduction to Machine Learning

<!doctype html>
<html>
<head>
  <meta charset="utf-8">
  <title>parsing</title>
  <style>
      * { box-sizing: border-box; }
      body {
        font-family: "Open Sans", sans-serif;
        margin: 0;
      }
      circle {
        fill: #008000;
        stroke: none;
      }
      #svg-wrap {
        margin: 10px auto;
        position: relative;
        overflow: scroll;
      }
      .bucket-top {
        stroke: none;
        fill: #999;
      }
      .bucket-barrel {
        stroke: none;
        fill: #e1e1e1;
      }
      .bucket-wire {
        stroke: #979797;
        stroke-linecap: round;
        fill: none;
      }
      .bucket-handle {
        stroke: none;
        fill: #4a4a4a;
      }
      .drop-path {
        stroke: none;
        fill: none;
      }
      .drop-guide {
        fill: none;
        stroke: #e1e1e1;
        stroke-width: 1.5px;
        stroke-linecap: round;
      }
     .count-text, count-label { fill: #444; }
  </style>
</head>
<body>
  <div id="svg-wrap">
  <svg version="1.1" xmlns="https://www.w3.org/2000/svg" xmlns:xlink="https://www.w3.org/1999/xlink">
      <g id="drop-paths"></g>
      <g id="drop-guides"></g>
      <g id="buckets"></g>
      <g id="counts"></g>
  </svg>
  </div>
  <script src="https://cdnjs.cloudflare.com/ajax/libs/d3/3.5.9/d3.min.js"></script>
  <script>
    var svg = d3.select('svg');
    var dropPathsGrp = svg.select('#drop-paths');
    var dropGuidesGrp = svg.select('#drop-guides');
    var bucketsGrp = svg.select('#buckets');
    var countsGrp = svg.select('#counts');
    var rwd = { x: 0, y: 0};
    var ballR = 0;
    var interval = 10000;
    var cats = [
      'processed',
      'parsed',
      'unique',
      'cat_1',
      'cat_2',
      'cat_3',
      'cat_4',
      'cat_5',
      'cat_6',
      'cat_7',
      'cat_8',
      'cat_9',
      'cat_10',
      'duplicates',
      'cat_11',
      'cat_12',
      'cat_13',
      'failed',
    ];

var stats = {
      'processed':  { count: 0, queued : 0, countText: {}, path: {} },
      'parsed':     { count: 0, queued : 0, countText: {}, path: {} },
      'unique':     { count: 0, queued : 0, countText: {}, path: {} },
      'cat_1':      { count: 0, queued : 0, countText: {}, path: {} },
      'cat_2':      { count: 0, queued : 0, countText: {}, path: {} },
      'cat_3':      { count: 0, queued : 0, countText: {}, path: {} },
      'cat_4':      { count: 0, queued : 0, countText: {}, path: {} },
      'cat_5':      { count: 0, queued : 0, countText: {}, path: {} },
      'cat_6':      { count: 0, queued : 0, countText: {}, path: {} },
      'cat_7':      { count: 0, queued : 0, countText: {}, path: {} },
      'cat_8':      { count: 0, queued : 0, countText: {}, path: {} },
      'cat_9':      { count: 0, queued : 0, countText: {}, path: {} },
      'cat_10':     { count: 0, queued : 0, countText: {}, path: {} },
      'duplicates': { count: 0, queued : 0, countText: {}, path: {} },
      'cat_11':     { count: 0, queued : 0, countText: {}, path: {} },
      'cat_12':     { count: 0, queued : 0, countText: {}, path: {} },
      'cat_13':     { count: 0, queued : 0, countText: {}, path: {} },
      'failed':     { count: 0, queued : 0, countText: {}, path: {} }
    };

var updates = [];
    var upIdx   = 0;
    var upLen   = 60;

function genData () {
      var upX = 0;
      var tots = initTots();

for (upX; upX < upLen; upX += 1 ) {
        updates.push(genRow());
      }

row[cat] = tots[cat] + catCt;
          tots[cat] = row[cat];
        });

return row;
      }

function initTots () {
        var tots = {};
        cats.forEach(function (cat) { tots[cat] = 0; });
        return tots;
      }
    }

function drawBuckets () {
      var barrel = [
            ['M',[40,4]],
            ['L',[36,38]],
            ['C',[36,40],' ',[28.84,42],' ',[20,42]],
            ['C',[11.163,42],' ',[4,40],' ',[4,38]],
            ['L',[0,4]],
            ['C',[0.5,6.3],' ',[9.275,8],' ',[20,8]],
            ['C',[30.724,8],' ',[39.477,6.3],' ',[40,4]],
            ['Z']
          ];
      var barrelD = calcD(barrel);

var wire = [
            ['M',[0,4]],
            ['C',[0,4],' ',[0,22],' ',[20,22]],
            ['C', [40,22],' ',[40,4],' ',[40,4]]
          ];
      var wireD = calcD(wire);

var handle = { x: 16, y: 20, width: 8, height: 4, rx: 2 };

var bucketY = to3(rwd.y * 403);

var buckets = {
        'processed':   28,
        'parsed':      92,
        'unique':     156,
        'cat_1':      220,
        'cat_2':      284,
        'cat_3':      348,
        'cat_4':      412,
        'cat_5':      476,
        'cat_6':      540,
        'cat_7':      604,
        'cat_8':      668,
        'cat_9':      732,
        'cat_10':     796,
        'duplicates': 860,
        'cat_11':     924,
        'cat_12':     988,
        'cat_13':    1052,
        'failed':    1116
      };

cats.forEach(function (cat) {
        var bucket = buckets[cat];
        drawTop(cat, bucket);
        drawBucket(cat, bucket);
      });

function drawTop (cat, bucket) {
        var bucketX = to3(buckets[cat] * rwd.x);
        var rx = to3(rwd.x * 20);
        var ry = to3(rwd.y * 4);
        var cx = bucketX + rx;
        var cy = bucketY + ry;

dropPathsGrp.append('ellipse')
          .attr('id', 'bucket-top-' + cat)
          .attr('class', 'bucket-top')
          .attr('cx', cx)
          .attr('cy', cy)
          .attr('rx', rx)
          .attr('ry', ry);
      }

function drawBucket (cat, bucket) {
        var bucketGrp = bucketsGrp.append('g')
              .attr('id', 'bucket-' + cat)
              .attr('transform', 'translate(' + to3(bucket * rwd.x) + ',' + bucketY + ')');

bucketGrp.append('path')
          .attr('class', 'bucket-barrel')
          .attr('d', barrelD);

bucketGrp.append('path')
          .attr('class', 'bucket-wire')
          .attr('d', wireD)
          .attr('stroke-linecap', 'round');

bucketGrp.append('rect')
          .attr('class', 'bucket-handle')
          .attr('x', to3(16 * rwd.x))
          .attr('y', to3(20 * rwd.y))
          .attr('width', to3(8 * rwd.x))
          .attr('height', to3(4 * rwd.y))
          .attr('rx', to3(2 * rwd.x));
      }
    }

function drawPaths () {
      var dropPaths = {
        'processed':    [ ['M',[16,16]], ['L',[296,32]], ['L',[48,112]], ['L',[48,406]] ],
        'parsed':       [ ['M',[16,16]], ['L',[296,32]], ['L',[560,48]], ['L',[112,208]], ['L',[112,406]] ],
        'unique':       [ ['M',[16,16]], ['L',[296,32]], ['L',[560,48]], ['L',[592,144]], ['L',[176,304]], ['L',[176,406]] ],
        'cat_1':        [ ['M',[16,16]], ['L',[296,32]], ['L',[560,48]], ['L',[592,144]], ['L',[508,176]], ['L',[525,225]], ['L',[240,336]], ['L',[240,406]] ],
        'cat_2':       [ ['M',[16,16]], ['L',[296,32]], ['L',[560,48]], ['L',[592,144]], ['L',[508,176]], ['L',[525,225]], ['L',[525,250]], ['L',[304,336]], ['L',[304,406]] ],
        'cat_3':       [ ['M',[16,16]], ['L',[296,32]], ['L',[560,48]], ['L',[592,144]], ['L',[508,176]], ['L',[525,225]], ['L',[525,260]], ['L',[368,336]], ['L',[368,406]] ],
        'cat_4':       [ ['M',[16,16]], ['L',[296,32]], ['L',[560,48]], ['L',[592,144]], ['L',[508,176]], ['L',[525,225]], ['L',[525,260]], ['L',[432,336]], ['L',[432,406]] ],
        'cat_5':   [ ['M',[16,16]], ['L',[296,32]], ['L',[560,48]], ['L',[592,144]], ['L',[508,176]], ['L',[525,225]], ['L',[525,260]], ['L',[496,336]], ['L',[496,406]] ],
        'cat_6':        [ ['M',[16,16]], ['L',[296,32]], ['L',[560,48]], ['L',[592,144]], ['L',[508,176]], ['L',[525,225]], ['L',[525,260]], ['L',[560,336]], ['L',[560,406]] ],
        'cat_7':     [ ['M',[16,16]], ['L',[296,32]], ['L',[560,48]], ['L',[592,144]], ['L',[508,176]], ['L',[525,225]], ['L',[525,260]],  ['L',[624,336]], ['L',[624,406]] ],
        'cat_8':     [ ['M',[16,16]], ['L',[296,32]], ['L',[560,48]], ['L',[592,144]], ['L',[508,176]], ['L',[525,225]], ['L',[525,260]], ['L',[688,336]], ['L',[688,406]] ],
        'cat_9':    [ ['M',[16,16]], ['L',[296,32]], ['L',[560,48]], ['L',[592,144]], ['L',[508,176]], ['L',[525,225]], ['L',[525,250]], ['L',[752,311]], ['L',[752,406]] ],
        'cat_10':  [ ['M',[16,16]], ['L',[296,32]], ['L',[560,48]], ['L',[592,144]], ['L',[508,176]], ['L',[525,225]], ['L',[592,244.4]], ['L',[816,304]], ['L',[816,406]] ],
        'duplicates':   [ ['M',[16,16]], ['L',[296,32]], ['L',[560,48]], ['L',[592,144]], ['L',[513,174.3]], ['L',[880,272]], ['L',[880,406]] ],
        'cat_11':          [ ['M',[16,16]], ['L',[296,32]], ['L',[560,48]], ['L',[592,144]], ['L',[1008,240]], ['L',[944,304]], ['L',[944,406]] ],
        'cat_12':        [ ['M',[16,16]], ['L',[296,32]], ['L',[560,48]], ['L',[592,144]], ['L',[1008,240]], ['L',[1008,406]] ],
        'cat_13':       [ ['M',[16,16]], ['L',[296,32]], ['L',[560,48]], ['L',[592,144]], ['L',[1008,240]], ['L',[1072,304]], ['L',[1072,406]] ],
        'failed':       [ ['M',[16,16]], ['L',[296,32]], ['L',[560,48]], ['L',[1136,187.5]], ['L',[1136,406]] ]
      };
      var dropGuides = [
        { id: 'init',         points: [ ['M',[9.75,22.5]], ['L',[258.5,37]] ] },
        { id: 'processed',    points: [ ['M',[526.5,52.75]], ['L',[296,39]], ['L',[54.75,117]], ['L',[54.75,144]] ] },
        { id: 'parsed',       points: [ ['M',[584.239,140.024]], ['L',[555.928,56.354]], ['L',[118.5,212.39]], ['L',[118.5,239.824]] ] },
        { id: 'query-table',  points: [ ['M',[535.032,173.16]], ['L',[592.484,151.051]], ['L',[994.548,243.828]] ] },
        { id: 'unique',       points: [ ['M',[516.5,221]], ['L',[503.818,185.111]], ['L',[182.75,308.75]], ['L',[182.75,336]] ] },
        { id: 'cat_4',       points: [ ['M',[438.75,368]], ['L',[438.75,339.089]], ['L',[508,282.75]] ] },
        { id: 'cat_1',       points: [ ['M',[246.5,368]], ['L',[246.5,340.308]], ['L',[518.418,234.626]] ] },
        { id: 'cat_2',       points: [ ['M',[310.75,368]], ['L',[310.75,340]], ['L',[470.811,278]] ] },
        { id: 'cat_3',       points: [ ['M',[374.75,368]], ['L',[374.75,338.688]], ['L',[473.877,292]] ] },
        { id: 'sub-cat_6',    points: [ ['M',[502.75,368]], ['L',[502.75,337.5]], ['L',[525.5,277.75]], ['L',[553.25,337.5]], ['L',[553.25,368]] ] },
        { id: 'cat_7',     points: [ ['M',[543.5,282.75]], ['L',[617.25,339.5]], ['L',[617.25,368]] ] },
        { id: 'cat_8',     points: [ ['M',[578.5,292]], ['L',[681.25,340]], ['L',[681.25,368]] ] },
        { id: 'cat_9',    points: [ ['M',[603.175,278]], ['L',[745.25,316.25]], ['L',[745.25,342]] ] },
        { id: 'cat_10',  points: [ ['M',[532,234]], ['L',[809.25,309.25]], ['L',[809.25,336]] ] },
        { id: 'duplicates',   points: [ ['M',[517.4,182.5]], ['L',[873.25,277.171]], ['L',[873.25,304]] ] },
        { id: 'cat_11',          points: [ ['M',[950.5,336]], ['L',[950.5,306.683]], ['L',[1001.03625,256.5]] ] },
        { id: 'cat_13',       points: [ ['M',[1014.9,256.5]], ['L',[1065.25,307]], ['L',[1065.25,336.188]] ] },
        { id: 'failed',       points: [ ['M',[570.22,57.4]], ['L',[1129.25,192.808]], ['L',[1129.25,222.135]] ] }
      ];

cats.forEach(function (cat) {
        var dropD = calcD(dropPaths[cat]);
        drawDrop(cat, dropD);
      });

dropGuides.forEach(function (dropGuide) {
        dropGuidesGrp.append('path')
          .attr('id', 'guide-' + dropGuide.id)
          .attr('class', 'drop-guide')
          .attr('d', calcD(dropGuide.points))
      });

function drawDrop(cat, dropD) {
        var dropPath = dropPathsGrp.append('path')
              .attr('id', 'drop-' + cat)
              .attr('class', 'drop-path')
              .attr('d', dropD);
        // TODO: append another path here for hover?

setStatProps(cat, dropPath);
      }
    }

function drawCounts () {
      var countY = to3(566 * rwd.y);
      var labelY = to3(488.5 * rwd.y);
      var counts = {
        'processed':  { x:    48, label: 'Processed' },
        'parsed':     { x:   112, label: 'Parsed' },
        'unique':     { x:   176, label: 'Unique' },
        'cat_1':      { x:   240, label: 'Cat 1' },
        'cat_2':      { x:   304, label: 'Cat 2' },
        'cat_3':      { x:   368, label: 'Cat 3' },
        'cat_4':      { x:   432, label: 'Cat 4' },
        'cat_5':      { x:   496, label: 'Cat 5' },
        'cat_6':      { x:   560, label: 'Cat 6' },
        'cat_7':      { x:   624, label: 'Cat 7' },
        'cat_8':      { x:   688, label: 'Cat 8' },
        'cat_9':      { x:   752, label: 'Cat 9' },
        'cat_10':     { x:   816, label: 'Cat 10' },
        'duplicates': { x:   880, label: 'Duplicates' },
        'cat_11':     { x:   944, label: 'Cat 11' },
        'cat_12':     { x:  1008, label: 'Cat 12' },
        'cat_13':     { x:  1072, label: 'Cat 13' },
        'failed':     { x:  1136, label: 'Failed' }
      }

cats.forEach(function(cat) {
        var count = counts[cat];
        drawCount(cat, count);
        procLabel(count);
      });

function drawCount(cat, count) {
        var countX = to3(count.x * rwd.x);
        var fontSize = to3(16 * rwd.y);
        var countText = countsGrp.append('text')
             .attr('id', 'count-' + cat)
             .attr('class', 'count-text')
             .attr('x', countX)
             .attr('y', countY)
             .attr('font-size', fontSize)
             .attr('text-anchor', 'middle')
             .text('0');

stats[cat].countText = countText;
      }

function procLabel(count) {
        var labelX = to3(count.x * rwd.x);

drawLabel(count.label, labelX, labelY);
      }

function drawLabel(label, labelX, labelY) {
        var fontSize = to3(12 * rwd.y);
        countsGrp.append('text')
         .attr('class', 'count-label')
         .attr('x', labelX)
         .attr('y', labelY)
         .attr('font-size', function () { return fontSize < 8 ? 8 : fontSize; })
         .attr('text-anchor', 'middle')
         .text(label)
         .attr('transform', 'translate(' + labelX + ',' + labelY + ') rotate(-45) translate(' + -labelX + ',' + -labelY + ')');
      }
    }

function load (idx) {
      var next = idx + 1;
      var data = updates[idx];

console.log('\nprocessing ' + next + ' of ' + upLen + '\n')

updateAll(data);

if (next < upLen) {
        upIdx = next;
        setTimeout(function() {
          load(upIdx);
        }, interval);
      }
    }

function updateAll (data) {
      // console.log('queued: ')
      cats.forEach(function(cat) {
        var stat = stats[cat];
        updateStat(cat, stat, data);
      });
    }

function updateCountText (stat) {
      var count = +stat.countText.text() + 1;
      stat.countText.text(count);
    }

function dropEnd (circle, stat) {
      updateCountText(stat);
      d3.select(circle).remove();
    }

function init () {
      genData();
      setDims();
      drawBuckets();
      drawPaths();
      drawCounts();
      load(upIdx);
      dequeue();
    }

function setDims () {
      var defaultWidth = 1184;
      var defaultHeight = 580;
      var minWidth = 560;
      var minHeight = 290;
      var svgWidth = 0;
      var svgHeight = 0;
      var width = window.innerWidth - 20;
      var height = window.innerHeight - 20;
      var svgWrap = document.getElementById('svg-wrap');
      var wrapBounds = {};

svgWrap.style.width = width + 'px';
      svgWrap.style.height =  height + 'px';
      wrapBounds = svgWrap.getBoundingClientRect();

svgWidth = width > minWidth ? width : minWidth;
      rwd.x = svgWidth / defaultWidth;
      svgHeight = height > minHeight ? height : minHeight;
      rwd.y = svgHeight / defaultHeight;

ballR = Math.round(5 * Math.min(rwd.x, rwd.y));

svg.attr('width', svgWidth).attr('height', svgHeight);
    }

function setStatProps (cat, path) {
      var stat = stats[cat];
      var node = path.node();
      stat.path = path;
      stat.pathStart = node.getPointAtLength(0);
      stat.pathEnd = node.getPointAtLength(node.getTotalLength());
      stat.pathLength = node.getTotalLength();
    }

function randRange (min, max) {
      return parseInt(Math.round(min + Math.random() * (max - min)));
    }

function to3 (n) { return parseFloat(n.toFixed(3)) }

function leftPad (num, padding, padChar) {
      // returns left-padded string from an int or float
      var padded = num + '',
          padLen = padding || 2,
          padder = padChar || '0';

if (padded.length < padLen) {
        while (padded.length < padLen) {
          padded = padder + padded;
        }
      }
      return padded;
    }

init();
  </script>
</body>
</html>

https://cdnjs.cloudflare.com/ajax/libs/d3/3.5.9/d3.min.js