All examples By author By category About

eesur

Full window Github gist

Sparse and missing data

Sparse data can be really tricky to represent when dealing with real time data —especially as you don't know when it will be sparse or dense, or even both in one representation.

These examples look at views with the same data. A chart showing reading per minute but the dataset doesn't have values for each minute; they would be zero.

This example takes from this super article Padding for zero values but uses the difference of the start and end time to create an array to map to, instead of using the ticks()

The charts are rendered using the awesome [D3 Financial Components (D3FC)](https://github.com/ScottLogic/d3-financial-components`

<!DOCTYPE html>
<html>
    <head>
        <meta charset="utf-8">
        <meta http-equiv="X-UA-Compatible" content="IE=edge">
        <title>Monitoring Components | FC testing</title>
        
        <meta name="viewport" content="width=device-width, initial-scale=1">

</head>
    <body>

<section id="chart-one">
        <p>normal chart, that doesn't change the data; suggesting connections/values where there shouldn't be any ↓</p>
    </section>
    <section id="chart-two">
        <p>function that inserts missing zero values; better, but the drama is misleading ↓</p>
    </section>
    <section id="chart-three">
        <p>Using NaN for values, and line.defined; as the values are often not next to each other, we don't get to make a path ↓</p>
    </section>
    <section id="chart-four">
        <p>Using points to view the data ↓</p>
    </section>
    <section id="table">
        <p>Table to see the actual data ↓</p>
    </section>

// function that inserts missing zero values
    (function valuesAdded() {
        // variables
        var data,
            parseDate = d3.time.format('%Y-%m-%d %H:%M:%S').parse,
            formatDate = d3.time.format("%H:%M:%S");

// JSON callback
        d3.json("sparse_test.json", function(error, json) {
            if (error) return console.warn(error);
            data = json;

var orginalData = data.data,
                newData = [];
             
            // need reference for start and end date 
            // note: won't be first and last item in array as missing values
            var startRef  = data.start_date;
            var endRef = data.end_date;

var duration = moment.utc(moment(endRef,"YYYY/MM/DD/ HH:mm:ss").diff(moment(startRef,"YYYY/MM/DD/ HH:mm:ss"))).format("HH:mm:ss");

if (orginalData.length > duration.length) {

var duration = moment.utc(moment(endRef,"YYYY/MM/DD/ HH:mm:ss").diff(moment(startRef,"YYYY/MM/DD/ HH:mm:ss"))).format("HH:mm:ss");

// need the minutes to create an array to map to and inset missing 0 values
                var durationMinutes = moment.duration(duration).asMinutes();

// console.log('durationMinutes:' + '\n' + durationMinutes);

var durationMinutesArray = [];

for (var i=0; i<=durationMinutes; i++) {
                    durationMinutesArray.push(moment(startRef).add(i, 'minutes').format('YYYY-MM-DD HH:mm:ss'));
                }

console.log('durationMinutesArray:' + '\n' + durationMinutesArray);

var newData = durationMinutesArray
                .map(function(_value) { 
                   return _.find(orginalData, 
                       { date: _value }) || 
                       { date: _value, total: NaN };
                });

// console.log('newData:' + '\n' + newData);

} else {
                newData = orginalData;
            }

dataFormat(newData);
           renderChart(newData);
        });

// Clean/format data
        function dataFormat(data) {
            data.forEach(function(d) {
                d.date = parseDate(d.date);
                d.total = +d.total;
            });
        }
        
        function renderChart(data) {

var chart = fc.charts.linearTimeSeries()
                .xDomain(d3.extent(data, function(d) { return d.date; }))
                .xNice()
                .xTicks(d3.time.minutes, 10)
                // .tickFormat(timeFormatter)
                .yDomain(fc.utilities.extent(data, 'total'))
                .yNice()
                .yTicks(5);

// Create the gridlines
            var gridlines = fc.scale.gridlines();

// Create the line series
            var line = fc.series.line()
                .yValue(function(d) { return d.total; });

var multi = fc.series.multi()
                .series([gridlines, line]);
            chart.plotArea(multi);

d3.select('#chart-three')
                .append('svg')
                .style({
                    height: '300px',
                    width: '900px'
                })
                .datum(data)
                .call(chart);
        }
    }());

/*  ______ ______ ______ ______ ______ ______ ______ ______ 
   |______|______|______|______|______|______|______|______|
*/

// JSON callback
        d3.json("sparse_test.json", function(error, json) {
            if (error) return console.warn(error);
            data = json;

var duration = moment.utc(moment(endRef,"YYYY/MM/DD/ HH:mm:ss").diff(moment(startRef,"YYYY/MM/DD/ HH:mm:ss"))).format("HH:mm:ss");

if (orginalData.length > duration.length) {

var duration = moment.utc(moment(endRef,"YYYY/MM/DD/ HH:mm:ss").diff(moment(startRef,"YYYY/MM/DD/ HH:mm:ss"))).format("HH:mm:ss");

// need the minutes to create an array to map to and inset missing 0 values
                var durationMinutes = moment.duration(duration).asMinutes();

// console.log('durationMinutes:' + '\n' + durationMinutes);

var durationMinutesArray = [];

for (var i=0; i<=durationMinutes; i++) {
                    durationMinutesArray.push(moment(startRef).add(i, 'minutes').format('YYYY-MM-DD HH:mm:ss'));
                }

console.log('durationMinutesArray:' + '\n' + durationMinutesArray);

var newData = durationMinutesArray
                .map(function(_value) { 
                   return _.find(orginalData, 
                       { date: _value }) || 
                       { date: _value, total: 0 };
                });

// console.log('newData:' + '\n' + newData);

} else {
                newData = orginalData;
            }

dataFormat(newData);
           renderChart(newData);
        });

// Create the gridlines
            var gridlines = fc.scale.gridlines();

// Create the line series
            var line = fc.series.line()
                .yValue(function(d) { return d.total; });

var multi = fc.series.multi()
                .series([gridlines, line]);
            chart.plotArea(multi);

d3.select('#chart-two')
                .append('svg')
                .style({
                    height: '300px',
                    width: '900px'
                })
                .datum(data)
                .call(chart);
        }
    }());

/*  ______ ______ ______ ______ ______ ______ ______ ______ 
   |______|______|______|______|______|______|______|______|
*/

// normal chart, that doesn't change the data

(function missingZeroValues() {
    //  variables
        var data,
            parseDate = d3.time.format('%Y-%m-%d %H:%M:%S').parse,
            formatDate = d3.time.format("%H:%M:%S");

// JSON callback
        d3.json("sparse_test.json", function(error, json) {
          if (error) return console.warn(error);
          data = json;

dataFormat(data.data);
          renderChart(data.data);
        });

var chart = fc.charts.linearTimeSeries()
            .xDomain(d3.extent(data, function(d) { return d.date; }))
            .xNice()
            .xTicks(d3.time.minutes, 10)
            .yDomain(fc.utilities.extent(data, 'total'))
            .yNice()
            .yTicks(5);

// Create the gridlines
            var gridlines = fc.scale.gridlines();

// Create the line series
            var line = fc.series.line()
                .yValue(function(d) { return d.total; });

var multi = fc.series.multi()
                .series([gridlines, line]);
            chart.plotArea(multi);

d3.select('#chart-one')
                .append('svg')
                .style({
                    height: '300px',
                    width: '900px'
                })
                .datum(data)
                .call(chart);
        }
    }());
                                        
                                                                                                                                                       
/*  ______ ______ ______ ______ ______ ______ ______ ______ 
   |______|______|______|______|______|______|______|______|
*/

// data as points

(function missingZeroValues() {
    //  variables
        var data,
            parseDate = d3.time.format('%Y-%m-%d %H:%M:%S').parse,
            formatDate = d3.time.format("%H:%M:%S");

// JSON callback
        d3.json("sparse_test.json", function(error, json) {
          if (error) return console.warn(error);
          data = json;

dataFormat(data.data);
          renderChart(data.data);
        });

// Create the gridlines
            var gridlines = fc.scale.gridlines();

// Create the line series
            var points = fc.series.point()
                .yValue(function(d) { return d.total; });

var multi = fc.series.multi()
                .series([gridlines, points]);
            chart.plotArea(multi);

d3.select('#chart-four')
                .append('svg')
                .style({
                    height: '300px',
                    width: '900px'
                })
                .datum(data)
                .call(chart);
        }
    }());

</script>

// view data in table for reference

(function(d3, fc) {
        'use strict';
        
        //  variables
        var data,
            parseDate = d3.time.format('%Y-%m-%d %H:%M:%S').parse,
            formatDate = d3.time.format("%H:%M:%S");

// Clean/format data
        function dataFormat(data) {
            data.forEach(function(d) {
                d.date = formatDate(parseDate(d.date));
                d.total = +d.total; // ensure numeric value
                d.avg = d3.format(".4f")(d.avg);
            });
        }

d3.json("sparse_test.json", function(error, json) {
            if (error) return console.warn(error);
            data = json;

dataFormat(data.data);
            renderChart(data.data);
        });

function renderChart(data) {

var chart = d3.select('#table');

// Create the table
            var table = fc.series.table()
                .columns(['date', 'total']); // or use d3.keys

chart
                .datum(data)
                .call(table);

}

})(d3, fc);

</script>

</body>
</html>

Modified http://dev.eesur.com/d3fc/Layout.js to a secure url

Modified http://dev.eesur.com/d3fc/d3-financial-components.min.js to a secure url

https://cdnjs.cloudflare.com/ajax/libs/d3/3.5.5/d3.min.js

https://dev.eesur.com/d3fc/Layout.js

https://dev.eesur.com/d3fc/d3-financial-components.min.js

https://cdnjs.cloudflare.com/ajax/libs/moment.js/2.10.3/moment.min.js

https://cdnjs.cloudflare.com/ajax/libs/lodash.js/3.9.0/lodash.min.js