timeline - seasonality detection (III)

This block is a continuation of this previous block. Both experiments how to detect if a timeline has a seasonality component, using a correlogram. This block explains why detrending the time serie before computing the correlogram is a must have.

Usages :

same usages as in this previous block
in the correlogram, detrending the time serie before computing the correlogram allows to detect very small seasonnality order of magnitude; detrending the time serie implies that coefficients of correlation for each lag no longer reflect any trend, and thus only reflect the seasonality component;
while using detrended time serie, increasing/decreasing trend has no longer any impact on the correlogram; chery on the cake, seasons are easier to detect.

Notes:

the previous block experiments season detection without detrending
another block experiments autocorrelation
another block experiments time series correlation
another block deals with the impact of seasonality when computing the trend of a timeline

Acknowledgments:

done with D3 v3.5.5
blockbuilder.org

<!DOCTYPE html>
<meta charset="utf-8">
<style>
  
  body {
    position: relative;
    background-color: #ddd;
    margin: auto;
  }
  
  #under-construction {
    display: none;
    position: absolute;
    top: 200px;
    left: 300px;
    font-size: 40px;
  }
  
  .controls {
    position: absolute;
    font: 11px arial;
  }
  #controls1 {
    top: 300px;
    left: 10px;
  }
  #controls2 {
    top: 300px;
    left: 450px;
  }
  #controls3 {
    top: 300px;
    right: 10px;
    text-align: right;
  }
  #controls4 {
    top: 5px;
    right: 10px;
    text-align: right;
  }
  
  .viz {
    position: absolute;
    background-color: white;
    border-radius: 10px;
    left: 5px;
  }
  .viz#timelines {
    top: 5px;
  }
  .viz#correlation {
    top: 355px;
  }
  
  .flow {
    position: absolute;
    font-size: 30px;
    color: darkgrey;
    top: 320px;
    right: 435px;
  }
  
  .axis path,
  .axis line {
    fill: none;
    stroke: black;
    shape-rendering: crispEdges;
  }
  .axis text {
    font-family: sans-serif;
    font-size: 11px;
  }
  .grid>line, .grid>.intersect {
    fill: none;
    stroke: #ddd;
    shape-rendering: crispEdges;
    vector-effect: non-scaling-stroke;
  }
  .legend {
    font-size: 12px;
  }
  
  .dot {
    fill: steelblue;
    stroke: white;
    stroke-width: 3px;
  }
  .dot.original {
    //fill: lightsteelblue;
    fill-opacity: 0.3;
  }
  .dot.draggable:hover, .dot.dragging {
    fill: pink;
    cursor: ns-resize;
  }
	
  .timeline {
    fill: none;
    stroke: lightsteelblue;
    stroke-width: 2px;
  }
  .timeline.original {
    stroke-opacity: 0.3;
  }
  .timeline.draggable:hover, .timeline.dragging {
  	stroke: pink;
    opacity: 1;
    cursor: ns-resize;
  }
  
  .correlation-bar {
    fill: grey;
  }
</style>
<body>
  <div id="timelines" class="viz">
    <div id="controls1" class="controls">
      update time serie with a seasonality's length of <a href="#" onclick="updateSeasonalityPeriod(2);">2</a> /<a href="#" onclick="updateSeasonalityPeriod(3);">3</a> / <a href="#" onclick="updateSeasonalityPeriod(4);">4</a> / <a href="#" onclick="updateSeasonalityPeriod(5);">5</a> / <a href="#" onclick="updateSeasonalityPeriod(6);">6</a> / <a href="#" onclick="updateSeasonalityPeriod(7);">7</a> / <a href="#" onclick="updateSeasonalityPeriod(8);">8</a> / <a href="#" onclick="updateSeasonalityPeriod(9);">9</a> / <a href="#" onclick="updateSeasonalityPeriod(10);">10</a> periods
    </div>
    <div id="controls2" class="controls">
      <a href="#" onclick="changeSeasonOrderOfMagnitude(1.6);">increase</a> / <a href="#"  onclick="changeSeasonOrderOfMagnitude(0.675);">decrease</a> seasonality's order of magnitude
    </div>
    <div id="controls3" class="controls">
      <a href="#" onclick="changeTrend(1.6);">increase</a> / <a href="#"  onclick="changeTrend(0.675);">decrease</a> timeline's trend
    </div>
  </div>
  <div id="correlation" class="viz">
  	<div id="controls4" class="controls">
      <label><input type='checkbox' onclick='handleDetrend(this);'>use detrended time serie</label>
    </div>
  </div>
  <div id="flow" class="flow">&#8615;</div>
  <div id="under-construction">
    UNDER CONSTRUCTION
  </div>
<script src="https://d3js.org/d3.v3.min.js"></script>
<script>
  var timeSerie = [];
  var timeSerieForAutocor = [];
  var randomness = [];
  var currentSeasonLength = 4;
  var currentSeasonOrderOfMagnitude = 8;
  var currentTrend = 1.4;
  var shouldDetrend = false;
  
  var WITH_TRANSITION = true;
  var WITHOUT_TRANSITION = false;
	var duration = 500;
  var NEW_RANDOMNESS = true;
  var PRESERVE_RANDOMNESS = false;
  
  var timelineVizDimension = {width: 960, height:340},
      correlationVizDimension = {width: 960, height:160},
      vizMargin = 5,
      flowWidth = 20
  		legendHeight = 20,
  		xAxisLabelHeight = 10,
  		yAxisLabelWidth = 10,
      correlationLeftShift = 110,
  		margin = {top: 20, right: 20, bottom: 20, left: 20},
      timelineSvgWidth = timelineVizDimension.width - 2*vizMargin,
      timelineSvgHeight = timelineVizDimension.height - 2*vizMargin - flowWidth/2,
      correlationSvgWidth = correlationVizDimension.width - 2*vizMargin,
      correlationSvgHeight = correlationVizDimension.height - 2*vizMargin - flowWidth/2,
      timelineWidth = timelineSvgWidth - margin.left - margin.right - yAxisLabelWidth,
      timelineHeight = timelineSvgHeight - margin.top - margin.bottom - xAxisLabelHeight - 1.5*legendHeight,
      correlationWidth = correlationSvgWidth - margin.left - margin.right - yAxisLabelWidth - correlationLeftShift,
      correlationHeight = correlationSvgHeight - margin.top - margin.bottom;
  var drag = d3.behavior.drag()
      .origin(function(d) { return d; })
      .on("dragstart", dragStarted)
      .on("drag", dragged1)
      .on("dragend", dragEnded);
  
  var x = d3.scale.linear()
				.domain([0, 20])
				.range([0, timelineWidth]);
  var y = d3.scale.linear()
  			.domain([0, 50])
				.range([0, -timelineHeight]);
  
  var xCorrelation = d3.scale.linear()
  			.domain([1, 11])
				.range([0, correlationWidth]);
  var yCorrelation = d3.scale.linear()
  			.domain([-1, 1])
				.range([0, -correlationHeight]);
  var xAxisDef = d3.svg.axis()
  	.scale(x)
  	.ticks(20);
  var yAxisDef = d3.svg.axis()
  	.scale(y)
  	.orient("left");
  
  var xAxisCorrelationDef = d3.svg.axis()
  	.scale(xCorrelation)
  	.tickValues([2,3,4,5,6,7,8,9,10])
  	.tickFormat("");
	var yAxisCorrelationDef = d3.svg.axis()
  	.scale(yCorrelation)
  	.ticks(5)
  	.orient("left");;
  
  var svg = d3.select("#timelines").append("svg")
      .attr("width", timelineSvgWidth)
      .attr("height", timelineSvgHeight)
  	.append("g")
  		.attr("transform", "translate(" + [margin.left, margin.top] + ")");
  var container = svg.append("g")
  	.attr("id", "graph")
  	.attr("transform", "translate(" + [yAxisLabelWidth, timelineHeight] + ")");
	
  var grid = container.append("g")
      .attr("class", "grid");
  var intersects = [];
  d3.range(1, x.invert(timelineWidth)+1, 1).forEach(function(a) { d3.range(5, y.invert(-timelineHeight)+5,5).forEach(function(b) { intersects.push([a,b])})});
  grid.selectAll(".intersect")
  	.data(intersects)
  	.enter().append("path")
			.classed("intersect", true)
  		.attr("d", function(d) { return "M"+[x(d[0])-1,y(d[1])]+"h3M"+[x(d[0]),y(d[1])-1]+"v3"});
  
  container.append("text")
  	.attr("transform", "translate(" + [timelineWidth/2, -timelineHeight] + ")")
  	.attr("text-anchor", "middle")
    .text("Timeline");
  
  container.append("g")
    .attr("class", "axis x")
    .call(xAxisDef)
    .append("text")
      .attr("x", timelineWidth)
      .attr("y", -6)
      .style("text-anchor", "end")
      .text("Time");
  
  container.append("g")
    .attr("class", "axis y")
    .call(yAxisDef)
    .append("text")
  		.attr("transform", "rotate(-90)")
      .attr("x", timelineHeight)
      .attr("y", 16)
      .style("text-anchor", "end")
      .text("Amount");
  
  var timeline = container.append("path")
  	.datum(1)
  	.classed("timeline original", true)
  	.attr("d", line)
  var timelineForAutocor = container.append("path")
  	.datum(2)
  	.classed("timeline for-autocor", true)
  	.attr("d", line)
  
  var dotContainer = container.append("g")
        .classed("dots", true);
	
  svg = d3.select("#correlation").append("svg")
      .attr("width", correlationSvgWidth)
      .attr("height", correlationSvgHeight)
  	.append("g")
  		.attr("transform", "translate(" + [margin.left, margin.top] + ")");
  
  container = svg.append("g")
  	.attr("id", "graph correlation")
  	.attr("transform", "translate(" + [yAxisLabelWidth + correlationLeftShift, correlationHeight] + ")");
  
  var correlationTitle = container.append("text")
  	.attr("transform", "translate(" + [correlationWidth/2, -correlationHeight] + ")")
  	.attr("text-anchor", "middle")
    .text("Correlogram");
  
  grid = container.append("g")
      .attr("class", "grid");
  intersects = [];
  d3.range(2, xCorrelation.invert(correlationWidth), 1).forEach(function(a) { d3.range(-1, yCorrelation.invert(-correlationHeight)+0.5,0.5).forEach(function(b) { intersects.push([a,b])})});
  grid.selectAll(".intersect")
  	.data(intersects)
  	.enter().append("path")
			.classed("intersect", true)
  		.attr("d", function(d) { return "M"+[xCorrelation(d[0])-1,yCorrelation(d[1])]+"h3M"+[xCorrelation(d[0]),yCorrelation(d[1])-1]+"v3"});
  
  container.append("g")
    .attr("class", "axis y")
    .call(yAxisCorrelationDef);
  var xAxisContainer = container.append("g")
    .attr("class", "axis x");
  xAxisContainer.append("line")
  	.attr("x1", 0)
	  .attr("y1", yCorrelation(0))
	  .attr("x2", correlationWidth)
	  .attr("y2", yCorrelation(0))
  xAxisContainer.append("text")
  	.attr("transform", "translate("+[xCorrelation(2)-40,yCorrelation(-1)+15]+")")
  	.attr("text-anchor", "end")
    .text("Coefficient of autocorrelation for ... ")
  var xTicks = container.select(".axis.x").selectAll(".tick-label")
    .data([2,3,4,5,6,7,8,9,10])
  	.enter()
  		.append("g")
  		.classed("tick-label", true)
  		.attr("transform", function(d) { return "translate("+[xCorrelation(d),0]+")"});
  xTicks.append("line")
  	.attr("x1", 0)
	  .attr("y1", yCorrelation(0)-3)
	  .attr("x2", 0)
	  .attr("y2", yCorrelation(0)+4)
  xTicks.append("text")
  	.attr("transform", "translate("+[0,yCorrelation(-1)+15]+")")
  	.attr("text-anchor", "middle")
    .text(function(d) { return d+"-periods lag"});
  
  var barContainer = container.append("g")
  	.attr("id", "bar-conatiner");
  
  d3.csv("timeserie.csv", dottype, function(error, dots) {
    updateTimeSeries(PRESERVE_RANDOMNESS);
    
    updateDots(WITHOUT_TRANSITION);
    updateTimelines(WITHOUT_TRANSITION);
    updateAutocorrelations(WITHOUT_TRANSITION);
  });
  
  function dottype(d) {
    d.x = +d.x;
    d.y = +d.y+(+d.random);
    timeSerie.push(d);
    randomness.push(+d.random);
    return d;
  }
  
  var line = d3.svg.line()
    .x(function(d) { return x(d.x); })
    .y(function(d) { return y(d.y); });
  
  function updateSeasonalityPeriod(newSeasonLength) {
    currentSeasonLength = newSeasonLength;
    updateTimeSeries(NEW_RANDOMNESS);
    
    updateDots(WITH_TRANSITION);
    updateTimelines(WITH_TRANSITION);
    updateAutocorrelations(WITH_TRANSITION);
  }
  
  function changeTrend(scale) {
    currentTrend *= scale;
    updateTimeSeries(PRESERVE_RANDOMNESS);
    
    updateDots(WITH_TRANSITION);
    updateTimelines(WITH_TRANSITION);
    updateAutocorrelations(WITH_TRANSITION);
  }
  
  function changeSeasonOrderOfMagnitude(scale) {
    currentSeasonOrderOfMagnitude *= scale;
    updateTimeSeries(PRESERVE_RANDOMNESS);
    
    updateDots(WITH_TRANSITION);
    updateTimelines(WITH_TRANSITION);
    updateAutocorrelations(WITH_TRANSITION);
  }
  
  function handleDetrend(cb) {
    shouldDetrend = cb.checked;
    updateTimeSeries(PRESERVE_RANDOMNESS);
    
    updateDots(WITH_TRANSITION);
    updateTimelines(WITH_TRANSITION);
    updateAutocorrelations(WITH_TRANSITION);
  }
  
  function updateTimeSeries(reRandom) {
    timeSerieForAutocor = [];
    var intercept = 10;
    
    var expected;
    timeSerie.forEach(function(d,i) {
      expected = intercept;
      switch (i%currentSeasonLength) {
        case 0: expected -= currentSeasonOrderOfMagnitude; break;
        case (currentSeasonLength-1): expected += currentSeasonOrderOfMagnitude; break;
      }
      
      if (reRandom) {
        randomness[i] = 3*(Math.random()-0.5);
      }
      d.y = expected + currentTrend*d.x + randomness[i];
      
      timeSerieForAutocor.push({
        x: d.x,
        y: shouldDetrend? (d.y) : expected
      })
    })
    updateTimeSerieForAutocor()
  }
  
  function updateTimeSerieForAutocor() {
    timeSerieForAutocor = shouldDetrend? computeDetrendedSerie(timeSerie) : timeSerie;
  }
  
  function computeDetrendedSerie(serie) {
    var serieLength = serie.length;
    var periodMetrics = computePeriodMetrics(serieLength);
    var valueMetrics = computeSerieMetrics(serie.map(function(d) { return d.y; }))
      
    var correlationMetrics = computeCorrelationMetrics(periodMetrics,
                                          valueMetrics,
                                          serie.map(function(d) { return d.x; }),
                                          serie.map(function(d) { return d.y; })
                                         )
    
    var detrended = [];
    serie.forEach(function(d){
      detrended.push({ x: d.x, y: d.y-d.x*correlationMetrics.correlatedTrend });
    });
    
    return detrended;
  }
  
  function computePeriodMetrics(periodCount) {
    // compute metrics for the serie representing Time [1,2,...,periodCount]
  	var periodSum = periodCount*(periodCount+1)/2;
    var squarePeriodSum = periodCount*(periodCount+1)*(2*periodCount+1)/6;
    var mean = (periodCount+1)/2;
    var variance = squarePeriodSum/periodCount - Math.pow(mean, 2);
    var standardDeviation = Math.pow(variance, 0.5);
    
    return {
      countSum: periodSum,
      squareCountSum: squarePeriodSum,
      mean: mean,
      variance: variance,
      standardDeviation: standardDeviation
    }
  }
  
  function computeSerieMetrics(serie) {
    // compute metrics for the serie representing amounts
    var serieLength = serie.length;
    var valueSum = 0;
    var squareValueSum = 0;

serie.forEach(function(d) {
      valueSum += d;
    	squareValueSum += Math.pow(d, 2);
    });

https://d3js.org/d3.v3.min.js