Javascript / D3.js - draw large data set - improve the speed of zoom and pan in svg chart ploted by d3.js

Question

Edit

Just found the post plotting 50 million points with d3.js.

Sluggish interaction with zoom and pan are due to too many elements in the svg. The key is to use hierarchical levels of detail, just like the image pyramid. , to limit the maximum elements in svg.

Original post

I am trying to read some data points from csv/excel file and plot them using d3.js.

The data set contains 100,000s of rows, each row contains a time stamp and a value at that time.

Time stamp, pressure
12/17/2019 12:00:00 AM, 600

I followed this example to plot the time-pressure chart with zoom and pan.

There is no issue and worked perfectly.

One issue is that when working with large data set, say 500,000 of data points, the interaction with the chart is sluggish.

The chart with 500,000 data points shows an overall shape, and the details would only come up when zoomed in at large scale.

When zoomed in, all the data points are re-plotted and clipped out by the clip path. Would there be some room to improve the speed?

Updated Code

function draw(res){
        //clear the current content in the div
      document.getElementById("spectrum-fig").innerHTML = '';    

      var fullwidth = d3.select('#spectrum-fig').node().getBoundingClientRect().width;
      fullwidth = fullwidth < 500? 500:fullwidth;
      var fullheight = 500;
      var resLevelOne = getWindowed(res, 1);
      var resLevelTwo = getWindowed(res, 2);

      var designMax= getMaxPressureKPa();
      var resMax = getPsiTopTen(res);
      const SMYSKPa = getSMYSPressureKPa();
      const avePsi = getAvePsi(res);

      var psiRange = d3.max(res, d=>d.psi) - d3.min(res, d=>d.psi);
      var resSmallChart = getWindowed(res, 2);//
      //filtSpectrum(res, 0.05*psiRange); //0.05 magic numbers
      //var resSmallChart = res;
      //margin for focus chart, margin for small chart
      var margin = {left:50, right: 50, top: 30, bottom:170},
          margin2 = {left:50, right: 50, top: 360, bottom:30}, 
          width = fullwidth - margin.left - margin.right,
          height = fullheight - margin.top - margin.bottom,
          height2 = fullheight - margin2.top-margin2.bottom;
      
      //x, y, for big chart, x2, y2 for small chart
      var x = d3.scaleTime().domain(d3.extent(res, d => d.Time)).range([0, width]),
          x2 = d3.scaleTime().domain(d3.extent(res, d => d.Time)).range([0, width]),
          y  = d3.scaleLinear().domain([0, SMYSKPa]).range([height, 0]),
          y2 = d3.scaleLinear().domain([0, SMYSKPa]).range([height2, 0]);

      //clear the content in Spectrum-fig div before drawring
      //avoid multiple drawings;
      var xAxis =d3.axisBottom(x).tickFormat(d3.timeFormat("%m-%d")),
          xAxis2 = d3.axisBottom(x2).tickFormat(d3.timeFormat("%b")),
          yAxis = d3.axisLeft(y);

      var brush = d3.brushX()                   // Add the brush feature using the d3.brush function
          .extent( [ [0,0], [width,height2] ] )  // initialise the brush area: start at 0,0 and finishes at width,height: it means I select the whole graph area
          .on("brush end", brushed);               // trigger the brushed function 
        
      var zoom = d3.zoom()
          .scaleExtent([1, 100]) //defined the scale extend 
          .translateExtent([[0, 0], [width, height]])
          .extent([[0, 0], [width, height]])
          .on("zoom", zoomed); //at the zoom end trigger zoomed function
      
          
      //line for big chart line
      var line = d3.line()
                .x(function(d) { return x(d.Time) })
                .y(function(d) { return y(d.psi) });

      //line2 for small chart line
      var line2 = d3.line()
                  .x(function(d) { return x2(d.Time) })
                  .y(function(d) { return y2(d.psi) });

      var svg = d3.select("#spectrum-fig")
                  .append("svg")
                  .attr("width", fullwidth)
                  .attr("height", fullheight);
      
      svg.append("defs").append("clipPath")
            .attr("id", "clip")
        .append("rect")
            .attr("width", width)
            .attr("height", height);

      var focus = svg.append("g")
            .attr("class", "focus")
            .attr("transform", "translate(" + margin.left + "," + margin.top + ")");
              
      var context = svg.append("g")
            .attr("class", "context")
            .attr("transform", "translate(" + margin2.left + "," + margin2.top + ")");
      
      focus.append("g")
      .attr("class", "axis axis--x")
      .attr("transform", "translate (0," + height +")")
      .call(xAxis);

      focus.append("g")
      .attr("class", "axis axis--y")
      .call(yAxis);

      focus.append("g")
      .attr("transform", "translate (" +  width + ", 0)")
      .call(d3.axisRight(y).tickFormat('').tickSize(0));

      focus.append("g")
      .attr("transform", "translate (0, 0)")
      .call(d3.axisTop(x).tickFormat('').tickSize(0));

      // Add the line
      focus.insert("path")
        //.datum(res)
        .attr("class", "line")  // I add the class line to be able to modify this line later on.
        .attr("fill", "none")
        .attr('clip-path', 'url(#clip)')
        .attr("stroke", "steelblue")
        .attr("stroke-width", 1.5)
        .attr("d", line(resLevelTwo));

      context.insert("path")
        //.datum(resSmallChart)
        .attr("class", "line")
        .attr("stroke", "steelblue")
        .attr("stroke-width", 1.5)
        .attr("fill", "none")
        .attr("d", line2(resSmallChart));
  
      context.append("g")
        .attr("class", "axis axis--x")
        .attr("transform", "translate(0," + height2 + ")")
        .call(xAxis2);
  
      context.append("g")
        .attr("class", "brush")
        .call(brush)
        .call(brush.move, x.range());
  
      svg.append("rect")
        .attr("class", "zoom")
        .attr('fill', 'none')
        .attr('cursor', 'move')
        .attr('pointer-events', 'all')
        .attr("width", width)
        .attr("height", height)
        .attr("transform", "translate(" + margin.left + "," + margin.top + ")")
        .call(zoom);

      function getWindowed(arr, level){ 
        var windowed = new Array();
        var arrLength = arr.length;
        var windowSize =Math.pow(16, level); //set the window size
        for(let i = 0; i * windowSize < arrLength; i++ ){ //each to be the window size
          let startIndex = i * windowSize;
          let endIndex = (i+1) * windowSize;
          endIndex = endIndex >= arrLength ? arrLength-1 : endIndex;
          let localExtreme = findLocalExtreme(arr.slice(startIndex, endIndex));
          if (localExtreme.Max.Time.getTime() === localExtreme.Min.Time.getTime()){ //anything include = need getTime
            windowed.push(localExtreme.Max)
          }else if(localExtreme.Max.Time  < localExtreme.Min.Time){
            windowed.push(localExtreme.Max);
            windowed.push(localExtreme.Min);
          }else{
            windowed.push(localExtreme.Min);
            windowed.push(localExtreme.Max);
          }
        }
        let firstElement = {...arr[0]};
        let lastElement = {...arr[arr.length-1]};
        if(firstElement.Time.getTime() != windowed[0].Time.getTime()){ //insert to the position zero
          windowed.unshift(firstElement);
        }
        if(lastElement.Time.getTime() != windowed[windowed.length-1].Time.getTime()){
          windowed.push(lastElement);
        }//insert to the end last member;
        return windowed;
      }

      function findLocalExtreme(slicedArr){
        if(slicedArr === undefined || slicedArr.length == 0){
          throw 'error: no array members';
        } 
        let slicedArrLength = slicedArr.length;
        let tempMax = {...slicedArr[0]};
        let tempMin = {...slicedArr[0]};
        if(slicedArrLength === 1){
          return {
            Max: tempMax,
            Min: tempMin
          }
        }
        for (let i = 1; i < slicedArrLength; i++){
          if (slicedArr[i].psi > tempMax.psi){
            tempMax = {...slicedArr[i]};
          }
          if (slicedArr[i].psi < tempMin.psi){
            tempMin = {...slicedArr[i]};
          }
        }
        return {
          Max: tempMax,
          Min: tempMin
        }
      }

      function getDataToDraw(timeRange){ //timeRange [0,1] , [startTime, endTime]
        const bisect = d3.bisector(d => d.Time).left;
        const startIndex = bisect(res, timeRange[0]);
        const endIndex = bisect(res, timeRange[1]);
        const numberInOriginal = endIndex-startIndex+1;
        const windowSize =16;
        const maxNumber = 8000;
        let level = Math.ceil(Math.log(numberInOriginal/maxNumber ) / Math.log(windowSize));
        if(level <=0 ) level =0;
        console.log(endIndex, startIndex, endIndex-startIndex+1, level);
        if(level === 0){
          return res.slice(startIndex, endIndex);
        }if(level === 1){
          let start_i = bisect(resLevelOne, timeRange[0]);
          let end_i =bisect(resLevelOne, timeRange[1]);
          return resLevelOne.slice(start_i, end_i);
        }else { //if level 2 or higher, never happen
          let start_i = bisect(resLevelTwo, timeRange[0]);
          let end_i =bisect(resLevelTwo, timeRange[1]);
          return resLevelTwo.slice(start_i, end_i);
        }
      }

      function brushed() {
          if (d3.event.sourceEvent && d3.event.sourceEvent.type === "zoom") return; // ignore brush-by-zoom
          var s = d3.event.selection || x2.range();
          x.domain(s.map(x2.invert, x2));
          focus.select(".line").attr("d", line(getDataToDraw(x.domain())));
          focus.select(".axis--x").call(xAxis);
          svg.select(".zoom").call(zoom.transform, d3.zoomIdentity
              .scale(width / (s[1] - s[0]))
              .translate(-s[0], 0));
      }
        
      function zoomed() {
          if (d3.event.sourceEvent && d3.event.sourceEvent.type === "brush") return; // ignore zoom-by-brush
          var t = d3.event.transform;
          //console.log(t);
          x.domain(t.rescaleX(x2).domain());
          focus.select(".line").attr("d", line(getDataToDraw(t.rescaleX(x2).domain())));
          focus.select(".axis--x").call(xAxis);
          context.select(".brush").call(brush.move, x.range().map(t.invertX, t));
      }
}

Answer 1

Here's my thoughts.

Re-plot seems a must-have, because how could you expect to have same position when you zoom in the points ?

However there's some frequency of the replot you can control. For example, people use debounce to decrease the number of firing below 50ms during any event (ex. pan especailly). Debounce is a general solution, you can check lodash library for some implementation.

  .on("zoom", debounced(zoomed)) // lower the chance if you get 5 calls under 500ms

Also if there's any animation involved, you can defer the animation until the last stage of the zoom (or pan), which is similar to debounce concept. Or just simply disable animation.

Note: React does support another mode called concurrent, it's not enabled by default, not yet. However what it does is that, assuming each plot is captured by a small component, and it spends 1ms for render, then after it renders 16 components, it believes it spend too much time in this rendering, and give the response back to the browser to handle other things, ex. user input etc. This way you can start to scroll your page or move your mouse. And in the next cycle it can pick up the next 16 components. Assuming you have 1000 components, it'll take couple of cycles before it can finish all the rendering. And if you zooms again in the middle, it'll skip the first 16 components and move to the new render all over again. Hope you get the idea. It might help your problem with the latest React 18.

Answer 2

Refer to the post plotting 50 million points with d3.js.

Sluggish interaction with zoom and pan are due to too many elements in the svg. The key is to use hierarchical levels of detail, to limit the maximum elements in svg.