Trying to load JSON into d3 treemap without using a GET

Question

I am using http://bost.ocks.org/mike/treemap/ to attempt to incorporate a D3 treemap into Splunk. However, it errors on the d3.JSON("flare.json") as it can't find the file. I have tried putting the JSON array right into the js and calling root = JSON.parse(myjson), but then then it arrays with unexpected character JSON.parse. If you look at the js from Bostick's page, you can see that I can't just remove the d3.JSON, because it calls back to the functions that actually render the treemap.

Do you have any ideas on how I can fix this?

renderResults: function($super, results) {

    if(!results) {
    this.resultsContainer.html('No content available.');
        return;
    }
    var margin = {top: 20, right: 0, bottom: 0, left: 0},
        width = 960,
        height = 500 - margin.top - margin.bottom,
        formatNumber = d3.format(",d"),
        transitioning;

    var x = d3.scale.linear()
        .domain([0, width])
        .range([0, width]);

    var y = d3.scale.linear()
        .domain([0, height])
        .range([0, height]);

    var treemap = d3.layout.treemap()
        .children(function(d, depth) { return depth ? null : d._children; })
        .sort(function(a, b) { return a.value - b.value; })
        .ratio(height / width * 0.5 * (1 + Math.sqrt(5)))
        .round(false);

    var svg = d3.select("#chart").append("svg")
        .attr("width", width + margin.left + margin.right)
        .attr("height", height + margin.bottom + margin.top)
        .style("margin-left", -margin.left + "px")
        .style("margin.right", -margin.right + "px")
      .append("g")
        .attr("transform", "translate(" + margin.left + "," + margin.top + ")")
        .style("shape-rendering", "crispEdges");

    var grandparent = svg.append("g")
        .attr("class", "grandparent");

    grandparent.append("rect")
        .attr("y", -margin.top)
        .attr("width", width)
        .attr("height", margin.top);

    grandparent.append("text")
        .attr("x", 6)
        .attr("y", 6 - margin.top)
        .attr("dy", ".75em");

    var myjson = not including the actual array to save space

            root = JSON.parse(myjson); 




    d3.json("flare.json", function(root) { 
      initialize(root);
      accumulate(root);
      layout(root);
      display(root); 

      function initialize(root) {
        root.x = root.y = 0;
        root.dx = width;
        root.dy = height;
        root.depth = 0;
      } 

      // Aggregate the values for internal nodes. This is normally done by the
      // treemap layout, but not here because of our custom implementation.
      // We also take a snapshot of the original children (_children) to avoid
      // the children being overwritten when when layout is computed.
      function accumulate(d) {
        return (d._children = d.children)
            ? d.value = d.children.reduce(function(p, v) { return p + accumulate(v); }, 0)
            : d.value;
      }

      // Compute the treemap layout recursively such that each group of siblings
      // uses the same size (1×1) rather than the dimensions of the parent cell.
      // This optimizes the layout for the current zoom state. Note that a wrapper
      // object is created for the parent node for each group of siblings so that
      // the parent’s dimensions are not discarded as we recurse. Since each group
      // of sibling was laid out in 1×1, we must rescale to fit using absolute
      // coordinates. This lets us use a viewport to zoom.
      function layout(d) {
        if (d._children) {
          treemap.nodes({_children: d._children});
          d._children.forEach(function(c) {
            c.x = d.x + c.x * d.dx;
            c.y = d.y + c.y * d.dy;
            c.dx *= d.dx;
            c.dy *= d.dy;
            c.parent = d;
            layout(c);
          });
        }
      }

      function display(d) {
        grandparent
            .datum(d.parent)
            .on("click", transition)
          .select("text")
            .text(name(d));

        var g1 = svg.insert("g", ".grandparent")
            .datum(d)
            .attr("class", "depth");

        var g = g1.selectAll("g")
            .data(d._children)
          .enter().append("g");

        g.filter(function(d) { return d._children; })
            .classed("children", true)
            .on("click", transition);

        g.selectAll(".child")
            .data(function(d) { return d._children || [d]; })
          .enter().append("rect")
            .attr("class", "child")
            .call(rect);

        g.append("rect")
            .attr("class", "parent")
            .call(rect)
          .append("title")
            .text(function(d) { return formatNumber(d.value); });

        g.append("text")
            .attr("dy", ".75em")
            .text(function(d) { return d.name; })
            .call(text);

        function transition(d) {
          if (transitioning || !d) return;
          transitioning = true;

          var g2 = display(d),
              t1 = g1.transition().duration(750),
              t2 = g2.transition().duration(750);

          // Update the domain only after entering new elements.
          x.domain([d.x, d.x + d.dx]);
          y.domain([d.y, d.y + d.dy]);

          // Enable anti-aliasing during the transition.
          svg.style("shape-rendering", null);

          // Draw child nodes on top of parent nodes.
          svg.selectAll(".depth").sort(function(a, b) { return a.depth - b.depth; });

          // Fade-in entering text.
          g2.selectAll("text").style("fill-opacity", 0);

          // Transition to the new view.
          t1.selectAll("text").call(text).style("fill-opacity", 0);
          t2.selectAll("text").call(text).style("fill-opacity", 1);
          t1.selectAll("rect").call(rect);
          t2.selectAll("rect").call(rect);

          // Remove the old node when the transition is finished.
          t1.remove().each("end", function() {
            svg.style("shape-rendering", "crispEdges");
            transitioning = false;
          });
        }

        return g;
      }

      function text(text) {
        text.attr("x", function(d) { return x(d.x) + 6; })
            .attr("y", function(d) { return y(d.y) + 6; });
      }

      function rect(rect) {
        rect.attr("x", function(d) { return x(d.x); })
            .attr("y", function(d) { return y(d.y); })
            .attr("width", function(d) { return x(d.x + d.dx) - x(d.x); })
            .attr("height", function(d) { return y(d.y + d.dy) - y(d.y); });
      }

      function name(d) {
        return d.parent
            ? name(d.parent) + "." + d.name
            : d.name;
      }
    };

} 

I am still not 100% sure how these two even are able to interact. I have a little JavaScript experience, but no Python experience. The way Splunk integrates all these scripts together is baffling.

import cherrypy
import controllers.module as module

import splunk, splunk.search, splunk.util, splunk.entity
import json
from splunk.appserver.mrsparkle.lib import jsonresponse
import lib.util as util
import lib.i18n as i18n

import logging
logger = logging.getLogger('splunk.module.TreeMap1')

import math
import cgi

class TreeMap1(module.ModuleHandler):

def generateResults(self, host_app, client_app, sid, count=1000, 
        offset=0, entity_name='results'):

    count = max(int(count), 0)
    offset = max(int(offset), 0)
    if not sid:
        raise Exception('TreeMap1.generateResults - sid not passed!')

    try:
        job = splunk.search.getJob(sid)
    except splunk.ResourceNotFound, e:
        logger.error('TreeMap could not find job %s. Exception: %s' % (sid, e))
        return _('<p class="resultStatusMessage">Could not get search data.</p>')

    dataset = getattr(job, entity_name)[offset: offset+count]

    outputJSON = {}
    for i, result in enumerate(dataset):
        tdict = {}
        tdict[str(result.get('itemName', None))] = str(result.get('totalCPU', None))
        name = str(result.get('itemCat', None))
        if name not in outputJSON:
            outputJSON[name] = dict()
        outputJSON[name].update(tdict)

    cherrypy.response.headers['Content-Type'] = 'text/json'
    return json.dumps(outputJSON, sort_keys=True)

def render_json(self, response_data, set_mime='text/json'):
    cherrypy.response.headers['Content-Type'] = set_mime

    if isinstance(response_data, jsonresponse.JsonResponse):
        response = response_data.toJson().replace("</", "<\\/")
    else:
        response = json.dumps(response_data).replace("</", "<\\/")

    return ' ' * 256  + '\n' + response

Answer 1

D3's d3.JSON(url, ...) literally performs a GET request and attempts to parse the response as JSON*; it is similar to jQuery's $.getJSON(url, ...). If you already have the JSON necessary to construct the tree map, just ignore the call altogether and go straight to the callback. If you already have an array/object, you don't need JSON.parse; JSON.parse turns a valid JSON string into an array/object.

*See the documentation on d3.json for more...

Along the same lines, if you already have the data, you can just use:

  ...

  var myJSON = [ ... ] //The actual array you're loading.
  initialize(myJSON);
  accumulate(myJSON);
  layout(myJSON);
  display(myJSON);

  ... 

However, notice what you're actually doing in the code above. You're passing an array into initialize -- which appears to want an object:

  function initialize(root) { //Root in this case is myVar -- an ARRAY.
    root.x = root.y = 0;  //Arrays don't have fields. This is BAD.
    root.dx = width;
    root.dy = height;
    root.depth = 0;
  } 

Thus, what you should do is figure out what type of structure myJSON should use.

---

If the error that you want to fix is actually that the "flare.json" file can't be found, then your problem is actually server-side. How are you serving the flare.json file? Try accessing it in your browser to make sure it is located where you think it is.