Get all child spans associated with OpenTelemetry trace

Question

I am able to get the current active context, span, and associated trace ID with the following code. Is there anyway to get the rest of the child spans that are associated with the trace (go from traceId to all of the associated spans)?

const opentelemetryApi = require('@opentelemetry/api');

const activeCtx = opentelemetryApi.context.active();
const span = opentelemetryApi.trace.getSpan(activeCtx);
const traceId = span.spanContext().traceId;

Spans have a child -> parent relationship and define the traceId and parentSpanId that they belong to. But if I have the root span, I don't see a way to navigate downwards for the child spans. Is there a way to navigate the DAG of spans in a trace?

Can I get a list of all of the spans? Maybe it's possible to setup a passthrough collector or processor to capture the spans that way? Maybe a way to listen just for spans from a certain traceId?

Related to Is there a way to get the full trace of a request given a starting point from anywhere in the lifecycle of the trace? although there is no practical example of how to get your hands on a trace and all of the spans in that question.

---

You can see my full OpenTelemetry tracing.js setup in https://github.com/matrix-org/matrix-public-archive/pull/27

My goal is to add a performance bar to the top of the page served to the client that shows off the slow API requests that happened in the backend. I would like to pull and re-use the data from the OpenTelemetry spans/traces.

require('./tracing.js');

const opentelemetryApi = require('@opentelemetry/api');
const express = require('express');

// Instrumented by `@opentelemetry/instrumentation-express`
const app = express();

app.get('/test', (req, res) => {
  // Make some external API HTTP requests which create child spans thanks to `@opentelemetry/instrumentation-http`
  const res = await fetch('https://matrix-client.matrix.org/_matrix/client/versions');

  const activeCtx = opentelemetryApi.context.active();
  const span = opentelemetryApi.trace.getSpan(activeCtx);
  const traceId = span.spanContext().traceId;
  // TODO: Find all spans associated with the `traceId`

  res.send(`<section><h1>Performance bar</h1> <p>(Trace data)</p></section> other page HTML`);
});

app.listen(3000);

Answer 1

I solved this by making a span processor that captures all of spans for a given trace. And then an Express middleware to to tell the span processor which traceId to keep track of during the lifetime of the request.

Would be nice if this kind of functionality was built-in though. Seems like a sane default to keep around the spans for the trace that corresponds to the current active context. I've created an issue to propose making this possible, https://github.com/open-telemetry/opentelemetry-js-api/issues/167

I've summarized the new moving pieces but you can see my full result in https://github.com/matrix-org/matrix-public-archive/pull/27

capture-span-processor.js

const { suppressTracing } = require('@opentelemetry/core');
const { context } = require('@opentelemetry/api');

// 1. Keeps track of all spans for a given trace after calling
//    `trackSpansInTrace(traceId)` (call this in a middleware before any other
//    routes).
// 2. Then during the request, you can see all spans for a given trace with
//    `getSpansInTrace(traceId)`.
// 3. Don't forget to clean up with `dropSpansInTrace(traceId)` after you're
//    done with the spans (should be done in the `res.on('finish', ...)`
//    callback).
class CaptureSpanProcessor {
  // Map from traceId to spans in the trace
  traceMap = {};

  // We capture when the span starts so that we get any ongoing spans if the
  // request times out and we want to show what it was stuck on.
  onStart(span /*, ctx*/) {
    // prevent downstream exporter calls from generating spans
    context.with(suppressTracing(context.active()), () => {
      const traceIdsToTrack = Object.keys(this.traceMap);

      const traceId = span.spanContext().traceId;

      if (traceIdsToTrack.includes(traceId)) {
        this.traceMap[traceId].push(span);
      }
    });
  }

  onEnd(/*span*/) {
    /* noop */
  }

  shutdown() {
    /* noop */
    return Promise.resolve();
  }
  forceFlush() {
    /* noop */
    return Promise.resolve();
  }

  // Get all spans for a given trace.
  getSpansInTrace(traceId) {
    return this.traceMap[traceId];
  }

  // Keeps track of all spans for a given trace after calling
  // `trackSpansInTrace(traceId)` (call this in a middleware before any other
  // routes).
  trackSpansInTrace(traceId) {
    this.traceMap[traceId] = [];
  }

  // Don't forget to clean up with `dropSpansInTrace(traceId)` after you're done
  // with the spans (should be done in the `res.on('finish', ...)` callback).
  //
  // alias: Dispose
  dropSpansInTrace(traceId) {
    delete this.traceMap[traceId];
  }
}

module.exports = CaptureSpanProcessor;

tracing.js: Add the captureSpanProcessor to the provider so all of the spans are available to process.

// ...

// Add the capture span processor
const captureSpanProcessor = new CaptureSpanProcessor();
provider.addSpanProcessor(captureSpanProcessor);

// ...

module.exports = {
  captureSpanProcessor
};

app.js: Setup the tracing middleware so captureSpanProcessor knows what to track.

require('./tracing.js');

const opentelemetryApi = require('@opentelemetry/api');
const express = require('express');
const { captureSpanProcessor } = require('./tracing');

function getActiveTraceId() {
  const activeCtx = opentelemetryApi.context.active();
  const span = opentelemetryApi.trace.getSpan(activeCtx);
  const traceId = span.spanContext().traceId;
  return traceId;
}

function handleTracingMiddleware(req, res, next) {
  const traceId = getActiveTraceId();

  // Add the OpenTelemetry trace ID to the `X-Trace-Id` response header so
  // we can cross-reference. We can use this to lookup the request in
  // Jaeger.
  res.set('X-Trace-Id', traceId);

  // Start keeping track of all of spans that happen during the request
  captureSpanProcessor.trackSpansInTrace(traceId);

  // Cleanup after the request is done
  res.on('finish', function () {
    captureSpanProcessor.dropSpansInTrace(traceId);
  });

  next();
}

// Instrumented by `@opentelemetry/instrumentation-express`
const app = express();

app.use(handleTracingMiddleware);

app.get('/test', (req, res) => {
  // Make some external API HTTP requests which create child spans thanks to `@opentelemetry/instrumentation-http`
  const res = await fetch('https://matrix-client.matrix.org/_matrix/client/versions');

  const traceId = getActiveTraceId();
  const spans = captureSpanProcessor.getSpansInTrace(traceId)

  res.send(`<section><h1>Performance bar</h1> <p>${spans}</p></section> other page HTML`);
});

app.listen(3000);