CODEWITHSUNDEEP
javascripttensorflowtensorflow.js
Rylyn
Rylyn

Reputation: 333

Tensorflowjs - optimize.minimize cannot find a connection between any variable and the resulr of the loss function

I am trying to adapt Daniel Shiffman's linear regression exemple with tensorflowjs (https://www.youtube.com/watch?v=NZR-N_dhK2M) to use a polynomial equation instead of a linear equation. But I am struggling with the predict function. In my first version (see below), the optimize.minimze function does not find a link between my function and my tf.variables (that are stored in my coefficients array). On the other hand, my second version works, but have a memory leak that I have not been able to fix

Here's the non-working version : 

const WIDTH = 800, HEIGHT = 400;

const x_vals = [];
const y_vals = [];

let coefficients = [];
let degree = 5;

let lr = 0.2;
let optimizer = tf.train.adamax(lr);

function setup() {
  createCanvas(WIDTH, HEIGHT);
  background(0);
  initCoeffs();

  let up = false;
  for (let i = 0; i < WIDTH; i += WIDTH / 10) {
    x_vals.push(map(i, 0, WIDTH, -1, 1));
    y_vals.push(map((up) ? 0 : HEIGHT, 0, HEIGHT, -1, 1));
    up = !up;
  }
}

function initCoeffs() {
  for (let i = 0; i < degree; i++)
    coefficients.push(tf.variable(tf.scalar(random(1))));
}

function loss(pred, labels) {
  return tf.losses.meanSquaredError(labels, pred);
}

function predict(x) {
  const xs = tf.tensor1d(x);
  const ys = tf.variable(tf.zerosLike(xs));
  for (let i = 0; i < degree; i++) {
    const coef = coefficients[i];
    const pow_ts = tf.fill(xs.shape, degree - i);
    const sum = tf.add(ys, coef.mul(xs.pow(pow_ts)));
    ys.assign(sum);
  }
  ys.print();
  return ys;
}

function draw() {
  noFill();
  background(0);
  stroke(255);
  strokeWeight(8);
  for (let i = 0; i < x_vals.length; i++) {
    point(map(x_vals[i], -1, 1, 0, WIDTH), map(y_vals[i], -1, 1, 0, HEIGHT));
  }
  strokeWeight(4);

  if (x_vals.length > 0) {
    tf.tidy(() => {
      const ys = tf.tensor1d(y_vals);
      optimizer.minimize(() => loss(predict(x_vals), ys));
    });
  }


  let lineX = [];
  for (let x = -1.1; x <= 1.1; x += 0.01)
    lineX.push(x);
  const ys = tf.tidy(() => predict(lineX));
  let lineY = ys.dataSync();
  ys.dispose();

  beginShape();
  for (let i = 0; i < lineY.length; i++)
    curveVertex(map(lineX[i], -1, 1, 0, WIDTH), map(lineY[i], -1, 1, 0, HEIGHT));
  endShape();

  for (let i = 0; i < lineY.length; i++) {
    stroke(200, 100, 100);
    point(map(lineX[i], -1, 1, 0, WIDTH), map(lineY[i], -1, 1, 0, HEIGHT));
  }
}

function mousePressed() {
  x_vals.push(map(mouseX, 0, WIDTH, -1, 1));
  y_vals.push(map(mouseY, 0, HEIGHT, -1, 1));
}
<script src="https://cdnjs.cloudflare.com/ajax/libs/p5.js/0.5.7/p5.min.js"></script>
<script src="https://cdnjs.cloudflare.com/ajax/libs/tensorflow/0.11.2/tf.min.js"></script>

As you can see, I have this error in console :

Cannot find a connection between any variable and the result of the loss function y=f(x). Please make sure the operations that use variables are inside the function f passed to minimize().

But if I change my predict function like this, it works :

function predict(x) {
  const xs = tf.tensor1d(x);
  let ys = tf.variable(tf.zerosLike(xs));
  for (let i = 0; i < degree; i++) {
    const coef = coefficients[i];
    const pow_ts = tf.fill(xs.shape, degree - i);
    const sum = tf.add(ys, coef.mul(xs.pow(pow_ts)));
    ys = sum;
  }
  ys.print();
  return ys;
}

The problem is that this second version creates a memory leak as I use let to declare my ys tf.variable.

How can I fix my code to avoid a memory leak, without having the optimize.minimizer error ?

Thanks

Upvotes: 0

Views: 524

Answers (1)

Rylyn
Rylyn

Reputation: 333

I managed to get my code to work without memory leak by manually disposing ys variable before assigning it to the result of the tf.add function.

Here's my working solution 

const WIDTH = 800, HEIGHT = 400;

const x_vals = [];
const y_vals = [];

let coefficients = [];
let degree = 15;

let lr = 0.2;
let optimizer = tf.train.adamax(lr);

function setup() {
  createCanvas(WIDTH, HEIGHT);
  background(0);
  initCoeffs();

  let up = false;
  for (let i = 0; i < WIDTH; i += WIDTH / 10) {
    x_vals.push(map(i, 0, WIDTH, -1, 1));
    y_vals.push(map((up) ? 0 : HEIGHT, 0, HEIGHT, -1, 1));
    up = !up;
  }
}

function initCoeffs() {
  for (let i = 0; i < degree; i++)
    coefficients.push(tf.variable(tf.scalar(random(1))));
}

function loss(pred, labels) {
  return tf.losses.meanSquaredError(labels, pred);
}

function predict(x) {
  const xs = tf.tensor1d(x);
  let ys = tf.variable(tf.zerosLike(xs));
  for (let i = 0; i < degree; i++) {
    const coef = coefficients[i];
    const pow_ts = tf.fill(xs.shape, degree - i);
    const sum = tf.add(ys, coefficients[i].mul(xs.pow(pow_ts)));
    ys.dispose();
    ys = sum.clone();
  }
  return ys;
}

function draw() {
  noFill();
  background(0);
  stroke(255);
  strokeWeight(8);
  for (let i = 0; i < x_vals.length; i++) {
    point(map(x_vals[i], -1, 1, 0, WIDTH), map(y_vals[i], -1, 1, 0, HEIGHT));
  }
  strokeWeight(4);

  if (x_vals.length > 0) {
    tf.tidy(() => {
      const ys = tf.tensor1d(y_vals);
      optimizer.minimize(() => loss(predict(x_vals), ys), coefficients);
    });
  }


  let lineX = [];
  for (let x = -1.1; x <= 1.1; x += 0.01)
    lineX.push(x);
  const ys = tf.tidy(() => predict(lineX));
  let lineY = ys.dataSync();
  ys.dispose();

  beginShape();
  for (let i = 0; i < lineY.length; i++)
    curveVertex(map(lineX[i], -1, 1, 0, WIDTH), map(lineY[i], -1, 1, 0, HEIGHT));
  endShape();

  for (let i = 0; i < lineY.length; i++) {
    stroke(200, 100, 100);
    point(map(lineX[i], -1, 1, 0, WIDTH), map(lineY[i], -1, 1, 0, HEIGHT));
  }
  //console.log(tf.memory().numTensors);
}

function mousePressed() {
  x_vals.push(map(mouseX, 0, WIDTH, -1, 1));
  y_vals.push(map(mouseY, 0, HEIGHT, -1, 1));
}
<script src="https://cdnjs.cloudflare.com/ajax/libs/p5.js/0.5.7/p5.min.js"></script>
<script src="https://cdnjs.cloudflare.com/ajax/libs/tensorflow/0.11.2/tf.min.js"></script>

I'm not sure if those are bugs :

  • using ys = tf.add() creates a new tensor that is not disposed by tidy()
  • using ys.assign(tf.add()) prevents optimizer.minimize() function to find relation with variables

Upvotes: 1

Related Questions