Service invoked too many times for one day: gmail read

Question

I'm running across the various rate limits with gmail, and I'm not sure what I can do to work around them. What I'm trying to accomplish is to search for any messages in gmail which have certain "expire" labels on them (with various times in minutes). My intent is to have mail rules apply these labels on messages that become irrelevant after a certain amount of time. I want these messages to stay in my inbox until they expire, and then automatically get archived. I based the general technique off of a post I saw at http://googleappsdeveloper.blogspot.com/2011/07/gmail-snooze-with-apps-script.html

I have this on a trigger that runs the "expire" function every five minutes.

So, every five minutes, it makes 10 calls to GmailApp.getUserLabelByName, and one to getThreads(). Then depending on how many matching threads it finds, for each one it:

1 x getFirstMessageSubject 1 x getLastMessageDate 1 x addLabel 1 x moveToArchive 1 x removeLabel

Which is 5 calls per thread.

So, I would think, that I'm making 10 call / 5 minutes, or 2880 calls a day just looking for messages. Then, assuming I find 1,000 messages over the course of the day, an additional 5,000 calls. Combined that would be 7,880 calls, which is under what I thought were the quota limits, but I'm a little unclear there. According to https://docs.google.com/a/macros/latinschool.org/dashboard under Quota Limits, it would appear that for a Google Apps for Edu account, like mine, I should be able to do 10,000 GMail Read and 10,000 GMail Write actions per day. I think all of my calls would fall under one of those two.

I can reduce the number of labels/times I'm using, and not get the subject (which I'm using for debugging), but beyond that, I'm not sure how I can make this script more efficient. And I'm totally unclear as to how I'm hitting the quota (see math above).

Here's my code:

/**
 * Expire mail after set time by archiving based on labels
 * based on http://googleappsdeveloper.blogspot.com/2011/07/gmail-snooze-with-apps-script.html
 * For more information on interacting with GMail labels, see
 * https://developers.google.com/apps-script/class_gmaillabel
 */

var expiretimes = [5,15,30,60,120,180,240,480,720];  // times in minutes to expire messages


function getLabelName(i) {
  return "expire/expire " + i;
}

function setup() {
  // Create the labels we’ll need for expiring
  GmailApp.createLabel("expire");
  for (var expt = 0; expt < expiretimes.length; expt++) {
    GmailApp.createLabel(getLabelName(expt));
  }
    GmailApp.createLabel("expired");
}


function expiremsgs(expiremin) {
  // get current date
  var nowdate = new Date();
  Logger.log(nowdate);

  // get the label for given name
  var label = GmailApp.getUserLabelByName(getLabelName(expiremin));
  var expiredlabel = GmailApp.getUserLabelByName("expired");
  var threads = label.getThreads();
  Utilities.sleep(1000);
  for (var i = 0; i < threads.length; i++) {
    Logger.log("i: " + i);
    Logger.log(threads[i].getFirstMessageSubject());
    var lastmsgdate = threads[i].getLastMessageDate();
    Logger.log(lastmsgdate);
    var minold = (nowdate-lastmsgdate)/60000; // convert from ms to minutes
    Logger.log(minold);
    if (minold > expiremin) {
      threads[i].addLabel(expiredlabel);
      threads[i].moveToArchive();
      threads[i].removeLabel(label);
      Logger.log("Archived");
    }
  Utilities.sleep(1500);
  }
};

function expire() {
  for (var expt = 0; expt < expiretimes.length; expt++) {
    expiremsgs(expt);
  }
}

Answer 1

I took a good long look at what your were doing, and it does look like your math is correct. However, I know that using batch actions is best practice when working with spreadsheets. So, I did some looking in the reference materials and I found a few key functions:

GmailApp.moveThreadsToArchive(GmailThreads[]);
label.removeFromThreads(GmailThreads[]);
label.addToThreads(GmailThreads[]);

These functions, I assume, will reduce the number of read/write calls you are making with the API. (Google OPs please correct me if I'm wrong.)

Nonetheless, while I was playing with your code, I discovered a that these functions can only handle 100 threads at a time, so if there are more than that, then the operation must be batched. I also checked the speed and scalability of calls to the GmailApp service, and discovered that the delay for most calls is loosely independent of the number of items involved. Batch archiving 6 threads takes about (within a second or two) the same time as 100, at least in my testing. I also made an effort to do as much as possible in the script before going out the Gmail. Finally, I modified the way in which you were interacting with the timeouts. I couldn't figure out for the life of me how you were getting the value in the array, so I modified that as well as the labels. They will now read "expires/in 15min" etc. The script takes about 30 seconds to process 300 or so messages which actually have tags.

Another approach might be to run a single search looking for all of the labels in one go, then checking the labels afterwards. This, however, might still call too many calls back to Gmail as you check each thread's properties.

With all of that out of the way, here is my iteration of your code:

var expiretimes = [5,15,30,60,120,180,240,480,720];

function getLabelName(i) {
  return "expire/in " + expiretimes[i]  + " min";
}

function setup() {
  GmailApp.createLabel("expired");
  GmailApp.createLabel("expire");
  for (var expt = 0; expt < expiretimes.length; expt++) {
    GmailApp.createLabel(getLabelName(expt));
  }
}

function expire() {
  var allExpThreads = [];
  for (var expt = 0; expt < expiretimes.length; expt++) allExpThreads = allExpThreads.concat(expiremsgs(expt));
  if(allExpThreads.length > 100) batchExpire(allExpThreads);
  else GmailApp.moveThreadsToArchive(allExpThreads).getUserLabelByName("expired").addToThreads(allExpThreads);
}

function expiremsgs(expiremin) {
  var label = GmailApp.getUserLabelByName(getLabelName(expiremin));   
  var threads = label.getThreads();
  if(threads.length > 0){   
    var threadsToExpire = threads.filter(msgExpired, {date: new Date(), min: expiretimes[expiremin]});
    if(threadsToExpire.length > 100) batchRemoveLabel(threadsToExpire,label);
    else if(threadsToExpire.length > 0) label.removeFromThreads(threadsToExpire);
    return threadsToExpire;
  }
  return [];
}

function msgExpired(thread) {
  return this.date - thread.getLastMessageDate() / 6000 > this.min;
}

function batchRemoveLabel(threads,label) {
  var start = 0, end = 100;
  do {
    label.removeFromThreads(threads.slice(start,end));
    start = end;
    end = end > threads.length - 100 ? threads.length : end + 100;
  } while (start < threads.length);
}

function batchExpire(threads) {
  var start = 0, end = 100;
  var expLabel = GmailApp.getUserLabelByName("expired");
  do {
    GmailApp.moveThreadsToArchive(threads.slice(start,end));
    expLabel.addToThreads(threads.slice(start,end));
    start = end;
    end = end > threads.length - 100 ? threads.length : end + 100;
  } while (start < threads.length);
}