douglasrcjames
Reputation: 1645
Cannot send more than a few SendGrid emails with Node.js & Firebase
I am working on an app using Node.js as a backend on Firebase Functions and SendGrid to send mass emails to customers at one time when the tracking for a bulk of orders is ready. I can upload a CSV to the system with a list of 1-3, maybe a few more and will all work properly, but when I have as many as 100 or 1000, the system will not send and I get the error:
Error with sending tracking emails: Error: socket hang up followed by Error setting processed to true: Error: 4 DEADLINE_EXCEEDED: Deadline exceeded on Firebase Function's Node.js server logs. Is there a maximum for sending emails in bulk like this? Any clues on where the issue may lie?
Relevant Code Snippet:
"use strict";
import functions = require('firebase-functions');
import * as uuid from 'uuid';
import admin = require("firebase-admin");
import { DocumentSnapshot } from '@google-cloud/firestore';
import { RESPONSE_TYPES } from './common';
admin.initializeApp(functions.config().firebase);
const request = require("request");
const Papa = require('papaparse');
const sgMail = require('@sendgrid/mail')
sgMail.setApiKey(functions.config().sendgrid_api.key)
export const onUploadCreated = functions.firestore.document('users/{userId}/uploads/{uploadId}')
.onCreate(async (snap: DocumentSnapshot, context: functions.EventContext) => {
const newValue = snap.data();
if (newValue === null || newValue === undefined) {
return;
}
try {
const allPromises: Array<Promise<any>> = [];
console.log("Upload type set to set shipment tracking info...")
const orders: any = [];
// Parse through uploaded CSV
const options = {
download: true,
header: true,
worker: true
};
const parseStream = await Papa.parse(Papa.NODE_STREAM_INPUT, options);
allPromises.push(parseStream);
const dataStream = await request.get(newValue.fileUrl).pipe(parseStream);
allPromises.push(dataStream);
let parseError = "";
allPromises.push(
await parseStream.on("data", async (chunk: any) => {
if(
chunk['Order ID'] && chunk['Order ID'] !== undefined && chunk['Order ID'] !== null &&
chunk['Order Number'] && chunk['Order Number'] !== undefined && chunk['Order Number'] !== null &&
chunk['First Name'] && chunk['First Name'] !== undefined && chunk['First Name'] !== null &&
chunk['Last Name'] && chunk['Last Name'] !== undefined && chunk['Last Name'] !== null &&
chunk['Email'] && chunk['Email'] !== undefined && chunk['Email'] !== null &&
chunk['Tracking'] && chunk['Tracking'] !== undefined && chunk['Tracking'] !== null
){
// If in e notation, throw error
if(/e\+|E\+/g.test(chunk['Tracking'])){
parseError = "Tracking is in E notation, please reformat as outlined in NOTE 3 and re-upload."
} else {
await orders.push({
id: chunk['Order ID'],
number: chunk['Order Number'],
productName: chunk['Product Name'] || '',
productSize: chunk['Product Size'] || '',
productVariant: chunk['Product Variant'] || '',
firstName: chunk['First Name'],
lastName: chunk['Last Name'],
email: chunk['Email'],
tracking: chunk['Tracking'],
carrier: chunk['Carrier'] || '',
method: chunk['Method'] || '',
})
}
} else {
parseError = "Required columns were not properly defined properly. Please check your file follows the NOTES and re-upload."
}
})
);
// TODO: if multiple products, we need to just create more rows for each product like BS does
allPromises.push(
await dataStream.on("finish", async () => {
if(orders.length > 1000){
parseError = "You uploaded over 1000 orders to this upload. Please retry by only submitting 1000 orders at a time so we don't overwhelm the servers!"
}
if(parseError.length === 0){
console.log("Parsing complete, no errors. Number of orders processed: " + orders.length + ". Starting to add orders to Firestore Order docs now... ")
const emailMessages: { to: any; from: string; subject: string; text: string; html: string; }[] = [];
let shopData: FirebaseFirestore.DocumentData | any = null;
await admin.firestore().collection('public').doc("shop").get().then((shopDoc) => {
if (shopDoc.exists) {
let docWithMore = Object.assign({}, shopDoc.data());
docWithMore.id = shopDoc.id;
shopData = docWithMore;
} else {
console.error("Shop doc doesn't exist!")
}
}).catch((error) => {
console.log("Error getting shop document:", error);
})
const batchArray: any = [];
batchArray.push(admin.firestore().batch());
console.log("batchArray.length: " + batchArray.length);
let operationCounter = 0;
let batchIndex = 0;
let ordersProcessed = 0;
allPromises.push(
await orders.forEach((order: any) => {
const orderRef = admin.firestore().collection('orders').doc(order.id)
batchArray[batchIndex].set(
orderRef,
{
shipment: {
tracking: order.tracking,
carrier: order.carrier || '',
method: order.method || ''
}
},
{ merge: true }
);
const htmlEmail =
`
<div style="width: 100%; font-family: Arial, Helvetica, sans-serif">
${shopData?.nav?.showLogo ?
`
<div style="text-align: center;">
<img
alt="company logo"
src="${shopData?.logoUrl}"
width="${shopData?.nav?.logoSize || "200"}" height="auto"
/>
</div>
`
: ""
}
${shopData?.nav?.showTitle ? `<h1 style="margin: 20px 0 0 0; text-align: center;">${shopData?.name}</h1>` : ""}
<div style="margin: auto; width: 70%; padding: 1%;">
<h2>Great news ${order.firstName}!</h2>
<p>
Your order #${order.number} has shipped with the tracking # <b>${order.tracking}</b>.
</p>
${order?.productName ? `<p><b>Product Name:</b> ${order.productName}</p>` : ""}
${order?.productSize ? `<p><b>Product Size:</b> ${order.productSize}</p>` : ""}
${order?.productVariant ? `<p><b>Product Variant:</b> ${order.productVariant}</p>` : ""}
${order?.method ? `<p><b>Shipping Method:</b> ${order.method}</p>` : ""}
${order?.carrier ? `<p><b>Shipping Carrier:</b> ${order.carrier}</p>` : ""}
${shopData?.emails?.order?.shipment ?? ''}
<p>
Feel free to reach out to <a href="mailto:${shopData?.emails?.support ?? `[email protected]`}">${shopData?.emails?.support ?? `[email protected]`}</a> if you have any questions!
</p>
</div>
</div>
`
const msg = {
to: order.email,
from: `[email protected]`,
subject: `${shopData?.name} Order #${order.number} Shipped!`,
text: `
Great news! We shipped out your order with the tracking #${order.tracking}!
${order?.method ? ` Shipping Method: ${order.method} ` : ""}
${order?.carrier ? ` Shipping Carrier: ${order.carrier} ` : ""}
`,
html: htmlEmail,
}
emailMessages.push(msg)
operationCounter++;
ordersProcessed++;
if (operationCounter === 499) {
console.log("operationCounter: " + operationCounter)
batchArray.push(admin.firestore().batch());
batchIndex++;
operationCounter = 0;
}
})
);
// Attempt to send out emails, if it fails, do not continue with adding changes to Firestore! Sending email is the main purpose of this process.
allPromises.push(
sgMail.send(emailMessages).then(async () => {
console.log(`${emailMessages.length} tracking emails sent successfully!`);
console.log("Starting to push batchArray.length: " + batchArray.length);
let batchesProcessed = 0;
allPromises.push(
await batchArray.forEach(async (batch: any) => {
console.log("batchesProcessed: " + batchesProcessed);
batchesProcessed++;
allPromises.push(await batch.commit())
if(batchesProcessed === batchArray.length){
console.log("Done pushing " + batchesProcessed + " batches!");
allPromises.push(
admin.firestore().collection('users').doc(context.params.userId).collection('uploads').doc(context.params.uploadId).set({
processed: true,
response: {
type: RESPONSE_TYPES.SUCCESS,
message: `Successfully sent ${emailMessages.length} tracking emails and updated ${ordersProcessed} orders on the database with this tracking info.`,
read: false,
timestamp: Date.now()
}
}, { merge: true }).then(() => {
console.log("Set upload processed to true.")
})
);
}
})
);
}).catch((error: any) => {
console.log("Error with sending tracking emails: " + error);
allPromises.push(
admin.firestore().collection('users').doc(context.params.userId).collection('uploads').doc(context.params.uploadId).set({
processed: true,
response: {
type: RESPONSE_TYPES.ERROR,
message: `Error sending tracking emails: ${error}`,
read: false,
timestamp: Date.now()
}
}, { merge: true }).then(() => {
console.log("Set upload processed to true.")
}).catch(err => {
console.error("Error setting processed to true: " + err)
})
);
})
)
} else {
console.error("Error parsing shipments CSV: " + parseError)
allPromises.push(
admin.firestore().collection('users').doc(context.params.userId).collection('uploads').doc(context.params.uploadId).set({
processed: true,
response: {
type: RESPONSE_TYPES.ERROR,
message: `Error parsing shipments CSV: ${parseError}`,
read: false,
timestamp: Date.now()
}
}, { merge: true }).then(() => {
console.log("Set upload processed to true and parseError")
}).catch(error => {
console.error("Error setting processed to true and parseError: " + error)
})
);
}
})
);
return Promise.all(allPromises)
} catch(error) {
console.error("Error: " + error);
return;
}
});
Upvotes: 1
Views: 414
Answers (1)
Gregorio Palamà
Reputation: 1952
The error you get has nothing to do with Firebase. This is a SendGrid limit for Free Accounts, as you can see on this page.
The limit is 100 emails per day. To send more emails you'll need to upgrade your account to a paid one.
Upvotes: 1
Related Questions
- how to write a Cloud Function for send email using SendGrid
- SendGrid unable to send email, firebase functions. (code: 403)
- Sending emails with sendgrid on firestore error
- Google Cloud Function - SendGrid - 400 Bad Request
- NodeJS Sendgrid Issue in sending email to multiple recipients
- How to get incoming sendgrid email on firebase functions
- Firebase SendGrid error on sending email via Firebase functions - Error: Bad Request
- Transnational email with SendGrid and Firebase No Errors but no emails (Used NodeMailer as answer)
- want to send an email through cloud functions using sendgrid
- SendGrid not mailing, but not throwing any errors