401 Unauthorised when processing a Webhook

Question

I'm trying to process a webhook sent by the auth provider Clerk when a user is created. In order to test this procedure locally, I tried localtunnel which did not work and then ngrok.
When the webhook is sent to the https://13f1-...-859.ngrok-free.app/api/webhooks/clerk provided from ngrok I get the following output:

Web Interface                 http://127.0.0.1:4040
https://13f1-...-859.ngrok-free.app -> http://localhost:3000

Connections                   ttl     opn     rt1     rt5     p50     p90
                              5       0       0.00    0.00    0.03    0.04
HTTP Requests
-------------

POST /api/webhooks/clerk                401 Unauthorized

Using Nextjs13's new app router, I wrote the following route to handle the webhook:

(app/api/webhooks/clerk/route.ts):

import { db } from "@/db/db";
import { playlists, users } from "@/db/schema";
import type { User } from "@clerk/nextjs/api";
import { headers } from "next/headers";
import { Webhook } from "svix";
import { eq, isNull, inArray } from "drizzle-orm";

type UnwantedKeys = "primaryEmailAddressId" | "primaryPhoneNumberId" | "phoneNumbers";

interface UserInterface extends Omit<User, UnwantedKeys> {
    email_addresses: {
        email_address: string;
        id: string;
    }[];
    primary_email_address_id: string;
    first_name: string;
    last_name: string;
    primary_phone_number_id: string;
    phone_numbers: {
        phone_number: string;
        id: string;
    }[];
}

const webhookSecret: string = process.env.WEBHOOK_SECRET || "";

export async function POST(req: Request) {
    const payload = await req.json()
    const payloadString = JSON.stringify(payload);
    const headerPayload = headers();
    const svixId = headerPayload.get("svix-id");
    const svixIdTimeStamp = headerPayload.get("svix-timestamp");
    const svixSignature = headerPayload.get("svix-signature");
    if (!svixId || !svixIdTimeStamp || !svixSignature) {
        console.log("svixId", svixId)
        console.log("svixIdTimeStamp", svixIdTimeStamp)
        console.log("svixSignature", svixSignature)
        return new Response("Error occured", {
            status: 400,
        })
    }
    const svixHeaders = {
        "svix-id": svixId,
        "svix-timestamp": svixIdTimeStamp,
        "svix-signature": svixSignature,
    };
    const wh = new Webhook(webhookSecret);
    let evt: Event | null = null;
    try {
        evt = wh.verify(payloadString, svixHeaders) as Event;
    } catch (_) {
        console.log("error")
        return new Response("Error occured", {
            status: 400,
        })
    }
    // Handle the webhook
    const eventType: EventType = evt.type;
    const { id, first_name, last_name, emailAddresses } = evt.data;
    if (eventType === "user.created") {
        const email = emailAddresses[0].emailAddress;
        try {
            await db.insert(users).values({
                id,
                first_name,
                last_name,
                email,
            });
            return new Response("OK", { status: 200 });
        } catch (error) {
            console.log(error);
            return new Response("Error handling user creation in the database", {
                status: 400,
            })
        }
    } else if (eventType == "user.deleted") {
        try {
            await db.delete(users).where(eq(users.id, id));
            const recordsToDelete = (await db.select().from(playlists).leftJoin(users, eq(playlists.user_id, users.id)).where(isNull(users.id)));
            const idsToDelete = recordsToDelete.map(record => record.playlists.id);
            await db
                .delete(playlists).where(inArray(playlists.id, idsToDelete));
            return new Response("OK", { status: 200 });
        } catch (error) {
            console.error(error);
            throw new Error(`Failed to insert user into database`);
        }
    } else {
        console.log("eventType", eventType)
        return new Response("Invalid event type", {
            status: 201,
        })
    }
}


type Event = {
    data: UserInterface;
    object: "event";
    type: EventType;
};

type EventType = "user.created" | "user.deleted" | "*";

Answer 1

Had the same problem for a while, albeit with NextJS 13.4.2 Pages Router. Banged my head around for a while - turned out it was my middleware.ts setup.

export const config was not registering my publicRoute or ignoredRoutes. Very weird issue but got it working now.

Maybe some newb mistake but I couldn't find a solution online so thought I would share.

This is all tested on my local development server with ngrok http 3000 to tunnel it to an online host so my webhook can work. Not tested in production. ngrok v3.3.4, free plan.

/src/middleware.ts below

import { authMiddleware } from "@clerk/nextjs";
export default authMiddleware({ 
  ignoredRoutes: ["/((?!api|trpc))(_next|.+\..+)(.*)"],  
  publicRoutes: ["/api/webhooks/user/route"],
});

export const config = {
  matcher: ["/((?!.*\\..*|_next).*)", "/", "/(api|trpc)(.*)"],
};

/pages/api/webhooks/user/route.ts below

import { IncomingHttpHeaders } from "http";
import type { WebhookEvent } from "@clerk/clerk-sdk-node";
import type { NextApiRequest, NextApiResponse } from 'next';
import { Webhook, WebhookRequiredHeaders } from "svix";
// FROM https://github.com/clerkinc/clerk-nextjs-examples/blob/main/examples/widget/pages/api/webhooks/user.ts

type NextApiRequestWithSvixRequiredHeaders = NextApiRequest & {
    headers: IncomingHttpHeaders & WebhookRequiredHeaders;
};



// Disable the bodyParser so we can access the raw
// request body for verification.
export const config = {
    api: {
        bodyParser: false,
    },
};

async function buffer(readable: NextApiRequestWithSvixRequiredHeaders) {
    const chunks = [];
    for await (const chunk of readable) {
        chunks.push(typeof chunk === 'string' ? Buffer.from(chunk) : chunk);
    }

    return Buffer.concat(chunks);
}

if (!process.env.WEBHOOK_SECRET) {
    throw new Error("WEBHOOK_SECRET is not set.");
}

const webhookSecret: string = process.env.WEBHOOK_SECRET || "";

export default async function handler(req: NextApiRequestWithSvixRequiredHeaders, res: NextApiResponse) {
    console.log("got hit at wh")
    // Verify the webhook signature
    // See https://docs.svix.com/receiving/verifying-payloads/how
    const payload = (await buffer(req)).toString();
    console.log(payload, "payload")
    const headers = req.headers;
    const wh = new Webhook(webhookSecret);
    let evt: WebhookEvent | null = null;

    try {
        evt = wh.verify(payload, headers) as WebhookEvent;
    } catch (_) {
        return res.status(400).json({});
    }

    // Handle the webhook
    const eventType = evt.type;
    // console.log(eventType);

    res.json({});
}

Note: async function buffer() was from another stackoverflow answer... can't seem to find the post. This is supposed to help with deploying to serverless Vercel. TBD.

Answer 2

Restart your ngrok agent by running the command, replacing {ws endpoint signing secret} with the Webhook endpoint signing secret: ngrok http 3000 --verify-webhook clerk --verify-webhook-secret {ws endpoint signing secret}

from this guide: https://ngrok.com/docs/integrations/clerk/webhooks/#security

they have updated ngrok agent, so you might update your local agent's version too. you can do this by starting the old agent with default command: ngrok http 3000. and see the upgrade message in CLI.

What actually worked for me:

• last version of Next.js with Server Components and App folder. previously I was on older version with Pages folder.
• make my WS endpoint public in middleware.ts:

export default authMiddleware({
    publicRoutes: ["/api/webhooks/clerk"]
});

• problem wasn't with ngrok it works fine with: ngrok http 3000

Answer 3

this is my /webhooks/user/route file

import { headers } from "next/headers";
import { IncomingHttpHeaders } from "http";
import { NextResponse } from "next/server";
import { Webhook, WebhookRequiredHeaders } from "svix";

type EventType = "user.created" | "user.updated" | "*";

type Event = {
    data: Record<string, string | number>,
    object: "event",
    type: EventType,
};

const webhookSecret = process.env.WEBHOOK_SECRET || "";

// console.log(`Webhook secret: ${webhookSecret}`);

async function handler(request: Request) {
    try {
        const payload = await request.json();
        const headersList = request.headers;
        const heads = {
            "svix-id": headersList.get("svix-id"),
            "svix-timestamp": headersList.get("svix-timestamp"),
            "svix-signature": headersList.get("svix-signature"),
        };

        const wh = new Webhook(webhookSecret);
        let evt: Event | null = null;

        try {
            evt = wh.verify(
                JSON.stringify(payload),
                heads as IncomingHttpHeaders & WebhookRequiredHeaders
            ) as Event;
        } catch (err) {
            console.error(`Verification error: ${(err as Error).message}`);
            console.error(err);
            return NextResponse.json({}, { status: 400 });
        }

        const eventType: EventType = evt.type;
        if (eventType === "user.created" || eventType === "user.updated") {
            const { id, ...attributes } = evt.data;
            // console.log(id);
            // console.log(attributes);
        }

        // Added this line to return a response in case of no errors
        return NextResponse.json({ message: 'Handled successfully' }, { status: 200 });
    } catch (err) {
        console.error(`Unexpected error: ${err}`);
        return NextResponse.json({ error: 'Something went wrong' }, { status: 500 });
    }
}


export const GET = handler;
export const POST = handler;
export const PUT = handler;

---

// I defined the Clerk middleware function as such to enable the API route as a public route, so that it is not being blocked by CLerk middleware

const clerkAuth = clerkAuthMiddleware({
  publicRoutes: ["/", "/admin", "/api/webhooks/user"]
});

---

then used this for ngork instead

ngrok http 3000 --verify-webhook clerk --verify-webhook-secret {endpoint signing secret}

---

Make sure your Clerk Webhook is set the right Ngork url