How to collect and send file from S3 to external SFTP using Lambda + Java?

Question

I have files that will be placed inside an S3 bucket on a daily basis, I need to create a lambda function to forward this file to an external SFTP. (Using java)

My problem is that I am not sure how to establish this connection with the S3 from my lambda to collect the file (eventually edit it, for renaming for example) and then forwarding to the SFTP. Could it be invoked like we do if we need to call another Lambda Function? Example. Or would I have to connect as if I were outside AWS environment?

If you had some advice or maybe some simple example of implementation that is even close to this, that would be nice!

ᴛʜᴇᴘᴀᴛᴇʟ · Accepted Answer

Here's a full example of getting a file from S3 and saving it to SFTP.

I've added the following libraries to make this Java module work:

Lambda Core
Google Gson
AWS S3 SDK

SSHJ

import com.amazonaws.regions.Regions;
import com.amazonaws.services.lambda.runtime.Context;
import com.amazonaws.services.lambda.runtime.LambdaLogger;
import com.amazonaws.services.lambda.runtime.RequestStreamHandler;
import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.s3.AmazonS3ClientBuilder;
import com.amazonaws.services.s3.model.S3Object;
import com.google.gson.Gson;
import com.google.gson.GsonBuilder;
import net.schmizz.sshj.SSHClient;
import net.schmizz.sshj.sftp.SFTPClient;
import net.schmizz.sshj.xfer.FileSystemFile;

import java.io.*;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.StandardCopyOption;

public class S3ToSFTPTest implements RequestStreamHandler {

    private LambdaLogger logger;

    @Override
    public void handleRequest(InputStream input, OutputStream output, Context context) throws IOException {
        logger = context.getLogger();

        AmazonS3 s3Client = AmazonS3ClientBuilder.standard().withRegion(Regions.US_EAST_1).build();

        // s3 client
        if (s3Client == null) {
            logger.log("S3 Client is null - can't continue!");
            sendResponse(context, new S3ToSFTPTestResponse(false), output);
            return;
        }

        String bucketName = "==== S3 BUCKET NAME ====";

        // s3 bucket - make sure it exist
        if (!s3Client.doesBucketExistV2(bucketName)) {
            logger.log("S3 Bucket does not exists - can't continue!");
            sendResponse(context, new S3ToSFTPTestResponse(false), output);
            return;
        }

        String fileName = "==== S3 FILE NAME ====";
        File localFile = null;

        try {
            localFile = File.createTempFile(fileName, "");

            // get S3Object
            S3Object s3Object = s3Client.getObject(bucketName, fileName);

            // get stream from S3Object
            InputStream inputStream = s3Object.getObjectContent();

            // write S3Object stream into a temp file
            Files.copy(inputStream, localFile.toPath(), StandardCopyOption.REPLACE_EXISTING);
        } catch (Exception e) {
            logger.log("Failed to get file from S3: " + e.toString());
        }

        if(localFile == null) {
            sendResponse(context, new S3ToSFTPTestResponse(false), output);
            return;
        }


        // now, you have the file stored locally
        // modify it as you need to
        // .....



        // finally, send the file to SFTP
        boolean fileSaved = saveFilesToSFTP(context, localFile);
        logger.log("fileSaved: " + fileSaved);

        sendResponse(context, new S3ToSFTPTestResponse(true), output);
    }


    public boolean saveFilesToSFTP(Context context, File... files) {
        // this is for test only - In real application, I would suggest that
        // do NOT store these information in the code.
        // You should use service like Secrets Manager or Parameter Store
        final String sftpHostname = "==== SFTP Hostname ====";
        final String sftpUsername = "==== SFTP Username ====";
        final String sftpPassword = "==== SFTP Password ====";

        String remoteFolderPath = "/root/S3Files/";

        try {
            SSHClient ssh = new SSHClient();
            ssh.addHostKeyVerifier((hostname1, port, key) -> true);

            ssh.connect(sftpHostname);
            logger.log("SSHClient Connected!");

            try {
                ssh.authPassword(sftpUsername, sftpPassword);
                logger.log("SSHClient Authenticated!");

                try (SFTPClient sftp = ssh.newSFTPClient()) {
                    for(File file : files) {
                        sftp.put(new FileSystemFile(file), remoteFolderPath);
                    }
                } catch (Exception e) {
                    logger.log("failed to get SFTPClient: " + e.toString());
                    return false;
                }
            } finally {
                ssh.disconnect();
            }
        } catch (Exception e) {
            logger.log("SFTP connection failed: " + e.toString());
            return false;
        }

        return true;
    }


    public void sendResponse(Context context, S3ToSFTPTestResponse response, OutputStream output) {
        Gson gson = new GsonBuilder().setPrettyPrinting().create();
        String responseStr = gson.toJson(response);

        logger.log("response:
" + responseStr);

        try {
            OutputStreamWriter writer = new OutputStreamWriter(output, StandardCharsets.UTF_8);
            writer.write(responseStr);
            writer.close();
        } catch (Exception e) {
            logger.log("failed to send response: " + e.toString());
        }
    }


    public static class S3ToSFTPTestResponse implements Serializable {
        private boolean success;

        public S3ToSFTPTestResponse() {
        }

        public S3ToSFTPTestResponse(boolean success) {
            this.success = success;
        }

        public boolean isSuccess() {
            return success;
        }
    }
}

How to collect and send file from S3 to external SFTP using Lambda + Java?

Answers (2)

Related Questions