CODEWITHSUNDEEP
phpamazon-s3cakephptwilio
Fokrule
Fokrule

Reputation: 854

How to upload files to S3 bucket from url directly

I am getting some MMS messages from my users. Those MMS are coming via twilio. So twilio storing those files into their server and I can visit those files from twilio. But in my case, I need to store those files into S3 and show into our system from S3. I can store those files into my local folder or my server. But I am not finding any way to store file into the S3 directly from the url. This is what I have done to store into the local directory from url.

// url of my file. Mostly it will be image.
$url = 'urlofmyfile';
// Path where I am saving. Keeping for jpg for now
$img = 'file/sms/file.jpg';
// saving the file into the folder
file_put_contents($img, file_get_contents($url));

And this is how I am saving my files into S3 if anyone want to upload it directly into my system. For example if any user want to upload their profile picture.

public function saveToS3Bucket($uploadFileName, $imageTmpName) {
    $s3Client = new \Aws\S3\S3Client([
        'version' => env('S3_BUCKET_VERSION'),
        'region'  => env('S3_BUCKET_REGION'),
        'credentials' => array(
            'key'    => env('S3_BUCKET_KEY'),
            'secret' => env('S3_BUCKET_SECRET'),
        )
    ]);
    try {
        $s3Client->putObject([
            'Bucket' => env('S3_BUCKET_NAME'),
            'Key'    => $uploadFileName,
            'SourceFile' => $imageTmpName,
            'StorageClass' => 'REDUCED_REDUNDANCY',
            'ACL' => 'public-read'
        ]);
        return true;
    } catch (S3Exception $e) {
        echo $e->getMessage() . PHP_EOL;
        return false;
    }
}

Those above codes are working fine. But I am not finding any way to store into S3 from url. Please note I am writing code in CakePHP.

Upvotes: 2

Views: 4650

Answers (2)

Phung Dao
Phung Dao

Reputation: 11

Since I couldn't really find an answer that worked, I stumbled upon two methods that finally worked :)

One is with a package that's deprecated now (request) and the other is with axios.

Using Axios

const AWS = require('aws-sdk');
const axios = require('axios');

const s3 = new AWS.S3({
  accessKeyId: 'YOUR_ACCESS_KEY_ID',
  secretAccessKey: 'YOUR_SECRET_ACCESS_KEY',
});

const uploadImageFromUrlToS3 = async (imageUrl, bucketName, fileName) => {
  try {
    const response = await axios.get(imageUrl, {
      responseType: 'arraybuffer'
    });
    const buffer = Buffer.from(response.data, 'binary');

    const params = {
      Bucket: bucketName,
      Key: fileName,
      Body: buffer,
      ACL: 'public-read',
      ContentType: response.headers['content-type']
    };

    const data = await s3.upload(params).promise();
    console.log(`Successfully uploaded image to S3: ${data.Location}`);
  } catch (error) {
    console.error(`Failed to upload image to S3: ${error}`);
  }
};

const imageUrl = 'https://example.com/image.jpg';
const bucketName = 'YOUR_BUCKET_NAME';
const fileName = 'example.jpg';
uploadImageFromUrlToS3(imageUrl, bucketName, fileName);

Using Request

const AWS = require('aws-sdk');
const request = require('request');

const s3 = new AWS.S3({
  accessKeyId: 'YOUR_ACCESS_KEY_ID',
  secretAccessKey: 'YOUR_SECRET_ACCESS_KEY',
});

const uploadImageFromUrlToS3 = (imageUrl, bucketName, fileName) => {
  request({
    url: imageUrl,
    encoding: null
  }, (error, response, body) => {
    if (error || response.statusCode !== 200) {
      console.error(`Failed to download image: ${imageUrl}`);
      return;
    }
    s3.upload({
      Bucket: bucketName,
      Key: fileName,
      Body: body,
      ACL: 'public-read', 
      ContentType: response.headers['content-type']
    }, (error, data) => {
      if (error) {
        console.error(`Failed to upload image to S3: ${error}`);
        return;
      }
      console.log(`Successfully uploaded image to S3: ${data.Location}`);
    });
  });
};

const imageUrl = 'https://example.com/image.jpg';
const bucketName = 'YOUR_BUCKET_NAME';
const fileName = 'example.jpg';
uploadImageFromUrlToS3(imageUrl, bucketName, fileName);

Upvotes: 1

Alan
Alan

Reputation: 10801

Have a look at the Twilio Function below, it should point you in the right direction.

It was derived from this Twilio Blog:

Encrypting and Storing Twilio Flex Recordings Off-site

const axios = require('axios');
let AWS = require('aws-sdk');
const S3UploadStream = require('s3-upload-stream');

exports.handler = async function(context, event, callback) {

// Set the region
AWS.config.update({region: 'us-west-2'});
AWS.config.update({ accessKeyId: context.AWSaccessKeyId, secretAccessKey: context.AWSsecretAccessKey });

// The name of the bucket that you have created
const BUCKET_NAME = 'winston';

const fileUrl = "https://a.b.twil.io/assets/KittehWinston.jpg";
const fileName = "winston.jpg";

const s3Stream = S3UploadStream(new AWS.S3());

// call S3 to retrieve upload file to specified bucket
let upload = s3Stream.upload({Bucket: BUCKET_NAME, Key: fileName, ContentType: 'image/jpeg', ACL: 'public-read'   });

const fileUpload = await uploadFile(fileUrl, upload)
.then(result => callback(null, `success: ${JSON.stringify(result)}`))
.catch(err => callback(err.message));

async function uploadFile (url, upload) {

    const response = await axios({
      url,
      method: 'GET',
      responseType: 'stream'
    })
  
    response.data.pipe(upload);
  
    return new Promise((resolve, reject) => {
      upload.on('uploaded', resolve)
      upload.on('error', reject)
    })
  }
};

Upvotes: 5

Related Questions