CODEWITHSUNDEEP
goamazon-s3
Rome Torres
Rome Torres

Reputation: 1081

Golang amazon s3 how can I upload images faster

I have some images that I am uploading to my s3 account using the official amazon s3 package . My images are typically around 250 - 350 KB so small images, however they take around 8 or 9 seconds to upload which seems excessive any suggestions on improving speed would be great. This is my code and if I take the resizing image code off it still takes a good 8 or 9 seconds still .

      func UploadStreamImage(w http.ResponseWriter, r *http.Request) {


          r.ParseForm()
    var buff bytes.Buffer


    wg := sync.WaitGroup{}

    wg.Add(1)
    go func() {
        defer wg.Done()

        var buf bytes.Buffer
        sess, _ := session.NewSession(&aws.Config{
            Region:      aws.String("us-west-2"),
            Credentials: credentials.NewStaticCredentials(aws_access_key_id, aws_secret_access_key, ""),

        })

        svc := s3.New(sess)

        file, handler, err := r.FormFile("file")
        if err != nil {
            log_errors(" error on upload",err.Error(),w)
            fmt.Println("Error Uploading Image")
            return
        }
        defer file.Close()

        img,err := imaging.Decode(file)
        if err != nil {
            print("Imaging Open error")
            log_errors("Error decoding",err.Error(),w)
            return
        }

                imgSize,err := strconv.Atoi(r.FormValue("imgsize"))

        if err != nil {
            println("Error converting to integer")
            log_errors("Error converting to integer",err.Error(),w)
            return
        }

        b := img.Bounds()
        heightImg := b.Max.Y
        widthImg := b.Max.X


        // resize image
        height := int(float64(heightImg) * .23)
        width := int(float64(widthImg) * .23)


        if imgSize < 401 {
            height = int(float64(heightImg) * 1)
            width = int(float64(widthImg) * 1)

        } else if imgSize  >= 401 && imgSize < 900 {
               height = int(float64(heightImg) * .68)
               width = int(float64(widthImg) * .68)
               println("Middle Image")
                  } else if  imgSize  >= 900 && imgSize < 1300 {
            height = int(float64(heightImg) * .45)
            width = int(float64(widthImg) * .45)
           } else if  imgSize  >= 1301 && imgSize < 1700 {
            height = int(float64(heightImg) * .40)
            width = int(float64(widthImg) * .40)
        } 


        new_img := imaging.Resize(img,width,height, imaging.Lanczos)
       // end resize

        err = imaging.Encode(&buf,new_img, imaging.JPEG)
        if err != nil {
            log.Println(err)
            log_errors(" error encoding file",err.Error(),w)
                        return
        }


        r := bytes.NewReader(buf.Bytes())



        read_file,err := ioutil.ReadAll(r)
        if err != nil {
            fmt.Println("Error Reading file")
            log_errors(" error reading file",err.Error(),w)
            return

        }




        file.Read(read_file)
        fileBytes := bytes.NewReader(read_file)
        fileSize, err := buff.ReadFrom(fileBytes)
        if err != nil {
            log_errors(" error fileSize",err.Error(),w)
            return
        }
        fileType := http.DetectContentType(read_file)
        path := handler.Filename
        params := &s3.PutObjectInput{
            Bucket: aws.String("bucket name"),
            Key: aws.String(path),
            Body: fileBytes,

            ContentLength: aws.Int64(fileSize),
            ContentType: aws.String(fileType),
        }

        resp, err := svc.PutObject(params)
        if err != nil {
            fmt.Printf("bad response: %s", err)
            log_errors("error in putObject",err.Error(),w)
            return
        }

        fmt.Println(w,"Done")


    }()
    wg.Wait()

}

Upvotes: 2

Views: 2700

Answers (1)

jeevatkm
jeevatkm

Reputation: 4791

Improving speed? this is subjective one; depends on many factors such as server uplink, client uplink, etc.

Instead, I will provide my inputs to improve your code snippets:

  • Let's start with sync.WaitGroup - I do not see a benefit as per your code flow, you just create and wait for goroutine to complete. Instead use without WaitGroup. Tip: Do not use feature for sake/buzz; use it when needed.
  • Do not create multiple bytes.Buffer and Reader, when you can accomplish without it. For e.g. getting fileSize after the resize, just do buf.Len().
  • If you want to offload the AWS upload then extract upload code part into separate func and call it as goroutine.

I have updated your code snippet (I have not tested your code, so please fix/improve it as required):

func UploadStreamImage(w http.ResponseWriter, r *http.Request) {
    file, handler, err := r.FormFile("file")
    if err != nil {
        log_errors(" error on upload", err.Error(), w)
        fmt.Println("Error Uploading Image")
        return
    }
    defer file.Close()

    // Suggestion: You can calculate the file size from bytes.
    // instead getting it from form; since after resize image will change
    imgSize, err := strconv.Atoi(r.FormValue("imgsize"))
    if err != nil {
        println("Error converting to integer")
        log_errors("Error converting to integer", err.Error(), w)
        return
    }

    img, err := imaging.Decode(file)
    if err != nil {
        print("Imaging Open error")
        log_errors("Error decoding", err.Error(), w)
        return
    }

    b := img.Bounds()
    heightImg := b.Max.Y
    widthImg := b.Max.X

    // resize image
    height := int(float64(heightImg) * .23)
    width := int(float64(widthImg) * .23)

    if imgSize < 401 {
        height = int(float64(heightImg) * 1)
        width = int(float64(widthImg) * 1)

    } else if imgSize >= 401 && imgSize < 900 {
        height = int(float64(heightImg) * .68)
        width = int(float64(widthImg) * .68)
        println("Middle Image")
    } else if imgSize >= 900 && imgSize < 1300 {
        height = int(float64(heightImg) * .45)
        width = int(float64(widthImg) * .45)
    } else if imgSize >= 1301 && imgSize < 1700 {
        height = int(float64(heightImg) * .40)
        width = int(float64(widthImg) * .40)
    }

    new_img := imaging.Resize(img, width, height, imaging.Lanczos)
    // end resize

    var buf bytes.Buffer
    err = imaging.Encode(&buf, new_img, imaging.JPEG)
    if err != nil {
        log.Println(err)
        log_errors(" error encoding file", err.Error(), w)
        return
    }

    fileType := http.DetectContentType(buf.Bytes())
    fileSize := buf.Len()
    path := handler.Filename
    params := &s3.PutObjectInput{
        Bucket: aws.String("bucket name"),
        Key:    aws.String(path),
        Body:   bytes.NewReader(buf.Bytes()),

        ContentLength: aws.Int64(fileSize),
        ContentType:   aws.String(fileType),
    }

    resp, err := svc.PutObject(params)
    if err != nil {
        fmt.Printf("bad response: %s", err)
        log_errors("error in putObject", err.Error(), w)
        return
    }

    fmt.Println("Done", resp)
}

Upvotes: 4

Related Questions