How to fix "Only one auth mechanism allowed; only the X-Amz-Algorithm query parameter..."

Question

Amazon API's often give you a URL to resource, which gives a redirect URL with 30s life as redirect. However, usually the headers from the original request are carried over resulting in this error. The redirect is processed with authentication from signed redirect url, and the authentication header.

Answer 1

Here's how I did this in Java.

    // Set up the initial GET request with the required Amazon authorization headers
    HttpGet httpGet1 = new HttpGet(downloadUrl);
    httpGet1.setHeaders(authHeaders);

    // We need to tell HttpClient NOT to follow redirects because we need to handle this ourselves
    HttpParams params = new BasicHttpParams();
    params.setParameter(ClientPNames.HANDLE_REDIRECTS, false);
    httpGet1.setParams(params);

    String location = null; // this will hold the location where you can download the file

    try (CloseableHttpResponse response = httpClient.execute(httpGet1)) {

        // Check if we have a redirect
        if (response.getStatusLine().getStatusCode() == 307) {
            // The Location header has the download URL we need
            Header hdr = response.getFirstHeader("Location");

            // This is the URL we are looking for 
            location = hdr.getValue();
        }
    }

    // Now that we have the S3 download location we can build a GET request to fetch the contents
    // Do NOT include auth headers because the query string contains the auth parameters
    // This is a very short lived auth grant so we need to send the request within 30 secs

    HttpGet httpGet2 = new HttpGet(location);

    try (CloseableHttpResponse response = httpClient.execute(httpGet2)) {

        HttpEntity entity = response.getEntity();

        // Handle the stream. In this case it will be gzipped
        GZIPInputStream gis = new GZIPInputStream(entity.getContent());

        // Do what you need to do with the data

        // ...
    }

Answer 2

Using the static function below solves the problem.

import org.apache.http.*;
import org.apache.http.client.HttpClient;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.HttpClientBuilder;
import org.apache.http.message.BasicHeader;
import org.apache.http.protocol.HttpContext;

import java.io.*;
import java.net.HttpURLConnection;
import java.net.URL;


import static org.apache.commons.io.FileUtils.copyURLToFile;

/**  To intercept AMS redirects to S3, which requires removing authorization header since redirect is signed
 *
 *   Counteracts the error message
 *   <Error><Code>InvalidArgument</Code><Message>Only one auth mechanism allowed; only the X-Amz-Algorithm query parameter, Signature query string parameter or the Authorization header should be specified</Message>
 **/
public class AMSDownloadInterceptor implements HttpResponseInterceptor {

    String filePath;
    int count;

    public AMSDownloadInterceptor(String filenameAndPath) {
        filePath = filenameAndPath;
        count = 0;
    }

    /**
     *      * This function call has done, HttpURLConnection.setFollowRedirects(false);
     *      * you may need to HttpURLConnection.setFollowRedirects(true);
     * @param httpResponse
     * @param httpContext
     * @throws HttpException
     * @throws IOException
     */
    @Override
    public void process(HttpResponse httpResponse, HttpContext httpContext) throws HttpException, IOException {

        //do nothing on the subsequent redirectURL call
        if (count == 0) {
            //TODO is this too hack-a-licious to not be robust?
            String redirectUrl = httpContext.getAttribute("http.response").toString().split(",")[5].substring(11);
            System.err.println("\t\t\tDownloading to\t"+filePath+"\tfrom\t" +redirectUrl);
            copyURLToFile(new URL(redirectUrl), new File(filePath));
            HttpURLConnection.setFollowRedirects(false);
        }
        count++;
    }

    /**
     * Fetch a file requiring authentication to a temporary redirect (307)
     * Response modified to return status 200, but is not perfect
     *

     *
     * @param amazonS3URL
     * @param filenameAndPath
     * @param accessToken       authentication token
     * @param AMS_API_SCOPE     currently "1903489005170091"
     * @return  modified HttpResponse, overriding the failed second http call to malformed redirect URL
     */
    public static HttpResponse downloadViaRedirect(String amazonS3URL, String filenameAndPath, String accessToken, String AMS_API_SCOPE) throws IOException {

        HttpResponse result = null;
        AMSDownloadInterceptor downloader = new AMSDownloadInterceptor(filenameAndPath);
        HttpClient clientFileInterceptor = HttpClientBuilder.create().addInterceptorFirst(downloader).build();
        HttpGet get = new HttpGet(amazonS3URL);
        get.setHeader(new BasicHeader(HttpHeaders.AUTHORIZATION, "Bearer " + accessToken));
        get.setHeader(new BasicHeader("Amazon-Advertising-API-Scope", AMS_API_SCOPE));

        //HttpURLConnection.setFollowRedirects(false);
        HttpResponse hr = clientFileInterceptor.execute(get);
        HttpURLConnection.setFollowRedirects(true);

        hr.setStatusCode(200);
        hr.setReasonPhrase("URL redirect downloaded "+amazonS3URL);
        hr.setEntity(new HttpEntity() {
            String result = "{\"statusDetails\":\"File successfully downloaded.\",\"status\":\"SUCCESS\"}";
            @Override
            public boolean isRepeatable() {
                return true;
            }

            @Override
            public boolean isChunked() {
                return false;
            }

            @Override
            public long getContentLength() {
                return result.length();
            }

            @Override
            public Header getContentType() {
                return null;
            }

            @Override
            public Header getContentEncoding() {
                return null;
            }

            @Override
            public InputStream getContent() throws IOException, UnsupportedOperationException {
                return new ByteArrayInputStream(result.getBytes("UTF-8"));
            }

            @Override
            public void writeTo(OutputStream outputStream) throws IOException {

            }

            @Override
            public boolean isStreaming() {
                return false;
            }

            @Override
            public void consumeContent() throws IOException {

            }
        });

        return result;
    }
}