OAuth 1.0 Signature Calculation for E*TRADE API

Question

The ETRADE Developer Platform uses the OAuth authorization protocol, version 1.0a. The eTrade developers guides (https://developer.etrade.com/getting-started/developer-guides) contains the following example:

Item	Value
Key	c5bb4dcb7bd6826c7c4340df3f791188
Secret	7d30246211192cda43ede3abd9b393b9
Access Token	VbiNYl63EejjlKdQM6FeENzcnrLACrZ2JYD6NQROfVI=
Access Secret	XCF9RzyQr4UEPloA+WlC06BnTfYC1P0Fwr3GUw/B0Es=
Timestamp	1344885636
Nonce	0bba225a40d1bbac2430aa0c6163ce44
HTTP Method	GET
URL	https://api.etrade.com/v1/accounts/list

The expected signature is: UOnPVdzExTAgHkcGWLLfeTaaMSM%3D

This is the signature string and key I have built:

field	value
base string	GET&https%3A%2F%2Fapi.etrade.com%2Fv1%2Faccounts%2Flist&oauth_consumer_key%3Dc5bb4dcb7bd6826c7c4340df3f791188%26oauth_nonce%3D0bba225a40d1bbac2430aa0c6163ce44%26oauth_signature_method%3DHMAC-SHA1%26oauth_timestamp%3D1344885636%26oauth_token%3DVbiNYl63EejjlKdQM6FeENzcnrLACrZ2JYD6NQROfVI%3D
Key	7d30246211192cda43ede3abd9b393b9&XCF9RzyQr4UEPloA+WlC06BnTfYC1P0Fwr3GUw/B0Es=

But my resulting Signature is: 8alOEOXdzxx+N7+77VRyducKWJM=, which is different than the expected value. What am I doing wrong?

To hash the base string I call (this is a DELPHI class for HMAC_SHA1 hash)

result := TNetEncoding.Base64.EncodeBytesToString(THashSHA1.GetHMACAsBytes(AData, AKey));

I believe that this hash is correct as it returns the correct signature for another OAuth signature example (as shown below) at ttps://oauth.net/core/1.0a/#sig_base_example where they provided the base string and key.

field	value
base string	GET&http%3A%2F%2Fphotos.example.net%2Fphotos&file%3Dvacation.jpg%26oauth_consumer_key%3Ddpf43f3p2l4k3l03%26oauth_nonce%3Dkllo9940pd9333jh%26oauth_signature_method%3DHMAC-SHA1%26oauth_timestamp%3D1191242096%26oauth_token%3Dnnch734d00sl2jdk%26oauth_version%3D1.0%26size%3Doriginal
key	kd94hf93k423kf44&pfkkdhi9sl3r4s00
signature	tR3+Ty81lMeYAr/Fid0kMTYa/WM=

I think I just missing some little detail in the base string and possibly the key.

// ETrade test case from https://developer.etrade.com/getting-started/developer-guides
// Following the Signture example
//    Item          Value
//    Key   c         5bb4dcb7bd6826c7c4340df3f791188
//    Secret          7d30246211192cda43ede3abd9b393b9
//    Access Token  VbiNYl63EejjlKdQM6FeENzcnrLACrZ2JYD6NQROfVI=
//    Access Secret XCF9RzyQr4UEPloA+WlC06BnTfYC1P0Fwr3GUw/B0Es=
//    Timestamp     1344885636
//    Nonce         0bba225a40d1bbac2430aa0c6163ce44
//    HTTP Method     GET
//    URL             https://api.etrade.com/v1/accounts/list
//    Resulting signature   UOnPVdzExTAgHkcGWLLfeTaaMSM%3D

URL := URIEncode('https://api.etrade.com/v1/accounts/list');
URL := 'GET&' + URL + '&';
Data.Add('oauth_consumer_key=c5bb4dcb7bd6826c7c4340df3f791188');
Data.Add('&oauth_nonce=0bba225a40d1bbac2430aa0c6163ce44');
Data.Add('&oauth_signature_method=HMAC-SHA1');
Data.Add('&oauth_timestamp=1344885636');
Data.Add('&oauth_token=VbiNYl63EejjlKdQM6FeENzcnrLACrZ2JYD6NQROfVI=');
Data.Add('&oauth_version=1.0');
AData := Data[0];
for i:= 1 to Data.Count - 1 do
  AData := AData + Data[i];
AData := URL + URIEncode(AData);
AKey := '7d30246211192cda43ede3abd9b393b9&XCF9RzyQr4UEPloA+WlC06BnTfYC1P0Fwr3GUw/B0Es=';
result := TNetEncoding.Base64.EncodeBytesToString(THashSHA1.GetHMACAsBytes(AData, AKey));
// the incorrect result is returned

// an example from https://oauth.net/core/1.0a/#sig_base_example
AString :=
  'GET&http%3A%2F%2Fphotos.example.net%2Fphotos&file%3Dvacation.jpg' +
  '%26oauth_consumer_key%3Ddpf43f3p2l4k3l03' +
  '%26oauth_nonce%3Dkllo9940pd9333jh' +
  '%26oauth_signature_method%3DHMAC-SHA1' +
  '%26oauth_timestamp%3D1191242096' +
  '%26oauth_token%3Dnnch734d00sl2jdk' +
  '%26oauth_version%3D1.0' +
  '%26size%3Doriginal';
result := EncodeBase64(THashSHA1.GetHMACAsBytes(AString,
                      'kd94hf93k423kf44&pfkkdhi9sl3r4s00'));
// returns the correct hash

Answer 1

With the help I received I was able to create a class which correctly signs the headers for authorization and to get quotes

unit MyOauth;

interface

uses
  SysUtils, Classes, Rest.Utils, DateUtils, System.Hash, System.NetEncoding,
  AuthSharedMem;

type
  TOathSignature = class(TObject)
  private
    FURL: string;
    FTimeStamp: string;
    FNonce: string;
    FToken: string;
    FCallback: string;
    FSignature: string;
    FConsumerKey: string;
    FConsumerSecret: string;
    FAccessSecret: string;
    FAction: string;
    FData: string;
    FKey: string;
    FRealm: string;
    FVerifier: string;
    FTokenSecret: string;

    function GetTimestamp: int64;
  public
    constructor Create(const AAction, AURL: string);

    function GetSignature(): string;
    function GetAuthorization: string;

    property Action: string read FAction write FAction;
    property AccessSecret: string read FAccessSecret write FAccessSecret;
    property Callback: string read FCallback write FCallback;
    property ConsumerKey: string read FConsumerKey write FConsumerKey;
    property ConsumerSecret: string read FConsumerSecret write FConsumerSecret;
    property URL: string read FURL write FURL;
    property Nonce: string read FNonce write FNonce;
    property Signature: string read FSignature write FSignature;
    property Timestamp: string read FTimeStamp write fTimestamp;
    property Token: string read FToken write FToken;
    property TokenSecret: string read FTokenSecret write FTokenSecret;
    property Data: string read FData;
    property Key: string read FKey;
    property Realm: string read FRealm write FRealm;
    property Verifier: string read FVerifier write FVerifier;
  end;


implementation

uses
  DebugMemory;

{ TOathSignature }

constructor TOathSignature.Create(const AAction, AURL: string);
begin
  Action := AAction;
  URL := AURL;
end;

function TOathSignature.GetSignature: string;
var
  URLEncoded: string;
  Data: TStringList;
  AData: string;
  AKey: string;
  i: integer;
  BaseURL: string;
  q: integer;
  Parameters: string;

begin
  if TimeStamp = '' then
    Timestamp := IntToStr(GetTimeStamp div 1000);
  if Nonce = '' then
    Nonce := THashMD5.GetHashString(TimeStamp + IntToStr(Random(MAXINT)));
  if ConsumerKey = '' then
    ConsumerKey := etrade_consumer_key;
  if ConsumerSecret = '' then
    ConsumerSecret := etrade_consumer_secret;

  q := URL.IndexOf('?');
  if q < 0 then
  begin
    baseURL := URL;
    parameters := '';
  end
  else
  begin
    baseURL := URL.Substring(0, q);
    parameters := URL.Substring(q+1);
  end;
  URLEncoded := Action + '&' + URIEncode(BaseURL) + '&';
  Data := TStringList.Create('~', '&');
  try
    Data.StrictDelimiter := true;
    if parameters <> '' then
      Data.DelimitedText := parameters;
    if callback <> '' then
      Data.Add('oauth_callback=' + callback);
    Data.Add('oauth_consumer_key=' + ConsumerKey);
    Data.Add('oauth_nonce=' + Nonce);
    Data.Add('oauth_signature_method=' + 'HMAC-SHA1');
    Data.Add('oauth_timestamp=' + Timestamp);
    if Token <> '' then
      Data.Add('oauth_token=' + URIEncode(Token));
    if Verifier <> '' then
      Data.Add('oauth_verifier=' + Verifier);
    Data.Sort;
    AData := Data[0];
    for i:= 1 to Data.Count - 1 do
      AData := AData + '&' + Data[i];

    FData := URLEncoded + URIEncode(AData);

    FKey := URIEncode(ConsumerSecret) + '&';
    if AccessSecret <> '' then
      FKey := FKey + URIEncode(AccessSecret)
    else if TokenSecret <> '' then
      FKey := FKey + URIEncode(TokenSecret);
    signature := TNetEncoding.Base64.EncodeBytesToString(THashSHA1.GetHMACAsBytes(FData, FKey));
    result := signature;
    OutputDebugString(FData);
  finally
    data.Free;
  end;
end;

function TOathSignature.GetTimestamp: int64;
var
  epochStart: TDateTime;
  UDate: TDateTime;

begin
  epochStart := UnixDateDelta; // StrToDate('1/1/1970');
  UDate := TTimeZone.Local.ToUniversalTime(Now);
  result := round((UDate - epochStart) * 86400000.0);

  //OutputDebugString('%s %s', [IntToStr(DateTimeToUnix(UDate)), IntToStr(result div 1000)]);
end;

function TOathSignature.GetAuthorization: string;
var
  SignatureEncoded: string;
begin
  if Signature = '' then
    GetSignature;

  SignatureEncoded := URIEncode(Signature);

  if Callback = 'oob' then
    result := format('realm="",oauth_callback="oob",oauth_signature="%s",oauth_nonce="%s",oauth_signature_method="HMAC-SHA1",oauth_consumer_key="%s",oauth_timestamp="%s"',
                     [SignatureEncoded, Nonce, URIEncode(Consumerkey), Timestamp])
  else if Verifier <> '' then
  begin
    result := format('realm="",' +
                     'oauth_signature="%s",' +
                     'oauth_nonce="%s",' +
                     'oauth_signature_method="HMAC-SHA1",' +
                     'oauth_consumer_key="%s",' +
                     'oauth_timestamp="%s",' +
                     'oauth_verifier="%s",' +
                     'oauth_token="%s"',
            [SignatureEncoded, Nonce, UriEncode(ConsumerKey), Timestamp, Verifier, URIEncode(Token)]);
  end
  else
  begin
    result := format('oauth_signature="%s",' +
                     'oauth_nonce="%s",' +
                     'oauth_signature_method="HMAC-SHA1",' +
                     'oauth_consumer_key="%s",' +
                     'oauth_timestamp="%s",' +
                     'oauth_token="%s"',
            [SignatureEncoded, Nonce, UriEncode(ConsumerKey), Timestamp, URIEncode(Token)]);
  end;
  outputdebugstring(result);
end;

end.

Answer 2

One thing obviously wrong in your code is the key. It should be the consumer secret and access secret concatenated with an "&". But each of them should be URL encoded first. The good news is that I can get a matching signature given by the etrade developer guides (https://developer.etrade.com/getting-started/developer-guides). The bad news is that this may be wrong according to the oauth spec (https://oauth.net/core/1.0a/).

Here is the key and message to be encoded:

String access_token ="VbiNYl63EejjlKdQM6FeENzcnrLACrZ2JYD6NQROfVI=";
String oauth_nonce="0bba225a40d1bbac2430aa0c6163ce44";
String consumer_key="c5bb4dcb7bd6826c7c4340df3f791188";
String consumer_secret="7d30246211192cda43ede3abd9b393b9";
String access_secret="XCF9RzyQr4UEPloA+WlC06BnTfYC1P0Fwr3GUw/B0Es=";
String key = URLEncoder.encode(consumer_secret, "UTF-8") + "&" + URLEncoder.encode(access_secret, "UTF-8");

String msg = 
    "GET&https%3A%2F%2Fapi.etrade.com%2Fv1%2Faccounts%2Flist" + "&" +
    "oauth_consumer_key" + "%3D" + consumer_key + "%26" +
    "oauth_nonce" + "%3D" + oauth_nonce + "%26" +
    "oauth_signature_method%3DHMAC-SHA1%26" +
    "oauth_timestamp%3D1344885636%26" + 
    "oauth_token" + "%3D" + URLEncoder.encode(URLEncoder.encode(access_token, "UTF-8"), "UTF-8");

As you can see that the token is URL encoded twice. The oauth_version is missing from the parameter list. According to the oauth v1 spec, it should be there. But this seems to be the only way to get a matching signature.

For everyone's convenience, here is a complete listing of a Java program to show the calculations.

import java.net.URLEncoder;
import java.security.InvalidKeyException;
import java.security.NoSuchAlgorithmException;
import java.security.SignatureException;
import java.util.Base64;
import javax.crypto.Mac;
import javax.crypto.spec.SecretKeySpec;

public class Oauth1Signature {
    private static final String HMAC_SHA1_ALGORITHM = "HmacSHA1";
    //private static final String ENC = "UTF-8";

    private static String access_token ="VbiNYl63EejjlKdQM6FeENzcnrLACrZ2JYD6NQROfVI=";
    private static String access_secret="XCF9RzyQr4UEPloA+WlC06BnTfYC1P0Fwr3GUw/B0Es=";
    private static String oauth_nonce="0bba225a40d1bbac2430aa0c6163ce44";
    
    private static String consumer_key="c5bb4dcb7bd6826c7c4340df3f791188";
    private static String consumer_secret="7d30246211192cda43ede3abd9b393b9";

    public static String hmacSha1Base64(String data, String key)
            throws SignatureException, NoSuchAlgorithmException, InvalidKeyException    {
        SecretKeySpec signingKey = new SecretKeySpec(key.getBytes(), HMAC_SHA1_ALGORITHM);
        Mac mac = Mac.getInstance(HMAC_SHA1_ALGORITHM);
        mac.init(signingKey);
        return Base64.getEncoder().encodeToString((mac.doFinal(data.getBytes())));
    }

    
    public static void main(String[] args) throws Exception {
        String hmac;
        
        // example from https://developer.etrade.com/getting-started/developer-guides
        // Expected signature: UOnPVdzExTAgHkcGWLLfeTaaMSM=
        // Note: can get this signature only if oauth_version is not included in the msg to be hashed and 
        //       token encoded twice, contradicting oauth 1.0 spec
        String key = URLEncoder.encode(consumer_secret, "UTF-8") + "&" + URLEncoder.encode(access_secret, "UTF-8");

        String msg = 
            "GET&https%3A%2F%2Fapi.etrade.com%2Fv1%2Faccounts%2Flist" + "&" +
            "oauth_consumer_key" + "%3D" + consumer_key + "%26" +
            "oauth_nonce" + "%3D" + oauth_nonce + "%26" +
            "oauth_signature_method%3DHMAC-SHA1%26" +
            "oauth_timestamp%3D1344885636%26" + 
            "oauth_token" + "%3D" + URLEncoder.encode(URLEncoder.encode(access_token, "UTF-8"), "UTF-8");
            // + "%26" + "oauth_version" + "%3D" + "1.0";

        hmac = hmacSha1Base64(msg, key);
        System.out.println("msg=" + msg);
        System.out.println("key=" + key);
        System.out.println(hmac);
        //output: UOnPVdzExTAgHkcGWLLfeTaaMSM=

        //example from https://oauth.net/core/1.0a/#anchor46
        // working as expected
        String key2 = "kd94hf93k423kf44&pfkkdhi9sl3r4s00";
        String msg2 = 
            "GET&http%3A%2F%2Fphotos.example.net%2Fphotos" + "&" + 
            "file%3Dvacation.jpg" + "%26" +
            "oauth_consumer_key%3Ddpf43f3p2l4k3l03" + "%26" + 
            "oauth_nonce%3Dkllo9940pd9333jh" + "%26" + 
            "oauth_signature_method%3DHMAC-SHA1" + "%26" +
            "oauth_timestamp%3D1191242096" + "%26" +
            "oauth_token%3Dnnch734d00sl2jdk" + "%26" +
            "oauth_version%3D1.0" + "%26" +
            "size%3Doriginal";
        
        hmac = hmacSha1Base64(msg2, key2);
        System.out.println("msg2=" + msg2);
        System.out.println("key2=" + key2);
        System.out.println(hmac);
        // match expected result: tR3+Ty81lMeYAr/Fid0kMTYa/WM=
    }
}