Error in downloading csv from new Yahoo Finance historical data

Question

I've tried to make some code work for the new yahoo finances historical data now that the old API is officially dead. Code is as follows:

Private Function DownloadData(ByVal StockSymbol As String)

    Try

        'https://query1.finance.yahoo.com/v7/finance/download/CPG.TO?period1=1492824351&period2=1495416351&interval=1d&events=history&crumb=M3Ig5MvcK77

        Dim Period1 As Integer = (New DateTime(2007, 1, 1, 0, 0, 0) - New DateTime(1980, 1, 1, 0, 0, 0)).TotalSeconds
        Dim Period2 As Integer = (DateTime.UtcNow - New DateTime(1980, 1, 1, 0, 0, 0)).TotalSeconds

        Dim csvData As String = String.Empty

        Using Web As New WebClient

            Dim strm As Stream = Web.OpenRead("https://finance.yahoo.com/quote/" & StockSymbol & "/history?p=" & StockSymbol)

            Dim Crumb As String = ScrapeCrumb(strm)

            Dim s As String = "https://query1.finance.yahoo.com/v7/finance/download/" & StockSymbol &
                                "?period1=" & Period1.ToString & "&period2=" & Period2.ToString & "&interval=1d&events=history&crumb=" & Crumb
            csvData = Web.DownloadString(s)

        End Using

        Return csvData

    Catch ex As Exception

        Return String.Empty

    End Try

End Function
Private Function ScrapeCrumb(ByVal strm As Stream) As String

    Try

        Dim Output As String = String.Empty

        Using sr As New StreamReader(strm)
            Output = sr.ReadToEnd()
            ' Close and clean up the StreamReader
            sr.Close()
        End Using

        Dim Result As String = Output.Remove(0, Output.IndexOf("CrumbStore"))
        Result = Result.Remove(0, 22)
        Result = Result.Remove(Result.IndexOf("}"), Result.Length - Result.IndexOf("}"))
        Result = Result.Remove(Result.Length - 1, 1)

        Return Result

    Catch ex As Exception

        Return String.Empty

    End Try

End Function

But I keep getting the error

The remote server returned an error: (401) Unauthorized.

I feel like it has to do with the cookies/crumb deal but I am scraping it so I'm not sure why this is not giving permission to download the file. Does anybody have any ideas?

EDIT: Using vb language

Answer 1

You got error "(401) Unauthorized" because you do not have a valid cookie (cookie "B") value.

I managed to work out a .NET class to obtain valid token (cookie and crumb) from Yahoo Finance

For complete API library in fetching historical data from new Yahoo Finance, you may visit YahooFinanceAPI in Github

Here is the class to grab the cookie and crumb

Token.cs

using System;
using System.Diagnostics;
using System.Net;
using System.IO;
using System.Text.RegularExpressions;

namespace YahooFinanceAPI
{
    /// <summary>
    /// Class for fetching token (cookie and crumb) from Yahoo Finance
    /// Copyright Dennis Lee
    /// 19 May 2017
    /// 
    /// </summary>
    public class Token
    {
        public static string Cookie { get; set; }
        public static string Crumb { get; set; }

        private static Regex regex_crumb;
        /// <summary>
        /// Refresh cookie and crumb value Yahoo Fianance
        /// </summary>
        /// <param name="symbol">Stock ticker symbol</param>
        /// <returns></returns>
        public static bool Refresh(string symbol = "SPY")
        {

            try
            {
                Token.Cookie = "";
                Token.Crumb = "";

                string url_scrape = "https://finance.yahoo.com/quote/{0}?p={0}";
                //url_scrape = "https://finance.yahoo.com/quote/{0}/history"

                string url = string.Format(url_scrape, symbol);

                HttpWebRequest request = (HttpWebRequest)HttpWebRequest.Create(url);

                request.CookieContainer = new CookieContainer();
                request.Method = "GET";

                using (HttpWebResponse response = (HttpWebResponse)request.GetResponse())
                {

                    string cookie = response.GetResponseHeader("Set-Cookie").Split(';')[0];

                    string html = "";

                    using (Stream stream = response.GetResponseStream())
                    {
                        html = new StreamReader(stream).ReadToEnd();
                    }

                    if (html.Length < 5000)
                        return false;
                    string crumb = getCrumb(html);
                    html = "";

                    if (crumb != null)
                    {
                        Token.Cookie = cookie;
                        Token.Crumb = crumb;
                        Debug.Print("Crumb: '{0}', Cookie: '{1}'", crumb, cookie);
                        return true;
                    }

                }

            }
            catch (Exception ex)
            {
                Debug.Print(ex.Message);
            }

            return false;

        }

        /// <summary>
        /// Get crumb value from HTML
        /// </summary>
        /// <param name="html">HTML code</param>
        /// <returns></returns>
        private static string getCrumb(string html)
        {

            string crumb = null;

            try
            {
                //initialize on first time use
                if (regex_crumb == null)
                    regex_crumb = new Regex("CrumbStore\":{\"crumb\":\"(?<crumb>\\w+)\"}", 
                    RegexOptions.CultureInvariant | RegexOptions.Compiled, TimeSpan.FromSeconds(5));

                MatchCollection matches = regex_crumb.Matches(html);

                if (matches.Count > 0)
                {
                    crumb = matches[0].Groups["crumb"].Value;
                }
                else
                {
                    Debug.Print("Regex no match");
                }

                //prevent regex memory leak
                matches = null;

            }
            catch (Exception ex)
            {
                Debug.Print(ex.Message);
            }

            GC.Collect();
            return crumb;

        }

    }
}