Download multiple files simultaneously from FTP using FluentFTP

Question

I'm using FluentFTP for connecting, downloading, etc. from FTP
https://github.com/robinrodricks/FluentFTP/wiki

I would like to download files simultaneously from List. There is no problem downloading them one by one.

This is how my code looks like:

Downloading function:

public async Task<bool> DownloadFileAsync(string RemoteUrl, string AppName, Progress<FtpProgress> progress = null)
    {
        return await Task.Run(async() =>
        {

            using (FileStream read = new FileStream("settings.xml", FileMode.Open, FileAccess.ReadWrite, FileShare.ReadWrite))
            {
                if (ftpClient.IsConnected)
                {
                    if (File.Exists("settings.xml"))
                    {
                        Information info = (Information)xs.Deserialize(read);
                    
                        if (Directory.Exists(info.Downloads))
                        {
                            bool DownloadFinished = await ftpClient.DownloadFileAsync(info.Downloads + "\\" + AppName, RemoteUrl, FtpLocalExists.Overwrite, FtpVerify.Retry, progress);
                            if (DownloadFinished == true)
                            {
                                loger.LogWrite("File " + RemoteUrl + " downloaded succesfully.");
                                //read.Dispose();
                                return true;
                            }
                            else
                            {
                                loger.LogWrite("File" + RemoteUrl + " download failed.");
                                //read.Dispose();
                                return false;
                            }
                        }
                        else
                        {
                            loger.LogWrite("Could not locate folder " + info.Downloads + " downloading terminated.");
                            return false;
                        }
                    }
                    else
                    {
                        MessageBox.Show("settings.xml file is missing.");
                        loger.LogWrite("settings.xml file is missing.");
                        read.Dispose();
                        return false;
                    }
                }
                else
                {
                    loger.LogWrite("FTP Client is not connected could not download: " + RemoteUrl);
                    read.Dispose();
                    return false;
                }
            }
        });

    }

How I fill the list:

Arta_Variables.ArtaSoftware.Add(new Software() { RemoteUrl = "Ultra_Script/Basic_SW/Adobe_Reader.exe", SoftwareName = "Adobe_Reader.exe", FileExistsOnRemoteUrl = null, Downloaded = null });

This is how downloading them one by one:

if(Arta_Variables.DAAOChecked == false)
{
    if (CheckFinished == true)
    {
        using (FileStream read = new FileStream("settings.xml", FileMode.Open, FileAccess.ReadWrite, FileShare.ReadWrite))
        {

            XmlSerializer xs = new XmlSerializer(typeof(Information));
            Information info = (Information)xs.Deserialize(read);

            AddBlackLine("");
            AddBlackLine("Downloading all available files.");
            AddBlackLine("");

            foreach (Software software1 in ArtaChosenSW)
            {

                string item = software1.SoftwareName;
                int index = ArtaChosenSW.FindIndex(p => p.SoftwareName == item);

                if (software1.FileExistsOnRemoteUrl == true)
                {
                    AddBlackLine("Downloading " + software1.SoftwareName);

                    Dispatcher.Invoke(() =>
                    {
                        DWGProgressLab.Visibility = Visibility.Visible;
                        DP_ProgressPercentage.Visibility = Visibility.Visible;
                    });

                    Progress<FtpProgress> prog = new Progress<FtpProgress>(x =>
                    {
                        int ConvertedInt = (int)x.Progress;
                        DP_ProgressPercentage.Dispatcher.BeginInvoke((Action)(() => DP_ProgressPercentage.Content = ConvertedInt + "%"));
                    });

                    bool DWFinished = await ftp.DownloadFileAsync(software1.RemoteUrl, software1.SoftwareName, prog);

                    if (DWFinished == true)
                    {
                        AddGreenLine("Download of " + software1.SoftwareName + " succesfull.");
                        ArtaChosenSW[index].Downloaded = true;
                        ArtaChosenSW[index].LocalUrl = info.Downloads;

                        Dispatcher.Invoke(() =>
                        {
                            DWGProgressLab.Visibility = Visibility.Hidden;
                            DP_ProgressPercentage.Visibility = Visibility.Hidden;
                        });

                    }
                    else
                    {
                        AddRedLine("Download of " + software1.SoftwareName + " failed");
                        ArtaChosenSW[index].FileExistsOnRemoteUrl = false;
                    }
                }
                else
                {
                    ArtaChosenSW[index].FileExistsOnRemoteUrl = true;
                    AddBlackLine("File " + software1.SoftwareName + " did not found on ftp. Could not download.");
                    loger.LogWrite("File " + software1.SoftwareName + " did not found on ftp. Could not download.");
                }
            }
        }
    }
}

My try for simultaneous download:

foreach(Software software in ArtaChosenSW)
{
    var tasks = ArtaChosenSW.Select(c => Task.Factory.StartNew(() => ftp.DownloadFileAsync(c.RemoteUrl, c.SoftwareName))).ToArray();
    Task.WaitAll(tasks);
}

Sadly what it does it's creating blank files in local url with 0kb but no downloading is happening.

I'm not much experienced in async programming so I'll be glad for all answers or some better approaches.

Answer 1

It seems that you are using one FtpClient instance for all your transfers.

The FtpClient represents one connection to an FTP server. FTP protocol does not allow multiple parallel transfers over one connection. You have to open a new connection for every parallel transfer.

For an example of an implementation, see Download multiple files concurrently from FTP using FluentFTP with a maximum value.

Answer 2

Based on @Martin Prikryl answer i created new function for concurrent downloads:

Works flawlesly:

public async Task DownloadMultipleFilesConcurrentFromFTP(int NumberOfConnections, string AppName, string RemoteUrl)
{
    await Task.Run(async() =>
    {
        NetworkCredential networkCredential = new NetworkCredential(FTPUsername, FTPPassword);
        List<FtpClient> ftpClients = new List<FtpClient>();

        for (int i = 0; i < NumberOfConnections; i++)
        {
            ftpClients.Add(new FtpClient(FTPHost, networkCredential));
            foreach(FtpClient ftp in ftpClients)
            {
                ftp.ConnectTimeout = ConnectTimeout;
                ftp.SocketPollInterval = SocketPollInterval;
                ftp.ReadTimeout = ReadTimeout;
                ftp.DataConnectionConnectTimeout = DataConnectionConnectTimeout;
                ftp.DataConnectionReadTimeout = DataConnectionReadTimeout;
                ftp.Connect();
                if (ftp.IsConnected)
                {
                    using (FileStream read = new FileStream("settings.xml", FileMode.Open, FileAccess.ReadWrite, FileShare.ReadWrite))
                    {
                        if(ftp.IsConnected == true)
                        {
                            if (File.Exists("settings.xml"))
                            {
                                Information info = (Information)xs.Deserialize(read);
                                if (Directory.Exists(info.Downloads))
                                {
                                    await ftp.DownloadFileAsync(info.Downloads + "\\" + AppName, RemoteUrl, FtpLocalExists.Overwrite, FtpVerify.Retry);
                                }
                            }
                        }
                    }
                }
            }
        }
    });
}

Now i just need figure out how return download progress of all files.

Answer 3

Taking your code from the answer you posted, this should handle multiple downloads without any thread-pool or Task wrapper overhead.

public async Task DownloadMultipleFilesConcurrentFromFTP(int NumberOfConnections, string AppName, string RemoteUrl)
{
    var downloadTasks = new List<Task>();
    for (int i = 0; i < NumberOfConnections; i++){
        downloadTasks.Add(DownloadFile(AppName, RemoteUrl));
    }
    await Task.WhenAll(downloadTasks);
}
 
public async Task DownloadFile(string AppName, string RemoteUrl)
{
    var ftp = new FtpClient(FTPHost, networkCredential);
    ftp.ConnectTimeout = ConnectTimeout;
    ftp.SocketPollInterval = SocketPollInterval;
    ftp.ReadTimeout = ReadTimeout;
    ftp.DataConnectionConnectTimeout = DataConnectionConnectTimeout;
    ftp.DataConnectionReadTimeout = DataConnectionReadTimeout;
    await ftp.ConnectAsync();
    if (ftp.IsConnected)
    {
        using (FileStream read = new FileStream("settings.xml", FileMode.Open, FileAccess.ReadWrite, FileShare.ReadWrite))
        {
            if (ftp.IsConnected == true)
            {
                if (File.Exists("settings.xml"))
                {
                    Information info = (Information)xs.Deserialize(read);
                    if (Directory.Exists(info.Downloads))
                    {
                        await ftp.DownloadFileAsync(info.Downloads + "\\" + AppName, RemoteUrl, FtpLocalExists.Overwrite, FtpVerify.Retry);
                    }
                }
            }
        }
    }
}

It creates a single Task for each download, and then calls Task.WhenAll on the list of tasks to wait for all files to complete before returning.

I haven't modified any of your file handling code, but you should consider using async versions of your calls there also, as accessing the file system using blocking calls can cause responsiveness issues.